@mndrk/agx 2.0.32 → 2.0.34

package/cloud-runtime/standalone/lib/orchestrator/chat-processor.ts

@@ -9,6 +9,7 @@ import {
   loadChatRunActivity,
   updateChatRunActivity,
 } from "@/lib/orchestrator/chat-activities";
+import { writeDebugLog } from "@/lib/debug-log";
 
 function isChatRunPayload(value: unknown): value is ChatRunPayload {
   if (!value || typeof value !== "object") return false;
@@ -21,15 +22,31 @@ function isChatRunPayload(value: unknown): value is ChatRunPayload {
 }
 
 async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
+  writeDebugLog("chat-processor.start.received", {
+    jobId: job.id,
+    chatRunId: job.data.chatRunId,
+    signal: job.data.signal,
+  });
   const chatRun = await loadChatRunActivity(job.data.chatRunId);
   if (!chatRun) {
+    writeDebugLog("chat-processor.start.missing", {
+      chatRunId: job.data.chatRunId,
+      jobId: job.id,
+    });
     console.warn(`[chat-processor] Chat run ${job.data.chatRunId} not found, skipping`);
     return;
   }
   if (chatRun.status === "completed" || chatRun.status === "failed" || chatRun.status === "cancelled") {
+    writeDebugLog("chat-processor.start.skip_terminal", {
+      chatRunId: chatRun.id,
+      status: chatRun.status,
+    });
     return;
   }
   if (!isChatRunPayload(chatRun.payload)) {
+    writeDebugLog("chat-processor.start.invalid_payload", {
+      chatRunId: chatRun.id,
+    });
     await updateChatRunActivity({
       id: chatRun.id,
       status: "failed",
@@ -59,6 +76,12 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
   });
 
   try {
+    writeDebugLog("chat-processor.start.running", {
+      chatRunId: chatRun.id,
+      threadId: chatRun.threadId,
+      rootMessageId: chatRun.rootMessageId,
+      participantIds: chatRun.payload.participantIds,
+    });
     const library = await loadDbParticipants();
     const participantIdSet = new Set(chatRun.payload.participantIds);
     const participants = chatRun.payload.participantIds
@@ -66,6 +89,10 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
       .filter((participant): participant is NonNullable<typeof participant> => Boolean(participant));
 
     if (participants.length === 0) {
+      writeDebugLog("chat-processor.start.no_participants_resolved", {
+        chatRunId: chatRun.id,
+        participantIds: chatRun.payload.participantIds,
+      });
       throw new Error("No participants resolved for chat run");
     }
 
@@ -90,10 +117,17 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
     });
 
     const reader = stream.getReader();
+    writeDebugLog("chat-processor.stream.open", {
+      chatRunId: chatRun.id,
+      participantCount: participants.length,
+    });
     while (true) {
       const { done } = await reader.read();
       if (done) break;
     }
+    writeDebugLog("chat-processor.stream.complete", {
+      chatRunId: chatRun.id,
+    });
 
     await completeChatRunStepActivity({
       stepId: step.id,
@@ -112,8 +146,17 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
         rootMessageId: chatRun.payload.rootMessageId,
       },
     });
+    writeDebugLog("chat-processor.complete", {
+      chatRunId: chatRun.id,
+      rootMessageId: chatRun.payload.rootMessageId,
+    });
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
+    writeDebugLog("chat-processor.error", {
+      chatRunId: chatRun.id,
+      error,
+      message,
+    });
     await completeChatRunStepActivity({
       stepId: step.id,
       status: "failed",
@@ -130,6 +173,10 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
 }
 
 async function handleCancel(job: Job<ChatRunJobData>): Promise<void> {
+  writeDebugLog("chat-processor.cancel.received", {
+    jobId: job.id,
+    chatRunId: job.data.chatRunId,
+  });
   const chatRun = await loadChatRunActivity(job.data.chatRunId);
   if (!chatRun) return;
 
@@ -146,6 +193,10 @@ async function handleCancel(job: Job<ChatRunJobData>): Promise<void> {
     completedAt: Date.now(),
     result: { cancelled: true },
   });
+  writeDebugLog("chat-processor.cancel.complete", {
+    chatRunId: chatRun.id,
+    threadId: chatRun.threadId,
+  });
 }
 
 async function processSingleJob(job: Job<ChatRunJobData>): Promise<void> {

package/cloud-runtime/standalone/lib/orchestrator/runtime.ts

@@ -1,6 +1,7 @@
 import { getQueue, QUEUE_NAMES } from "@/lib/queue/boss";
 import { taskProcessor } from "@/lib/orchestrator/processor";
 import { chatProcessor } from "@/lib/orchestrator/chat-processor";
+import { writeDebugLog } from "@/lib/debug-log";
 
 let bootstrapPromise: Promise<void> | null = null;
 
@@ -8,10 +9,15 @@ export async function ensureOrchestratorRuntime(): Promise<void> {
   if (bootstrapPromise) return bootstrapPromise;
 
   bootstrapPromise = (async () => {
+    writeDebugLog("orchestrator.bootstrap.start");
     const queue = await getQueue();
     await queue.work(QUEUE_NAMES.TASK_PROCESS, taskProcessor, { batchSize: 5 });
     await queue.work(QUEUE_NAMES.CHAT_RUN_PROCESS, chatProcessor, { batchSize: 2 });
+    writeDebugLog("orchestrator.bootstrap.ready", {
+      queues: [QUEUE_NAMES.TASK_PROCESS, QUEUE_NAMES.CHAT_RUN_PROCESS],
+    });
   })().catch((error) => {
+    writeDebugLog("orchestrator.bootstrap.error", { error });
     bootstrapPromise = null;
     throw error;
   });

package/cloud-runtime/standalone/lib/queue/sqlite-adapter.ts

@@ -6,8 +6,15 @@ import os from "os";
 import { QueueAdapter, QueueOptions, WorkerOptions, Job } from "./adapter";
 import { validateSQLiteEnvironment, REQUIRED_PRAGMAS } from "../startup";
 import { WriteRateMonitor, WRITE_RATE_SAMPLE_WINDOW_MS } from "../limits";
+import { writeDebugLog } from "../debug-log";
 
 const AGX_DATA_DIR = process.env.AGX_DATA_DIR || path.join(os.homedir(), ".agx");
+const DEFAULT_STALE_JOB_TIMEOUT_MS = 10 * 60 * 1000;
+
+function getStaleJobTimeoutMs(): number {
+  const raw = Number(process.env.AGX_QUEUE_STALE_MS);
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_STALE_JOB_TIMEOUT_MS;
+}
 
 interface SQLiteJobRow {
   id: string;
@@ -120,6 +127,14 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     );
 
     this.writeMonitor.record();
+    writeDebugLog("queue.send", {
+      queue,
+      jobId: id,
+      startAfter,
+      retryLimit: options?.retryLimit ?? 3,
+      priority: options?.priority ?? 0,
+      data,
+    });
     return id;
   }
 
@@ -128,6 +143,12 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     handler: (jobs: Job<T>[]) => Promise<void>,
     options?: WorkerOptions
   ): Promise<void> {
+    writeDebugLog("queue.worker.register", {
+      queue,
+      batchSize: options?.batchSize ?? 1,
+      pollInterval: options?.pollInterval ?? null,
+      workerId: this.workerId,
+    });
     this.workers.set(queue, {
       handler,
       options: options || {},
@@ -163,9 +184,20 @@ export class SQLiteQueueAdapter implements QueueAdapter {
   private processQueue(queue: string, worker: { handler: Function; options: WorkerOptions }) {
     const batchSize = worker.options.batchSize || 1;
     const now = Date.now();
+    const staleBefore = now - getStaleJobTimeoutMs();
 
     // Transactional claim
     const jobs = this.db.transaction(() => {
+      const reapedJobs = this.reapStaleJobs(queue, now, staleBefore);
+      if (reapedJobs.length > 0) {
+        writeDebugLog("queue.reap_stale", {
+          queue,
+          workerId: this.workerId,
+          staleBefore,
+          jobs: reapedJobs,
+        });
+      }
+
       // 1. Find candidates
       const candidates = this.db.prepare(`
         SELECT id, data FROM agx_jobs
@@ -198,6 +230,11 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     })();
 
     if (jobs.length > 0) {
+      writeDebugLog("queue.claim", {
+        queue,
+        workerId: this.workerId,
+        jobIds: jobs.map((job) => job.id),
+      });
       // Execute handler
       // Note: We don't await here to avoid blocking the poll loop, 
       // but in single-threaded Node, we are effectively blocking if the handler is sync-heavy.
@@ -214,9 +251,16 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       await handler(jobs);
       // Success
       this.completeJobs(jobs.map(j => j.id));
+      writeDebugLog("queue.complete", {
+        jobIds: jobs.map((job) => job.id),
+      });
     } catch (err: any) {
       // Failure (batch failure — all jobs in the batch fail together)
       this.failJobs(jobs.map(j => j.id), err.message || String(err));
+      writeDebugLog("queue.fail", {
+        jobIds: jobs.map((job) => job.id),
+        error: err,
+      });
     }
   }
 
@@ -224,7 +268,7 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     const now = Date.now();
     const stmt = this.db.prepare(`
       UPDATE agx_jobs
-      SET status = 'completed', completed_at = ?
+      SET status = 'completed', completed_at = ?, worker_id = NULL
       WHERE id = ?
     `);
     this.db.transaction(() => {
@@ -243,13 +287,15 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       UPDATE agx_jobs
       SET status = 'retry', retry_count = retry_count + 1, 
           start_after = ? + (retry_count * 1000 * 2), -- Exponential backoff (kinda)
+          started_at = NULL,
+          worker_id = NULL,
           error = ?
       WHERE id = ?
     `);
 
     const updateFailed = this.db.prepare(`
       UPDATE agx_jobs
-      SET status = 'failed', failed_at = ?, error = ?
+      SET status = 'failed', failed_at = ?, started_at = NULL, worker_id = NULL, error = ?
       WHERE id = ?
     `);
 
@@ -268,4 +314,73 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       }
     })();
   }
+
+  private reapStaleJobs(queue: string, now: number, staleBefore: number): Array<{
+    id: string;
+    fromStatus: "active";
+    toStatus: "retry" | "failed";
+    retryCount: number;
+  }> {
+    const staleJobs = this.db.prepare(`
+      SELECT id, retry_count, retry_limit
+      FROM agx_jobs
+      WHERE queue = ?
+        AND status = 'active'
+        AND started_at IS NOT NULL
+        AND started_at <= ?
+      ORDER BY started_at ASC
+    `).all(queue, staleBefore) as Array<Pick<SQLiteJobRow, "id" | "retry_count" | "retry_limit">>;
+
+    if (staleJobs.length === 0) return [];
+
+    const updateRetry = this.db.prepare(`
+      UPDATE agx_jobs
+      SET status = 'retry',
+          retry_count = retry_count + 1,
+          start_after = ?,
+          started_at = NULL,
+          worker_id = NULL,
+          error = ?
+      WHERE id = ?
+    `);
+    const updateFailed = this.db.prepare(`
+      UPDATE agx_jobs
+      SET status = 'failed',
+          failed_at = ?,
+          started_at = NULL,
+          worker_id = NULL,
+          error = ?
+      WHERE id = ?
+    `);
+
+    const reaped: Array<{
+      id: string;
+      fromStatus: "active";
+      toStatus: "retry" | "failed";
+      retryCount: number;
+    }> = [];
+
+    for (const job of staleJobs) {
+      const error = `Job exceeded active timeout of ${getStaleJobTimeoutMs()}ms`;
+      if (job.retry_count < job.retry_limit) {
+        updateRetry.run(now, error, job.id);
+        reaped.push({
+          id: job.id,
+          fromStatus: "active",
+          toStatus: "retry",
+          retryCount: job.retry_count + 1,
+        });
+      } else {
+        updateFailed.run(now, error, job.id);
+        reaped.push({
+          id: job.id,
+          fromStatus: "active",
+          toStatus: "failed",
+          retryCount: job.retry_count,
+        });
+      }
+    }
+
+    return reaped;
+  }
 }