@mndrk/agx 2.0.32 → 2.0.33

package/cloud-runtime/standalone/lib/history-store.ts

@@ -22,6 +22,8 @@ const REACTION_TYPES = new Set<ReactionType>([
   "clarify",
   "blocked",
 ]);
+const ACTIVE_AGENT_PROCESS_STALE_MS = 10 * 60 * 1000;
+const ACTIVE_CHAT_RUN_STALE_MS = 10 * 60 * 1000;
 const ALLOWED_TRANSITIONS: Record<ReactionType, Set<ReactionType>> = {
   ack: new Set<ReactionType>(["ack", "working"]),
   working: new Set<ReactionType>(["ack", "working", "done", "clarify", "blocked"]),
@@ -223,6 +225,16 @@ function hasSqliteObject(db: Database.Database, type: "table" | "trigger", name:
   return Boolean(row);
 }
 
+function isPidAlive(pid: number | null | undefined): boolean {
+  if (!Number.isFinite(pid) || !pid || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function initializeMessageSearch(db: Database.Database): void {
   const hasFtsTable = hasSqliteObject(db, "table", "messages_fts");
   db.exec(`
@@ -804,7 +816,7 @@ export async function getProjectThreadIds(projectId: string): Promise<string[]>
   const { getSQLiteDb } = await import("@/lib/sqlite-query-adapter");
   const db = getSQLiteDb();
   const rows = db
-    .prepare("SELECT thread_id FROM project_threads WHERE project_id = ?")
+    .prepare("SELECT thread_id FROM project_threads WHERE project_id = ? ORDER BY created_at ASC")
     .all(projectId) as { thread_id: string }[];
   return rows.map((r) => r.thread_id);
 }
@@ -979,7 +991,7 @@ export async function getThreadStatusSnapshot(
     // These may not have an assistant message yet, so the primary query misses them.
     const activeProcessRows = db
       .prepare(
-        `SELECT ap.agent_id, ap.pid, ap.state, ap.since_message_id, ap.started_at, ap.last_activity,
+        `SELECT ap.id, ap.agent_id, ap.pid, ap.state, ap.since_message_id, ap.started_at, ap.last_activity,
                 m.content as response_to,
                 m.role as response_to_role,
                 m.participant_id as response_to_participant_id
@@ -990,6 +1002,7 @@ export async function getThreadStatusSnapshot(
            AND ap.state IN ('running', 'spawning')`
       )
       .all(scopedRootMessageId) as Array<{
+      id: number;
       agent_id: string;
       pid: number;
       state: string;
@@ -1001,10 +1014,36 @@ export async function getThreadStatusSnapshot(
       response_to_participant_id: string | null;
     }>;
 
+    const now = Date.now();
+    const staleActiveProcessIds = activeProcessRows
+      .filter((ap) => {
+        const lastSeenAt = Math.max(ap.last_activity || 0, ap.started_at || 0);
+        if (lastSeenAt > now - ACTIVE_AGENT_PROCESS_STALE_MS) return false;
+        return !isPidAlive(ap.pid);
+      })
+      .map((ap) => ap.id);
+    const staleActiveProcessIdSet = new Set(staleActiveProcessIds);
+
+    if (staleActiveProcessIds.length > 0) {
+      const markStale = db.prepare(
+        `UPDATE agent_processes
+         SET state = 'error',
+             last_activity = ?
+         WHERE id = ?`
+      );
+      const markManyStale = db.transaction((ids: number[]) => {
+        for (const id of ids) {
+          markStale.run(now, id);
+        }
+      });
+      markManyStale(staleActiveProcessIds);
+    }
+
     // Merge: only add active processes not already represented
     const existingPids = new Set(processes.filter(p => p.processId != null).map(p => p.processId));
     const existingAgents = new Set(processes.filter(p => p.status === "running").map(p => p.agent));
     for (const ap of activeProcessRows) {
+      if (staleActiveProcessIdSet.has(ap.id)) continue;
       if (existingPids.has(ap.pid) || existingAgents.has(ap.agent_id)) continue;
       processes.push({
         processId: ap.pid || null,
@@ -1146,7 +1185,43 @@ export async function createChatRun(input: {
   });
 }
 
+export async function sweepStaleChatRuns(input?: {
+  threadId?: string | null;
+  olderThanMs?: number;
+}): Promise<number> {
+  const normalizedThreadId = normalizeOptionalText(input?.threadId ?? null);
+  const thresholdMs = Number.isFinite(input?.olderThanMs)
+    ? Math.max(input!.olderThanMs!, 1)
+    : ACTIVE_CHAT_RUN_STALE_MS;
+  const now = Date.now();
+  const cutoff = now - thresholdMs;
+
+  return withDatabase((db) => {
+    const clauses = ["status = 'running'", "updated_at <= ?"];
+    const params: Array<string | number> = [cutoff];
+
+    if (normalizedThreadId) {
+      clauses.push("thread_id = ?");
+      params.push(normalizedThreadId);
+    }
+
+    const result = db
+      .prepare(
+        `UPDATE chat_runs
+         SET status = 'failed',
+             last_error = COALESCE(NULLIF(last_error, ''), 'Worker process died mid-execution'),
+             updated_at = ?,
+             completed_at = COALESCE(completed_at, ?)
+         WHERE ${clauses.join(" AND ")}`
+      )
+      .run(now, now, ...params);
+
+    return result.changes;
+  });
+}
+
 export async function getChatRun(chatRunId: string): Promise<ChatRunRecord | null> {
+  await sweepStaleChatRuns();
   return withDatabase((db) => {
     const row = db.prepare("SELECT * FROM chat_runs WHERE id = ? LIMIT 1").get(chatRunId) as ChatRunRow | undefined;
     return row ? toChatRunRecord(row) : null;
@@ -1158,6 +1233,7 @@ export async function listChatRuns(input: {
   status?: ChatRunStatus | "active";
   limit?: number;
 }): Promise<ChatRunRecord[]> {
+  await sweepStaleChatRuns({ threadId: input.threadId });
   const normalizedThreadId = normalizeOptionalText(input.threadId ?? null);
   const limit = clampPositiveInt(input.limit, 20, 100);
   return withDatabase((db) => {

package/cloud-runtime/standalone/lib/orchestrator/chat-processor.ts

@@ -9,6 +9,7 @@ import {
   loadChatRunActivity,
   updateChatRunActivity,
 } from "@/lib/orchestrator/chat-activities";
+import { writeDebugLog } from "@/lib/debug-log";
 
 function isChatRunPayload(value: unknown): value is ChatRunPayload {
   if (!value || typeof value !== "object") return false;
@@ -21,15 +22,31 @@ function isChatRunPayload(value: unknown): value is ChatRunPayload {
 }
 
 async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
+  writeDebugLog("chat-processor.start.received", {
+    jobId: job.id,
+    chatRunId: job.data.chatRunId,
+    signal: job.data.signal,
+  });
   const chatRun = await loadChatRunActivity(job.data.chatRunId);
   if (!chatRun) {
+    writeDebugLog("chat-processor.start.missing", {
+      chatRunId: job.data.chatRunId,
+      jobId: job.id,
+    });
     console.warn(`[chat-processor] Chat run ${job.data.chatRunId} not found, skipping`);
     return;
   }
   if (chatRun.status === "completed" || chatRun.status === "failed" || chatRun.status === "cancelled") {
+    writeDebugLog("chat-processor.start.skip_terminal", {
+      chatRunId: chatRun.id,
+      status: chatRun.status,
+    });
     return;
   }
   if (!isChatRunPayload(chatRun.payload)) {
+    writeDebugLog("chat-processor.start.invalid_payload", {
+      chatRunId: chatRun.id,
+    });
     await updateChatRunActivity({
       id: chatRun.id,
       status: "failed",
@@ -59,6 +76,12 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
   });
 
   try {
+    writeDebugLog("chat-processor.start.running", {
+      chatRunId: chatRun.id,
+      threadId: chatRun.threadId,
+      rootMessageId: chatRun.rootMessageId,
+      participantIds: chatRun.payload.participantIds,
+    });
     const library = await loadDbParticipants();
     const participantIdSet = new Set(chatRun.payload.participantIds);
     const participants = chatRun.payload.participantIds
@@ -66,6 +89,10 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
       .filter((participant): participant is NonNullable<typeof participant> => Boolean(participant));
 
     if (participants.length === 0) {
+      writeDebugLog("chat-processor.start.no_participants_resolved", {
+        chatRunId: chatRun.id,
+        participantIds: chatRun.payload.participantIds,
+      });
       throw new Error("No participants resolved for chat run");
     }
 
@@ -90,10 +117,17 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
     });
 
     const reader = stream.getReader();
+    writeDebugLog("chat-processor.stream.open", {
+      chatRunId: chatRun.id,
+      participantCount: participants.length,
+    });
     while (true) {
       const { done } = await reader.read();
       if (done) break;
     }
+    writeDebugLog("chat-processor.stream.complete", {
+      chatRunId: chatRun.id,
+    });
 
     await completeChatRunStepActivity({
       stepId: step.id,
@@ -112,8 +146,17 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
         rootMessageId: chatRun.payload.rootMessageId,
       },
     });
+    writeDebugLog("chat-processor.complete", {
+      chatRunId: chatRun.id,
+      rootMessageId: chatRun.payload.rootMessageId,
+    });
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
+    writeDebugLog("chat-processor.error", {
+      chatRunId: chatRun.id,
+      error,
+      message,
+    });
     await completeChatRunStepActivity({
       stepId: step.id,
       status: "failed",
@@ -130,6 +173,10 @@ async function handleStart(job: Job<ChatRunJobData>): Promise<void> {
 }
 
 async function handleCancel(job: Job<ChatRunJobData>): Promise<void> {
+  writeDebugLog("chat-processor.cancel.received", {
+    jobId: job.id,
+    chatRunId: job.data.chatRunId,
+  });
   const chatRun = await loadChatRunActivity(job.data.chatRunId);
   if (!chatRun) return;
 
@@ -146,6 +193,10 @@ async function handleCancel(job: Job<ChatRunJobData>): Promise<void> {
     completedAt: Date.now(),
     result: { cancelled: true },
   });
+  writeDebugLog("chat-processor.cancel.complete", {
+    chatRunId: chatRun.id,
+    threadId: chatRun.threadId,
+  });
 }
 
 async function processSingleJob(job: Job<ChatRunJobData>): Promise<void> {

package/cloud-runtime/standalone/lib/orchestrator/runtime.ts

@@ -1,6 +1,7 @@
 import { getQueue, QUEUE_NAMES } from "@/lib/queue/boss";
 import { taskProcessor } from "@/lib/orchestrator/processor";
 import { chatProcessor } from "@/lib/orchestrator/chat-processor";
+import { writeDebugLog } from "@/lib/debug-log";
 
 let bootstrapPromise: Promise<void> | null = null;
 
@@ -8,10 +9,15 @@ export async function ensureOrchestratorRuntime(): Promise<void> {
   if (bootstrapPromise) return bootstrapPromise;
 
   bootstrapPromise = (async () => {
+    writeDebugLog("orchestrator.bootstrap.start");
     const queue = await getQueue();
     await queue.work(QUEUE_NAMES.TASK_PROCESS, taskProcessor, { batchSize: 5 });
     await queue.work(QUEUE_NAMES.CHAT_RUN_PROCESS, chatProcessor, { batchSize: 2 });
+    writeDebugLog("orchestrator.bootstrap.ready", {
+      queues: [QUEUE_NAMES.TASK_PROCESS, QUEUE_NAMES.CHAT_RUN_PROCESS],
+    });
   })().catch((error) => {
+    writeDebugLog("orchestrator.bootstrap.error", { error });
     bootstrapPromise = null;
     throw error;
   });

package/cloud-runtime/standalone/lib/queue/sqlite-adapter.ts

@@ -6,8 +6,15 @@ import os from "os";
 import { QueueAdapter, QueueOptions, WorkerOptions, Job } from "./adapter";
 import { validateSQLiteEnvironment, REQUIRED_PRAGMAS } from "../startup";
 import { WriteRateMonitor, WRITE_RATE_SAMPLE_WINDOW_MS } from "../limits";
+import { writeDebugLog } from "../debug-log";
 
 const AGX_DATA_DIR = process.env.AGX_DATA_DIR || path.join(os.homedir(), ".agx");
+const DEFAULT_STALE_JOB_TIMEOUT_MS = 10 * 60 * 1000;
+
+function getStaleJobTimeoutMs(): number {
+  const raw = Number(process.env.AGX_QUEUE_STALE_MS);
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_STALE_JOB_TIMEOUT_MS;
+}
 
 interface SQLiteJobRow {
   id: string;
@@ -120,6 +127,14 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     );
 
     this.writeMonitor.record();
+    writeDebugLog("queue.send", {
+      queue,
+      jobId: id,
+      startAfter,
+      retryLimit: options?.retryLimit ?? 3,
+      priority: options?.priority ?? 0,
+      data,
+    });
     return id;
   }
 
@@ -128,6 +143,12 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     handler: (jobs: Job<T>[]) => Promise<void>,
     options?: WorkerOptions
   ): Promise<void> {
+    writeDebugLog("queue.worker.register", {
+      queue,
+      batchSize: options?.batchSize ?? 1,
+      pollInterval: options?.pollInterval ?? null,
+      workerId: this.workerId,
+    });
     this.workers.set(queue, {
       handler,
       options: options || {},
@@ -163,9 +184,20 @@ export class SQLiteQueueAdapter implements QueueAdapter {
   private processQueue(queue: string, worker: { handler: Function; options: WorkerOptions }) {
     const batchSize = worker.options.batchSize || 1;
     const now = Date.now();
+    const staleBefore = now - getStaleJobTimeoutMs();
 
     // Transactional claim
     const jobs = this.db.transaction(() => {
+      const reapedJobs = this.reapStaleJobs(queue, now, staleBefore);
+      if (reapedJobs.length > 0) {
+        writeDebugLog("queue.reap_stale", {
+          queue,
+          workerId: this.workerId,
+          staleBefore,
+          jobs: reapedJobs,
+        });
+      }
+
       // 1. Find candidates
       const candidates = this.db.prepare(`
         SELECT id, data FROM agx_jobs
@@ -198,6 +230,11 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     })();
 
     if (jobs.length > 0) {
+      writeDebugLog("queue.claim", {
+        queue,
+        workerId: this.workerId,
+        jobIds: jobs.map((job) => job.id),
+      });
       // Execute handler
       // Note: We don't await here to avoid blocking the poll loop, 
       // but in single-threaded Node, we are effectively blocking if the handler is sync-heavy.
@@ -214,9 +251,16 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       await handler(jobs);
       // Success
       this.completeJobs(jobs.map(j => j.id));
+      writeDebugLog("queue.complete", {
+        jobIds: jobs.map((job) => job.id),
+      });
     } catch (err: any) {
       // Failure (batch failure — all jobs in the batch fail together)
       this.failJobs(jobs.map(j => j.id), err.message || String(err));
+      writeDebugLog("queue.fail", {
+        jobIds: jobs.map((job) => job.id),
+        error: err,
+      });
     }
   }
 
@@ -224,7 +268,7 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     const now = Date.now();
     const stmt = this.db.prepare(`
       UPDATE agx_jobs
-      SET status = 'completed', completed_at = ?
+      SET status = 'completed', completed_at = ?, worker_id = NULL
       WHERE id = ?
     `);
     this.db.transaction(() => {
@@ -243,13 +287,15 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       UPDATE agx_jobs
       SET status = 'retry', retry_count = retry_count + 1, 
           start_after = ? + (retry_count * 1000 * 2), -- Exponential backoff (kinda)
+          started_at = NULL,
+          worker_id = NULL,
           error = ?
       WHERE id = ?
     `);
 
     const updateFailed = this.db.prepare(`
       UPDATE agx_jobs
-      SET status = 'failed', failed_at = ?, error = ?
+      SET status = 'failed', failed_at = ?, started_at = NULL, worker_id = NULL, error = ?
       WHERE id = ?
     `);
 
@@ -268,4 +314,73 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       }
     })();
   }
+
+  private reapStaleJobs(queue: string, now: number, staleBefore: number): Array<{
+    id: string;
+    fromStatus: "active";
+    toStatus: "retry" | "failed";
+    retryCount: number;
+  }> {
+    const staleJobs = this.db.prepare(`
+      SELECT id, retry_count, retry_limit
+      FROM agx_jobs
+      WHERE queue = ?
+        AND status = 'active'
+        AND started_at IS NOT NULL
+        AND started_at <= ?
+      ORDER BY started_at ASC
+    `).all(queue, staleBefore) as Array<Pick<SQLiteJobRow, "id" | "retry_count" | "retry_limit">>;
+
+    if (staleJobs.length === 0) return [];
+
+    const updateRetry = this.db.prepare(`
+      UPDATE agx_jobs
+      SET status = 'retry',
+          retry_count = retry_count + 1,
+          start_after = ?,
+          started_at = NULL,
+          worker_id = NULL,
+          error = ?
+      WHERE id = ?
+    `);
+    const updateFailed = this.db.prepare(`
+      UPDATE agx_jobs
+      SET status = 'failed',
+          failed_at = ?,
+          started_at = NULL,
+          worker_id = NULL,
+          error = ?
+      WHERE id = ?
+    `);
+
+    const reaped: Array<{
+      id: string;
+      fromStatus: "active";
+      toStatus: "retry" | "failed";
+      retryCount: number;
+    }> = [];
+
+    for (const job of staleJobs) {
+      const error = `Job exceeded active timeout of ${getStaleJobTimeoutMs()}ms`;
+      if (job.retry_count < job.retry_limit) {
+        updateRetry.run(now, error, job.id);
+        reaped.push({
+          id: job.id,
+          fromStatus: "active",
+          toStatus: "retry",
+          retryCount: job.retry_count + 1,
+        });
+      } else {
+        updateFailed.run(now, error, job.id);
+        reaped.push({
+          id: job.id,
+          fromStatus: "active",
+          toStatus: "failed",
+          retryCount: job.retry_count,
+        });
+      }
+    }
+
+    return reaped;
+  }
 }