@agent-native/core 0.7.19 → 0.7.21

package/dist/agent/run-manager.js

@@ -4,6 +4,13 @@ const activeRuns = new Map();
 const threadToRun = new Map();
 /** How long to keep completed runs in memory before cleanup (5 min) */
 const CLEANUP_DELAY_MS = 5 * 60 * 1000;
+const DEFAULT_RUN_SOFT_TIMEOUT_MS = 75_000;
+function getRunSoftTimeoutMs() {
+    const raw = Number(process.env.AGENT_RUN_SOFT_TIMEOUT_MS);
+    if (Number.isFinite(raw) && raw >= 0)
+        return raw;
+    return DEFAULT_RUN_SOFT_TIMEOUT_MS;
+}
 /**
  * Start a new agent run in the background.
  * `runFn` receives a `send` callback and an `AbortSignal`.
@@ -18,6 +25,7 @@ export function startRun(runId, threadId, runFn, onComplete) {
         abortRun(existingRunId);
     }
     const abort = new AbortController();
+    let softTimedOut = false;
     const run = {
         runId,
         threadId,
@@ -38,6 +46,22 @@ export function startRun(runId, threadId, runFn, onComplete) {
     const heartbeatTimer = setInterval(() => {
         updateRunHeartbeat(runId).catch(() => { });
     }, 1500);
+    const softTimeoutMs = getRunSoftTimeoutMs();
+    const softTimeoutTimer = softTimeoutMs > 0
+        ? setTimeout(() => {
+            if (run.status !== "running" || abort.signal.aborted)
+                return;
+            softTimedOut = true;
+            send({
+                type: "error",
+                error: "The agent reached the run time limit before it could finish.",
+                errorCode: "run_timeout",
+                recoverable: true,
+                details: `The run exceeded ${Math.round(softTimeoutMs / 1000)} seconds. Partial output and tool calls were preserved so you can continue or retry.`,
+            });
+            abort.abort();
+        }, softTimeoutMs)
+        : null;
     // Periodic SQL abort check interval (for cross-isolate abort on Workers)
     let lastAbortCheck = Date.now();
     const send = (event) => {
@@ -69,12 +93,12 @@ export function startRun(runId, threadId, runFn, onComplete) {
     // Run in background — intentionally detached from any HTTP connection
     const runPromise = runFn(send, abort.signal)
         .then(() => {
-        run.status = "completed";
+        run.status = softTimedOut ? "errored" : "completed";
     })
         .catch((err) => {
         // Don't surface abort errors — the run was intentionally stopped
         if (abort.signal.aborted) {
-            run.status = "completed";
+            run.status = softTimedOut ? "errored" : "aborted";
             return;
         }
         run.status = "errored";
@@ -142,10 +166,16 @@ export function startRun(runId, threadId, runFn, onComplete) {
         }
         // 3. Stop the heartbeat — all liveness writes are done.
         clearInterval(heartbeatTimer);
+        if (softTimeoutTimer)
+            clearTimeout(softTimeoutTimer);
         // 4. Persist final status to SQL. If the completion callback threw,
         //    we'd rather mark the run errored than claim success with
         //    incomplete thread_data.
-        const finalStatus = run.status === "errored" || completionError ? "errored" : "completed";
+        const finalStatus = run.status === "aborted"
+            ? "aborted"
+            : run.status === "errored" || completionError
+                ? "errored"
+                : "completed";
         try {
             await updateRunStatus(runId, finalStatus);
         }
@@ -194,8 +224,22 @@ export function subscribeToRun(runId, fromSeq) {
 function subscribeInMemory(run, fromSeq) {
     const encoder = new TextEncoder();
     let subscriberRef = null;
+    let pingTimer = null;
     return new ReadableStream({
         start(controller) {
+            const ping = () => {
+                try {
+                    controller.enqueue(encoder.encode(`: ping ${Date.now()}\n\n`));
+                }
+                catch {
+                    if (subscriberRef)
+                        run.subscribers.delete(subscriberRef);
+                    if (pingTimer)
+                        clearInterval(pingTimer);
+                }
+            };
+            ping();
+            pingTimer = setInterval(ping, 10_000);
             // Replay buffered events from fromSeq
             for (let i = fromSeq; i < run.events.length; i++) {
                 try {
@@ -207,6 +251,8 @@ function subscribeInMemory(run, fromSeq) {
             }
             // If run is already done, close immediately
             if (run.status !== "running") {
+                if (pingTimer)
+                    clearInterval(pingTimer);
                 controller.close();
                 return;
             }
@@ -220,6 +266,8 @@ function subscribeInMemory(run, fromSeq) {
                         event.event.type === "missing_api_key" ||
                         event.event.type === "loop_limit") {
                         run.subscribers.delete(subscriberRef);
+                        if (pingTimer)
+                            clearInterval(pingTimer);
                         controller.close();
                     }
                 }
@@ -233,6 +281,8 @@ function subscribeInMemory(run, fromSeq) {
             // Only unsubscribe — do NOT abort the agent run
             if (subscriberRef)
                 run.subscribers.delete(subscriberRef);
+            if (pingTimer)
+                clearInterval(pingTimer);
         },
     });
 }
@@ -241,9 +291,22 @@ function subscribeFromSQL(runId, fromSeq) {
     const encoder = new TextEncoder();
     let cancelled = false;
     let pollTimer = null;
+    let pingTimer = null;
     return new ReadableStream({
         async start(controller) {
             let lastSeq = fromSeq;
+            const ping = () => {
+                try {
+                    controller.enqueue(encoder.encode(`: ping ${Date.now()}\n\n`));
+                }
+                catch {
+                    cancelled = true;
+                    if (pingTimer)
+                        clearInterval(pingTimer);
+                }
+            };
+            ping();
+            pingTimer = setInterval(ping, 10_000);
             const poll = async () => {
                 if (cancelled)
                     return;
@@ -271,6 +334,8 @@ function subscribeFromSQL(runId, fromSeq) {
                             parsed.type === "error" ||
                             parsed.type === "missing_api_key" ||
                             parsed.type === "loop_limit") {
+                            if (pingTimer)
+                                clearInterval(pingTimer);
                             controller.close();
                             return;
                         }
@@ -301,6 +366,8 @@ function subscribeFromSQL(runId, fromSeq) {
                                     return;
                                 }
                             }
+                            if (pingTimer)
+                                clearInterval(pingTimer);
                             controller.close();
                             return;
                         }
@@ -313,6 +380,8 @@ function subscribeFromSQL(runId, fromSeq) {
                 catch {
                     // SQL error — close stream
                     try {
+                        if (pingTimer)
+                            clearInterval(pingTimer);
                         controller.close();
                     }
                     catch { }
@@ -322,6 +391,8 @@ function subscribeFromSQL(runId, fromSeq) {
             try {
                 const run = await getRunById(runId);
                 if (!run) {
+                    if (pingTimer)
+                        clearInterval(pingTimer);
                     controller.close();
                     return;
                 }
@@ -336,6 +407,8 @@ function subscribeFromSQL(runId, fromSeq) {
             cancelled = true;
             if (pollTimer)
                 clearTimeout(pollTimer);
+            if (pingTimer)
+                clearInterval(pingTimer);
         },
     });
 }

package/dist/agent/run-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-manager.js","sourceRoot":"","sources":["../../src/agent/run-manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,SAAS,EACT,cAAc,EACd,eAAe,EACf,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,WAAW,GACZ,MAAM,gBAAgB,CAAC;AAYxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;AAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE9C,uEAAuE;AACvE,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvC;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,QAAgB,EAChB,KAGkB,EAClB,UAAqD;IAErD,qDAAqD;IACrD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,aAAa,EAAE,CAAC;QAClB,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,GAAG,GAAc;QACrB,KAAK;QACL,QAAQ;QACR,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,IAAI,GAAG,EAAE;QACtB,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC3B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEjC,kEAAkE;IAClE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE3C,wEAAwE;IACxE,sEAAsE;IACtE,mEAAmE;IACnE,0DAA0D;IAC1D,MAAM,cAAc,GAAmC,WAAW,CAAC,GAAG,EAAE;QACtE,kBAAkB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,EAAE,IAAI,CAAC,CAAC;IAET,yEAAyE;IACzE,IAAI,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEhC,MAAM,IAAI,GAAG,CAAC,KAAqB,EAAE,EAAE;QACrC,MAAM,QAAQ,GAAa,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC7D,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1B,yDAAyD;QACzD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE3E,oDAAoD;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,cAAc,GAAG,IAAI,EAAE,CAAC;YAChC,cAAc,GAAG,GAAG,CAAC;YACrB,YAAY,CAAC,KAAK,CAAC;iBAChB,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAChB,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO;oBAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACtD,CAAC,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IAEF,sEAAsE;IACtE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;SACzC,IAAI,CAAC,GAAG,EAAE;QACT,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;IAC3B,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,iEAAiE;QACjE,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;YACzB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,eAAe;YACtC,GAAG,CAAC,GAAG,YAAY,WAAW,IAAI,GAAG,CAAC,SAAS;gBAC7C,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE;gBAC9B,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,GAAG,YAAY,WAAW,IAAI,GAAG,CAAC,UAAU;gBAC9C,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC,CAAC;SACD,OAAO,CAAC,KAAK,IAAI,EAAE;QAClB,gEAAgE;QAChE,mEAAmE;QACnE,kEAAkE;QAClE,kEAAkE;QAClE,gEAAgE;QAChE,sCAAsC;QAEtC,gEAAgE;QAChE,kEAAkE;QAClE,sDAAsD;QACtD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAa;gBACzB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;gBACtB,KAAK,EACH,GAAG,CAAC,MAAM,KAAK,SAAS;oBACtB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,8BAA8B,EAAE;oBAC1D,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;aACvB,CAAC;YACF,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,IACE,CAAC,IAAI;gBACL,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM;oBACzB,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;oBAC3B,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB;oBACrC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC,EACnC,CAAC;gBACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC1B,cAAc,CACZ,KAAK,EACL,QAAQ,CAAC,GAAG,EACZ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC/B,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAClB,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;oBACzC,IAAI,CAAC;wBACH,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACvB,CAAC;oBAAC,MAAM,CAAC;wBACP,+CAA+C;oBACjD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACrC,CAAC;QAED,oEAAoE;QACpE,mEAAmE;QACnE,8DAA8D;QAC9D,IAAI,eAAe,GAAY,IAAI,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,eAAe,GAAG,GAAG,CAAC;gBACtB,OAAO,CAAC,KAAK,CACX,0CAA0C,EAC1C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,wDAAwD;QACxD,aAAa,CAAC,cAAc,CAAC,CAAC;QAE9B,oEAAoE;QACpE,8DAA8D;QAC9D,6BAA6B;QAC7B,MAAM,WAAW,GACf,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;QACxE,IAAI,CAAC;YACH,MAAM,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,4BAA4B;QAC9B,CAAC;QAED,iEAAiE;QACjE,UAAU,CAAC,GAAG,EAAE;YACd,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;gBACxC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrB,cAAc,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEL,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC;QAClC,IAAI,KAAK,EAAE,SAAS,EAAE,CAAC;YACrB,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,KAAa,EACb,OAAe;IAEf,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,OAAO,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IACD,qDAAqD;IACrD,OAAO,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,uDAAuD;AACvD,SAAS,iBAAiB,CACxB,GAAc,EACd,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,aAAa,GAAuC,IAAI,CAAC;IAE7D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,UAAU;YACd,sCAAsC;YACtC,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAClF,CACF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;gBACT,CAAC;YACH,CAAC;YAED,4CAA4C;YAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC7B,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,2BAA2B;YAC3B,aAAa,GAAG,CAAC,KAAe,EAAE,EAAE;gBAClC,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAClE,CACF,CAAC;oBACF,qCAAqC;oBACrC,IACE,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM;wBAC3B,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;wBAC5B,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB;wBACtC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,EACjC,CAAC;wBACD,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;wBACvC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC,CAAC;YAEF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACrC,CAAC;QACD,MAAM;YACJ,gDAAgD;YAChD,IAAI,aAAa;gBAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AACtD,SAAS,gBAAgB,CACvB,KAAa,EACb,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,SAAS,GAAyC,IAAI,CAAC;IAE3D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,KAAK,CAAC,UAAU;YACpB,IAAI,OAAO,GAAG,OAAO,CAAC;YAEtB,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;gBACtB,IAAI,SAAS;oBAAE,OAAO;gBACtB,IAAI,CAAC;oBACH,2BAA2B;oBAC3B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBACvD,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,MAAM,EAAE,CAAC;wBACxC,IAAI,MAAW,CAAC;wBAChB,IAAI,CAAC;4BACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;wBACjC,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS;wBACX,CAAC;wBACD,IAAI,CAAC;4BACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;wBACJ,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS,GAAG,IAAI,CAAC;4BACjB,OAAO;wBACT,CAAC;wBACD,OAAO,GAAG,GAAG,CAAC;wBAEd,2BAA2B;wBAC3B,IACE,MAAM,CAAC,IAAI,KAAK,MAAM;4BACtB,MAAM,CAAC,IAAI,KAAK,OAAO;4BACvB,MAAM,CAAC,IAAI,KAAK,iBAAiB;4BACjC,MAAM,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;4BACD,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACxB,gEAAgE;wBAChE,kEAAkE;wBAClE,WAAW;wBACX,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;wBACzC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;wBACpC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;4BACrC,kDAAkD;4BAClD,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;4BAC5D,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,WAAW,EAAE,CAAC;gCAC7C,IAAI,MAAW,CAAC;gCAChB,IAAI,CAAC;oCACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gCACjC,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS;gCACX,CAAC;gCACD,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;4BACD,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,qBAAqB;oBACrB,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBACpC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,2BAA2B;oBAC3B,IAAI,CAAC;wBACH,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC;YAEF,yCAAyC;YACzC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO;gBACT,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;QACD,MAAM;YACJ,SAAS,GAAG,IAAI,CAAC;YACjB,IAAI,SAAS;gBAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,QAAgB;IAM/D,uEAAuE;IACvE,yEAAyE;IACzE,qDAAqD;IACrD,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACxE,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mEAAmE;YACnE,8DAA8D;YAC9D,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;SACxB,CAAC;IACJ,CAAC;IACD,mBAAmB;IACnB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC1C,iEAAiE;YACjE,gEAAgE;YAChE,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;YAC/D,IAAI,MAAM;gBAAE,OAAO,IAAI,CAAC;YACxB,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS;aACpD,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sBAAsB;AACtB,MAAM,UAAU,MAAM,CAAC,KAAa;IAClC,OAAO,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,QAAQ,CAAC,KAAa;IACpC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IACD,mEAAmE;IACnE,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtC,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC","sourcesContent":["import type { AgentChatEvent, RunEvent, RunStatus } from \"./types.js\";\nimport { EngineError } from \"./engine/types.js\";\nimport {\n  insertRun,\n  insertRunEvent,\n  updateRunStatus,\n  markRunAborted,\n  isRunAborted,\n  getRunEventsSince,\n  getRunById,\n  getRunByThread,\n  cleanupOldRuns,\n  updateRunHeartbeat,\n  reapIfStale,\n} from \"./run-store.js\";\n\nexport interface ActiveRun {\n  runId: string;\n  threadId: string;\n  events: RunEvent[];\n  status: RunStatus;\n  subscribers: Set<(event: RunEvent) => void>;\n  abort: AbortController;\n  startedAt: number;\n}\n\nconst activeRuns = new Map<string, ActiveRun>();\nconst threadToRun = new Map<string, string>();\n\n/** How long to keep completed runs in memory before cleanup (5 min) */\nconst CLEANUP_DELAY_MS = 5 * 60 * 1000;\n\n/**\n * Start a new agent run in the background.\n * `runFn` receives a `send` callback and an `AbortSignal`.\n * The run continues even if all SSE subscribers disconnect.\n *\n * Events are persisted to SQL for cross-isolate access (Cloudflare Workers).\n */\nexport function startRun(\n  runId: string,\n  threadId: string,\n  runFn: (\n    send: (event: AgentChatEvent) => void,\n    signal: AbortSignal,\n  ) => Promise<void>,\n  onComplete?: (run: ActiveRun) => void | Promise<void>,\n): ActiveRun {\n  // If there's already a run for this thread, abort it\n  const existingRunId = threadToRun.get(threadId);\n  if (existingRunId) {\n    abortRun(existingRunId);\n  }\n\n  const abort = new AbortController();\n  const run: ActiveRun = {\n    runId,\n    threadId,\n    events: [],\n    status: \"running\",\n    subscribers: new Set(),\n    abort,\n    startedAt: Date.now(),\n  };\n\n  activeRuns.set(runId, run);\n  threadToRun.set(threadId, runId);\n\n  // Persist run to SQL (fire-and-forget — don't block the response)\n  insertRun(runId, threadId).catch(() => {});\n\n  // Heartbeat: bump heartbeat_at every 1.5s so watchers can detect a dead\n  // producer (process crash, HMR restart, isolate eviction) quickly and\n  // reap the row. Paired with RUN_STALE_MS (6s) — 4x the interval to\n  // tolerate transient DB slowness without false positives.\n  const heartbeatTimer: ReturnType<typeof setInterval> = setInterval(() => {\n    updateRunHeartbeat(runId).catch(() => {});\n  }, 1500);\n\n  // Periodic SQL abort check interval (for cross-isolate abort on Workers)\n  let lastAbortCheck = Date.now();\n\n  const send = (event: AgentChatEvent) => {\n    const runEvent: RunEvent = { seq: run.events.length, event };\n    run.events.push(runEvent);\n\n    // Notify in-memory subscribers (same isolate, fast path)\n    for (const subscriber of run.subscribers) {\n      try {\n        subscriber(runEvent);\n      } catch {\n        run.subscribers.delete(subscriber);\n      }\n    }\n\n    // Persist event to SQL (fire-and-forget)\n    insertRunEvent(runId, runEvent.seq, JSON.stringify(event)).catch(() => {});\n\n    // Check SQL for cross-isolate abort every 3 seconds\n    const now = Date.now();\n    if (now - lastAbortCheck > 3000) {\n      lastAbortCheck = now;\n      isRunAborted(runId)\n        .then((aborted) => {\n          if (aborted && !abort.signal.aborted) abort.abort();\n        })\n        .catch(() => {});\n    }\n  };\n\n  // Run in background — intentionally detached from any HTTP connection\n  const runPromise = runFn(send, abort.signal)\n    .then(() => {\n      run.status = \"completed\";\n    })\n    .catch((err) => {\n      // Don't surface abort errors — the run was intentionally stopped\n      if (abort.signal.aborted) {\n        run.status = \"completed\";\n        return;\n      }\n      run.status = \"errored\";\n      send({\n        type: \"error\",\n        error: err?.message ?? \"Unknown error\",\n        ...(err instanceof EngineError && err.errorCode\n          ? { errorCode: err.errorCode }\n          : {}),\n        ...(err instanceof EngineError && err.upgradeUrl\n          ? { upgradeUrl: err.upgradeUrl }\n          : {}),\n      });\n    })\n    .finally(async () => {\n      // Ordering matters here — this is the atomic-complete boundary.\n      // Invariant: once agent_runs.status flips to \"completed\"/\"errored\"\n      // in SQL, thread_data for this turn is already durable. This lets\n      // reconnecting clients trust the simple rule \"status != running →\n      // fetch thread_data\" without polling/retrying for a race window\n      // where onComplete was still pending.\n\n      // 1. Emit terminal event to live subscribers + SQL event log so\n      //    in-flight SSE streams close promptly. Thread-data save below\n      //    runs in parallel with subscribers disconnecting.\n      if (run.status === \"errored\" || run.status === \"completed\") {\n        const terminal: RunEvent = {\n          seq: run.events.length,\n          event:\n            run.status === \"errored\"\n              ? { type: \"error\", error: \"Agent run ended unexpectedly\" }\n              : { type: \"done\" },\n        };\n        const last = run.events[run.events.length - 1];\n        if (\n          !last ||\n          (last.event.type !== \"done\" &&\n            last.event.type !== \"error\" &&\n            last.event.type !== \"missing_api_key\" &&\n            last.event.type !== \"loop_limit\")\n        ) {\n          run.events.push(terminal);\n          insertRunEvent(\n            runId,\n            terminal.seq,\n            JSON.stringify(terminal.event),\n          ).catch(() => {});\n          for (const subscriber of run.subscribers) {\n            try {\n              subscriber(terminal);\n            } catch {\n              // ignore — subscriber will be cleaned up below\n            }\n          }\n        }\n      }\n      for (const subscriber of run.subscribers) {\n        run.subscribers.delete(subscriber);\n      }\n\n      // 2. Await the completion callback (thread_data save). Heartbeat is\n      //    still ticking so the run doesn't look stale to any concurrent\n      //    /runs/active check while we wait for SQL writes to land.\n      let completionError: unknown = null;\n      if (onComplete) {\n        try {\n          await onComplete(run);\n        } catch (err) {\n          completionError = err;\n          console.error(\n            \"[run-manager] onComplete callback error:\",\n            err instanceof Error ? err.message : err,\n          );\n        }\n      }\n\n      // 3. Stop the heartbeat — all liveness writes are done.\n      clearInterval(heartbeatTimer);\n\n      // 4. Persist final status to SQL. If the completion callback threw,\n      //    we'd rather mark the run errored than claim success with\n      //    incomplete thread_data.\n      const finalStatus =\n        run.status === \"errored\" || completionError ? \"errored\" : \"completed\";\n      try {\n        await updateRunStatus(runId, finalStatus);\n      } catch {\n        // Best-effort — reapIfStale will eventually clean this up via\n        // the heartbeat-stale path.\n      }\n\n      // 5. Schedule in-memory cleanup + opportunistic old-run pruning.\n      setTimeout(() => {\n        activeRuns.delete(runId);\n        if (threadToRun.get(threadId) === runId) {\n          threadToRun.delete(threadId);\n        }\n      }, CLEANUP_DELAY_MS);\n      cleanupOldRuns(30 * 60 * 1000).catch(() => {});\n    });\n\n  // On Cloudflare Workers, keep the isolate alive for this run\n  try {\n    const cfCtx = globalThis.__cf_ctx;\n    if (cfCtx?.waitUntil) {\n      cfCtx.waitUntil(runPromise);\n    }\n  } catch {\n    // Not on Workers — ignore\n  }\n\n  return run;\n}\n\n/**\n * Subscribe to a run's events starting from `fromSeq`.\n * Returns a ReadableStream that replays buffered events then live-tails.\n * Cancelling the stream only unsubscribes — does NOT abort the agent.\n *\n * Falls back to SQL polling when the run is not in local memory\n * (cross-isolate reconnection on Workers).\n */\nexport function subscribeToRun(\n  runId: string,\n  fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n  const run = activeRuns.get(runId);\n  if (run) {\n    return subscribeInMemory(run, fromSeq);\n  }\n  // Not in local memory — try SQL (cross-isolate path)\n  return subscribeFromSQL(runId, fromSeq);\n}\n\n/** In-memory subscription (same isolate, fast path) */\nfunction subscribeInMemory(\n  run: ActiveRun,\n  fromSeq: number,\n): ReadableStream<Uint8Array> {\n  const encoder = new TextEncoder();\n  let subscriberRef: ((event: RunEvent) => void) | null = null;\n\n  return new ReadableStream({\n    start(controller) {\n      // Replay buffered events from fromSeq\n      for (let i = fromSeq; i < run.events.length; i++) {\n        try {\n          controller.enqueue(\n            encoder.encode(\n              `data: ${JSON.stringify({ ...run.events[i].event, seq: run.events[i].seq })}\\n\\n`,\n            ),\n          );\n        } catch {\n          return;\n        }\n      }\n\n      // If run is already done, close immediately\n      if (run.status !== \"running\") {\n        controller.close();\n        return;\n      }\n\n      // Subscribe to live events\n      subscriberRef = (event: RunEvent) => {\n        try {\n          controller.enqueue(\n            encoder.encode(\n              `data: ${JSON.stringify({ ...event.event, seq: event.seq })}\\n\\n`,\n            ),\n          );\n          // Close stream after terminal events\n          if (\n            event.event.type === \"done\" ||\n            event.event.type === \"error\" ||\n            event.event.type === \"missing_api_key\" ||\n            event.event.type === \"loop_limit\"\n          ) {\n            run.subscribers.delete(subscriberRef!);\n            controller.close();\n          }\n        } catch {\n          run.subscribers.delete(subscriberRef!);\n        }\n      };\n\n      run.subscribers.add(subscriberRef);\n    },\n    cancel() {\n      // Only unsubscribe — do NOT abort the agent run\n      if (subscriberRef) run.subscribers.delete(subscriberRef);\n    },\n  });\n}\n\n/** SQL-based subscription (cross-isolate, polling) */\nfunction subscribeFromSQL(\n  runId: string,\n  fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n  const encoder = new TextEncoder();\n  let cancelled = false;\n  let pollTimer: ReturnType<typeof setTimeout> | null = null;\n\n  return new ReadableStream({\n    async start(controller) {\n      let lastSeq = fromSeq;\n\n      const poll = async () => {\n        if (cancelled) return;\n        try {\n          // Read new events from SQL\n          const events = await getRunEventsSince(runId, lastSeq);\n          for (const { seq, eventData } of events) {\n            let parsed: any;\n            try {\n              parsed = JSON.parse(eventData);\n            } catch {\n              continue;\n            }\n            try {\n              controller.enqueue(\n                encoder.encode(\n                  `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n                ),\n              );\n            } catch {\n              cancelled = true;\n              return;\n            }\n            lastSeq = seq;\n\n            // Close on terminal events\n            if (\n              parsed.type === \"done\" ||\n              parsed.type === \"error\" ||\n              parsed.type === \"missing_api_key\" ||\n              parsed.type === \"loop_limit\"\n            ) {\n              controller.close();\n              return;\n            }\n          }\n\n          // Check if run completed (no terminal event but status changed)\n          if (events.length === 0) {\n            // Opportunistically reap a stale producer before trusting SQL's\n            // \"running\" status — otherwise a crashed server leaves us polling\n            // forever.\n            await reapIfStale(runId).catch(() => {});\n            const run = await getRunById(runId);\n            if (!run || run.status !== \"running\") {\n              // Run ended — do one final event read, then close\n              const finalEvents = await getRunEventsSince(runId, lastSeq);\n              for (const { seq, eventData } of finalEvents) {\n                let parsed: any;\n                try {\n                  parsed = JSON.parse(eventData);\n                } catch {\n                  continue;\n                }\n                try {\n                  controller.enqueue(\n                    encoder.encode(\n                      `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n                    ),\n                  );\n                } catch {\n                  cancelled = true;\n                  return;\n                }\n              }\n              controller.close();\n              return;\n            }\n          }\n\n          // Schedule next poll\n          if (!cancelled) {\n            pollTimer = setTimeout(poll, 500);\n          }\n        } catch {\n          // SQL error — close stream\n          try {\n            controller.close();\n          } catch {}\n        }\n      };\n\n      // Verify run exists before starting poll\n      try {\n        const run = await getRunById(runId);\n        if (!run) {\n          controller.close();\n          return;\n        }\n      } catch {\n        controller.close();\n        return;\n      }\n\n      await poll();\n    },\n    cancel() {\n      cancelled = true;\n      if (pollTimer) clearTimeout(pollTimer);\n    },\n  });\n}\n\n/** Get the active run for a thread (if any) — checks memory then SQL */\nexport function getActiveRunForThread(threadId: string): ActiveRun | null {\n  const runId = threadToRun.get(threadId);\n  if (runId) {\n    const run = activeRuns.get(runId);\n    if (run) return run;\n  }\n  return null;\n}\n\n/**\n * Async version that also checks SQL — for cross-isolate access.\n * Used by the /runs/active endpoint.\n *\n * Returns `heartbeatAt` so the client can independently decide a run is\n * dead even before the server-side stale reap has fired.\n */\nexport async function getActiveRunForThreadAsync(threadId: string): Promise<{\n  runId: string;\n  threadId: string;\n  status: string;\n  heartbeatAt: number;\n} | null> {\n  // Check memory first — return both running AND recently-completed runs\n  // that still have events in memory. This allows sub-agent tabs to replay\n  // the full conversation from completed runs via SSE.\n  const memRun = getActiveRunForThread(threadId);\n  if (memRun && (memRun.status === \"running\" || memRun.events.length > 0)) {\n    return {\n      runId: memRun.runId,\n      threadId: memRun.threadId,\n      status: memRun.status,\n      // In-memory means this isolate is the producer. By definition, the\n      // heartbeat is fresh as of \"now\" — the client can trust this.\n      heartbeatAt: Date.now(),\n    };\n  }\n  // Fall back to SQL\n  try {\n    const sqlRun = await getRunByThread(threadId);\n    if (sqlRun && sqlRun.status === \"running\") {\n      // If the producer is dead (no recent heartbeat), reap before the\n      // client can see a stale \"running\" status and enter a reconnect\n      // loop it can never exit.\n      const reaped = await reapIfStale(sqlRun.id).catch(() => false);\n      if (reaped) return null;\n      return {\n        runId: sqlRun.id,\n        threadId: sqlRun.threadId,\n        status: sqlRun.status,\n        heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,\n      };\n    }\n  } catch {\n    // SQL error — fall through\n  }\n  return null;\n}\n\n/** Get a run by ID */\nexport function getRun(runId: string): ActiveRun | null {\n  return activeRuns.get(runId) ?? null;\n}\n\n/** Explicitly abort a run (e.g. Stop button) */\nexport function abortRun(runId: string): boolean {\n  const run = activeRuns.get(runId);\n  if (run) {\n    run.abort.abort();\n  }\n  // Also mark as aborted in SQL (for cross-isolate abort on Workers)\n  markRunAborted(runId).catch(() => {});\n  return !!run;\n}\n"]}
+{"version":3,"file":"run-manager.js","sourceRoot":"","sources":["../../src/agent/run-manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,SAAS,EACT,cAAc,EACd,eAAe,EACf,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,WAAW,GACZ,MAAM,gBAAgB,CAAC;AAYxB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;AAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE9C,uEAAuE;AACvE,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACvC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAE3C,SAAS,mBAAmB;IAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IACjD,OAAO,2BAA2B,CAAC;AACrC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,KAAa,EACb,QAAgB,EAChB,KAGkB,EAClB,UAAqD;IAErD,qDAAqD;IACrD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,aAAa,EAAE,CAAC;QAClB,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,GAAG,GAAc;QACrB,KAAK;QACL,QAAQ;QACR,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,IAAI,GAAG,EAAE;QACtB,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC3B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEjC,kEAAkE;IAClE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE3C,wEAAwE;IACxE,sEAAsE;IACtE,mEAAmE;IACnE,0DAA0D;IAC1D,MAAM,cAAc,GAAmC,WAAW,CAAC,GAAG,EAAE;QACtE,kBAAkB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,EAAE,IAAI,CAAC,CAAC;IACT,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAC;IAC5C,MAAM,gBAAgB,GACpB,aAAa,GAAG,CAAC;QACf,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE;YACd,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO;gBAAE,OAAO;YAC7D,YAAY,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC;gBACH,IAAI,EAAE,OAAO;gBACb,KAAK,EACH,8DAA8D;gBAChE,SAAS,EAAE,aAAa;gBACxB,WAAW,EAAE,IAAI;gBACjB,OAAO,EAAE,oBAAoB,IAAI,CAAC,KAAK,CACrC,aAAa,GAAG,IAAI,CACrB,sFAAsF;aACxF,CAAC,CAAC;YACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC,EAAE,aAAa,CAAC;QACnB,CAAC,CAAC,IAAI,CAAC;IAEX,yEAAyE;IACzE,IAAI,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEhC,MAAM,IAAI,GAAG,CAAC,KAAqB,EAAE,EAAE;QACrC,MAAM,QAAQ,GAAa,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAC7D,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1B,yDAAyD;QACzD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE3E,oDAAoD;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,cAAc,GAAG,IAAI,EAAE,CAAC;YAChC,cAAc,GAAG,GAAG,CAAC;YACrB,YAAY,CAAC,KAAK,CAAC;iBAChB,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAChB,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO;oBAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACtD,CAAC,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IAEF,sEAAsE;IACtE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;SACzC,IAAI,CAAC,GAAG,EAAE;QACT,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;IACtD,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,iEAAiE;QACjE,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAClD,OAAO;QACT,CAAC;QACD,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,eAAe;YACtC,GAAG,CAAC,GAAG,YAAY,WAAW,IAAI,GAAG,CAAC,SAAS;gBAC7C,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE;gBAC9B,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,GAAG,YAAY,WAAW,IAAI,GAAG,CAAC,UAAU;gBAC9C,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC,CAAC;SACD,OAAO,CAAC,KAAK,IAAI,EAAE;QAClB,gEAAgE;QAChE,mEAAmE;QACnE,kEAAkE;QAClE,kEAAkE;QAClE,gEAAgE;QAChE,sCAAsC;QAEtC,gEAAgE;QAChE,kEAAkE;QAClE,sDAAsD;QACtD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAa;gBACzB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;gBACtB,KAAK,EACH,GAAG,CAAC,MAAM,KAAK,SAAS;oBACtB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,8BAA8B,EAAE;oBAC1D,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;aACvB,CAAC;YACF,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,IACE,CAAC,IAAI;gBACL,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM;oBACzB,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;oBAC3B,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB;oBACrC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC,EACnC,CAAC;gBACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC1B,cAAc,CACZ,KAAK,EACL,QAAQ,CAAC,GAAG,EACZ,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAC/B,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAClB,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;oBACzC,IAAI,CAAC;wBACH,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACvB,CAAC;oBAAC,MAAM,CAAC;wBACP,+CAA+C;oBACjD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;YACzC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACrC,CAAC;QAED,oEAAoE;QACpE,mEAAmE;QACnE,8DAA8D;QAC9D,IAAI,eAAe,GAAY,IAAI,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,eAAe,GAAG,GAAG,CAAC;gBACtB,OAAO,CAAC,KAAK,CACX,0CAA0C,EAC1C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,wDAAwD;QACxD,aAAa,CAAC,cAAc,CAAC,CAAC;QAC9B,IAAI,gBAAgB;YAAE,YAAY,CAAC,gBAAgB,CAAC,CAAC;QAErD,oEAAoE;QACpE,8DAA8D;QAC9D,6BAA6B;QAC7B,MAAM,WAAW,GACf,GAAG,CAAC,MAAM,KAAK,SAAS;YACtB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,eAAe;gBAC3C,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,WAAW,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,4BAA4B;QAC9B,CAAC;QAED,iEAAiE;QACjE,UAAU,CAAC,GAAG,EAAE;YACd,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;gBACxC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrB,cAAc,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEL,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC;QAClC,IAAI,KAAK,EAAE,SAAS,EAAE,CAAC;YACrB,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,KAAa,EACb,OAAe;IAEf,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,OAAO,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IACD,qDAAqD;IACrD,OAAO,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,uDAAuD;AACvD,SAAS,iBAAiB,CACxB,GAAc,EACd,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,aAAa,GAAuC,IAAI,CAAC;IAC7D,IAAI,SAAS,GAA0C,IAAI,CAAC;IAE5D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,UAAU;YACd,MAAM,IAAI,GAAG,GAAG,EAAE;gBAChB,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,aAAa;wBAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACzD,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,CAAC;YACF,IAAI,EAAE,CAAC;YACP,SAAS,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEtC,sCAAsC;YACtC,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAClF,CACF,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;gBACT,CAAC;YACH,CAAC;YAED,4CAA4C;YAC5C,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC7B,IAAI,SAAS;oBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,2BAA2B;YAC3B,aAAa,GAAG,CAAC,KAAe,EAAE,EAAE;gBAClC,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAClE,CACF,CAAC;oBACF,qCAAqC;oBACrC,IACE,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM;wBAC3B,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO;wBAC5B,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,iBAAiB;wBACtC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,EACjC,CAAC;wBACD,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;wBACvC,IAAI,SAAS;4BAAE,aAAa,CAAC,SAAS,CAAC,CAAC;wBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC,CAAC;YAEF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACrC,CAAC;QACD,MAAM;YACJ,gDAAgD;YAChD,IAAI,aAAa;gBAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzD,IAAI,SAAS;gBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AACtD,SAAS,gBAAgB,CACvB,KAAa,EACb,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,SAAS,GAAyC,IAAI,CAAC;IAC3D,IAAI,SAAS,GAA0C,IAAI,CAAC;IAE5D,OAAO,IAAI,cAAc,CAAC;QACxB,KAAK,CAAC,KAAK,CAAC,UAAU;YACpB,IAAI,OAAO,GAAG,OAAO,CAAC;YACtB,MAAM,IAAI,GAAG,GAAG,EAAE;gBAChB,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,GAAG,IAAI,CAAC;oBACjB,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,CAAC;YACF,IAAI,EAAE,CAAC;YACP,SAAS,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEtC,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;gBACtB,IAAI,SAAS;oBAAE,OAAO;gBACtB,IAAI,CAAC;oBACH,2BAA2B;oBAC3B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBACvD,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,MAAM,EAAE,CAAC;wBACxC,IAAI,MAAW,CAAC;wBAChB,IAAI,CAAC;4BACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;wBACjC,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS;wBACX,CAAC;wBACD,IAAI,CAAC;4BACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;wBACJ,CAAC;wBAAC,MAAM,CAAC;4BACP,SAAS,GAAG,IAAI,CAAC;4BACjB,OAAO;wBACT,CAAC;wBACD,OAAO,GAAG,GAAG,CAAC;wBAEd,2BAA2B;wBAC3B,IACE,MAAM,CAAC,IAAI,KAAK,MAAM;4BACtB,MAAM,CAAC,IAAI,KAAK,OAAO;4BACvB,MAAM,CAAC,IAAI,KAAK,iBAAiB;4BACjC,MAAM,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;4BACD,IAAI,SAAS;gCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;4BACxC,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBACxB,gEAAgE;wBAChE,kEAAkE;wBAClE,WAAW;wBACX,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;wBACzC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;wBACpC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;4BACrC,kDAAkD;4BAClD,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;4BAC5D,KAAK,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,WAAW,EAAE,CAAC;gCAC7C,IAAI,MAAW,CAAC;gCAChB,IAAI,CAAC;oCACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gCACjC,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS;gCACX,CAAC;gCACD,IAAI,CAAC;oCACH,UAAU,CAAC,OAAO,CAChB,OAAO,CAAC,MAAM,CACZ,SAAS,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAClD,CACF,CAAC;gCACJ,CAAC;gCAAC,MAAM,CAAC;oCACP,SAAS,GAAG,IAAI,CAAC;oCACjB,OAAO;gCACT,CAAC;4BACH,CAAC;4BACD,IAAI,SAAS;gCAAE,aAAa,CAAC,SAAS,CAAC,CAAC;4BACxC,UAAU,CAAC,KAAK,EAAE,CAAC;4BACnB,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,qBAAqB;oBACrB,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBACpC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,2BAA2B;oBAC3B,IAAI,CAAC;wBACH,IAAI,SAAS;4BAAE,aAAa,CAAC,SAAS,CAAC,CAAC;wBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC;YAEF,yCAAyC;YACzC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,IAAI,SAAS;wBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;oBACxC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO;gBACT,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;YAED,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;QACD,MAAM;YACJ,SAAS,GAAG,IAAI,CAAC;YACjB,IAAI,SAAS;gBAAE,YAAY,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,SAAS;gBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,QAAgB;IAM/D,uEAAuE;IACvE,yEAAyE;IACzE,qDAAqD;IACrD,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACxE,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,mEAAmE;YACnE,8DAA8D;YAC9D,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;SACxB,CAAC;IACJ,CAAC;IACD,mBAAmB;IACnB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC1C,iEAAiE;YACjE,gEAAgE;YAChE,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;YAC/D,IAAI,MAAM;gBAAE,OAAO,IAAI,CAAC;YACxB,OAAO;gBACL,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS;aACpD,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sBAAsB;AACtB,MAAM,UAAU,MAAM,CAAC,KAAa;IAClC,OAAO,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,QAAQ,CAAC,KAAa;IACpC,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,EAAE,CAAC;QACR,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IACD,mEAAmE;IACnE,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACtC,OAAO,CAAC,CAAC,GAAG,CAAC;AACf,CAAC","sourcesContent":["import type { AgentChatEvent, RunEvent, RunStatus } from \"./types.js\";\nimport { EngineError } from \"./engine/types.js\";\nimport {\n  insertRun,\n  insertRunEvent,\n  updateRunStatus,\n  markRunAborted,\n  isRunAborted,\n  getRunEventsSince,\n  getRunById,\n  getRunByThread,\n  cleanupOldRuns,\n  updateRunHeartbeat,\n  reapIfStale,\n} from \"./run-store.js\";\n\nexport interface ActiveRun {\n  runId: string;\n  threadId: string;\n  events: RunEvent[];\n  status: RunStatus;\n  subscribers: Set<(event: RunEvent) => void>;\n  abort: AbortController;\n  startedAt: number;\n}\n\nconst activeRuns = new Map<string, ActiveRun>();\nconst threadToRun = new Map<string, string>();\n\n/** How long to keep completed runs in memory before cleanup (5 min) */\nconst CLEANUP_DELAY_MS = 5 * 60 * 1000;\nconst DEFAULT_RUN_SOFT_TIMEOUT_MS = 75_000;\n\nfunction getRunSoftTimeoutMs(): number {\n  const raw = Number(process.env.AGENT_RUN_SOFT_TIMEOUT_MS);\n  if (Number.isFinite(raw) && raw >= 0) return raw;\n  return DEFAULT_RUN_SOFT_TIMEOUT_MS;\n}\n\n/**\n * Start a new agent run in the background.\n * `runFn` receives a `send` callback and an `AbortSignal`.\n * The run continues even if all SSE subscribers disconnect.\n *\n * Events are persisted to SQL for cross-isolate access (Cloudflare Workers).\n */\nexport function startRun(\n  runId: string,\n  threadId: string,\n  runFn: (\n    send: (event: AgentChatEvent) => void,\n    signal: AbortSignal,\n  ) => Promise<void>,\n  onComplete?: (run: ActiveRun) => void | Promise<void>,\n): ActiveRun {\n  // If there's already a run for this thread, abort it\n  const existingRunId = threadToRun.get(threadId);\n  if (existingRunId) {\n    abortRun(existingRunId);\n  }\n\n  const abort = new AbortController();\n  let softTimedOut = false;\n  const run: ActiveRun = {\n    runId,\n    threadId,\n    events: [],\n    status: \"running\",\n    subscribers: new Set(),\n    abort,\n    startedAt: Date.now(),\n  };\n\n  activeRuns.set(runId, run);\n  threadToRun.set(threadId, runId);\n\n  // Persist run to SQL (fire-and-forget — don't block the response)\n  insertRun(runId, threadId).catch(() => {});\n\n  // Heartbeat: bump heartbeat_at every 1.5s so watchers can detect a dead\n  // producer (process crash, HMR restart, isolate eviction) quickly and\n  // reap the row. Paired with RUN_STALE_MS (6s) — 4x the interval to\n  // tolerate transient DB slowness without false positives.\n  const heartbeatTimer: ReturnType<typeof setInterval> = setInterval(() => {\n    updateRunHeartbeat(runId).catch(() => {});\n  }, 1500);\n  const softTimeoutMs = getRunSoftTimeoutMs();\n  const softTimeoutTimer =\n    softTimeoutMs > 0\n      ? setTimeout(() => {\n          if (run.status !== \"running\" || abort.signal.aborted) return;\n          softTimedOut = true;\n          send({\n            type: \"error\",\n            error:\n              \"The agent reached the run time limit before it could finish.\",\n            errorCode: \"run_timeout\",\n            recoverable: true,\n            details: `The run exceeded ${Math.round(\n              softTimeoutMs / 1000,\n            )} seconds. Partial output and tool calls were preserved so you can continue or retry.`,\n          });\n          abort.abort();\n        }, softTimeoutMs)\n      : null;\n\n  // Periodic SQL abort check interval (for cross-isolate abort on Workers)\n  let lastAbortCheck = Date.now();\n\n  const send = (event: AgentChatEvent) => {\n    const runEvent: RunEvent = { seq: run.events.length, event };\n    run.events.push(runEvent);\n\n    // Notify in-memory subscribers (same isolate, fast path)\n    for (const subscriber of run.subscribers) {\n      try {\n        subscriber(runEvent);\n      } catch {\n        run.subscribers.delete(subscriber);\n      }\n    }\n\n    // Persist event to SQL (fire-and-forget)\n    insertRunEvent(runId, runEvent.seq, JSON.stringify(event)).catch(() => {});\n\n    // Check SQL for cross-isolate abort every 3 seconds\n    const now = Date.now();\n    if (now - lastAbortCheck > 3000) {\n      lastAbortCheck = now;\n      isRunAborted(runId)\n        .then((aborted) => {\n          if (aborted && !abort.signal.aborted) abort.abort();\n        })\n        .catch(() => {});\n    }\n  };\n\n  // Run in background — intentionally detached from any HTTP connection\n  const runPromise = runFn(send, abort.signal)\n    .then(() => {\n      run.status = softTimedOut ? \"errored\" : \"completed\";\n    })\n    .catch((err) => {\n      // Don't surface abort errors — the run was intentionally stopped\n      if (abort.signal.aborted) {\n        run.status = softTimedOut ? \"errored\" : \"aborted\";\n        return;\n      }\n      run.status = \"errored\";\n      send({\n        type: \"error\",\n        error: err?.message ?? \"Unknown error\",\n        ...(err instanceof EngineError && err.errorCode\n          ? { errorCode: err.errorCode }\n          : {}),\n        ...(err instanceof EngineError && err.upgradeUrl\n          ? { upgradeUrl: err.upgradeUrl }\n          : {}),\n      });\n    })\n    .finally(async () => {\n      // Ordering matters here — this is the atomic-complete boundary.\n      // Invariant: once agent_runs.status flips to \"completed\"/\"errored\"\n      // in SQL, thread_data for this turn is already durable. This lets\n      // reconnecting clients trust the simple rule \"status != running →\n      // fetch thread_data\" without polling/retrying for a race window\n      // where onComplete was still pending.\n\n      // 1. Emit terminal event to live subscribers + SQL event log so\n      //    in-flight SSE streams close promptly. Thread-data save below\n      //    runs in parallel with subscribers disconnecting.\n      if (run.status === \"errored\" || run.status === \"completed\") {\n        const terminal: RunEvent = {\n          seq: run.events.length,\n          event:\n            run.status === \"errored\"\n              ? { type: \"error\", error: \"Agent run ended unexpectedly\" }\n              : { type: \"done\" },\n        };\n        const last = run.events[run.events.length - 1];\n        if (\n          !last ||\n          (last.event.type !== \"done\" &&\n            last.event.type !== \"error\" &&\n            last.event.type !== \"missing_api_key\" &&\n            last.event.type !== \"loop_limit\")\n        ) {\n          run.events.push(terminal);\n          insertRunEvent(\n            runId,\n            terminal.seq,\n            JSON.stringify(terminal.event),\n          ).catch(() => {});\n          for (const subscriber of run.subscribers) {\n            try {\n              subscriber(terminal);\n            } catch {\n              // ignore — subscriber will be cleaned up below\n            }\n          }\n        }\n      }\n      for (const subscriber of run.subscribers) {\n        run.subscribers.delete(subscriber);\n      }\n\n      // 2. Await the completion callback (thread_data save). Heartbeat is\n      //    still ticking so the run doesn't look stale to any concurrent\n      //    /runs/active check while we wait for SQL writes to land.\n      let completionError: unknown = null;\n      if (onComplete) {\n        try {\n          await onComplete(run);\n        } catch (err) {\n          completionError = err;\n          console.error(\n            \"[run-manager] onComplete callback error:\",\n            err instanceof Error ? err.message : err,\n          );\n        }\n      }\n\n      // 3. Stop the heartbeat — all liveness writes are done.\n      clearInterval(heartbeatTimer);\n      if (softTimeoutTimer) clearTimeout(softTimeoutTimer);\n\n      // 4. Persist final status to SQL. If the completion callback threw,\n      //    we'd rather mark the run errored than claim success with\n      //    incomplete thread_data.\n      const finalStatus =\n        run.status === \"aborted\"\n          ? \"aborted\"\n          : run.status === \"errored\" || completionError\n            ? \"errored\"\n            : \"completed\";\n      try {\n        await updateRunStatus(runId, finalStatus);\n      } catch {\n        // Best-effort — reapIfStale will eventually clean this up via\n        // the heartbeat-stale path.\n      }\n\n      // 5. Schedule in-memory cleanup + opportunistic old-run pruning.\n      setTimeout(() => {\n        activeRuns.delete(runId);\n        if (threadToRun.get(threadId) === runId) {\n          threadToRun.delete(threadId);\n        }\n      }, CLEANUP_DELAY_MS);\n      cleanupOldRuns(30 * 60 * 1000).catch(() => {});\n    });\n\n  // On Cloudflare Workers, keep the isolate alive for this run\n  try {\n    const cfCtx = globalThis.__cf_ctx;\n    if (cfCtx?.waitUntil) {\n      cfCtx.waitUntil(runPromise);\n    }\n  } catch {\n    // Not on Workers — ignore\n  }\n\n  return run;\n}\n\n/**\n * Subscribe to a run's events starting from `fromSeq`.\n * Returns a ReadableStream that replays buffered events then live-tails.\n * Cancelling the stream only unsubscribes — does NOT abort the agent.\n *\n * Falls back to SQL polling when the run is not in local memory\n * (cross-isolate reconnection on Workers).\n */\nexport function subscribeToRun(\n  runId: string,\n  fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n  const run = activeRuns.get(runId);\n  if (run) {\n    return subscribeInMemory(run, fromSeq);\n  }\n  // Not in local memory — try SQL (cross-isolate path)\n  return subscribeFromSQL(runId, fromSeq);\n}\n\n/** In-memory subscription (same isolate, fast path) */\nfunction subscribeInMemory(\n  run: ActiveRun,\n  fromSeq: number,\n): ReadableStream<Uint8Array> {\n  const encoder = new TextEncoder();\n  let subscriberRef: ((event: RunEvent) => void) | null = null;\n  let pingTimer: ReturnType<typeof setInterval> | null = null;\n\n  return new ReadableStream({\n    start(controller) {\n      const ping = () => {\n        try {\n          controller.enqueue(encoder.encode(`: ping ${Date.now()}\\n\\n`));\n        } catch {\n          if (subscriberRef) run.subscribers.delete(subscriberRef);\n          if (pingTimer) clearInterval(pingTimer);\n        }\n      };\n      ping();\n      pingTimer = setInterval(ping, 10_000);\n\n      // Replay buffered events from fromSeq\n      for (let i = fromSeq; i < run.events.length; i++) {\n        try {\n          controller.enqueue(\n            encoder.encode(\n              `data: ${JSON.stringify({ ...run.events[i].event, seq: run.events[i].seq })}\\n\\n`,\n            ),\n          );\n        } catch {\n          return;\n        }\n      }\n\n      // If run is already done, close immediately\n      if (run.status !== \"running\") {\n        if (pingTimer) clearInterval(pingTimer);\n        controller.close();\n        return;\n      }\n\n      // Subscribe to live events\n      subscriberRef = (event: RunEvent) => {\n        try {\n          controller.enqueue(\n            encoder.encode(\n              `data: ${JSON.stringify({ ...event.event, seq: event.seq })}\\n\\n`,\n            ),\n          );\n          // Close stream after terminal events\n          if (\n            event.event.type === \"done\" ||\n            event.event.type === \"error\" ||\n            event.event.type === \"missing_api_key\" ||\n            event.event.type === \"loop_limit\"\n          ) {\n            run.subscribers.delete(subscriberRef!);\n            if (pingTimer) clearInterval(pingTimer);\n            controller.close();\n          }\n        } catch {\n          run.subscribers.delete(subscriberRef!);\n        }\n      };\n\n      run.subscribers.add(subscriberRef);\n    },\n    cancel() {\n      // Only unsubscribe — do NOT abort the agent run\n      if (subscriberRef) run.subscribers.delete(subscriberRef);\n      if (pingTimer) clearInterval(pingTimer);\n    },\n  });\n}\n\n/** SQL-based subscription (cross-isolate, polling) */\nfunction subscribeFromSQL(\n  runId: string,\n  fromSeq: number,\n): ReadableStream<Uint8Array> | null {\n  const encoder = new TextEncoder();\n  let cancelled = false;\n  let pollTimer: ReturnType<typeof setTimeout> | null = null;\n  let pingTimer: ReturnType<typeof setInterval> | null = null;\n\n  return new ReadableStream({\n    async start(controller) {\n      let lastSeq = fromSeq;\n      const ping = () => {\n        try {\n          controller.enqueue(encoder.encode(`: ping ${Date.now()}\\n\\n`));\n        } catch {\n          cancelled = true;\n          if (pingTimer) clearInterval(pingTimer);\n        }\n      };\n      ping();\n      pingTimer = setInterval(ping, 10_000);\n\n      const poll = async () => {\n        if (cancelled) return;\n        try {\n          // Read new events from SQL\n          const events = await getRunEventsSince(runId, lastSeq);\n          for (const { seq, eventData } of events) {\n            let parsed: any;\n            try {\n              parsed = JSON.parse(eventData);\n            } catch {\n              continue;\n            }\n            try {\n              controller.enqueue(\n                encoder.encode(\n                  `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n                ),\n              );\n            } catch {\n              cancelled = true;\n              return;\n            }\n            lastSeq = seq;\n\n            // Close on terminal events\n            if (\n              parsed.type === \"done\" ||\n              parsed.type === \"error\" ||\n              parsed.type === \"missing_api_key\" ||\n              parsed.type === \"loop_limit\"\n            ) {\n              if (pingTimer) clearInterval(pingTimer);\n              controller.close();\n              return;\n            }\n          }\n\n          // Check if run completed (no terminal event but status changed)\n          if (events.length === 0) {\n            // Opportunistically reap a stale producer before trusting SQL's\n            // \"running\" status — otherwise a crashed server leaves us polling\n            // forever.\n            await reapIfStale(runId).catch(() => {});\n            const run = await getRunById(runId);\n            if (!run || run.status !== \"running\") {\n              // Run ended — do one final event read, then close\n              const finalEvents = await getRunEventsSince(runId, lastSeq);\n              for (const { seq, eventData } of finalEvents) {\n                let parsed: any;\n                try {\n                  parsed = JSON.parse(eventData);\n                } catch {\n                  continue;\n                }\n                try {\n                  controller.enqueue(\n                    encoder.encode(\n                      `data: ${JSON.stringify({ ...parsed, seq })}\\n\\n`,\n                    ),\n                  );\n                } catch {\n                  cancelled = true;\n                  return;\n                }\n              }\n              if (pingTimer) clearInterval(pingTimer);\n              controller.close();\n              return;\n            }\n          }\n\n          // Schedule next poll\n          if (!cancelled) {\n            pollTimer = setTimeout(poll, 500);\n          }\n        } catch {\n          // SQL error — close stream\n          try {\n            if (pingTimer) clearInterval(pingTimer);\n            controller.close();\n          } catch {}\n        }\n      };\n\n      // Verify run exists before starting poll\n      try {\n        const run = await getRunById(runId);\n        if (!run) {\n          if (pingTimer) clearInterval(pingTimer);\n          controller.close();\n          return;\n        }\n      } catch {\n        controller.close();\n        return;\n      }\n\n      await poll();\n    },\n    cancel() {\n      cancelled = true;\n      if (pollTimer) clearTimeout(pollTimer);\n      if (pingTimer) clearInterval(pingTimer);\n    },\n  });\n}\n\n/** Get the active run for a thread (if any) — checks memory then SQL */\nexport function getActiveRunForThread(threadId: string): ActiveRun | null {\n  const runId = threadToRun.get(threadId);\n  if (runId) {\n    const run = activeRuns.get(runId);\n    if (run) return run;\n  }\n  return null;\n}\n\n/**\n * Async version that also checks SQL — for cross-isolate access.\n * Used by the /runs/active endpoint.\n *\n * Returns `heartbeatAt` so the client can independently decide a run is\n * dead even before the server-side stale reap has fired.\n */\nexport async function getActiveRunForThreadAsync(threadId: string): Promise<{\n  runId: string;\n  threadId: string;\n  status: string;\n  heartbeatAt: number;\n} | null> {\n  // Check memory first — return both running AND recently-completed runs\n  // that still have events in memory. This allows sub-agent tabs to replay\n  // the full conversation from completed runs via SSE.\n  const memRun = getActiveRunForThread(threadId);\n  if (memRun && (memRun.status === \"running\" || memRun.events.length > 0)) {\n    return {\n      runId: memRun.runId,\n      threadId: memRun.threadId,\n      status: memRun.status,\n      // In-memory means this isolate is the producer. By definition, the\n      // heartbeat is fresh as of \"now\" — the client can trust this.\n      heartbeatAt: Date.now(),\n    };\n  }\n  // Fall back to SQL\n  try {\n    const sqlRun = await getRunByThread(threadId);\n    if (sqlRun && sqlRun.status === \"running\") {\n      // If the producer is dead (no recent heartbeat), reap before the\n      // client can see a stale \"running\" status and enter a reconnect\n      // loop it can never exit.\n      const reaped = await reapIfStale(sqlRun.id).catch(() => false);\n      if (reaped) return null;\n      return {\n        runId: sqlRun.id,\n        threadId: sqlRun.threadId,\n        status: sqlRun.status,\n        heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,\n      };\n    }\n  } catch {\n    // SQL error — fall through\n  }\n  return null;\n}\n\n/** Get a run by ID */\nexport function getRun(runId: string): ActiveRun | null {\n  return activeRuns.get(runId) ?? null;\n}\n\n/** Explicitly abort a run (e.g. Stop button) */\nexport function abortRun(runId: string): boolean {\n  const run = activeRuns.get(runId);\n  if (run) {\n    run.abort.abort();\n  }\n  // Also mark as aborted in SQL (for cross-isolate abort on Workers)\n  markRunAborted(runId).catch(() => {});\n  return !!run;\n}\n"]}

package/dist/agent/run-store.d.ts

@@ -14,7 +14,7 @@ export declare function updateRunHeartbeat(runId: string): Promise<void>;
  * Returns true if the row was reaped.
  */
 export declare function reapIfStale(runId: string, maxStaleMs?: number): Promise<boolean>;
-export declare function updateRunStatus(runId: string, status: "completed" | "errored"): Promise<void>;
+export declare function updateRunStatus(runId: string, status: "completed" | "errored" | "aborted"): Promise<void>;
 export declare function markRunAborted(runId: string): Promise<void>;
 export declare function isRunAborted(runId: string): Promise<boolean>;
 export declare function insertRunEvent(runId: string, seq: number, eventData: string): Promise<void>;

package/dist/agent/run-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-store.d.ts","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AASA;;;;;GAKG;AACH,eAAO,MAAM,YAAY,OAAQ,CAAC;AA2ClC,wBAAsB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQ3E;AAED,+EAA+E;AAC/E,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrE;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAqB,GAChC,OAAO,CAAC,OAAO,CAAC,CAalB;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAC9B,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjE;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAUlE;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAWpD;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,CAoBR;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,GAAG,IAAI,CAAC,CAsBR;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAYxD;AAED;;oDAEoD;AACpD,wBAAsB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8BvE"}
+{"version":3,"file":"run-store.d.ts","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AASA;;;;;GAKG;AACH,eAAO,MAAM,YAAY,OAAQ,CAAC;AAqDlC,wBAAsB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQ3E;AAED,+EAA+E;AAC/E,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOrE;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAqB,GAChC,OAAO,CAAC,OAAO,CAAC,CAiBlB;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,SAAS,GAC1C,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjE;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAUlE;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAWpD;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,CAoBR;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,GAAG,IAAI,CAAC,CAsBR;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAwBxD;AAED;;oDAEoD;AACpD,wBAAsB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0CvE"}

package/dist/agent/run-store.js

@@ -12,6 +12,13 @@ let _initPromise;
  * stranding the user on "Thinking..." for up to 90s after a process death.
  */
 export const RUN_STALE_MS = 6_000;
+const STALE_RUN_ERROR_EVENT = {
+    type: "error",
+    error: "The agent stopped before it could finish. It may have hit a server timeout or the worker may have been interrupted.",
+    errorCode: "stale_run",
+    recoverable: true,
+    details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.",
+};
 async function ensureRunTables() {
     if (!_initPromise) {
         _initPromise = (async () => {
@@ -85,7 +92,11 @@ export async function reapIfStale(runId, maxStaleMs = RUN_STALE_MS) {
             AND COALESCE(heartbeat_at, started_at) < ?`,
         args: [Date.now(), runId, cutoff],
     });
-    return (rowsAffected ?? 0) > 0;
+    const reaped = (rowsAffected ?? 0) > 0;
+    if (reaped) {
+        await appendTerminalRunEvent(runId, STALE_RUN_ERROR_EVENT).catch(() => { });
+    }
+    return reaped;
 }
 export async function updateRunStatus(runId, status) {
     await ensureRunTables();
@@ -177,6 +188,12 @@ export async function reapAllStaleRuns() {
     await ensureRunTables();
     const client = getDbExec();
     const heartbeatCutoff = Date.now() - RUN_STALE_MS;
+    const stale = await client.execute({
+        sql: `SELECT id FROM agent_runs
+          WHERE status = 'running'
+            AND COALESCE(heartbeat_at, started_at) < ?`,
+        args: [heartbeatCutoff],
+    });
     const { rowsAffected } = await client.execute({
         sql: `UPDATE agent_runs
           SET status = 'errored', completed_at = ?
@@ -184,6 +201,12 @@ export async function reapAllStaleRuns() {
             AND COALESCE(heartbeat_at, started_at) < ?`,
         args: [Date.now(), heartbeatCutoff],
     });
+    for (const row of stale.rows) {
+        const id = row.id;
+        if (typeof id === "string") {
+            await appendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT).catch(() => { });
+        }
+    }
     return rowsAffected ?? 0;
 }
 /** Delete completed/errored runs older than the given threshold,
@@ -195,12 +218,18 @@ export async function cleanupOldRuns(olderThanMs) {
     const cutoff = Date.now() - olderThanMs;
     // Expire stale running rows on the absolute-age threshold — safety net
     // for runs that never received a heartbeat (very old deployments).
+    const heartbeatCutoff = Date.now() - RUN_STALE_MS;
+    const stale = await client.execute({
+        sql: `SELECT id FROM agent_runs
+          WHERE status = 'running'
+            AND COALESCE(heartbeat_at, started_at) < ?`,
+        args: [heartbeatCutoff],
+    });
     await client.execute({
         sql: `UPDATE agent_runs SET status = 'errored', completed_at = ? WHERE status = 'running' AND started_at < ?`,
         args: [Date.now(), cutoff],
     });
     // Also expire runs whose heartbeat is stale — producer has died.
-    const heartbeatCutoff = Date.now() - RUN_STALE_MS;
     await client.execute({
         sql: `UPDATE agent_runs
           SET status = 'errored', completed_at = ?
@@ -208,6 +237,12 @@ export async function cleanupOldRuns(olderThanMs) {
             AND COALESCE(heartbeat_at, started_at) < ?`,
         args: [Date.now(), heartbeatCutoff],
     });
+    for (const row of stale.rows) {
+        const id = row.id;
+        if (typeof id === "string") {
+            await appendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT).catch(() => { });
+        }
+    }
     // Delete events for old non-running runs
     await client.execute({
         sql: `DELETE FROM agent_run_events WHERE run_id IN (
@@ -220,4 +255,32 @@ export async function cleanupOldRuns(olderThanMs) {
         args: [cutoff],
     });
 }
+async function appendTerminalRunEvent(runId, event) {
+    await ensureRunTables();
+    const client = getDbExec();
+    const { rows } = await client.execute({
+        sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,
+        args: [runId],
+    });
+    const last = rows[0];
+    if (last?.event_data) {
+        try {
+            const parsed = JSON.parse(last.event_data);
+            if (parsed?.type === "done" ||
+                parsed?.type === "error" ||
+                parsed?.type === "missing_api_key" ||
+                parsed?.type === "loop_limit") {
+                return;
+            }
+        }
+        catch {
+            // Ignore malformed rows and append the terminal event.
+        }
+    }
+    const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;
+    await client.execute({
+        sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?)`,
+        args: [runId, nextSeq, JSON.stringify(event)],
+    });
+}
 //# sourceMappingURL=run-store.js.map

package/dist/agent/run-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-store.js","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEjE,IAAI,YAAuC,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC;AAElC,KAAK,UAAU,eAAe;IAC5B,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;;;;;uBAKJ,OAAO,EAAE;yBACP,OAAO,EAAE;yBACT,OAAO,EAAE;;OAE3B,CAAC,CAAC;YACH,8CAA8C;YAC9C,IAAI,CAAC;gBACH,IAAI,UAAU,EAAE,EAAE,CAAC;oBACjB,MAAM,MAAM,CAAC,OAAO,CAClB,gEAAgE,OAAO,EAAE,EAAE,CAC5E,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC,OAAO,CAClB,kDAAkD,OAAO,EAAE,EAAE,CAC9D,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,MAAM,MAAM,CAAC,OAAO,CAAC;;;gBAGX,OAAO,EAAE;;;;OAIlB,CAAC,CAAC;QACL,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAU,EAAE,QAAgB;IAC1D,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yGAAyG;QAC9G,IAAI,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;KAC/B,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAa;IACpD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,qDAAqD;QAC1D,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,aAAqB,YAAY;IAEjC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;IACvC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;;uDAI8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,MAA+B;IAE/B,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,iEAAiE;QACtE,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAClC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAa;IAChD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yEAAyE;QAC9E,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa;IAC9C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,4CAA4C;QACjD,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,OAAO,CACL,IAAI,CAAC,MAAM,GAAG,CAAC,IAAK,IAAI,CAAC,CAAC,CAAwB,CAAC,MAAM,KAAK,SAAS,CACxE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,GAAW,EACX,SAAiB;IAEjB,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yEAAyE;QAC9E,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,OAAe;IAEf,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,6FAA6F;QAClG,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;KACvB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,MAAM,GAAG,GAAG,CAAiD,CAAC;QAC9D,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa;IAM5C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,uEAAuE;QAC5E,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAKf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IAOnD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,oJAAoJ;QACzJ,IAAI,EAAE,CAAC,QAAQ,CAAC;KACjB,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAMf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/B,WAAW,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;KACpE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,OAAO,YAAY,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;oDAEoD;AACpD,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IACxC,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,wGAAwG;QAC7G,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IACH,iEAAiE;IACjE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,yCAAyC;IACzC,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;MAEH;QACF,IAAI,EAAE,CAAC,MAAM,CAAC;KACf,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,uEAAuE;QAC5E,IAAI,EAAE,CAAC,MAAM,CAAC;KACf,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL persistence for agent runs and events.\n * Enables cross-isolate access on Cloudflare Workers and\n * reliable reconnection after page refreshes.\n */\nimport { getDbExec, intType, isPostgres } from \"../db/client.js\";\n\nlet _initPromise: Promise<void> | undefined;\n\n/**\n * Max time without a heartbeat before a \"running\" run is considered dead.\n * The run-manager heartbeats every 1.5s, so 6s tolerates 3 missed writes.\n * Short window is what makes reload recovery feel instant instead of\n * stranding the user on \"Thinking...\" for up to 90s after a process death.\n */\nexport const RUN_STALE_MS = 6_000;\n\nasync function ensureRunTables(): Promise<void> {\n  if (!_initPromise) {\n    _initPromise = (async () => {\n      const client = getDbExec();\n      await client.execute(`\n        CREATE TABLE IF NOT EXISTS agent_runs (\n          id TEXT PRIMARY KEY,\n          thread_id TEXT NOT NULL,\n          status TEXT NOT NULL DEFAULT 'running',\n          started_at ${intType()} NOT NULL,\n          completed_at ${intType()},\n          heartbeat_at ${intType()}\n        )\n      `);\n      // Backfill heartbeat_at on older deployments.\n      try {\n        if (isPostgres()) {\n          await client.execute(\n            `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS heartbeat_at ${intType()}`,\n          );\n        } else {\n          await client.execute(\n            `ALTER TABLE agent_runs ADD COLUMN heartbeat_at ${intType()}`,\n          );\n        }\n      } catch {\n        // Column already exists — ignore\n      }\n      await client.execute(`\n        CREATE TABLE IF NOT EXISTS agent_run_events (\n          run_id TEXT NOT NULL,\n          seq ${intType()} NOT NULL,\n          event_data TEXT NOT NULL,\n          PRIMARY KEY (run_id, seq)\n        )\n      `);\n    })();\n  }\n  return _initPromise;\n}\n\nexport async function insertRun(id: string, threadId: string): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `INSERT INTO agent_runs (id, thread_id, status, started_at, heartbeat_at) VALUES (?, ?, 'running', ?, ?)`,\n    args: [id, threadId, now, now],\n  });\n}\n\n/** Update the run's liveness heartbeat. Called periodically by run-manager. */\nexport async function updateRunHeartbeat(runId: string): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `UPDATE agent_runs SET heartbeat_at = ? WHERE id = ?`,\n    args: [Date.now(), runId],\n  });\n}\n\n/**\n * If the given run is marked \"running\" in SQL but its heartbeat is stale\n * (producer likely crashed), flip it to \"errored\" so watchers stop waiting.\n * Returns true if the row was reaped.\n */\nexport async function reapIfStale(\n  runId: string,\n  maxStaleMs: number = RUN_STALE_MS,\n): Promise<boolean> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const cutoff = Date.now() - maxStaleMs;\n  const { rowsAffected } = await client.execute({\n    sql: `UPDATE agent_runs\n          SET status = 'errored', completed_at = ?\n          WHERE id = ?\n            AND status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [Date.now(), runId, cutoff],\n  });\n  return (rowsAffected ?? 0) > 0;\n}\n\nexport async function updateRunStatus(\n  runId: string,\n  status: \"completed\" | \"errored\",\n): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `UPDATE agent_runs SET status = ?, completed_at = ? WHERE id = ?`,\n    args: [status, Date.now(), runId],\n  });\n}\n\nexport async function markRunAborted(runId: string): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `UPDATE agent_runs SET status = 'aborted', completed_at = ? WHERE id = ?`,\n    args: [Date.now(), runId],\n  });\n}\n\nexport async function isRunAborted(runId: string): Promise<boolean> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT status FROM agent_runs WHERE id = ?`,\n    args: [runId],\n  });\n  return (\n    rows.length > 0 && (rows[0] as { status: string }).status === \"aborted\"\n  );\n}\n\nexport async function insertRunEvent(\n  runId: string,\n  seq: number,\n  eventData: string,\n): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?)`,\n    args: [runId, seq, eventData],\n  });\n}\n\nexport async function getRunEventsSince(\n  runId: string,\n  fromSeq: number,\n): Promise<Array<{ seq: number; eventData: string }>> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? AND seq >= ? ORDER BY seq ASC`,\n    args: [runId, fromSeq],\n  });\n  return rows.map((r) => {\n    const row = r as { seq: number | string; event_data: string };\n    return { seq: Number(row.seq), eventData: row.event_data };\n  });\n}\n\nexport async function getRunById(runId: string): Promise<{\n  id: string;\n  threadId: string;\n  status: string;\n  startedAt: number;\n} | null> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT id, thread_id, status, started_at FROM agent_runs WHERE id = ?`,\n    args: [runId],\n  });\n  if (rows.length === 0) return null;\n  const r = rows[0] as {\n    id: string;\n    thread_id: string;\n    status: string;\n    started_at: number | string;\n  };\n  return {\n    id: r.id,\n    threadId: r.thread_id,\n    status: r.status,\n    startedAt: Number(r.started_at),\n  };\n}\n\nexport async function getRunByThread(threadId: string): Promise<{\n  id: string;\n  threadId: string;\n  status: string;\n  startedAt: number;\n  heartbeatAt: number | null;\n} | null> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT id, thread_id, status, started_at, heartbeat_at FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`,\n    args: [threadId],\n  });\n  if (rows.length === 0) return null;\n  const r = rows[0] as {\n    id: string;\n    thread_id: string;\n    status: string;\n    started_at: number | string;\n    heartbeat_at: number | string | null;\n  };\n  return {\n    id: r.id,\n    threadId: r.thread_id,\n    status: r.status,\n    startedAt: Number(r.started_at),\n    heartbeatAt: r.heartbeat_at == null ? null : Number(r.heartbeat_at),\n  };\n}\n\n/**\n * Expire any \"running\" rows whose heartbeat is stale — producer died.\n * Safe to call at server startup on multi-isolate deployments: only rows\n * without a fresh heartbeat get reaped, so runs owned by OTHER live\n * isolates (which keep heartbeating) are left alone.\n */\nexport async function reapAllStaleRuns(): Promise<number> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n  const { rowsAffected } = await client.execute({\n    sql: `UPDATE agent_runs\n          SET status = 'errored', completed_at = ?\n          WHERE status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [Date.now(), heartbeatCutoff],\n  });\n  return rowsAffected ?? 0;\n}\n\n/** Delete completed/errored runs older than the given threshold,\n *  and expire stale \"running\" rows that haven't had activity\n *  (e.g. worker crashed before updating status). */\nexport async function cleanupOldRuns(olderThanMs: number): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const cutoff = Date.now() - olderThanMs;\n  // Expire stale running rows on the absolute-age threshold — safety net\n  // for runs that never received a heartbeat (very old deployments).\n  await client.execute({\n    sql: `UPDATE agent_runs SET status = 'errored', completed_at = ? WHERE status = 'running' AND started_at < ?`,\n    args: [Date.now(), cutoff],\n  });\n  // Also expire runs whose heartbeat is stale — producer has died.\n  const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n  await client.execute({\n    sql: `UPDATE agent_runs\n          SET status = 'errored', completed_at = ?\n          WHERE status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [Date.now(), heartbeatCutoff],\n  });\n  // Delete events for old non-running runs\n  await client.execute({\n    sql: `DELETE FROM agent_run_events WHERE run_id IN (\n      SELECT id FROM agent_runs WHERE status != 'running' AND completed_at < ?\n    )`,\n    args: [cutoff],\n  });\n  await client.execute({\n    sql: `DELETE FROM agent_runs WHERE status != 'running' AND completed_at < ?`,\n    args: [cutoff],\n  });\n}\n"]}
+{"version":3,"file":"run-store.js","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEjE,IAAI,YAAuC,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC;AAElC,MAAM,qBAAqB,GAAG;IAC5B,IAAI,EAAE,OAAO;IACb,KAAK,EACH,qHAAqH;IACvH,SAAS,EAAE,WAAW;IACtB,WAAW,EAAE,IAAI;IACjB,OAAO,EACL,gIAAgI;CAC1H,CAAC;AAEX,KAAK,UAAU,eAAe;IAC5B,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;;;;;uBAKJ,OAAO,EAAE;yBACP,OAAO,EAAE;yBACT,OAAO,EAAE;;OAE3B,CAAC,CAAC;YACH,8CAA8C;YAC9C,IAAI,CAAC;gBACH,IAAI,UAAU,EAAE,EAAE,CAAC;oBACjB,MAAM,MAAM,CAAC,OAAO,CAClB,gEAAgE,OAAO,EAAE,EAAE,CAC5E,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC,OAAO,CAClB,kDAAkD,OAAO,EAAE,EAAE,CAC9D,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,MAAM,MAAM,CAAC,OAAO,CAAC;;;gBAGX,OAAO,EAAE;;;;OAIlB,CAAC,CAAC;QACL,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAU,EAAE,QAAgB;IAC1D,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yGAAyG;QAC9G,IAAI,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;KAC/B,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAa;IACpD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,qDAAqD;QAC1D,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,aAAqB,YAAY;IAEjC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;IACvC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;;uDAI8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,sBAAsB,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,MAA2C;IAE3C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,iEAAiE;QACtE,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAClC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAa;IAChD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yEAAyE;QAC9E,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa;IAC9C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,4CAA4C;QACjD,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,OAAO,CACL,IAAI,CAAC,MAAM,GAAG,CAAC,IAAK,IAAI,CAAC,CAAC,CAAwB,CAAC,MAAM,KAAK,SAAS,CACxE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,GAAW,EACX,SAAiB;IAEjB,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yEAAyE;QAC9E,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,OAAe;IAEf,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,6FAA6F;QAClG,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;KACvB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,MAAM,GAAG,GAAG,CAAiD,CAAC;QAC9D,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa;IAM5C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,uEAAuE;QAC5E,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAKf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IAOnD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,oJAAoJ;QACzJ,IAAI,EAAE,CAAC,QAAQ,CAAC;KACjB,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAMf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/B,WAAW,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;KACpE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACjC,GAAG,EAAE;;uDAE8C;QACnD,IAAI,EAAE,CAAC,eAAe,CAAC;KACxB,CAAC,CAAC;IACH,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAI,GAAwB,CAAC,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,sBAAsB,CAAC,EAAE,EAAE,qBAAqB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,OAAO,YAAY,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;oDAEoD;AACpD,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IACxC,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACjC,GAAG,EAAE;;uDAE8C;QACnD,IAAI,EAAE,CAAC,eAAe,CAAC;KACxB,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,wGAAwG;QAC7G,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IACH,iEAAiE;IACjE,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAI,GAAwB,CAAC,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,sBAAsB,CAAC,EAAE,EAAE,qBAAqB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;MAEH;QACF,IAAI,EAAE,CAAC,MAAM,CAAC;KACf,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,uEAAuE;QAC5E,IAAI,EAAE,CAAC,MAAM,CAAC;KACf,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,KAAa,EACb,KAA8B;IAE9B,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,yFAAyF;QAC9F,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAEN,CAAC;IACd,IAAI,IAAI,EAAE,UAAU,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3C,IACE,MAAM,EAAE,IAAI,KAAK,MAAM;gBACvB,MAAM,EAAE,IAAI,KAAK,OAAO;gBACxB,MAAM,EAAE,IAAI,KAAK,iBAAiB;gBAClC,MAAM,EAAE,IAAI,KAAK,YAAY,EAC7B,CAAC;gBACD,OAAO;YACT,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;QACzD,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yEAAyE;QAC9E,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;KAC9C,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL persistence for agent runs and events.\n * Enables cross-isolate access on Cloudflare Workers and\n * reliable reconnection after page refreshes.\n */\nimport { getDbExec, intType, isPostgres } from \"../db/client.js\";\n\nlet _initPromise: Promise<void> | undefined;\n\n/**\n * Max time without a heartbeat before a \"running\" run is considered dead.\n * The run-manager heartbeats every 1.5s, so 6s tolerates 3 missed writes.\n * Short window is what makes reload recovery feel instant instead of\n * stranding the user on \"Thinking...\" for up to 90s after a process death.\n */\nexport const RUN_STALE_MS = 6_000;\n\nconst STALE_RUN_ERROR_EVENT = {\n  type: \"error\",\n  error:\n    \"The agent stopped before it could finish. It may have hit a server timeout or the worker may have been interrupted.\",\n  errorCode: \"stale_run\",\n  recoverable: true,\n  details:\n    \"The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.\",\n} as const;\n\nasync function ensureRunTables(): Promise<void> {\n  if (!_initPromise) {\n    _initPromise = (async () => {\n      const client = getDbExec();\n      await client.execute(`\n        CREATE TABLE IF NOT EXISTS agent_runs (\n          id TEXT PRIMARY KEY,\n          thread_id TEXT NOT NULL,\n          status TEXT NOT NULL DEFAULT 'running',\n          started_at ${intType()} NOT NULL,\n          completed_at ${intType()},\n          heartbeat_at ${intType()}\n        )\n      `);\n      // Backfill heartbeat_at on older deployments.\n      try {\n        if (isPostgres()) {\n          await client.execute(\n            `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS heartbeat_at ${intType()}`,\n          );\n        } else {\n          await client.execute(\n            `ALTER TABLE agent_runs ADD COLUMN heartbeat_at ${intType()}`,\n          );\n        }\n      } catch {\n        // Column already exists — ignore\n      }\n      await client.execute(`\n        CREATE TABLE IF NOT EXISTS agent_run_events (\n          run_id TEXT NOT NULL,\n          seq ${intType()} NOT NULL,\n          event_data TEXT NOT NULL,\n          PRIMARY KEY (run_id, seq)\n        )\n      `);\n    })();\n  }\n  return _initPromise;\n}\n\nexport async function insertRun(id: string, threadId: string): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `INSERT INTO agent_runs (id, thread_id, status, started_at, heartbeat_at) VALUES (?, ?, 'running', ?, ?)`,\n    args: [id, threadId, now, now],\n  });\n}\n\n/** Update the run's liveness heartbeat. Called periodically by run-manager. */\nexport async function updateRunHeartbeat(runId: string): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `UPDATE agent_runs SET heartbeat_at = ? WHERE id = ?`,\n    args: [Date.now(), runId],\n  });\n}\n\n/**\n * If the given run is marked \"running\" in SQL but its heartbeat is stale\n * (producer likely crashed), flip it to \"errored\" so watchers stop waiting.\n * Returns true if the row was reaped.\n */\nexport async function reapIfStale(\n  runId: string,\n  maxStaleMs: number = RUN_STALE_MS,\n): Promise<boolean> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const cutoff = Date.now() - maxStaleMs;\n  const { rowsAffected } = await client.execute({\n    sql: `UPDATE agent_runs\n          SET status = 'errored', completed_at = ?\n          WHERE id = ?\n            AND status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [Date.now(), runId, cutoff],\n  });\n  const reaped = (rowsAffected ?? 0) > 0;\n  if (reaped) {\n    await appendTerminalRunEvent(runId, STALE_RUN_ERROR_EVENT).catch(() => {});\n  }\n  return reaped;\n}\n\nexport async function updateRunStatus(\n  runId: string,\n  status: \"completed\" | \"errored\" | \"aborted\",\n): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `UPDATE agent_runs SET status = ?, completed_at = ? WHERE id = ?`,\n    args: [status, Date.now(), runId],\n  });\n}\n\nexport async function markRunAborted(runId: string): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `UPDATE agent_runs SET status = 'aborted', completed_at = ? WHERE id = ?`,\n    args: [Date.now(), runId],\n  });\n}\n\nexport async function isRunAborted(runId: string): Promise<boolean> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT status FROM agent_runs WHERE id = ?`,\n    args: [runId],\n  });\n  return (\n    rows.length > 0 && (rows[0] as { status: string }).status === \"aborted\"\n  );\n}\n\nexport async function insertRunEvent(\n  runId: string,\n  seq: number,\n  eventData: string,\n): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  await client.execute({\n    sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?)`,\n    args: [runId, seq, eventData],\n  });\n}\n\nexport async function getRunEventsSince(\n  runId: string,\n  fromSeq: number,\n): Promise<Array<{ seq: number; eventData: string }>> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? AND seq >= ? ORDER BY seq ASC`,\n    args: [runId, fromSeq],\n  });\n  return rows.map((r) => {\n    const row = r as { seq: number | string; event_data: string };\n    return { seq: Number(row.seq), eventData: row.event_data };\n  });\n}\n\nexport async function getRunById(runId: string): Promise<{\n  id: string;\n  threadId: string;\n  status: string;\n  startedAt: number;\n} | null> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT id, thread_id, status, started_at FROM agent_runs WHERE id = ?`,\n    args: [runId],\n  });\n  if (rows.length === 0) return null;\n  const r = rows[0] as {\n    id: string;\n    thread_id: string;\n    status: string;\n    started_at: number | string;\n  };\n  return {\n    id: r.id,\n    threadId: r.thread_id,\n    status: r.status,\n    startedAt: Number(r.started_at),\n  };\n}\n\nexport async function getRunByThread(threadId: string): Promise<{\n  id: string;\n  threadId: string;\n  status: string;\n  startedAt: number;\n  heartbeatAt: number | null;\n} | null> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT id, thread_id, status, started_at, heartbeat_at FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`,\n    args: [threadId],\n  });\n  if (rows.length === 0) return null;\n  const r = rows[0] as {\n    id: string;\n    thread_id: string;\n    status: string;\n    started_at: number | string;\n    heartbeat_at: number | string | null;\n  };\n  return {\n    id: r.id,\n    threadId: r.thread_id,\n    status: r.status,\n    startedAt: Number(r.started_at),\n    heartbeatAt: r.heartbeat_at == null ? null : Number(r.heartbeat_at),\n  };\n}\n\n/**\n * Expire any \"running\" rows whose heartbeat is stale — producer died.\n * Safe to call at server startup on multi-isolate deployments: only rows\n * without a fresh heartbeat get reaped, so runs owned by OTHER live\n * isolates (which keep heartbeating) are left alone.\n */\nexport async function reapAllStaleRuns(): Promise<number> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n  const stale = await client.execute({\n    sql: `SELECT id FROM agent_runs\n          WHERE status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [heartbeatCutoff],\n  });\n  const { rowsAffected } = await client.execute({\n    sql: `UPDATE agent_runs\n          SET status = 'errored', completed_at = ?\n          WHERE status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [Date.now(), heartbeatCutoff],\n  });\n  for (const row of stale.rows) {\n    const id = (row as { id?: unknown }).id;\n    if (typeof id === \"string\") {\n      await appendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT).catch(() => {});\n    }\n  }\n  return rowsAffected ?? 0;\n}\n\n/** Delete completed/errored runs older than the given threshold,\n *  and expire stale \"running\" rows that haven't had activity\n *  (e.g. worker crashed before updating status). */\nexport async function cleanupOldRuns(olderThanMs: number): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const cutoff = Date.now() - olderThanMs;\n  // Expire stale running rows on the absolute-age threshold — safety net\n  // for runs that never received a heartbeat (very old deployments).\n  const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n  const stale = await client.execute({\n    sql: `SELECT id FROM agent_runs\n          WHERE status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [heartbeatCutoff],\n  });\n  await client.execute({\n    sql: `UPDATE agent_runs SET status = 'errored', completed_at = ? WHERE status = 'running' AND started_at < ?`,\n    args: [Date.now(), cutoff],\n  });\n  // Also expire runs whose heartbeat is stale — producer has died.\n  await client.execute({\n    sql: `UPDATE agent_runs\n          SET status = 'errored', completed_at = ?\n          WHERE status = 'running'\n            AND COALESCE(heartbeat_at, started_at) < ?`,\n    args: [Date.now(), heartbeatCutoff],\n  });\n  for (const row of stale.rows) {\n    const id = (row as { id?: unknown }).id;\n    if (typeof id === \"string\") {\n      await appendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT).catch(() => {});\n    }\n  }\n  // Delete events for old non-running runs\n  await client.execute({\n    sql: `DELETE FROM agent_run_events WHERE run_id IN (\n      SELECT id FROM agent_runs WHERE status != 'running' AND completed_at < ?\n    )`,\n    args: [cutoff],\n  });\n  await client.execute({\n    sql: `DELETE FROM agent_runs WHERE status != 'running' AND completed_at < ?`,\n    args: [cutoff],\n  });\n}\n\nasync function appendTerminalRunEvent(\n  runId: string,\n  event: Record<string, unknown>,\n): Promise<void> {\n  await ensureRunTables();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,\n    args: [runId],\n  });\n  const last = rows[0] as\n    | { seq?: number | string; event_data?: string }\n    | undefined;\n  if (last?.event_data) {\n    try {\n      const parsed = JSON.parse(last.event_data);\n      if (\n        parsed?.type === \"done\" ||\n        parsed?.type === \"error\" ||\n        parsed?.type === \"missing_api_key\" ||\n        parsed?.type === \"loop_limit\"\n      ) {\n        return;\n      }\n    } catch {\n      // Ignore malformed rows and append the terminal event.\n    }\n  }\n  const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;\n  await client.execute({\n    sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?)`,\n    args: [runId, nextSeq, JSON.stringify(event)],\n  });\n}\n"]}

package/dist/agent/thread-data-builder.d.ts

@@ -20,6 +20,9 @@ export declare function buildAssistantMessage(events: RunEvent[], runId?: string
     status: {
         type: "complete";
         reason: "stop";
+    } | {
+        type: "incomplete";
+        reason: "error";
     };
     metadata: Record<string, unknown>;
 } | null;

package/dist/agent/thread-data-builder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"thread-data-builder.d.ts","sourceRoot":"","sources":["../../src/agent/thread-data-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,QAAQ,EAAE,EAClB,KAAK,CAAC,EAAE,MAAM,GACb;IACD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GAAG,IAAI,CA6DP;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,GAAG,GAAG;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB,CAyBA"}
+{"version":3,"file":"thread-data-builder.d.ts","sourceRoot":"","sources":["../../src/agent/thread-data-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,QAAQ,EAAE,EAClB,KAAK,CAAC,EAAE,MAAM,GACb;IACD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EACF;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GACpC;QAAE,IAAI,EAAE,YAAY,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAC5C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GAAG,IAAI,CAmHP;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,GAAG,GAAG;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB,CAyBA"}