@agent-native/core 0.24.7 → 0.24.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/engine/anthropic-engine.d.ts.map +1 -1
- package/dist/agent/engine/anthropic-engine.js +7 -2
- package/dist/agent/engine/anthropic-engine.js.map +1 -1
- package/dist/agent/engine/translate-anthropic.d.ts +23 -1
- package/dist/agent/engine/translate-anthropic.d.ts.map +1 -1
- package/dist/agent/engine/translate-anthropic.js +43 -5
- package/dist/agent/engine/translate-anthropic.js.map +1 -1
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +7 -1
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-manager.d.ts +38 -6
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +81 -12
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/agent/run-store.d.ts +33 -5
- package/dist/agent/run-store.d.ts.map +1 -1
- package/dist/agent/run-store.js +105 -13
- package/dist/agent/run-store.js.map +1 -1
- package/dist/agent/thread-data-builder.d.ts +27 -0
- package/dist/agent/thread-data-builder.d.ts.map +1 -1
- package/dist/agent/thread-data-builder.js +178 -8
- package/dist/agent/thread-data-builder.js.map +1 -1
- package/dist/agent/types.d.ts +8 -0
- package/dist/agent/types.d.ts.map +1 -1
- package/dist/agent/types.js.map +1 -1
- package/dist/client/PoweredByBadge.d.ts +1 -1
- package/dist/client/PoweredByBadge.js +2 -2
- package/dist/client/PoweredByBadge.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +20 -0
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/api-path.d.ts.map +1 -1
- package/dist/client/api-path.js +37 -2
- package/dist/client/api-path.js.map +1 -1
- package/dist/client/composer/TiptapComposer.d.ts +2 -0
- package/dist/client/composer/TiptapComposer.d.ts.map +1 -1
- package/dist/client/composer/TiptapComposer.js +13 -1
- package/dist/client/composer/TiptapComposer.js.map +1 -1
- package/dist/deploy/build.d.ts.map +1 -1
- package/dist/deploy/build.js +31 -9
- package/dist/deploy/build.js.map +1 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +13 -3
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/ssr-handler.d.ts.map +1 -1
- package/dist/server/ssr-handler.js +24 -8
- package/dist/server/ssr-handler.js.map +1 -1
- package/dist/shared/index.d.ts +1 -0
- package/dist/shared/index.d.ts.map +1 -1
- package/dist/shared/index.js +1 -0
- package/dist/shared/index.js.map +1 -1
- package/dist/shared/social-meta.d.ts +13 -0
- package/dist/shared/social-meta.d.ts.map +1 -0
- package/dist/shared/social-meta.js +26 -0
- package/dist/shared/social-meta.js.map +1 -0
- package/package.json +1 -1
package/dist/agent/run-store.js
CHANGED
|
@@ -33,7 +33,10 @@ async function ensureRunTables() {
|
|
|
33
33
|
started_at ${intType()} NOT NULL,
|
|
34
34
|
completed_at ${intType()},
|
|
35
35
|
heartbeat_at ${intType()},
|
|
36
|
-
last_progress_at ${intType()}
|
|
36
|
+
last_progress_at ${intType()},
|
|
37
|
+
turn_id TEXT,
|
|
38
|
+
error_code TEXT,
|
|
39
|
+
error_detail TEXT
|
|
37
40
|
)
|
|
38
41
|
`);
|
|
39
42
|
// Backfill heartbeat_at on older deployments.
|
|
@@ -72,6 +75,26 @@ async function ensureRunTables() {
|
|
|
72
75
|
catch {
|
|
73
76
|
// Column already exists — ignore
|
|
74
77
|
}
|
|
78
|
+
// Backfill turn_id / error_code / error_detail.
|
|
79
|
+
// turn_id = stable identity for one logical assistant turn that may
|
|
80
|
+
// span several continuation runs, so the durable record
|
|
81
|
+
// can be folded across runs instead of dropped per-run.
|
|
82
|
+
// error_code / error_detail = terminal failure classification captured
|
|
83
|
+
// at completion so errored/cut-off runs are queryable for
|
|
84
|
+
// pattern analysis (see listErroredRuns).
|
|
85
|
+
for (const col of ["turn_id", "error_code", "error_detail"]) {
|
|
86
|
+
try {
|
|
87
|
+
if (isPostgres()) {
|
|
88
|
+
await client.execute(`ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS ${col} TEXT`);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
await client.execute(`ALTER TABLE agent_runs ADD COLUMN ${col} TEXT`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
// Column already exists — ignore
|
|
96
|
+
}
|
|
97
|
+
}
|
|
75
98
|
await client.execute(`
|
|
76
99
|
CREATE TABLE IF NOT EXISTS agent_run_events (
|
|
77
100
|
run_id TEXT NOT NULL,
|
|
@@ -84,15 +107,39 @@ async function ensureRunTables() {
|
|
|
84
107
|
}
|
|
85
108
|
return _initPromise;
|
|
86
109
|
}
|
|
87
|
-
export async function insertRun(id, threadId) {
|
|
110
|
+
export async function insertRun(id, threadId, turnId) {
|
|
88
111
|
await ensureRunTables();
|
|
89
112
|
const client = getDbExec();
|
|
90
113
|
const now = Date.now();
|
|
91
114
|
await client.execute({
|
|
92
|
-
sql: `INSERT INTO agent_runs (id, thread_id, status, started_at, heartbeat_at, last_progress_at) VALUES (?, ?, 'running', ?, ?, ?)`,
|
|
93
|
-
args: [id, threadId, now, now, now],
|
|
115
|
+
sql: `INSERT INTO agent_runs (id, thread_id, status, started_at, heartbeat_at, last_progress_at, turn_id) VALUES (?, ?, 'running', ?, ?, ?, ?)`,
|
|
116
|
+
args: [id, threadId, now, now, now, turnId ?? id],
|
|
94
117
|
});
|
|
95
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* Record terminal failure classification for a run so cut-off / errored runs
|
|
121
|
+
* can be surfaced for pattern analysis (see listErroredRuns). Best-effort —
|
|
122
|
+
* never throws, since it runs on the completion path that must not fail the run.
|
|
123
|
+
*/
|
|
124
|
+
export async function setRunError(runId, errorCode, errorDetail) {
|
|
125
|
+
if (!errorCode && !errorDetail)
|
|
126
|
+
return;
|
|
127
|
+
try {
|
|
128
|
+
await ensureRunTables();
|
|
129
|
+
const client = getDbExec();
|
|
130
|
+
await client.execute({
|
|
131
|
+
sql: `UPDATE agent_runs SET error_code = ?, error_detail = ? WHERE id = ?`,
|
|
132
|
+
args: [
|
|
133
|
+
errorCode ?? null,
|
|
134
|
+
errorDetail ? errorDetail.slice(0, 2000) : null,
|
|
135
|
+
runId,
|
|
136
|
+
],
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
// Diagnostics are best-effort; never let them break completion.
|
|
141
|
+
}
|
|
142
|
+
}
|
|
96
143
|
/** Update the run's liveness heartbeat. Called periodically by run-manager. */
|
|
97
144
|
export async function updateRunHeartbeat(runId) {
|
|
98
145
|
await ensureRunTables();
|
|
@@ -270,13 +317,16 @@ export async function reapAllStaleRuns() {
|
|
|
270
317
|
}
|
|
271
318
|
return rowsAffected ?? 0;
|
|
272
319
|
}
|
|
273
|
-
/** Delete
|
|
274
|
-
*
|
|
275
|
-
*
|
|
276
|
-
|
|
320
|
+
/** Delete old runs and expire stale "running" rows that haven't had activity
|
|
321
|
+
* (e.g. worker crashed before updating status). Completed runs are pruned at
|
|
322
|
+
* `olderThanMs`; errored/aborted runs are kept until `erroredOlderThanMs` (a
|
|
323
|
+
* longer window, falling back to `olderThanMs`) so their event log survives
|
|
324
|
+
* for cut-off pattern analysis via listErroredRuns. */
|
|
325
|
+
export async function cleanupOldRuns(olderThanMs, erroredOlderThanMs) {
|
|
277
326
|
await ensureRunTables();
|
|
278
327
|
const client = getDbExec();
|
|
279
328
|
const cutoff = Date.now() - olderThanMs;
|
|
329
|
+
const erroredCutoff = Date.now() - Math.max(erroredOlderThanMs ?? 0, olderThanMs);
|
|
280
330
|
// Expire stale running rows on the absolute-age threshold — safety net
|
|
281
331
|
// for runs that never received a heartbeat (very old deployments). The
|
|
282
332
|
// SELECT covers BOTH UPDATE conditions so the terminal-event-append loop
|
|
@@ -310,16 +360,58 @@ export async function cleanupOldRuns(olderThanMs) {
|
|
|
310
360
|
await safeAppendTerminalRunEvent(id, STALE_RUN_ERROR_EVENT, "cleanup-old-runs");
|
|
311
361
|
}
|
|
312
362
|
}
|
|
313
|
-
// Delete events for old
|
|
363
|
+
// Delete events for old terminal runs. Completed runs prune at `cutoff`;
|
|
364
|
+
// errored/aborted runs are retained until the (longer) `erroredCutoff`.
|
|
314
365
|
await client.execute({
|
|
315
366
|
sql: `DELETE FROM agent_run_events WHERE run_id IN (
|
|
316
|
-
SELECT id FROM agent_runs
|
|
367
|
+
SELECT id FROM agent_runs
|
|
368
|
+
WHERE (status = 'completed' AND completed_at < ?)
|
|
369
|
+
OR (status IN ('errored', 'aborted') AND completed_at < ?)
|
|
317
370
|
)`,
|
|
318
|
-
args: [cutoff],
|
|
371
|
+
args: [cutoff, erroredCutoff],
|
|
319
372
|
});
|
|
320
373
|
await client.execute({
|
|
321
|
-
sql: `DELETE FROM agent_runs
|
|
322
|
-
|
|
374
|
+
sql: `DELETE FROM agent_runs
|
|
375
|
+
WHERE (status = 'completed' AND completed_at < ?)
|
|
376
|
+
OR (status IN ('errored', 'aborted') AND completed_at < ?)`,
|
|
377
|
+
args: [cutoff, erroredCutoff],
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* List recent errored/aborted runs for cut-off pattern analysis. Read-only,
|
|
382
|
+
* bounded, and ordered newest-first. Surfaced via the list-errored-runs action
|
|
383
|
+
* so the team can see why chats are failing (terminal error code, duration,
|
|
384
|
+
* turn linkage) instead of discovering it ad hoc.
|
|
385
|
+
*/
|
|
386
|
+
export async function listErroredRuns(options) {
|
|
387
|
+
await ensureRunTables();
|
|
388
|
+
const client = getDbExec();
|
|
389
|
+
const limit = Math.min(Math.max(Math.floor(options?.limit ?? 100), 1), 1000);
|
|
390
|
+
const since = options?.sinceMs && options.sinceMs > 0 ? Date.now() - options.sinceMs : 0;
|
|
391
|
+
const { rows } = await client.execute({
|
|
392
|
+
sql: `SELECT id, thread_id, turn_id, status, error_code, error_detail, started_at, completed_at
|
|
393
|
+
FROM agent_runs
|
|
394
|
+
WHERE status IN ('errored', 'aborted')
|
|
395
|
+
AND COALESCE(completed_at, started_at) >= ?
|
|
396
|
+
ORDER BY COALESCE(completed_at, started_at) DESC
|
|
397
|
+
LIMIT ${limit}`,
|
|
398
|
+
args: [since],
|
|
399
|
+
});
|
|
400
|
+
return rows.map((r) => {
|
|
401
|
+
const row = r;
|
|
402
|
+
const startedAt = Number(row.started_at);
|
|
403
|
+
const completedAt = row.completed_at == null ? null : Number(row.completed_at);
|
|
404
|
+
return {
|
|
405
|
+
id: row.id,
|
|
406
|
+
threadId: row.thread_id,
|
|
407
|
+
turnId: row.turn_id ?? null,
|
|
408
|
+
status: row.status,
|
|
409
|
+
errorCode: row.error_code ?? null,
|
|
410
|
+
errorDetail: row.error_detail ?? null,
|
|
411
|
+
startedAt,
|
|
412
|
+
completedAt,
|
|
413
|
+
durationMs: completedAt == null ? null : completedAt - startedAt,
|
|
414
|
+
};
|
|
323
415
|
});
|
|
324
416
|
}
|
|
325
417
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-store.js","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,IAAI,YAAuC,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC;AAElC,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,OAAO;IACb,KAAK,EACH,qHAAqH;IACvH,SAAS,EAAE,WAAW;IACtB,WAAW,EAAE,IAAI;IACjB,OAAO,EACL,gIAAgI;CAC1H,CAAC;AAEX,KAAK,UAAU,eAAe;IAC5B,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;;;;;;uBAMJ,OAAO,EAAE;yBACP,OAAO,EAAE;yBACT,OAAO,EAAE;6BACL,OAAO,EAAE;;OAE/B,CAAC,CAAC;YACH,8CAA8C;YAC9C,IAAI,CAAC;gBACH,IAAI,UAAU,EAAE,EAAE,CAAC;oBACjB,MAAM,MAAM,CAAC,OAAO,CAClB,gEAAgE,OAAO,EAAE,EAAE,CAC5E,CAAC;oBACF,MAAM,MAAM,CAAC,OAAO,CAClB,mEAAmE,CACpE,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC,OAAO,CAClB,kDAAkD,OAAO,EAAE,EAAE,CAC9D,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAClB,MAAM,MAAM,CAAC,OAAO,CAClB,qDAAqD,CACtD,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,kEAAkE;YAClE,sEAAsE;YACtE,wEAAwE;YACxE,iEAAiE;YACjE,IAAI,CAAC;gBACH,IAAI,UAAU,EAAE,EAAE,CAAC;oBACjB,MAAM,MAAM,CAAC,OAAO,CAClB,oEAAoE,OAAO,EAAE,EAAE,CAChF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC,OAAO,CAClB,sDAAsD,OAAO,EAAE,EAAE,CAClE,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,MAAM,MAAM,CAAC,OAAO,CAAC;;;gBAGX,OAAO,EAAE;;;;OAIlB,CAAC,CAAC;QACL,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAU,EAAE,QAAgB;IAC1D,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,8HAA8H;QACnI,IAAI,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;KACpC,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAa;IACpD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,qDAAqD;QAC1D,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAa;IACjD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yDAAyD;QAC9D,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,aAAqB,YAAY;IAEjC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;IACvC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;;uDAI8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,0BAA0B,CAC9B,KAAK,EACL,qBAAqB,EACrB,eAAe,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,MAA2C;IAE3C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,iEAAiE;QACtE,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAClC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,MAAe;IAEf,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,2FAA2F;QAChG,IAAI,EAAE,CAAC,MAAM,IAAI,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC5C,CAAC,CAAC;IACH,MAAM,0BAA0B,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa;IAC9C,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;AACjD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAa;IAEb,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,0DAA0D;QAC/D,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAqD,CAAC;IACxE,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACxD,OAAO;QACL,OAAO,EAAE,IAAI;QACb,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,GAAW,EACX,SAAiB;IAEjB,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,qEAAqE;IACrE,sEAAsE;IACtE,qEAAqE;IACrE,sEAAsE;IACtE,kEAAkE;IAClE,kEAAkE;IAClE,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,8GAA8G;QACnH,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,OAAe;IAEf,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,6FAA6F;QAClG,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;KACvB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,MAAM,GAAG,GAAG,CAAiD,CAAC;QAC9D,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa;IAM5C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,uEAAuE;QAC5E,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAKf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,OAAuC;IAUvC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,OAAO,EAAE,eAAe;QAClC,CAAC,CAAC,6JAA6J;QAC/J,CAAC,CAAC,oLAAoL,CAAC;IACzL,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACjE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAQf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/B,WAAW,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;QACnE,WAAW,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;QACnE,cAAc,EACZ,CAAC,CAAC,gBAAgB,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC;KACjE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACjC,GAAG,EAAE;;uDAE8C;QACnD,IAAI,EAAE,CAAC,eAAe,CAAC;KACxB,CAAC,CAAC;IACH,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAI,GAAwB,CAAC,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,0BAA0B,CAC9B,EAAE,EACF,qBAAqB,EACrB,gBAAgB,CACjB,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,YAAY,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;oDAEoD;AACpD,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,WAAmB;IACtD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IACxC,uEAAuE;IACvE,uEAAuE;IACvE,yEAAyE;IACzE,qEAAqE;IACrE,mEAAmE;IACnE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACjC,GAAG,EAAE;;;;;cAKK;QACV,IAAI,EAAE,CAAC,eAAe,EAAE,MAAM,CAAC;KAChC,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,wGAAwG;QAC7G,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IACH,iEAAiE;IACjE,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAI,GAAwB,CAAC,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,0BAA0B,CAC9B,EAAE,EACF,qBAAqB,EACrB,kBAAkB,CACnB,CAAC;QACJ,CAAC;IACH,CAAC;IACD,yCAAyC;IACzC,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;MAEH;QACF,IAAI,EAAE,CAAC,MAAM,CAAC;KACf,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,uEAAuE;QAC5E,IAAI,EAAE,CAAC,MAAM,CAAC;KACf,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAa,EACb,KAA8B;IAE9B,OAAO,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,0BAA0B,CACvC,KAAa,EACb,KAA8B,EAC9B,MAAc;IAEd,IAAI,UAAmB,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3C,OAAO;IACT,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,UAAU,GAAG,GAAG,CAAC;IACnB,CAAC;IACD,uEAAuE;IACvE,gDAAgD;IAChD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,QAAQ,EAAE,CAAC;QAClB,YAAY,CAAC,QAAQ,EAAE;YACrB,IAAI,EAAE;gBACJ,SAAS,EAAE,iBAAiB;gBAC5B,SAAS,EAAE,uBAAuB;gBAClC,MAAM;aACP;YACD,KAAK,EAAE;gBACL,KAAK;gBACL,SAAS,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW;gBACpE,UAAU,EACR,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;aACxE;SACF,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,KAAa,EACb,KAA8B;IAE9B,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,yFAAyF;QAC9F,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAEN,CAAC;IACd,IAAI,IAAI,EAAE,UAAU,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3C,IACE,MAAM,EAAE,IAAI,KAAK,MAAM;gBACvB,MAAM,EAAE,IAAI,KAAK,OAAO;gBACxB,MAAM,EAAE,IAAI,KAAK,iBAAiB;gBAClC,MAAM,EAAE,IAAI,KAAK,YAAY;gBAC7B,MAAM,EAAE,IAAI,KAAK,eAAe,EAChC,CAAC;gBACD,OAAO;YACT,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;QACzD,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,8GAA8G;QACnH,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;KAC9C,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL persistence for agent runs and events.\n * Enables cross-isolate access on Cloudflare Workers and\n * reliable reconnection after page refreshes.\n */\nimport { getDbExec, intType, isPostgres } from \"../db/client.js\";\nimport { captureError } from \"../server/capture-error.js\";\n\nlet _initPromise: Promise<void> | undefined;\n\n/**\n * Max time without a heartbeat before a \"running\" run is considered dead.\n * The run-manager heartbeats every 1.5s, so 6s tolerates 3 missed writes.\n * Short window is what makes reload recovery feel instant instead of\n * stranding the user on \"Thinking...\" for up to 90s after a process death.\n */\nexport const RUN_STALE_MS = 6_000;\n\nexport const STALE_RUN_ERROR_EVENT = {\n type: \"error\",\n error:\n \"The agent stopped before it could finish. It may have hit a server timeout or the worker may have been interrupted.\",\n errorCode: \"stale_run\",\n recoverable: true,\n details:\n \"The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.\",\n} as const;\n\nasync function ensureRunTables(): Promise<void> {\n if (!_initPromise) {\n _initPromise = (async () => {\n const client = getDbExec();\n await client.execute(`\n CREATE TABLE IF NOT EXISTS agent_runs (\n id TEXT PRIMARY KEY,\n thread_id TEXT NOT NULL,\n status TEXT NOT NULL DEFAULT 'running',\n abort_reason TEXT,\n started_at ${intType()} NOT NULL,\n completed_at ${intType()},\n heartbeat_at ${intType()},\n last_progress_at ${intType()}\n )\n `);\n // Backfill heartbeat_at on older deployments.\n try {\n if (isPostgres()) {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS heartbeat_at ${intType()}`,\n );\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS abort_reason TEXT`,\n );\n } else {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN heartbeat_at ${intType()}`,\n );\n }\n } catch {\n // Column already exists — ignore\n }\n try {\n if (!isPostgres()) {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN abort_reason TEXT`,\n );\n }\n } catch {\n // Column already exists — ignore\n }\n // Backfill last_progress_at — this is distinct from heartbeat_at.\n // heartbeat_at = \"the producer process is alive\" (bumped on a timer).\n // last_progress_at = \"the agent is actually emitting events\" (bumped on\n // each emit). The gap between them is the stuck-detector signal.\n try {\n if (isPostgres()) {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS last_progress_at ${intType()}`,\n );\n } else {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN last_progress_at ${intType()}`,\n );\n }\n } catch {\n // Column already exists — ignore\n }\n await client.execute(`\n CREATE TABLE IF NOT EXISTS agent_run_events (\n run_id TEXT NOT NULL,\n seq ${intType()} NOT NULL,\n event_data TEXT NOT NULL,\n PRIMARY KEY (run_id, seq)\n )\n `);\n })();\n }\n return _initPromise;\n}\n\nexport async function insertRun(id: string, threadId: string): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `INSERT INTO agent_runs (id, thread_id, status, started_at, heartbeat_at, last_progress_at) VALUES (?, ?, 'running', ?, ?, ?)`,\n args: [id, threadId, now, now, now],\n });\n}\n\n/** Update the run's liveness heartbeat. Called periodically by run-manager. */\nexport async function updateRunHeartbeat(runId: string): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET heartbeat_at = ? WHERE id = ?`,\n args: [Date.now(), runId],\n });\n}\n\n/**\n * Bump `last_progress_at` — call this whenever the agent actually emits an\n * event (token, tool call, message). Distinct from `heartbeat_at` so the\n * stuck-detector can tell \"process alive but nothing happening\" from\n * \"process dead.\" Callers should throttle (run-manager debounces to ~1/s).\n */\nexport async function bumpRunProgress(runId: string): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET last_progress_at = ? WHERE id = ?`,\n args: [Date.now(), runId],\n });\n}\n\n/**\n * If the given run is marked \"running\" in SQL but its heartbeat is stale\n * (producer likely crashed), flip it to \"errored\" so watchers stop waiting.\n * Returns true if the row was reaped.\n */\nexport async function reapIfStale(\n runId: string,\n maxStaleMs: number = RUN_STALE_MS,\n): Promise<boolean> {\n await ensureRunTables();\n const client = getDbExec();\n const cutoff = Date.now() - maxStaleMs;\n const { rowsAffected } = await client.execute({\n sql: `UPDATE agent_runs\n SET status = 'errored', completed_at = ?\n WHERE id = ?\n AND status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [Date.now(), runId, cutoff],\n });\n const reaped = (rowsAffected ?? 0) > 0;\n if (reaped) {\n await safeAppendTerminalRunEvent(\n runId,\n STALE_RUN_ERROR_EVENT,\n \"reap-if-stale\",\n );\n }\n return reaped;\n}\n\nexport async function updateRunStatus(\n runId: string,\n status: \"completed\" | \"errored\" | \"aborted\",\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET status = ?, completed_at = ? WHERE id = ?`,\n args: [status, Date.now(), runId],\n });\n}\n\nexport async function markRunAborted(\n runId: string,\n reason?: string,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET status = 'aborted', abort_reason = ?, completed_at = ? WHERE id = ?`,\n args: [reason ?? \"user\", Date.now(), runId],\n });\n await safeAppendTerminalRunEvent(runId, { type: \"done\" }, \"mark-aborted\");\n}\n\nexport async function isRunAborted(runId: string): Promise<boolean> {\n return (await getRunAbortState(runId)).aborted;\n}\n\nexport async function getRunAbortState(\n runId: string,\n): Promise<{ aborted: boolean; reason?: string }> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT status, abort_reason FROM agent_runs WHERE id = ?`,\n args: [runId],\n });\n if (rows.length === 0) return { aborted: false };\n const row = rows[0] as { status: string; abort_reason?: string | null };\n if (row.status !== \"aborted\") return { aborted: false };\n return {\n aborted: true,\n ...(row.abort_reason ? { reason: row.abort_reason } : {}),\n };\n}\n\nexport async function insertRunEvent(\n runId: string,\n seq: number,\n eventData: string,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n // ON CONFLICT DO NOTHING: a (runId, seq) collision can happen on the\n // soft-timeout / terminal-event path where `pendingTerminalEvent` was\n // assigned a seq that later gets reused by an event pushed after it.\n // It can also race with `appendTerminalRunEvent` (max-seq + 1) when a\n // run aborts at the same time the producer emits its final event.\n // Treat the second write as a no-op so the run completes cleanly.\n await client.execute({\n sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING`,\n args: [runId, seq, eventData],\n });\n}\n\nexport async function getRunEventsSince(\n runId: string,\n fromSeq: number,\n): Promise<Array<{ seq: number; eventData: string }>> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? AND seq >= ? ORDER BY seq ASC`,\n args: [runId, fromSeq],\n });\n return rows.map((r) => {\n const row = r as { seq: number | string; event_data: string };\n return { seq: Number(row.seq), eventData: row.event_data };\n });\n}\n\nexport async function getRunById(runId: string): Promise<{\n id: string;\n threadId: string;\n status: string;\n startedAt: number;\n} | null> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT id, thread_id, status, started_at FROM agent_runs WHERE id = ?`,\n args: [runId],\n });\n if (rows.length === 0) return null;\n const r = rows[0] as {\n id: string;\n thread_id: string;\n status: string;\n started_at: number | string;\n };\n return {\n id: r.id,\n threadId: r.thread_id,\n status: r.status,\n startedAt: Number(r.started_at),\n };\n}\n\nexport async function getRunByThread(\n threadId: string,\n options?: { includeTerminal?: boolean },\n): Promise<{\n id: string;\n threadId: string;\n status: string;\n startedAt: number;\n heartbeatAt: number | null;\n completedAt: number | null;\n lastProgressAt: number | null;\n} | null> {\n await ensureRunTables();\n const client = getDbExec();\n const sql = options?.includeTerminal\n ? `SELECT id, thread_id, status, started_at, heartbeat_at, completed_at, last_progress_at FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`\n : `SELECT id, thread_id, status, started_at, heartbeat_at, completed_at, last_progress_at FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;\n const { rows } = await client.execute({ sql, args: [threadId] });\n if (rows.length === 0) return null;\n const r = rows[0] as {\n id: string;\n thread_id: string;\n status: string;\n started_at: number | string;\n heartbeat_at: number | string | null;\n completed_at: number | string | null;\n last_progress_at: number | string | null;\n };\n return {\n id: r.id,\n threadId: r.thread_id,\n status: r.status,\n startedAt: Number(r.started_at),\n heartbeatAt: r.heartbeat_at == null ? null : Number(r.heartbeat_at),\n completedAt: r.completed_at == null ? null : Number(r.completed_at),\n lastProgressAt:\n r.last_progress_at == null ? null : Number(r.last_progress_at),\n };\n}\n\n/**\n * Expire any \"running\" rows whose heartbeat is stale — producer died.\n * Safe to call at server startup on multi-isolate deployments: only rows\n * without a fresh heartbeat get reaped, so runs owned by OTHER live\n * isolates (which keep heartbeating) are left alone.\n */\nexport async function reapAllStaleRuns(): Promise<number> {\n await ensureRunTables();\n const client = getDbExec();\n const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n const stale = await client.execute({\n sql: `SELECT id FROM agent_runs\n WHERE status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [heartbeatCutoff],\n });\n const { rowsAffected } = await client.execute({\n sql: `UPDATE agent_runs\n SET status = 'errored', completed_at = ?\n WHERE status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [Date.now(), heartbeatCutoff],\n });\n for (const row of stale.rows) {\n const id = (row as { id?: unknown }).id;\n if (typeof id === \"string\") {\n await safeAppendTerminalRunEvent(\n id,\n STALE_RUN_ERROR_EVENT,\n \"reap-all-stale\",\n );\n }\n }\n return rowsAffected ?? 0;\n}\n\n/** Delete completed/errored runs older than the given threshold,\n * and expire stale \"running\" rows that haven't had activity\n * (e.g. worker crashed before updating status). */\nexport async function cleanupOldRuns(olderThanMs: number): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n const cutoff = Date.now() - olderThanMs;\n // Expire stale running rows on the absolute-age threshold — safety net\n // for runs that never received a heartbeat (very old deployments). The\n // SELECT covers BOTH UPDATE conditions so the terminal-event-append loop\n // below catches every row we're about to flip — a 24h-old row with a\n // somehow-fresh heartbeat would slip past a heartbeat-only SELECT.\n const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n const stale = await client.execute({\n sql: `SELECT id FROM agent_runs\n WHERE status = 'running'\n AND (\n COALESCE(heartbeat_at, started_at) < ?\n OR started_at < ?\n )`,\n args: [heartbeatCutoff, cutoff],\n });\n await client.execute({\n sql: `UPDATE agent_runs SET status = 'errored', completed_at = ? WHERE status = 'running' AND started_at < ?`,\n args: [Date.now(), cutoff],\n });\n // Also expire runs whose heartbeat is stale — producer has died.\n await client.execute({\n sql: `UPDATE agent_runs\n SET status = 'errored', completed_at = ?\n WHERE status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [Date.now(), heartbeatCutoff],\n });\n for (const row of stale.rows) {\n const id = (row as { id?: unknown }).id;\n if (typeof id === \"string\") {\n await safeAppendTerminalRunEvent(\n id,\n STALE_RUN_ERROR_EVENT,\n \"cleanup-old-runs\",\n );\n }\n }\n // Delete events for old non-running runs\n await client.execute({\n sql: `DELETE FROM agent_run_events WHERE run_id IN (\n SELECT id FROM agent_runs WHERE status != 'running' AND completed_at < ?\n )`,\n args: [cutoff],\n });\n await client.execute({\n sql: `DELETE FROM agent_runs WHERE status != 'running' AND completed_at < ?`,\n args: [cutoff],\n });\n}\n\n/**\n * Idempotently append a terminal event to a run's event stream. No-op if the\n * stream already ends in a terminal event. Used by reapers AND by SSE\n * reconnect paths that discover an `errored` run row with no terminal event\n * (e.g. an earlier reaper's silent `.catch(() => {})` swallowed the append).\n *\n * Persisting from the reconnect path is what keeps the system self-healing:\n * subsequent reconnects replay the proper terminal event from SQL instead of\n * synthesizing a fresh one each time.\n */\nexport async function ensureTerminalRunEvent(\n runId: string,\n event: Record<string, unknown>,\n): Promise<void> {\n return appendTerminalRunEvent(runId, event);\n}\n\n/**\n * Append a terminal run event, retrying once on failure and reporting to\n * Sentry if both attempts fail. Background reaper paths can't surface errors\n * to a user, but they MUST eventually persist a terminal event — losing it\n * leaves reconnecting clients staring at a bare `status='errored'` row with\n * no payload to render. The previous `.catch(() => {})` callsites silently\n * dropped transient SQL blips and produced exactly that bug. Never throws.\n */\nasync function safeAppendTerminalRunEvent(\n runId: string,\n event: Record<string, unknown>,\n source: string,\n): Promise<void> {\n let firstError: unknown;\n try {\n await appendTerminalRunEvent(runId, event);\n return;\n } catch (err) {\n firstError = err;\n }\n // Brief backoff — most \"transient\" SQL failures (connection blip, lock\n // contention) clear within a couple hundred ms.\n await new Promise<void>((resolve) => setTimeout(resolve, 100));\n try {\n await appendTerminalRunEvent(runId, event);\n } catch (retryErr) {\n captureError(retryErr, {\n tags: {\n component: \"agent-run-store\",\n operation: \"append-terminal-event\",\n source,\n },\n extra: {\n runId,\n eventType: typeof event.type === \"string\" ? event.type : \"(unknown)\",\n firstError:\n firstError instanceof Error ? firstError.message : String(firstError),\n },\n });\n }\n}\n\nasync function appendTerminalRunEvent(\n runId: string,\n event: Record<string, unknown>,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,\n args: [runId],\n });\n const last = rows[0] as\n | { seq?: number | string; event_data?: string }\n | undefined;\n if (last?.event_data) {\n try {\n const parsed = JSON.parse(last.event_data);\n if (\n parsed?.type === \"done\" ||\n parsed?.type === \"error\" ||\n parsed?.type === \"missing_api_key\" ||\n parsed?.type === \"loop_limit\" ||\n parsed?.type === \"auto_continue\"\n ) {\n return;\n }\n } catch {\n // Ignore malformed rows and append the terminal event.\n }\n }\n const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;\n await client.execute({\n sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING`,\n args: [runId, nextSeq, JSON.stringify(event)],\n });\n}\n"]}
|
|
1
|
+
{"version":3,"file":"run-store.js","sourceRoot":"","sources":["../../src/agent/run-store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,IAAI,YAAuC,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC;AAElC,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,OAAO;IACb,KAAK,EACH,qHAAqH;IACvH,SAAS,EAAE,WAAW;IACtB,WAAW,EAAE,IAAI;IACjB,OAAO,EACL,gIAAgI;CAC1H,CAAC;AAEX,KAAK,UAAU,eAAe;IAC5B,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;;;;;;uBAMJ,OAAO,EAAE;yBACP,OAAO,EAAE;yBACT,OAAO,EAAE;6BACL,OAAO,EAAE;;;;;OAK/B,CAAC,CAAC;YACH,8CAA8C;YAC9C,IAAI,CAAC;gBACH,IAAI,UAAU,EAAE,EAAE,CAAC;oBACjB,MAAM,MAAM,CAAC,OAAO,CAClB,gEAAgE,OAAO,EAAE,EAAE,CAC5E,CAAC;oBACF,MAAM,MAAM,CAAC,OAAO,CAClB,mEAAmE,CACpE,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC,OAAO,CAClB,kDAAkD,OAAO,EAAE,EAAE,CAC9D,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBAClB,MAAM,MAAM,CAAC,OAAO,CAClB,qDAAqD,CACtD,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,kEAAkE;YAClE,sEAAsE;YACtE,wEAAwE;YACxE,iEAAiE;YACjE,IAAI,CAAC;gBACH,IAAI,UAAU,EAAE,EAAE,CAAC;oBACjB,MAAM,MAAM,CAAC,OAAO,CAClB,oEAAoE,OAAO,EAAE,EAAE,CAChF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,MAAM,CAAC,OAAO,CAClB,sDAAsD,OAAO,EAAE,EAAE,CAClE,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;YACD,gDAAgD;YAChD,yEAAyE;YACzE,uEAAuE;YACvE,uEAAuE;YACvE,yEAAyE;YACzE,yEAAyE;YACzE,yDAAyD;YACzD,KAAK,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,cAAc,CAAU,EAAE,CAAC;gBACrE,IAAI,CAAC;oBACH,IAAI,UAAU,EAAE,EAAE,CAAC;wBACjB,MAAM,MAAM,CAAC,OAAO,CAClB,mDAAmD,GAAG,OAAO,CAC9D,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,CAAC,OAAO,CAClB,qCAAqC,GAAG,OAAO,CAChD,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,iCAAiC;gBACnC,CAAC;YACH,CAAC;YACD,MAAM,MAAM,CAAC,OAAO,CAAC;;;gBAGX,OAAO,EAAE;;;;OAIlB,CAAC,CAAC;QACL,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAU,EACV,QAAgB,EAChB,MAAe;IAEf,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,0IAA0I;QAC/I,IAAI,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,IAAI,EAAE,CAAC;KAClD,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,SAA6B,EAC7B,WAA+B;IAE/B,IAAI,CAAC,SAAS,IAAI,CAAC,WAAW;QAAE,OAAO;IACvC,IAAI,CAAC;QACH,MAAM,eAAe,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;YACnB,GAAG,EAAE,qEAAqE;YAC1E,IAAI,EAAE;gBACJ,SAAS,IAAI,IAAI;gBACjB,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;gBAC/C,KAAK;aACN;SACF,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,gEAAgE;IAClE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAa;IACpD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,qDAAqD;QAC1D,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAa;IACjD,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yDAAyD;QAC9D,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,aAAqB,YAAY;IAEjC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;IACvC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;;uDAI8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,0BAA0B,CAC9B,KAAK,EACL,qBAAqB,EACrB,eAAe,CAChB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,MAA2C;IAE3C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,iEAAiE;QACtE,IAAI,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAClC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,MAAe;IAEf,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,2FAA2F;QAChG,IAAI,EAAE,CAAC,MAAM,IAAI,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC;KAC5C,CAAC,CAAC;IACH,MAAM,0BAA0B,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa;IAC9C,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;AACjD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAa;IAEb,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,0DAA0D;QAC/D,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAqD,CAAC;IACxE,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACxD,OAAO;QACL,OAAO,EAAE,IAAI;QACb,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,GAAW,EACX,SAAiB;IAEjB,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,qEAAqE;IACrE,sEAAsE;IACtE,qEAAqE;IACrE,sEAAsE;IACtE,kEAAkE;IAClE,kEAAkE;IAClE,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,8GAA8G;QACnH,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,OAAe;IAEf,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,6FAA6F;QAClG,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;KACvB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,MAAM,GAAG,GAAG,CAAiD,CAAC;QAC9D,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa;IAM5C,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,uEAAuE;QAC5E,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAKf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,OAAuC;IAUvC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,OAAO,EAAE,eAAe;QAClC,CAAC,CAAC,6JAA6J;QAC/J,CAAC,CAAC,oLAAoL,CAAC;IACzL,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACjE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAQf,CAAC;IACF,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,QAAQ,EAAE,CAAC,CAAC,SAAS;QACrB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/B,WAAW,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;QACnE,WAAW,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;QACnE,cAAc,EACZ,CAAC,CAAC,gBAAgB,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC;KACjE,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACjC,GAAG,EAAE;;uDAE8C;QACnD,IAAI,EAAE,CAAC,eAAe,CAAC;KACxB,CAAC,CAAC;IACH,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAC5C,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAI,GAAwB,CAAC,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,0BAA0B,CAC9B,EAAE,EACF,qBAAqB,EACrB,gBAAgB,CACjB,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,YAAY,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;;;wDAIwD;AACxD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,WAAmB,EACnB,kBAA2B;IAE3B,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IACxC,MAAM,aAAa,GACjB,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,EAAE,WAAW,CAAC,CAAC;IAC9D,uEAAuE;IACvE,uEAAuE;IACvE,yEAAyE;IACzE,qEAAqE;IACrE,mEAAmE;IACnE,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACjC,GAAG,EAAE;;;;;cAKK;QACV,IAAI,EAAE,CAAC,eAAe,EAAE,MAAM,CAAC;KAChC,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,wGAAwG;QAC7G,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IACH,iEAAiE;IACjE,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;;uDAG8C;QACnD,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAI,GAAwB,CAAC,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,0BAA0B,CAC9B,EAAE,EACF,qBAAqB,EACrB,kBAAkB,CACnB,CAAC;QACJ,CAAC;IACH,CAAC;IACD,yEAAyE;IACzE,wEAAwE;IACxE,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;;;MAIH;QACF,IAAI,EAAE,CAAC,MAAM,EAAE,aAAa,CAAC;KAC9B,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;wEAE+D;QACpE,IAAI,EAAE,CAAC,MAAM,EAAE,aAAa,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAGrC;IAaC,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7E,MAAM,KAAK,GACT,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;;;;;kBAKS,KAAK,EAAE;QACrB,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,MAAM,GAAG,GAAG,CASX,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,WAAW,GACf,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC7D,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,MAAM,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI;YAC3B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,SAAS,EAAE,GAAG,CAAC,UAAU,IAAI,IAAI;YACjC,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,IAAI;YACrC,SAAS;YACT,WAAW;YACX,UAAU,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,SAAS;SACjE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAa,EACb,KAA8B;IAE9B,OAAO,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,0BAA0B,CACvC,KAAa,EACb,KAA8B,EAC9B,MAAc;IAEd,IAAI,UAAmB,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC3C,OAAO;IACT,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,UAAU,GAAG,GAAG,CAAC;IACnB,CAAC;IACD,uEAAuE;IACvE,gDAAgD;IAChD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,QAAQ,EAAE,CAAC;QAClB,YAAY,CAAC,QAAQ,EAAE;YACrB,IAAI,EAAE;gBACJ,SAAS,EAAE,iBAAiB;gBAC5B,SAAS,EAAE,uBAAuB;gBAClC,MAAM;aACP;YACD,KAAK,EAAE;gBACL,KAAK;gBACL,SAAS,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW;gBACpE,UAAU,EACR,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;aACxE;SACF,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,KAAa,EACb,KAA8B;IAE9B,MAAM,eAAe,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,yFAAyF;QAC9F,IAAI,EAAE,CAAC,KAAK,CAAC;KACd,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAEN,CAAC;IACd,IAAI,IAAI,EAAE,UAAU,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3C,IACE,MAAM,EAAE,IAAI,KAAK,MAAM;gBACvB,MAAM,EAAE,IAAI,KAAK,OAAO;gBACxB,MAAM,EAAE,IAAI,KAAK,iBAAiB;gBAClC,MAAM,EAAE,IAAI,KAAK,YAAY;gBAC7B,MAAM,EAAE,IAAI,KAAK,eAAe,EAChC,CAAC;gBACD,OAAO;YACT,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;QACzD,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,8GAA8G;QACnH,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;KAC9C,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL persistence for agent runs and events.\n * Enables cross-isolate access on Cloudflare Workers and\n * reliable reconnection after page refreshes.\n */\nimport { getDbExec, intType, isPostgres } from \"../db/client.js\";\nimport { captureError } from \"../server/capture-error.js\";\n\nlet _initPromise: Promise<void> | undefined;\n\n/**\n * Max time without a heartbeat before a \"running\" run is considered dead.\n * The run-manager heartbeats every 1.5s, so 6s tolerates 3 missed writes.\n * Short window is what makes reload recovery feel instant instead of\n * stranding the user on \"Thinking...\" for up to 90s after a process death.\n */\nexport const RUN_STALE_MS = 6_000;\n\nexport const STALE_RUN_ERROR_EVENT = {\n type: \"error\",\n error:\n \"The agent stopped before it could finish. It may have hit a server timeout or the worker may have been interrupted.\",\n errorCode: \"stale_run\",\n recoverable: true,\n details:\n \"The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.\",\n} as const;\n\nasync function ensureRunTables(): Promise<void> {\n if (!_initPromise) {\n _initPromise = (async () => {\n const client = getDbExec();\n await client.execute(`\n CREATE TABLE IF NOT EXISTS agent_runs (\n id TEXT PRIMARY KEY,\n thread_id TEXT NOT NULL,\n status TEXT NOT NULL DEFAULT 'running',\n abort_reason TEXT,\n started_at ${intType()} NOT NULL,\n completed_at ${intType()},\n heartbeat_at ${intType()},\n last_progress_at ${intType()},\n turn_id TEXT,\n error_code TEXT,\n error_detail TEXT\n )\n `);\n // Backfill heartbeat_at on older deployments.\n try {\n if (isPostgres()) {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS heartbeat_at ${intType()}`,\n );\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS abort_reason TEXT`,\n );\n } else {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN heartbeat_at ${intType()}`,\n );\n }\n } catch {\n // Column already exists — ignore\n }\n try {\n if (!isPostgres()) {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN abort_reason TEXT`,\n );\n }\n } catch {\n // Column already exists — ignore\n }\n // Backfill last_progress_at — this is distinct from heartbeat_at.\n // heartbeat_at = \"the producer process is alive\" (bumped on a timer).\n // last_progress_at = \"the agent is actually emitting events\" (bumped on\n // each emit). The gap between them is the stuck-detector signal.\n try {\n if (isPostgres()) {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS last_progress_at ${intType()}`,\n );\n } else {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN last_progress_at ${intType()}`,\n );\n }\n } catch {\n // Column already exists — ignore\n }\n // Backfill turn_id / error_code / error_detail.\n // turn_id = stable identity for one logical assistant turn that may\n // span several continuation runs, so the durable record\n // can be folded across runs instead of dropped per-run.\n // error_code / error_detail = terminal failure classification captured\n // at completion so errored/cut-off runs are queryable for\n // pattern analysis (see listErroredRuns).\n for (const col of [\"turn_id\", \"error_code\", \"error_detail\"] as const) {\n try {\n if (isPostgres()) {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS ${col} TEXT`,\n );\n } else {\n await client.execute(\n `ALTER TABLE agent_runs ADD COLUMN ${col} TEXT`,\n );\n }\n } catch {\n // Column already exists — ignore\n }\n }\n await client.execute(`\n CREATE TABLE IF NOT EXISTS agent_run_events (\n run_id TEXT NOT NULL,\n seq ${intType()} NOT NULL,\n event_data TEXT NOT NULL,\n PRIMARY KEY (run_id, seq)\n )\n `);\n })();\n }\n return _initPromise;\n}\n\nexport async function insertRun(\n id: string,\n threadId: string,\n turnId?: string,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `INSERT INTO agent_runs (id, thread_id, status, started_at, heartbeat_at, last_progress_at, turn_id) VALUES (?, ?, 'running', ?, ?, ?, ?)`,\n args: [id, threadId, now, now, now, turnId ?? id],\n });\n}\n\n/**\n * Record terminal failure classification for a run so cut-off / errored runs\n * can be surfaced for pattern analysis (see listErroredRuns). Best-effort —\n * never throws, since it runs on the completion path that must not fail the run.\n */\nexport async function setRunError(\n runId: string,\n errorCode: string | undefined,\n errorDetail: string | undefined,\n): Promise<void> {\n if (!errorCode && !errorDetail) return;\n try {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET error_code = ?, error_detail = ? WHERE id = ?`,\n args: [\n errorCode ?? null,\n errorDetail ? errorDetail.slice(0, 2000) : null,\n runId,\n ],\n });\n } catch {\n // Diagnostics are best-effort; never let them break completion.\n }\n}\n\n/** Update the run's liveness heartbeat. Called periodically by run-manager. */\nexport async function updateRunHeartbeat(runId: string): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET heartbeat_at = ? WHERE id = ?`,\n args: [Date.now(), runId],\n });\n}\n\n/**\n * Bump `last_progress_at` — call this whenever the agent actually emits an\n * event (token, tool call, message). Distinct from `heartbeat_at` so the\n * stuck-detector can tell \"process alive but nothing happening\" from\n * \"process dead.\" Callers should throttle (run-manager debounces to ~1/s).\n */\nexport async function bumpRunProgress(runId: string): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET last_progress_at = ? WHERE id = ?`,\n args: [Date.now(), runId],\n });\n}\n\n/**\n * If the given run is marked \"running\" in SQL but its heartbeat is stale\n * (producer likely crashed), flip it to \"errored\" so watchers stop waiting.\n * Returns true if the row was reaped.\n */\nexport async function reapIfStale(\n runId: string,\n maxStaleMs: number = RUN_STALE_MS,\n): Promise<boolean> {\n await ensureRunTables();\n const client = getDbExec();\n const cutoff = Date.now() - maxStaleMs;\n const { rowsAffected } = await client.execute({\n sql: `UPDATE agent_runs\n SET status = 'errored', completed_at = ?\n WHERE id = ?\n AND status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [Date.now(), runId, cutoff],\n });\n const reaped = (rowsAffected ?? 0) > 0;\n if (reaped) {\n await safeAppendTerminalRunEvent(\n runId,\n STALE_RUN_ERROR_EVENT,\n \"reap-if-stale\",\n );\n }\n return reaped;\n}\n\nexport async function updateRunStatus(\n runId: string,\n status: \"completed\" | \"errored\" | \"aborted\",\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET status = ?, completed_at = ? WHERE id = ?`,\n args: [status, Date.now(), runId],\n });\n}\n\nexport async function markRunAborted(\n runId: string,\n reason?: string,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n await client.execute({\n sql: `UPDATE agent_runs SET status = 'aborted', abort_reason = ?, completed_at = ? WHERE id = ?`,\n args: [reason ?? \"user\", Date.now(), runId],\n });\n await safeAppendTerminalRunEvent(runId, { type: \"done\" }, \"mark-aborted\");\n}\n\nexport async function isRunAborted(runId: string): Promise<boolean> {\n return (await getRunAbortState(runId)).aborted;\n}\n\nexport async function getRunAbortState(\n runId: string,\n): Promise<{ aborted: boolean; reason?: string }> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT status, abort_reason FROM agent_runs WHERE id = ?`,\n args: [runId],\n });\n if (rows.length === 0) return { aborted: false };\n const row = rows[0] as { status: string; abort_reason?: string | null };\n if (row.status !== \"aborted\") return { aborted: false };\n return {\n aborted: true,\n ...(row.abort_reason ? { reason: row.abort_reason } : {}),\n };\n}\n\nexport async function insertRunEvent(\n runId: string,\n seq: number,\n eventData: string,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n // ON CONFLICT DO NOTHING: a (runId, seq) collision can happen on the\n // soft-timeout / terminal-event path where `pendingTerminalEvent` was\n // assigned a seq that later gets reused by an event pushed after it.\n // It can also race with `appendTerminalRunEvent` (max-seq + 1) when a\n // run aborts at the same time the producer emits its final event.\n // Treat the second write as a no-op so the run completes cleanly.\n await client.execute({\n sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING`,\n args: [runId, seq, eventData],\n });\n}\n\nexport async function getRunEventsSince(\n runId: string,\n fromSeq: number,\n): Promise<Array<{ seq: number; eventData: string }>> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? AND seq >= ? ORDER BY seq ASC`,\n args: [runId, fromSeq],\n });\n return rows.map((r) => {\n const row = r as { seq: number | string; event_data: string };\n return { seq: Number(row.seq), eventData: row.event_data };\n });\n}\n\nexport async function getRunById(runId: string): Promise<{\n id: string;\n threadId: string;\n status: string;\n startedAt: number;\n} | null> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT id, thread_id, status, started_at FROM agent_runs WHERE id = ?`,\n args: [runId],\n });\n if (rows.length === 0) return null;\n const r = rows[0] as {\n id: string;\n thread_id: string;\n status: string;\n started_at: number | string;\n };\n return {\n id: r.id,\n threadId: r.thread_id,\n status: r.status,\n startedAt: Number(r.started_at),\n };\n}\n\nexport async function getRunByThread(\n threadId: string,\n options?: { includeTerminal?: boolean },\n): Promise<{\n id: string;\n threadId: string;\n status: string;\n startedAt: number;\n heartbeatAt: number | null;\n completedAt: number | null;\n lastProgressAt: number | null;\n} | null> {\n await ensureRunTables();\n const client = getDbExec();\n const sql = options?.includeTerminal\n ? `SELECT id, thread_id, status, started_at, heartbeat_at, completed_at, last_progress_at FROM agent_runs WHERE thread_id = ? ORDER BY started_at DESC LIMIT 1`\n : `SELECT id, thread_id, status, started_at, heartbeat_at, completed_at, last_progress_at FROM agent_runs WHERE thread_id = ? AND status = 'running' ORDER BY started_at DESC LIMIT 1`;\n const { rows } = await client.execute({ sql, args: [threadId] });\n if (rows.length === 0) return null;\n const r = rows[0] as {\n id: string;\n thread_id: string;\n status: string;\n started_at: number | string;\n heartbeat_at: number | string | null;\n completed_at: number | string | null;\n last_progress_at: number | string | null;\n };\n return {\n id: r.id,\n threadId: r.thread_id,\n status: r.status,\n startedAt: Number(r.started_at),\n heartbeatAt: r.heartbeat_at == null ? null : Number(r.heartbeat_at),\n completedAt: r.completed_at == null ? null : Number(r.completed_at),\n lastProgressAt:\n r.last_progress_at == null ? null : Number(r.last_progress_at),\n };\n}\n\n/**\n * Expire any \"running\" rows whose heartbeat is stale — producer died.\n * Safe to call at server startup on multi-isolate deployments: only rows\n * without a fresh heartbeat get reaped, so runs owned by OTHER live\n * isolates (which keep heartbeating) are left alone.\n */\nexport async function reapAllStaleRuns(): Promise<number> {\n await ensureRunTables();\n const client = getDbExec();\n const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n const stale = await client.execute({\n sql: `SELECT id FROM agent_runs\n WHERE status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [heartbeatCutoff],\n });\n const { rowsAffected } = await client.execute({\n sql: `UPDATE agent_runs\n SET status = 'errored', completed_at = ?\n WHERE status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [Date.now(), heartbeatCutoff],\n });\n for (const row of stale.rows) {\n const id = (row as { id?: unknown }).id;\n if (typeof id === \"string\") {\n await safeAppendTerminalRunEvent(\n id,\n STALE_RUN_ERROR_EVENT,\n \"reap-all-stale\",\n );\n }\n }\n return rowsAffected ?? 0;\n}\n\n/** Delete old runs and expire stale \"running\" rows that haven't had activity\n * (e.g. worker crashed before updating status). Completed runs are pruned at\n * `olderThanMs`; errored/aborted runs are kept until `erroredOlderThanMs` (a\n * longer window, falling back to `olderThanMs`) so their event log survives\n * for cut-off pattern analysis via listErroredRuns. */\nexport async function cleanupOldRuns(\n olderThanMs: number,\n erroredOlderThanMs?: number,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n const cutoff = Date.now() - olderThanMs;\n const erroredCutoff =\n Date.now() - Math.max(erroredOlderThanMs ?? 0, olderThanMs);\n // Expire stale running rows on the absolute-age threshold — safety net\n // for runs that never received a heartbeat (very old deployments). The\n // SELECT covers BOTH UPDATE conditions so the terminal-event-append loop\n // below catches every row we're about to flip — a 24h-old row with a\n // somehow-fresh heartbeat would slip past a heartbeat-only SELECT.\n const heartbeatCutoff = Date.now() - RUN_STALE_MS;\n const stale = await client.execute({\n sql: `SELECT id FROM agent_runs\n WHERE status = 'running'\n AND (\n COALESCE(heartbeat_at, started_at) < ?\n OR started_at < ?\n )`,\n args: [heartbeatCutoff, cutoff],\n });\n await client.execute({\n sql: `UPDATE agent_runs SET status = 'errored', completed_at = ? WHERE status = 'running' AND started_at < ?`,\n args: [Date.now(), cutoff],\n });\n // Also expire runs whose heartbeat is stale — producer has died.\n await client.execute({\n sql: `UPDATE agent_runs\n SET status = 'errored', completed_at = ?\n WHERE status = 'running'\n AND COALESCE(heartbeat_at, started_at) < ?`,\n args: [Date.now(), heartbeatCutoff],\n });\n for (const row of stale.rows) {\n const id = (row as { id?: unknown }).id;\n if (typeof id === \"string\") {\n await safeAppendTerminalRunEvent(\n id,\n STALE_RUN_ERROR_EVENT,\n \"cleanup-old-runs\",\n );\n }\n }\n // Delete events for old terminal runs. Completed runs prune at `cutoff`;\n // errored/aborted runs are retained until the (longer) `erroredCutoff`.\n await client.execute({\n sql: `DELETE FROM agent_run_events WHERE run_id IN (\n SELECT id FROM agent_runs\n WHERE (status = 'completed' AND completed_at < ?)\n OR (status IN ('errored', 'aborted') AND completed_at < ?)\n )`,\n args: [cutoff, erroredCutoff],\n });\n await client.execute({\n sql: `DELETE FROM agent_runs\n WHERE (status = 'completed' AND completed_at < ?)\n OR (status IN ('errored', 'aborted') AND completed_at < ?)`,\n args: [cutoff, erroredCutoff],\n });\n}\n\n/**\n * List recent errored/aborted runs for cut-off pattern analysis. Read-only,\n * bounded, and ordered newest-first. Surfaced via the list-errored-runs action\n * so the team can see why chats are failing (terminal error code, duration,\n * turn linkage) instead of discovering it ad hoc.\n */\nexport async function listErroredRuns(options?: {\n limit?: number;\n sinceMs?: number;\n}): Promise<\n Array<{\n id: string;\n threadId: string;\n turnId: string | null;\n status: string;\n errorCode: string | null;\n errorDetail: string | null;\n startedAt: number;\n completedAt: number | null;\n durationMs: number | null;\n }>\n> {\n await ensureRunTables();\n const client = getDbExec();\n const limit = Math.min(Math.max(Math.floor(options?.limit ?? 100), 1), 1000);\n const since =\n options?.sinceMs && options.sinceMs > 0 ? Date.now() - options.sinceMs : 0;\n const { rows } = await client.execute({\n sql: `SELECT id, thread_id, turn_id, status, error_code, error_detail, started_at, completed_at\n FROM agent_runs\n WHERE status IN ('errored', 'aborted')\n AND COALESCE(completed_at, started_at) >= ?\n ORDER BY COALESCE(completed_at, started_at) DESC\n LIMIT ${limit}`,\n args: [since],\n });\n return rows.map((r) => {\n const row = r as {\n id: string;\n thread_id: string;\n turn_id: string | null;\n status: string;\n error_code: string | null;\n error_detail: string | null;\n started_at: number | string;\n completed_at: number | string | null;\n };\n const startedAt = Number(row.started_at);\n const completedAt =\n row.completed_at == null ? null : Number(row.completed_at);\n return {\n id: row.id,\n threadId: row.thread_id,\n turnId: row.turn_id ?? null,\n status: row.status,\n errorCode: row.error_code ?? null,\n errorDetail: row.error_detail ?? null,\n startedAt,\n completedAt,\n durationMs: completedAt == null ? null : completedAt - startedAt,\n };\n });\n}\n\n/**\n * Idempotently append a terminal event to a run's event stream. No-op if the\n * stream already ends in a terminal event. Used by reapers AND by SSE\n * reconnect paths that discover an `errored` run row with no terminal event\n * (e.g. an earlier reaper's silent `.catch(() => {})` swallowed the append).\n *\n * Persisting from the reconnect path is what keeps the system self-healing:\n * subsequent reconnects replay the proper terminal event from SQL instead of\n * synthesizing a fresh one each time.\n */\nexport async function ensureTerminalRunEvent(\n runId: string,\n event: Record<string, unknown>,\n): Promise<void> {\n return appendTerminalRunEvent(runId, event);\n}\n\n/**\n * Append a terminal run event, retrying once on failure and reporting to\n * Sentry if both attempts fail. Background reaper paths can't surface errors\n * to a user, but they MUST eventually persist a terminal event — losing it\n * leaves reconnecting clients staring at a bare `status='errored'` row with\n * no payload to render. The previous `.catch(() => {})` callsites silently\n * dropped transient SQL blips and produced exactly that bug. Never throws.\n */\nasync function safeAppendTerminalRunEvent(\n runId: string,\n event: Record<string, unknown>,\n source: string,\n): Promise<void> {\n let firstError: unknown;\n try {\n await appendTerminalRunEvent(runId, event);\n return;\n } catch (err) {\n firstError = err;\n }\n // Brief backoff — most \"transient\" SQL failures (connection blip, lock\n // contention) clear within a couple hundred ms.\n await new Promise<void>((resolve) => setTimeout(resolve, 100));\n try {\n await appendTerminalRunEvent(runId, event);\n } catch (retryErr) {\n captureError(retryErr, {\n tags: {\n component: \"agent-run-store\",\n operation: \"append-terminal-event\",\n source,\n },\n extra: {\n runId,\n eventType: typeof event.type === \"string\" ? event.type : \"(unknown)\",\n firstError:\n firstError instanceof Error ? firstError.message : String(firstError),\n },\n });\n }\n}\n\nasync function appendTerminalRunEvent(\n runId: string,\n event: Record<string, unknown>,\n): Promise<void> {\n await ensureRunTables();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,\n args: [runId],\n });\n const last = rows[0] as\n | { seq?: number | string; event_data?: string }\n | undefined;\n if (last?.event_data) {\n try {\n const parsed = JSON.parse(last.event_data);\n if (\n parsed?.type === \"done\" ||\n parsed?.type === \"error\" ||\n parsed?.type === \"missing_api_key\" ||\n parsed?.type === \"loop_limit\" ||\n parsed?.type === \"auto_continue\"\n ) {\n return;\n }\n } catch {\n // Ignore malformed rows and append the terminal event.\n }\n }\n const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;\n await client.execute({\n sql: `INSERT INTO agent_run_events (run_id, seq, event_data) VALUES (?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING`,\n args: [runId, nextSeq, JSON.stringify(event)],\n });\n}\n"]}
|
|
@@ -11,6 +11,13 @@ interface ContentPart {
|
|
|
11
11
|
}
|
|
12
12
|
interface BuildAssistantMessageOptions {
|
|
13
13
|
suppressInternalContinuation?: boolean;
|
|
14
|
+
/**
|
|
15
|
+
* Logical-turn identity. When set it is stamped onto the message metadata so
|
|
16
|
+
* continuation runs of the same turn can be folded onto a single durable
|
|
17
|
+
* assistant message (see foldAssistantTurn) instead of each run dropping or
|
|
18
|
+
* overwriting the others.
|
|
19
|
+
*/
|
|
20
|
+
turnId?: string;
|
|
14
21
|
}
|
|
15
22
|
type AssistantMessage = NonNullable<ReturnType<typeof buildAssistantMessage>>;
|
|
16
23
|
type UserMessage = ReturnType<typeof buildUserMessage>;
|
|
@@ -94,6 +101,26 @@ export declare function upsertUserMessage(repo: any, userMsg: UserMessage): any;
|
|
|
94
101
|
* turn.
|
|
95
102
|
*/
|
|
96
103
|
export declare function upsertAssistantMessage(repo: any, assistantMsg: AssistantMessage): any;
|
|
104
|
+
/**
|
|
105
|
+
* Fold a continuation run's assistant message onto the single durable message
|
|
106
|
+
* for its logical turn (identified by `turnId`), so a turn that spans several
|
|
107
|
+
* continuation runs accumulates into ONE message that only ever grows. This is
|
|
108
|
+
* the server-side analog of an append-only rollout: the durable transcript is
|
|
109
|
+
* a monotonic fold over every run in the turn, never a per-run snapshot that
|
|
110
|
+
* drops the earlier chunks.
|
|
111
|
+
*
|
|
112
|
+
* Idempotent and never-shrinking, so it is safe to run alongside the client's
|
|
113
|
+
* full-thread export (which may write the same turn from the other side):
|
|
114
|
+
* - First chunk of a turn → appended as a fresh message.
|
|
115
|
+
* - A run whose content is already represented (already folded, or the client
|
|
116
|
+
* saved it) → kept as-is, choosing whichever copy has more content.
|
|
117
|
+
* - A new chunk → appended onto the accumulated turn.
|
|
118
|
+
* Falls back to per-run upsert when no `turnId` is available (turn == run).
|
|
119
|
+
*/
|
|
120
|
+
export declare function foldAssistantTurn(repo: any, assistantMsg: AssistantMessage, options: {
|
|
121
|
+
turnId?: string;
|
|
122
|
+
runId?: string;
|
|
123
|
+
}): any;
|
|
97
124
|
export declare function normalizeThreadTitle(value: unknown): string;
|
|
98
125
|
/**
|
|
99
126
|
* Extract title and preview from a thread runtime export.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"thread-data-builder.d.ts","sourceRoot":"","sources":["../../src/agent/thread-data-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAChE,OAAO,EAEL,KAAK,wBAAwB,IAAI,4BAA4B,EAI9D,MAAM,yCAAyC,CAAC;AAEjD,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,4BAA4B;IACpC,4BAA4B,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"thread-data-builder.d.ts","sourceRoot":"","sources":["../../src/agent/thread-data-builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAChE,OAAO,EAEL,KAAK,wBAAwB,IAAI,4BAA4B,EAI9D,MAAM,yCAAyC,CAAC;AAEjD,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,4BAA4B;IACpC,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,gBAAgB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,qBAAqB,CAAC,CAAC,CAAC;AAC9E,KAAK,WAAW,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAwCvD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,QAAQ,EAAE,EAClB,KAAK,CAAC,EAAE,MAAM,EACd,OAAO,GAAE,4BAAiC,GACzC;IACD,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,IAAI,CAAC;IAChB,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EACF;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GACpC;QAAE,IAAI,EAAE,YAAY,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAC5C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GAAG,IAAI,CAsIP;AAwKD;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,CAqCxD;AAED,MAAM,WAAW,8BAA8B;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,4BAA4B,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,CAAC,EAAE,4BAA4B,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACrD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,6CAA6C;IAC5D,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,wBAAgB,sCAAsC,CACpD,MAAM,EAAE,SAAS,8BAA8B,EAAE,EACjD,OAAO,GAAE,6CAAkD,GAC1D,GAAG,CA4GL;AAaD;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACrC,8BAA8B,CAAC,EAAE,OAAO,CAAC;IACzC,4BAA4B,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,wBAAgB,4BAA4B,CAC1C,YAAY,EAAE,GAAG,EACjB,YAAY,EAAE,GAAG,EACjB,OAAO,GAAE,sBAA2B,OAkFrC;AAqFD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,IAAI,CAAC;CAClB,GAAG;IACF,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,IAAI,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,CAcA;AAyID,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,GAAG,GAAG,CAetE;AA+BD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,GAAG,EACT,YAAY,EAAE,gBAAgB,GAC7B,GAAG,CA0BL;AAkDD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,GAAG,EACT,YAAY,EAAE,gBAAgB,EAC9B,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3C,GAAG,CAkFL;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAG3D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,GAAG,GAAG;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB,CA0BA"}
|
|
@@ -117,19 +117,32 @@ export function buildAssistantMessage(events, runId, options = {}) {
|
|
|
117
117
|
}
|
|
118
118
|
// done, missing_api_key — terminal signals, not content
|
|
119
119
|
}
|
|
120
|
-
|
|
120
|
+
// Only a truly empty turn produces nothing to persist. A turn that ended at
|
|
121
|
+
// an internal continuation boundary (soft-timeout auto_continue, a
|
|
122
|
+
// recoverable gateway error, suppressed loop_limit) DID stream real content
|
|
123
|
+
// — persist it as a partial so the continuation run can fold the next chunk
|
|
124
|
+
// onto it (foldAssistantTurn) instead of the earlier text being dropped.
|
|
125
|
+
if (content.length === 0)
|
|
121
126
|
return null;
|
|
122
|
-
const
|
|
127
|
+
const continued = endedAtInternalContinuationBoundary;
|
|
128
|
+
const custom = {};
|
|
129
|
+
if (options.turnId)
|
|
130
|
+
custom.turnId = options.turnId;
|
|
123
131
|
if (runId)
|
|
124
|
-
|
|
132
|
+
custom.foldedRunIds = [runId];
|
|
133
|
+
if (continued)
|
|
134
|
+
custom.continued = true;
|
|
125
135
|
if (runError) {
|
|
126
|
-
|
|
127
|
-
runError
|
|
128
|
-
|
|
129
|
-
...(runId ? { runId } : {}),
|
|
130
|
-
},
|
|
136
|
+
custom.runError = {
|
|
137
|
+
...runError,
|
|
138
|
+
...(runId ? { runId } : {}),
|
|
131
139
|
};
|
|
132
140
|
}
|
|
141
|
+
const metadata = {};
|
|
142
|
+
if (runId)
|
|
143
|
+
metadata.runId = runId;
|
|
144
|
+
if (Object.keys(custom).length > 0)
|
|
145
|
+
metadata.custom = custom;
|
|
133
146
|
return {
|
|
134
147
|
id: `server-${runId ?? Date.now()}`,
|
|
135
148
|
createdAt: new Date(),
|
|
@@ -206,6 +219,12 @@ function messageIdentityKeys(message) {
|
|
|
206
219
|
const runId = getMessageRunId(message);
|
|
207
220
|
if (runId)
|
|
208
221
|
keys.push(`run:${runId}`);
|
|
222
|
+
// A logical turn is ONE durable assistant message even though it may span
|
|
223
|
+
// several continuation runs, so two messages sharing a turnId (e.g. the
|
|
224
|
+
// client export and the server fold of the same answer) must dedupe to one.
|
|
225
|
+
const turnId = turnIdOf(message);
|
|
226
|
+
if (turnId)
|
|
227
|
+
keys.push(`turn:${turnId}`);
|
|
209
228
|
// Normalize attachments through `normalizeAttachmentIdentity` so an
|
|
210
229
|
// explicit empty `[]` (assistant-ui's default for messages with no
|
|
211
230
|
// attachments) and an omitted/undefined `attachments` field hash to the
|
|
@@ -248,6 +267,27 @@ function messagesMatch(a, b) {
|
|
|
248
267
|
function chooseMergedMessageEntry(existingEntry, incomingEntry) {
|
|
249
268
|
const existing = getStoredMessage(existingEntry);
|
|
250
269
|
const incoming = getStoredMessage(incomingEntry);
|
|
270
|
+
// Same logical turn (client export vs server fold of one accumulating
|
|
271
|
+
// answer): never shrink — keep whichever side accumulated more content, so a
|
|
272
|
+
// stale/lossy export can't overwrite the richer folded turn. Ties prefer the
|
|
273
|
+
// terminal copy.
|
|
274
|
+
const existingTurn = turnIdOf(existing);
|
|
275
|
+
const incomingTurn = turnIdOf(incoming);
|
|
276
|
+
if (existing?.role === "assistant" &&
|
|
277
|
+
incoming?.role === "assistant" &&
|
|
278
|
+
existingTurn &&
|
|
279
|
+
existingTurn === incomingTurn) {
|
|
280
|
+
const existingWeight = assistantContentWeight(existing.content);
|
|
281
|
+
const incomingWeight = assistantContentWeight(incoming.content);
|
|
282
|
+
if (existingWeight > incomingWeight)
|
|
283
|
+
return existingEntry;
|
|
284
|
+
if (incomingWeight > existingWeight)
|
|
285
|
+
return incomingEntry;
|
|
286
|
+
return isTerminalAssistantStatus(existing?.status) &&
|
|
287
|
+
!isTerminalAssistantStatus(incoming?.status)
|
|
288
|
+
? existingEntry
|
|
289
|
+
: incomingEntry;
|
|
290
|
+
}
|
|
251
291
|
if (existing?.role === "assistant" &&
|
|
252
292
|
incoming?.role === "assistant" &&
|
|
253
293
|
isTerminalAssistantStatus(existing?.status) &&
|
|
@@ -740,6 +780,136 @@ export function upsertAssistantMessage(repo, assistantMsg) {
|
|
|
740
780
|
nextRepo.headId = assistantMsg.id;
|
|
741
781
|
return nextRepo;
|
|
742
782
|
}
|
|
783
|
+
function turnIdOf(message) {
|
|
784
|
+
const t = message?.metadata?.custom?.turnId;
|
|
785
|
+
return typeof t === "string" && t ? t : undefined;
|
|
786
|
+
}
|
|
787
|
+
function foldedRunIdsOf(message) {
|
|
788
|
+
const ids = message?.metadata?.custom?.foldedRunIds;
|
|
789
|
+
return Array.isArray(ids)
|
|
790
|
+
? ids.filter((x) => typeof x === "string")
|
|
791
|
+
: [];
|
|
792
|
+
}
|
|
793
|
+
/** Rough size of an assistant message's content, used only to pick the larger
|
|
794
|
+
* of two representations of the same chunk so a fold can never shrink. */
|
|
795
|
+
function assistantContentWeight(content) {
|
|
796
|
+
if (!Array.isArray(content))
|
|
797
|
+
return 0;
|
|
798
|
+
let weight = 0;
|
|
799
|
+
for (const part of content) {
|
|
800
|
+
if (part?.type === "text" && typeof part.text === "string") {
|
|
801
|
+
weight += part.text.length;
|
|
802
|
+
}
|
|
803
|
+
else {
|
|
804
|
+
weight += 1;
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
return weight;
|
|
808
|
+
}
|
|
809
|
+
/** Concatenate continuation content onto the accumulated turn, merging a
|
|
810
|
+
* trailing+leading text run so the resumed answer reads as one flowing
|
|
811
|
+
* message rather than two stacked fragments. */
|
|
812
|
+
function appendFoldedContent(existing, incoming) {
|
|
813
|
+
const merged = existing.map((p) => ({ ...p }));
|
|
814
|
+
for (const part of incoming) {
|
|
815
|
+
const last = merged[merged.length - 1];
|
|
816
|
+
if (part?.type === "text" &&
|
|
817
|
+
typeof part.text === "string" &&
|
|
818
|
+
last?.type === "text" &&
|
|
819
|
+
typeof last.text === "string") {
|
|
820
|
+
last.text = `${last.text}${part.text}`;
|
|
821
|
+
}
|
|
822
|
+
else {
|
|
823
|
+
merged.push({ ...part });
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
return merged;
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* Fold a continuation run's assistant message onto the single durable message
|
|
830
|
+
* for its logical turn (identified by `turnId`), so a turn that spans several
|
|
831
|
+
* continuation runs accumulates into ONE message that only ever grows. This is
|
|
832
|
+
* the server-side analog of an append-only rollout: the durable transcript is
|
|
833
|
+
* a monotonic fold over every run in the turn, never a per-run snapshot that
|
|
834
|
+
* drops the earlier chunks.
|
|
835
|
+
*
|
|
836
|
+
* Idempotent and never-shrinking, so it is safe to run alongside the client's
|
|
837
|
+
* full-thread export (which may write the same turn from the other side):
|
|
838
|
+
* - First chunk of a turn → appended as a fresh message.
|
|
839
|
+
* - A run whose content is already represented (already folded, or the client
|
|
840
|
+
* saved it) → kept as-is, choosing whichever copy has more content.
|
|
841
|
+
* - A new chunk → appended onto the accumulated turn.
|
|
842
|
+
* Falls back to per-run upsert when no `turnId` is available (turn == run).
|
|
843
|
+
*/
|
|
844
|
+
export function foldAssistantTurn(repo, assistantMsg, options) {
|
|
845
|
+
const turnId = options.turnId;
|
|
846
|
+
const runId = options.runId;
|
|
847
|
+
if (!turnId)
|
|
848
|
+
return upsertAssistantMessage(repo, assistantMsg);
|
|
849
|
+
const nextRepo = normalizeThreadRepository(repo);
|
|
850
|
+
const lastIndex = nextRepo.messages.length - 1;
|
|
851
|
+
const lastEntry = lastIndex >= 0 ? nextRepo.messages[lastIndex] : undefined;
|
|
852
|
+
const lastMsg = getStoredMessage(lastEntry);
|
|
853
|
+
const sameTurn = lastMsg?.role === "assistant" &&
|
|
854
|
+
(turnIdOf(lastMsg) === turnId ||
|
|
855
|
+
// A message the client wrote for one of this turn's runs before it
|
|
856
|
+
// carried a turnId stamp.
|
|
857
|
+
(!!runId && getMessageRunId(lastMsg) === runId));
|
|
858
|
+
if (!sameTurn) {
|
|
859
|
+
// First chunk of this turn (or the previous assistant belongs to an
|
|
860
|
+
// earlier turn) — append as a fresh message; buildAssistantMessage already
|
|
861
|
+
// stamped turnId + foldedRunIds onto it.
|
|
862
|
+
return upsertAssistantMessage(repo, assistantMsg);
|
|
863
|
+
}
|
|
864
|
+
const existingContent = Array.isArray(lastMsg.content) ? lastMsg.content : [];
|
|
865
|
+
const incomingContent = Array.isArray(assistantMsg.content)
|
|
866
|
+
? assistantMsg.content
|
|
867
|
+
: [];
|
|
868
|
+
const existingFolded = foldedRunIdsOf(lastMsg);
|
|
869
|
+
const runAlreadyFolded = !!runId &&
|
|
870
|
+
(existingFolded.includes(runId) || getMessageRunId(lastMsg) === runId);
|
|
871
|
+
// If this run's chunk is already represented in the turn (the client saved
|
|
872
|
+
// it, or we already folded it), do not re-append — keep the larger copy so
|
|
873
|
+
// the turn never shrinks. Otherwise fold this chunk onto the accumulated turn.
|
|
874
|
+
const mergedContent = runAlreadyFolded
|
|
875
|
+
? assistantContentWeight(incomingContent) >
|
|
876
|
+
assistantContentWeight(existingContent)
|
|
877
|
+
? incomingContent
|
|
878
|
+
: existingContent
|
|
879
|
+
: appendFoldedContent(existingContent, incomingContent);
|
|
880
|
+
const mergedFolded = Array.from(new Set([...existingFolded, ...(runId ? [runId] : [])]));
|
|
881
|
+
const existingCustom = lastMsg.metadata?.custom && typeof lastMsg.metadata.custom === "object"
|
|
882
|
+
? lastMsg.metadata.custom
|
|
883
|
+
: {};
|
|
884
|
+
const incomingCustom = assistantMsg.metadata?.custom &&
|
|
885
|
+
typeof assistantMsg.metadata.custom === "object"
|
|
886
|
+
? assistantMsg.metadata.custom
|
|
887
|
+
: {};
|
|
888
|
+
const mergedCustom = {
|
|
889
|
+
...existingCustom,
|
|
890
|
+
...incomingCustom,
|
|
891
|
+
turnId,
|
|
892
|
+
foldedRunIds: mergedFolded,
|
|
893
|
+
};
|
|
894
|
+
// Only the freshest chunk decides whether the turn is still continuing.
|
|
895
|
+
if (incomingCustom.continued !== true)
|
|
896
|
+
delete mergedCustom.continued;
|
|
897
|
+
const mergedMessage = {
|
|
898
|
+
...lastMsg,
|
|
899
|
+
content: mergedContent,
|
|
900
|
+
// The freshest chunk's status wins: a clean done supersedes a prior
|
|
901
|
+
// partial; a real error supersedes a partial.
|
|
902
|
+
status: assistantMsg.status ?? lastMsg.status,
|
|
903
|
+
metadata: {
|
|
904
|
+
...lastMsg.metadata,
|
|
905
|
+
runId: runId ?? lastMsg.metadata?.runId,
|
|
906
|
+
custom: mergedCustom,
|
|
907
|
+
},
|
|
908
|
+
};
|
|
909
|
+
nextRepo.messages[lastIndex] = { ...lastEntry, message: mergedMessage };
|
|
910
|
+
nextRepo.headId = mergedMessage.id ?? nextRepo.headId;
|
|
911
|
+
return nextRepo;
|
|
912
|
+
}
|
|
743
913
|
export function normalizeThreadTitle(value) {
|
|
744
914
|
if (typeof value !== "string")
|
|
745
915
|
return "";
|