@standardagents/builder 0.25.7 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/built-in-routes.js +246 -16
- package/dist/built-in-routes.js.map +1 -1
- package/dist/client/{ApiKeysView-B9XLopSb.js → ApiKeysView-jhfg43UH.js} +1 -1
- package/dist/client/CenteredContentView-CS0mV2EJ.js +1 -0
- package/dist/client/{CompositionView-DJOtNN7f.js → CompositionView-DhkLWmtE.js} +1 -1
- package/dist/client/{ConfirmDialog.vue_vue_type_script_setup_true_lang-55dXt19T.js → ConfirmDialog.vue_vue_type_script_setup_true_lang-D-WToRMy.js} +1 -1
- package/dist/client/{CopyButton.vue_vue_type_script_setup_true_lang-B6NqQmW9.js → CopyButton.vue_vue_type_script_setup_true_lang-Dhq0t0dd.js} +1 -1
- package/dist/client/{DataTable.vue_vue_type_script_setup_true_lang-0cbPVIIz.js → DataTable.vue_vue_type_script_setup_true_lang-D1MDCns-.js} +1 -1
- package/dist/client/{JsonViewer-B7yHhxqP.js → JsonViewer-BoHrE76S.js} +1 -1
- package/dist/client/{LoginView-mY9OytK6.js → LoginView-BpR2nMNf.js} +1 -1
- package/dist/client/{MarketplaceView-B9Z9YMuB.js → MarketplaceView-uUf1JYla.js} +1 -1
- package/dist/client/{Modal.vue_vue_type_script_setup_true_lang-DEVCRbXl.js → Modal.vue_vue_type_script_setup_true_lang-D-SDVFQI.js} +1 -1
- package/dist/client/{ModelModal.vue_vue_type_script_setup_true_lang-FxEBZ3A8.js → ModelModal.vue_vue_type_script_setup_true_lang-DE57Zr0r.js} +1 -1
- package/dist/client/{ModelsView-DTU8pk-T.js → ModelsView-CuM_OieH.js} +1 -1
- package/dist/client/{PromptEditView-BAKoULxX.js → PromptEditView-BWjrA2zr.js} +1 -1
- package/dist/client/{PromptModal-CoinSKIM.js → PromptModal-BFxhuG0Z.js} +1 -1
- package/dist/client/{PromptsView-DIkj3q-o.js → PromptsView-AlGsRB6k.js} +1 -1
- package/dist/client/{ProvidersView-B8A6ezDR.js → ProvidersView-CbXBs3SH.js} +1 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-Dm3d37hz.js +22 -0
- package/dist/client/{ThreadsView-DGCVdDej.js → ThreadsView-rN9rzJi1.js} +1 -1
- package/dist/client/{ToolsView-CC-WsZet.js → ToolsView-0Gpo_RFg.js} +1 -1
- package/dist/client/{UiCodeBlock.vue_vue_type_script_setup_true_lang-BH3WC9G_.js → UiCodeBlock.vue_vue_type_script_setup_true_lang-CNncALQo.js} +1 -1
- package/dist/client/{UsersView-U3Stwxui.js → UsersView-iXPbcguY.js} +1 -1
- package/dist/client/{VariablesView-0-a1F4cn.js → VariablesView-C9BB9h34.js} +1 -1
- package/dist/client/assets/{ThreadInspectorPane-ouC_RyKA.css → ThreadInspectorPane-CfpdDnUQ.css} +1 -1
- package/dist/client/assets/index-BTPsPULO.css +1 -0
- package/dist/client/{check-Dkzs1C3V.js → check-BdJVlUmI.js} +1 -1
- package/dist/client/{chevron-right-BGirddPg.js → chevron-right-D49O3XLU.js} +1 -1
- package/dist/client/{copy-C08OR8rY.js → copy-40uDyTgY.js} +1 -1
- package/dist/client/{formkit-YJ1Z7D5r.js → formkit-Bl-sERnF.js} +1 -1
- package/dist/client/index-BCoE48be.js +6 -0
- package/dist/client/index.html +4 -4
- package/dist/client/{package-guyzMTDG.js → package-1UgoERpx.js} +1 -1
- package/dist/client/{plus-DG3ZN5wg.js → plus-BzCjfYs0.js} +1 -1
- package/dist/client/{timeAgo-D7V247YG.js → timeAgo-cFwldMu-.js} +1 -1
- package/dist/client/{useActionFeedback-byARNQjQ.js → useActionFeedback-BVifffYQ.js} +1 -1
- package/dist/client/{vue-BqC6IZDY.js → vue-CmmJuziT.js} +1 -1
- package/dist/{index-CX9Ti98s.d.ts → index-DJ7B8CTp.d.ts} +63 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +532 -83
- package/dist/index.js.map +1 -1
- package/dist/plugin.js +27 -3
- package/dist/plugin.js.map +1 -1
- package/dist/runtime.d.ts +184 -6
- package/dist/runtime.js +505 -80
- package/dist/runtime.js.map +1 -1
- package/dist/test.d.ts +1 -1
- package/package.json +5 -5
- package/dist/client/CenteredContentView-Bwv4MVSC.js +0 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-CvssvzLz.js +0 -22
- package/dist/client/assets/index-Ca3he3fq.css +0 -1
- package/dist/client/index-4Zwz6zXD.js +0 -6
package/dist/runtime.js
CHANGED
|
@@ -1070,6 +1070,24 @@ function resolveWatchdogStaleCeilingMs(env) {
|
|
|
1070
1070
|
);
|
|
1071
1071
|
return Math.max(firstChunk, stall) + 6e4;
|
|
1072
1072
|
}
|
|
1073
|
+
async function awaitBounded(work, budgetMs, label) {
|
|
1074
|
+
let timer;
|
|
1075
|
+
work.catch(() => {
|
|
1076
|
+
});
|
|
1077
|
+
const timeout = new Promise((resolve) => {
|
|
1078
|
+
timer = setTimeout(() => {
|
|
1079
|
+
console.error(
|
|
1080
|
+
`[LLMRequest] ${label} did not settle within ${budgetMs}ms \u2014 continuing without it`
|
|
1081
|
+
);
|
|
1082
|
+
resolve(void 0);
|
|
1083
|
+
}, budgetMs);
|
|
1084
|
+
});
|
|
1085
|
+
try {
|
|
1086
|
+
return await Promise.race([work, timeout]);
|
|
1087
|
+
} finally {
|
|
1088
|
+
if (timer) clearTimeout(timer);
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1073
1091
|
function modelSupportsImageInput(modelDef) {
|
|
1074
1092
|
const supportsImages = modelDef.capabilities?.supportsImages;
|
|
1075
1093
|
if (typeof supportsImages === "boolean") {
|
|
@@ -1283,13 +1301,42 @@ function transformMessages(messages) {
|
|
|
1283
1301
|
arguments: typeof tc.function.arguments === "string" ? JSON.parse(tc.function.arguments) : tc.function.arguments,
|
|
1284
1302
|
extraContent: tc.extra_content && typeof tc.extra_content === "object" ? tc.extra_content : void 0
|
|
1285
1303
|
}));
|
|
1304
|
+
const storedDetails = msg.reasoning_details;
|
|
1305
|
+
const reasoningDetails = Array.isArray(
|
|
1306
|
+
storedDetails
|
|
1307
|
+
) ? storedDetails.flatMap((entry) => {
|
|
1308
|
+
if (!entry || typeof entry !== "object") return [];
|
|
1309
|
+
const raw = entry;
|
|
1310
|
+
const type = raw.type === "summary" || raw.type === "encrypted" ? raw.type : "text";
|
|
1311
|
+
return [
|
|
1312
|
+
{
|
|
1313
|
+
type,
|
|
1314
|
+
id: typeof raw.id === "string" ? raw.id : void 0,
|
|
1315
|
+
text: typeof raw.text === "string" ? raw.text : void 0,
|
|
1316
|
+
data: typeof raw.data === "string" ? raw.data : void 0,
|
|
1317
|
+
format: typeof raw.format === "string" ? raw.format : void 0
|
|
1318
|
+
}
|
|
1319
|
+
];
|
|
1320
|
+
}) : void 0;
|
|
1321
|
+
const storedReasoning = msg.reasoning_content;
|
|
1322
|
+
let reasoning = typeof storedReasoning === "string" && storedReasoning.length > 0 ? storedReasoning : void 0;
|
|
1323
|
+
if (!reasoning && Array.isArray(reasoningDetails)) {
|
|
1324
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1325
|
+
const texts = [];
|
|
1326
|
+
for (const d of reasoningDetails) {
|
|
1327
|
+
if (typeof d?.text === "string" && d.text.length > 0 && !seen.has(d.text)) {
|
|
1328
|
+
seen.add(d.text);
|
|
1329
|
+
texts.push(d.text);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
if (texts.length) reasoning = texts.join("\n");
|
|
1333
|
+
}
|
|
1286
1334
|
return {
|
|
1287
1335
|
role: "assistant",
|
|
1288
1336
|
content: typeof msg.content === "string" ? msg.content : null,
|
|
1289
1337
|
toolCalls: toolCalls?.length ? toolCalls : void 0,
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
reasoningDetails: msg.reasoning_details || void 0
|
|
1338
|
+
reasoning,
|
|
1339
|
+
reasoningDetails: reasoningDetails?.length ? reasoningDetails : void 0
|
|
1293
1340
|
};
|
|
1294
1341
|
}
|
|
1295
1342
|
case "tool":
|
|
@@ -1464,7 +1511,7 @@ function normalizeProviderMetadataValue(value) {
|
|
|
1464
1511
|
if (!trimmed || trimmed.toLowerCase() === "unknown") return void 0;
|
|
1465
1512
|
return trimmed;
|
|
1466
1513
|
}
|
|
1467
|
-
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, LLMRequest;
|
|
1514
|
+
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, HOOK_AWAIT_BUDGET_MS, STREAM_CLEANUP_BUDGET_MS, LLMRequest;
|
|
1468
1515
|
var init_LLMRequest = __esm({
|
|
1469
1516
|
"src/agents/LLMRequest.ts"() {
|
|
1470
1517
|
init_pricing();
|
|
@@ -1475,6 +1522,8 @@ var init_LLMRequest = __esm({
|
|
|
1475
1522
|
NON_VISION_PLACEHOLDER_TEXT = "[Image omitted: model does not support vision]";
|
|
1476
1523
|
STREAM_FIRST_CHUNK_TIMEOUT_MS = 12e4;
|
|
1477
1524
|
STREAM_STALL_TIMEOUT_MS = 45e3;
|
|
1525
|
+
HOOK_AWAIT_BUDGET_MS = 3e4;
|
|
1526
|
+
STREAM_CLEANUP_BUDGET_MS = 5e3;
|
|
1478
1527
|
LLMRequest = class {
|
|
1479
1528
|
/**
|
|
1480
1529
|
* Execute an LLM request with full retry/fallback logic
|
|
@@ -1958,7 +2007,14 @@ var init_LLMRequest = __esm({
|
|
|
1958
2007
|
parentSignal.removeEventListener("abort", onParentAbort);
|
|
1959
2008
|
}
|
|
1960
2009
|
try {
|
|
1961
|
-
|
|
2010
|
+
const cleanup = iterator.return?.();
|
|
2011
|
+
if (cleanup) {
|
|
2012
|
+
await awaitBounded(
|
|
2013
|
+
Promise.resolve(cleanup),
|
|
2014
|
+
STREAM_CLEANUP_BUDGET_MS,
|
|
2015
|
+
"stream iterator cleanup"
|
|
2016
|
+
);
|
|
2017
|
+
}
|
|
1962
2018
|
} catch {
|
|
1963
2019
|
}
|
|
1964
2020
|
}
|
|
@@ -2091,7 +2147,11 @@ var init_LLMRequest = __esm({
|
|
|
2091
2147
|
*/
|
|
2092
2148
|
static async resolveProgressArgument(state, toolName) {
|
|
2093
2149
|
try {
|
|
2094
|
-
const mod = await
|
|
2150
|
+
const mod = await awaitBounded(
|
|
2151
|
+
state.thread.instance.loadTool(toolName),
|
|
2152
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2153
|
+
`loadTool(${toolName}) for progress argument`
|
|
2154
|
+
);
|
|
2095
2155
|
const pa = mod?.progressArgument ?? mod?.default?.progressArgument;
|
|
2096
2156
|
return typeof pa === "string" && pa ? pa : void 0;
|
|
2097
2157
|
} catch {
|
|
@@ -2112,7 +2172,11 @@ var init_LLMRequest = __esm({
|
|
|
2112
2172
|
function: { name: entry.name, arguments: entry.argumentsDelta || "" }
|
|
2113
2173
|
};
|
|
2114
2174
|
try {
|
|
2115
|
-
await
|
|
2175
|
+
await awaitBounded(
|
|
2176
|
+
FlowEngine.runToolCallStartedHook(state, toolCall, progress),
|
|
2177
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2178
|
+
`tool_call_started hook for ${entry.name}`
|
|
2179
|
+
);
|
|
2116
2180
|
} catch (error) {
|
|
2117
2181
|
console.error("[LLMRequest] Error firing tool_call_started:", error);
|
|
2118
2182
|
}
|
|
@@ -12697,8 +12761,12 @@ var init_ThreadStateImpl = __esm({
|
|
|
12697
12761
|
options?.includeSilent,
|
|
12698
12762
|
options?.maxDepth
|
|
12699
12763
|
);
|
|
12764
|
+
const messages = result.messages.map(this._mapMessage);
|
|
12765
|
+
if (options?.order === "desc") {
|
|
12766
|
+
messages.reverse();
|
|
12767
|
+
}
|
|
12700
12768
|
return {
|
|
12701
|
-
messages
|
|
12769
|
+
messages,
|
|
12702
12770
|
total: result.total,
|
|
12703
12771
|
hasMore: result.hasMore
|
|
12704
12772
|
};
|
|
@@ -16906,6 +16974,11 @@ var DurableThread = class _DurableThread extends DurableObject {
|
|
|
16906
16974
|
if (this.currentAbortController) {
|
|
16907
16975
|
this.currentAbortController.abort();
|
|
16908
16976
|
}
|
|
16977
|
+
try {
|
|
16978
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
16979
|
+
} catch (error) {
|
|
16980
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
16981
|
+
}
|
|
16909
16982
|
const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
|
|
16910
16983
|
const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
|
|
16911
16984
|
await agentBuilder.updateThread(threadId, { terminated: terminatedAt });
|
|
@@ -18065,6 +18138,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
18065
18138
|
*/
|
|
18066
18139
|
async getMessages(limit = 100, offset = 0, order = "DESC", includeSilent = false, maxDepth = 0) {
|
|
18067
18140
|
await this.ensureMigrated();
|
|
18141
|
+
const normalizedOrder = String(order).toUpperCase() === "ASC" ? "ASC" : "DESC";
|
|
18068
18142
|
try {
|
|
18069
18143
|
const whereClauses = [];
|
|
18070
18144
|
if (!includeSilent) {
|
|
@@ -18072,7 +18146,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
18072
18146
|
}
|
|
18073
18147
|
whereClauses.push(`(depth <= ${maxDepth} OR depth IS NULL)`);
|
|
18074
18148
|
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
|
|
18075
|
-
const orderDirection =
|
|
18149
|
+
const orderDirection = normalizedOrder;
|
|
18076
18150
|
const combinedMessagesSql = `
|
|
18077
18151
|
WITH combined_messages AS (
|
|
18078
18152
|
SELECT
|
|
@@ -18198,7 +18272,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
18198
18272
|
return projection ? { ...message, ...projection } : message;
|
|
18199
18273
|
});
|
|
18200
18274
|
}
|
|
18201
|
-
if (
|
|
18275
|
+
if (normalizedOrder === "DESC") {
|
|
18202
18276
|
messages = messages.reverse();
|
|
18203
18277
|
}
|
|
18204
18278
|
const hasMore = offset + messages.length < total;
|
|
@@ -19031,6 +19105,14 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19031
19105
|
}
|
|
19032
19106
|
this.logSockets.clear();
|
|
19033
19107
|
this.messageSockets.clear();
|
|
19108
|
+
try {
|
|
19109
|
+
const threadId = this.ctx.id.name;
|
|
19110
|
+
if (threadId) {
|
|
19111
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
19112
|
+
}
|
|
19113
|
+
} catch (error) {
|
|
19114
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
19115
|
+
}
|
|
19034
19116
|
await this.ctx.storage.deleteAlarm();
|
|
19035
19117
|
await this.ctx.storage.deleteAll();
|
|
19036
19118
|
this.migratedToVersion = null;
|
|
@@ -19686,6 +19768,18 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19686
19768
|
noteExecutionProgress() {
|
|
19687
19769
|
this.lastProgressAt = Date.now();
|
|
19688
19770
|
}
|
|
19771
|
+
/**
|
|
19772
|
+
* The per-thread companion watchdog DO (ThreadWatchdog), derived from the
|
|
19773
|
+
* thread id. It exists because this thread's OWN alarm cannot fire while a
|
|
19774
|
+
* queue-dispatched turn runs inside alarm() — the companion delivers the
|
|
19775
|
+
* same check via RPC, which interleaves at await points even mid-turn.
|
|
19776
|
+
* Returns null on deployments whose generated config predates the binding.
|
|
19777
|
+
*/
|
|
19778
|
+
watchdogCompanion(threadId) {
|
|
19779
|
+
const namespace = this.env.AGENT_BUILDER_WATCHDOG;
|
|
19780
|
+
if (!namespace || typeof namespace.idFromName !== "function") return null;
|
|
19781
|
+
return namespace.get(namespace.idFromName(threadId));
|
|
19782
|
+
}
|
|
19689
19783
|
/** Arm the standing watchdog for an execution start. One pending row max. */
|
|
19690
19784
|
async armExecutionWatchdog(threadId) {
|
|
19691
19785
|
try {
|
|
@@ -19694,14 +19788,20 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19694
19788
|
const pending = (await this.ctx.storage.sql.exec(
|
|
19695
19789
|
`SELECT id FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'pending' LIMIT 1`
|
|
19696
19790
|
)).toArray();
|
|
19697
|
-
if (pending.length
|
|
19698
|
-
|
|
19791
|
+
if (pending.length === 0) {
|
|
19792
|
+
await this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
19793
|
+
}
|
|
19699
19794
|
} catch (error) {
|
|
19700
19795
|
console.error("[DurableThread] Failed to arm execution watchdog (non-fatal):", error);
|
|
19701
19796
|
}
|
|
19797
|
+
try {
|
|
19798
|
+
await this.watchdogCompanion(threadId)?.arm(threadId);
|
|
19799
|
+
} catch (error) {
|
|
19800
|
+
console.error("[DurableThread] Failed to arm watchdog companion (non-fatal):", error);
|
|
19801
|
+
}
|
|
19702
19802
|
}
|
|
19703
19803
|
/** Disarm on clean execution end. Completed check rows are noise, not history. */
|
|
19704
|
-
async disarmExecutionWatchdog() {
|
|
19804
|
+
async disarmExecutionWatchdog(threadId) {
|
|
19705
19805
|
try {
|
|
19706
19806
|
await this.ctx.storage.sql.exec(
|
|
19707
19807
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status IN ('pending', 'completed')`
|
|
@@ -19710,10 +19810,21 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19710
19810
|
} catch (error) {
|
|
19711
19811
|
console.error("[DurableThread] Failed to disarm execution watchdog (non-fatal):", error);
|
|
19712
19812
|
}
|
|
19813
|
+
if (threadId) {
|
|
19814
|
+
try {
|
|
19815
|
+
await this.watchdogCompanion(threadId)?.disarmHint();
|
|
19816
|
+
} catch (error) {
|
|
19817
|
+
console.error("[DurableThread] Failed to hint watchdog companion (non-fatal):", error);
|
|
19818
|
+
}
|
|
19819
|
+
}
|
|
19713
19820
|
}
|
|
19714
19821
|
/**
|
|
19715
|
-
* The standing check. Evaluates the thread's state AT
|
|
19716
|
-
* nothing about whichever request was in flight when it
|
|
19822
|
+
* The standing check, alarm-queue delivery. Evaluates the thread's state AT
|
|
19823
|
+
* FIRE TIME — it knows nothing about whichever request was in flight when it
|
|
19824
|
+
* was armed. NOTE this delivery path cannot fire while a queue-dispatched
|
|
19825
|
+
* turn runs (the turn owns this DO's single alarm invocation); the
|
|
19826
|
+
* ThreadWatchdog companion delivers the same assessment via watchdogProbe()
|
|
19827
|
+
* in exactly that window.
|
|
19717
19828
|
*/
|
|
19718
19829
|
async watchdogCheck(threadId) {
|
|
19719
19830
|
const rearm = () => this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
@@ -19721,80 +19832,190 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19721
19832
|
await this.ctx.storage.sql.exec(
|
|
19722
19833
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'completed'`
|
|
19723
19834
|
);
|
|
19724
|
-
const
|
|
19725
|
-
|
|
19726
|
-
)).toArray();
|
|
19727
|
-
let incomplete = await readIncomplete();
|
|
19728
|
-
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
19729
|
-
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
19730
|
-
if (this.isExecuting) {
|
|
19731
|
-
if (incomplete.length === 0) {
|
|
19732
|
-
this.watchdogWedgeStrikes = 0;
|
|
19733
|
-
await rearm();
|
|
19734
|
-
return;
|
|
19735
|
-
}
|
|
19736
|
-
if (silenceMs <= ceiling) {
|
|
19737
|
-
this.watchdogWedgeStrikes = 0;
|
|
19738
|
-
await rearm();
|
|
19739
|
-
return;
|
|
19740
|
-
}
|
|
19741
|
-
this.watchdogWedgeStrikes++;
|
|
19742
|
-
if (this.watchdogWedgeStrikes < 2) {
|
|
19743
|
-
console.warn(
|
|
19744
|
-
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
19745
|
-
);
|
|
19746
|
-
await rearm();
|
|
19747
|
-
return;
|
|
19748
|
-
}
|
|
19749
|
-
console.error(
|
|
19750
|
-
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
19751
|
-
);
|
|
19752
|
-
try {
|
|
19753
|
-
this.currentAbortController?.abort();
|
|
19754
|
-
} catch {
|
|
19755
|
-
}
|
|
19756
|
-
this.isExecuting = false;
|
|
19757
|
-
this.currentAbortController = null;
|
|
19758
|
-
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
19835
|
+
const result = await this.performWatchdogAssessment(threadId);
|
|
19836
|
+
if (result.verdict === "healthy" || result.verdict === "observing") {
|
|
19759
19837
|
await rearm();
|
|
19760
19838
|
return;
|
|
19761
19839
|
}
|
|
19762
|
-
|
|
19763
|
-
|
|
19840
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
19841
|
+
} catch (error) {
|
|
19842
|
+
console.error("[DurableThread] Watchdog check failed (non-fatal):", error);
|
|
19843
|
+
try {
|
|
19764
19844
|
await rearm();
|
|
19765
|
-
|
|
19766
|
-
}
|
|
19767
|
-
if (incomplete.length > 0) {
|
|
19768
|
-
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
19845
|
+
} catch {
|
|
19769
19846
|
}
|
|
19770
|
-
|
|
19771
|
-
|
|
19772
|
-
|
|
19773
|
-
|
|
19774
|
-
|
|
19775
|
-
|
|
19776
|
-
|
|
19847
|
+
}
|
|
19848
|
+
}
|
|
19849
|
+
/**
|
|
19850
|
+
* RPC delivery of the standing check — called by the ThreadWatchdog
|
|
19851
|
+
* companion (and the diagnostics revive endpoint). Runs in a fresh request
|
|
19852
|
+
* context, so it interleaves with a running turn at await points and its
|
|
19853
|
+
* own awaits/timers are alive even when a wedged turn's context is frozen.
|
|
19854
|
+
* Does NOT touch the internal watchdogCheck alarm rows — those belong to
|
|
19855
|
+
* the alarm delivery path.
|
|
19856
|
+
*/
|
|
19857
|
+
async watchdogProbe(threadId) {
|
|
19858
|
+
await this.ensureMigrated();
|
|
19859
|
+
return await this.performWatchdogAssessment(threadId);
|
|
19860
|
+
}
|
|
19861
|
+
/**
|
|
19862
|
+
* Reset this Durable Object instance on request. Used by the watchdog
|
|
19863
|
+
* companion after a wedged-executor takeover: the zombie turn may still own
|
|
19864
|
+
* this DO's single alarm invocation, which would block the just-enqueued
|
|
19865
|
+
* recovery alarm forever. Aborting guarantees a fresh instance whose alarm
|
|
19866
|
+
* chain re-arms from persisted queue state on first wake.
|
|
19867
|
+
*/
|
|
19868
|
+
async forceRestart(reason) {
|
|
19869
|
+
console.error(`[DurableThread] Force restart requested: ${reason}`);
|
|
19870
|
+
this.ctx.abort(reason);
|
|
19871
|
+
}
|
|
19872
|
+
/**
|
|
19873
|
+
* Execution-health snapshot for the diagnostics endpoint and `agents diag`.
|
|
19874
|
+
* Exposes the state the 2026-07-08 post-mortem could not see from outside:
|
|
19875
|
+
* the alarm queue (was a turn item pending/processing/failed? was the
|
|
19876
|
+
* watchdog row gone?), the volatile execution flags, and open request logs.
|
|
19877
|
+
* Read-only; safe to call at any time.
|
|
19878
|
+
*/
|
|
19879
|
+
async getDiagnostics() {
|
|
19880
|
+
await this.ensureMigrated();
|
|
19881
|
+
const incompleteLogs = (await this.ctx.storage.sql.exec(
|
|
19882
|
+
`SELECT id, model, provider, created_at FROM logs WHERE is_complete = 0 ORDER BY created_at DESC LIMIT 20`
|
|
19883
|
+
)).toArray();
|
|
19884
|
+
const alarmQueue = (await this.ctx.storage.sql.exec(
|
|
19885
|
+
`SELECT id, method, status, scheduled_at, created_at, attempts, error
|
|
19886
|
+
FROM alarm_queue
|
|
19887
|
+
ORDER BY CASE WHEN status IN ('pending', 'processing') THEN 0 ELSE 1 END ASC,
|
|
19888
|
+
scheduled_at DESC
|
|
19889
|
+
LIMIT 50`
|
|
19890
|
+
)).toArray();
|
|
19891
|
+
const messageQueueCount = (await this.ctx.storage.sql.exec(
|
|
19892
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
19893
|
+
)).toArray()[0]?.c ?? 0;
|
|
19894
|
+
let contextUsage = null;
|
|
19895
|
+
try {
|
|
19896
|
+
const row = (await this.ctx.storage.sql.exec(
|
|
19897
|
+
`SELECT value FROM thread_values WHERE key = 'context_usage' LIMIT 1`
|
|
19898
|
+
)).toArray()[0];
|
|
19899
|
+
if (row?.value) {
|
|
19900
|
+
const parsed = JSON.parse(row.value);
|
|
19901
|
+
contextUsage = this.normalizePlainRecord(parsed);
|
|
19777
19902
|
}
|
|
19778
|
-
|
|
19779
|
-
|
|
19780
|
-
|
|
19903
|
+
} catch {
|
|
19904
|
+
}
|
|
19905
|
+
return {
|
|
19906
|
+
executing: this.isExecuting,
|
|
19907
|
+
lastProgressAt: this.lastProgressAt > 0 ? this.lastProgressAt : null,
|
|
19908
|
+
silenceMs: this.lastProgressAt > 0 ? Date.now() - this.lastProgressAt : null,
|
|
19909
|
+
watchdogStrikes: this.watchdogWedgeStrikes,
|
|
19910
|
+
watchdogCeilingMs: resolveWatchdogStaleCeilingMs(this.env),
|
|
19911
|
+
stopped: await this.shouldStop(),
|
|
19912
|
+
incompleteLogs,
|
|
19913
|
+
alarmQueue,
|
|
19914
|
+
messageQueueCount,
|
|
19915
|
+
contextUsage
|
|
19916
|
+
};
|
|
19917
|
+
}
|
|
19918
|
+
/**
|
|
19919
|
+
* The shared watchdog decision core. Both deliveries (alarm + companion
|
|
19920
|
+
* probe) evaluate and ACT through this method; the caller only decides what
|
|
19921
|
+
* to do with its own scheduling based on the verdict.
|
|
19922
|
+
*/
|
|
19923
|
+
async performWatchdogAssessment(threadId) {
|
|
19924
|
+
const readIncomplete = async () => (await this.ctx.storage.sql.exec(
|
|
19925
|
+
`SELECT id FROM logs WHERE is_complete = 0`
|
|
19926
|
+
)).toArray();
|
|
19927
|
+
let incomplete = await readIncomplete();
|
|
19928
|
+
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
19929
|
+
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
19930
|
+
const reportedSilence = Number.isFinite(silenceMs) ? silenceMs : null;
|
|
19931
|
+
const report = (verdict, action, extra = {}) => ({
|
|
19932
|
+
verdict,
|
|
19933
|
+
action,
|
|
19934
|
+
executing: this.isExecuting,
|
|
19935
|
+
incompleteLogs: incomplete.length,
|
|
19936
|
+
queuedMessages: 0,
|
|
19937
|
+
silenceMs: reportedSilence,
|
|
19938
|
+
ceilingMs: ceiling,
|
|
19939
|
+
strikes: this.watchdogWedgeStrikes,
|
|
19940
|
+
requestRestart: false,
|
|
19941
|
+
...extra
|
|
19942
|
+
});
|
|
19943
|
+
let tookOver = false;
|
|
19944
|
+
if (this.isExecuting) {
|
|
19945
|
+
if (incomplete.length === 0) {
|
|
19946
|
+
this.watchdogWedgeStrikes = 0;
|
|
19947
|
+
return report("healthy", "none");
|
|
19948
|
+
}
|
|
19949
|
+
if (silenceMs <= ceiling) {
|
|
19950
|
+
this.watchdogWedgeStrikes = 0;
|
|
19951
|
+
return report("healthy", "none");
|
|
19952
|
+
}
|
|
19953
|
+
this.watchdogWedgeStrikes++;
|
|
19954
|
+
if (this.watchdogWedgeStrikes < 2) {
|
|
19781
19955
|
console.warn(
|
|
19782
|
-
`[DurableThread] Watchdog:
|
|
19783
|
-
);
|
|
19784
|
-
} else {
|
|
19785
|
-
console.error(
|
|
19786
|
-
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
19956
|
+
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
19787
19957
|
);
|
|
19788
|
-
|
|
19958
|
+
return report("observing", "none");
|
|
19789
19959
|
}
|
|
19790
|
-
|
|
19791
|
-
|
|
19792
|
-
|
|
19960
|
+
console.error(
|
|
19961
|
+
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
19962
|
+
);
|
|
19793
19963
|
try {
|
|
19794
|
-
|
|
19964
|
+
this.currentAbortController?.abort();
|
|
19795
19965
|
} catch {
|
|
19796
19966
|
}
|
|
19967
|
+
this.isExecuting = false;
|
|
19968
|
+
this.currentAbortController = null;
|
|
19969
|
+
tookOver = true;
|
|
19970
|
+
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
19971
|
+
return report("observing", "none");
|
|
19972
|
+
}
|
|
19973
|
+
incomplete = await readIncomplete();
|
|
19974
|
+
if (this.isExecuting) {
|
|
19975
|
+
return report("healthy", "none");
|
|
19976
|
+
}
|
|
19977
|
+
const finalizedCount = incomplete.length;
|
|
19978
|
+
if (incomplete.length > 0) {
|
|
19979
|
+
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
19980
|
+
}
|
|
19981
|
+
const stopped = await this.shouldStop();
|
|
19982
|
+
const queued = (await this.ctx.storage.sql.exec(
|
|
19983
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
19984
|
+
)).toArray()[0]?.c ?? 0;
|
|
19985
|
+
if (stopped || finalizedCount === 0 && queued === 0) {
|
|
19986
|
+
if (finalizedCount > 0) {
|
|
19987
|
+
return report("parked", "finalized_parked", {
|
|
19988
|
+
queuedMessages: queued,
|
|
19989
|
+
requestRestart: tookOver
|
|
19990
|
+
});
|
|
19991
|
+
}
|
|
19992
|
+
return report("idle", "none", { queuedMessages: queued });
|
|
19993
|
+
}
|
|
19994
|
+
if (await this.hasPendingTurnAlarm()) {
|
|
19995
|
+
return report("recovered", "finalized_deferred", {
|
|
19996
|
+
queuedMessages: queued,
|
|
19997
|
+
requestRestart: tookOver
|
|
19998
|
+
});
|
|
19999
|
+
}
|
|
20000
|
+
if (await this.consumeWatchdogResumeBudget()) {
|
|
20001
|
+
const side = await this.getLatestQueuedMessageSide() ?? "a";
|
|
20002
|
+
await this.alarmQueue.enqueue("continueExecution", { threadId, side }, 0);
|
|
20003
|
+
console.warn(
|
|
20004
|
+
`[DurableThread] Watchdog: lost execution recovered \u2014 resuming thread ${threadId}`
|
|
20005
|
+
);
|
|
20006
|
+
return report("recovered", tookOver ? "aborted_takeover_resumed" : "finalized_resumed", {
|
|
20007
|
+
queuedMessages: queued,
|
|
20008
|
+
requestRestart: tookOver
|
|
20009
|
+
});
|
|
19797
20010
|
}
|
|
20011
|
+
console.error(
|
|
20012
|
+
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
20013
|
+
);
|
|
20014
|
+
await this.notifyParentOfLostExecution(threadId);
|
|
20015
|
+
return report("parked", "finalized_parked", {
|
|
20016
|
+
queuedMessages: queued,
|
|
20017
|
+
requestRestart: tookOver
|
|
20018
|
+
});
|
|
19798
20019
|
}
|
|
19799
20020
|
/** Mark logs the dead executor abandoned, and tell connected clients. */
|
|
19800
20021
|
async finalizeLostExecutionLogs(logIds) {
|
|
@@ -19903,6 +20124,18 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
19903
20124
|
*/
|
|
19904
20125
|
async executeFlow(threadId, agentName, initial_messages, data, awaitCompletion = true) {
|
|
19905
20126
|
await this.assertNotTerminated();
|
|
20127
|
+
if (this.isExecuting) {
|
|
20128
|
+
const busyWaitBudgetMs = 3e4;
|
|
20129
|
+
const waitStartedAt = Date.now();
|
|
20130
|
+
while (this.isExecuting && Date.now() - waitStartedAt < busyWaitBudgetMs) {
|
|
20131
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
20132
|
+
}
|
|
20133
|
+
if (this.isExecuting) {
|
|
20134
|
+
throw new Error(
|
|
20135
|
+
`Thread ${threadId} is already executing; refusing to start a second concurrent flow`
|
|
20136
|
+
);
|
|
20137
|
+
}
|
|
20138
|
+
}
|
|
19906
20139
|
this.isExecuting = true;
|
|
19907
20140
|
this.currentAbortController = new AbortController();
|
|
19908
20141
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -20036,7 +20269,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20036
20269
|
} finally {
|
|
20037
20270
|
this.isExecuting = false;
|
|
20038
20271
|
this.currentAbortController = null;
|
|
20039
|
-
await this.disarmExecutionWatchdog();
|
|
20272
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
20040
20273
|
}
|
|
20041
20274
|
}
|
|
20042
20275
|
/**
|
|
@@ -20076,6 +20309,19 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20076
20309
|
*/
|
|
20077
20310
|
async processMessage(threadId, content, role = "user", attachments, queuedMessageId, queuedAt) {
|
|
20078
20311
|
await this.assertNotTerminated();
|
|
20312
|
+
if (this.isExecuting) {
|
|
20313
|
+
console.warn(
|
|
20314
|
+
`[DurableThread] processMessage while executing \u2014 diverting message to queue for thread ${threadId}`
|
|
20315
|
+
);
|
|
20316
|
+
await this.queueMessage(threadId, {
|
|
20317
|
+
role: role === "assistant" ? "assistant" : "user",
|
|
20318
|
+
content,
|
|
20319
|
+
attachments: this.parseAttachmentRefsJson(attachments ?? null) ?? [],
|
|
20320
|
+
silent: false,
|
|
20321
|
+
metadata: { diverted_from: "processMessage" }
|
|
20322
|
+
});
|
|
20323
|
+
return;
|
|
20324
|
+
}
|
|
20079
20325
|
if (role === "user") {
|
|
20080
20326
|
await this.ctx.storage.sql.exec(
|
|
20081
20327
|
`UPDATE execution_state SET value = 'false' WHERE key = 'stopped'`
|
|
@@ -20222,7 +20468,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20222
20468
|
} finally {
|
|
20223
20469
|
this.isExecuting = false;
|
|
20224
20470
|
this.currentAbortController = null;
|
|
20225
|
-
await this.disarmExecutionWatchdog();
|
|
20471
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
20226
20472
|
}
|
|
20227
20473
|
}
|
|
20228
20474
|
/**
|
|
@@ -20231,6 +20477,12 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20231
20477
|
*/
|
|
20232
20478
|
async doContinueExecution(threadId, side = "a") {
|
|
20233
20479
|
await this.assertNotTerminated();
|
|
20480
|
+
if (this.isExecuting) {
|
|
20481
|
+
console.warn(
|
|
20482
|
+
`[DurableThread] continueExecution while executing \u2014 skipping (running flow owns the queue) for thread ${threadId}`
|
|
20483
|
+
);
|
|
20484
|
+
return;
|
|
20485
|
+
}
|
|
20234
20486
|
this.isExecuting = true;
|
|
20235
20487
|
this.currentAbortController = new AbortController();
|
|
20236
20488
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -20342,7 +20594,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20342
20594
|
} finally {
|
|
20343
20595
|
this.isExecuting = false;
|
|
20344
20596
|
this.currentAbortController = null;
|
|
20345
|
-
await this.disarmExecutionWatchdog();
|
|
20597
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
20346
20598
|
const latestQueuedSide = await this.getLatestQueuedMessageSide();
|
|
20347
20599
|
if (latestQueuedSide) {
|
|
20348
20600
|
await this.ensureQueuedMessageExecution(threadId, latestQueuedSide);
|
|
@@ -24159,6 +24411,179 @@ var CodeExecutionBridge = class extends WorkerEntrypoint {
|
|
|
24159
24411
|
return thread.callCodeExecutionBridge(runId, functionId, encodedArgs);
|
|
24160
24412
|
}
|
|
24161
24413
|
};
|
|
24414
|
+
var CHECK_CADENCE_MS = 3e4;
|
|
24415
|
+
var DISARM_CONFIRM_DELAY_MS = 3e3;
|
|
24416
|
+
var PROBE_TIMEOUT_MS = 15e3;
|
|
24417
|
+
var MAX_UNREACHABLE_STRIKES = 40;
|
|
24418
|
+
var HISTORY_CAP = 20;
|
|
24419
|
+
var KEY_ARMED = "armed";
|
|
24420
|
+
var KEY_HISTORY = "history";
|
|
24421
|
+
var KEY_UNREACHABLE = "unreachable_strikes";
|
|
24422
|
+
var ThreadWatchdog = class extends DurableObject {
|
|
24423
|
+
/**
|
|
24424
|
+
* Arm (or re-arm) the standing check for a thread execution. Idempotent —
|
|
24425
|
+
* called at every execution start. Resets the unreachable counter because a
|
|
24426
|
+
* fresh arm proves the thread is answering RPCs right now.
|
|
24427
|
+
*/
|
|
24428
|
+
async arm(threadId) {
|
|
24429
|
+
await this.ctx.storage.put(KEY_ARMED, {
|
|
24430
|
+
threadId,
|
|
24431
|
+
armedAt: Date.now()
|
|
24432
|
+
});
|
|
24433
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
24434
|
+
const existing = await this.ctx.storage.getAlarm();
|
|
24435
|
+
if (existing === null) {
|
|
24436
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
24437
|
+
}
|
|
24438
|
+
}
|
|
24439
|
+
/**
|
|
24440
|
+
* Hint that the thread believes its execution ended cleanly. The companion
|
|
24441
|
+
* does NOT trust this (a bogus "clean end" was one of the incident's failure
|
|
24442
|
+
* modes) — it just moves the next check close so a genuinely clean thread
|
|
24443
|
+
* stands down in seconds instead of a full cadence.
|
|
24444
|
+
*/
|
|
24445
|
+
async disarmHint() {
|
|
24446
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
24447
|
+
if (!armed) return;
|
|
24448
|
+
await this.ctx.storage.setAlarm(Date.now() + DISARM_CONFIRM_DELAY_MS);
|
|
24449
|
+
}
|
|
24450
|
+
/** Diagnostics view for the thread inspector UI and the diag CLI. */
|
|
24451
|
+
async getState() {
|
|
24452
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
24453
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
24454
|
+
return {
|
|
24455
|
+
armed: Boolean(armed),
|
|
24456
|
+
threadId: armed?.threadId ?? null,
|
|
24457
|
+
armedAt: armed?.armedAt ?? null,
|
|
24458
|
+
history
|
|
24459
|
+
};
|
|
24460
|
+
}
|
|
24461
|
+
/**
|
|
24462
|
+
* Explicit cascade from the parent thread (delete/terminate). Cloudflare
|
|
24463
|
+
* has no automatic parent-child cleanup — this IS the cleanup.
|
|
24464
|
+
*/
|
|
24465
|
+
async destroy() {
|
|
24466
|
+
await this.ctx.storage.deleteAlarm();
|
|
24467
|
+
await this.ctx.storage.deleteAll();
|
|
24468
|
+
}
|
|
24469
|
+
async alarm() {
|
|
24470
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
24471
|
+
if (!armed) {
|
|
24472
|
+
return;
|
|
24473
|
+
}
|
|
24474
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
24475
|
+
const startedAt = Date.now();
|
|
24476
|
+
let checkin;
|
|
24477
|
+
let requestRestart = false;
|
|
24478
|
+
let standDown = false;
|
|
24479
|
+
try {
|
|
24480
|
+
const result = await this.probeThread(armed.threadId);
|
|
24481
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
24482
|
+
requestRestart = result.requestRestart;
|
|
24483
|
+
standDown = result.verdict === "idle" || result.verdict === "parked";
|
|
24484
|
+
checkin = {
|
|
24485
|
+
at: startedAt,
|
|
24486
|
+
verdict: result.verdict,
|
|
24487
|
+
action: describeAction(result),
|
|
24488
|
+
executing: result.executing,
|
|
24489
|
+
incompleteLogs: result.incompleteLogs,
|
|
24490
|
+
silenceMs: result.silenceMs,
|
|
24491
|
+
probeMs: Date.now() - startedAt
|
|
24492
|
+
};
|
|
24493
|
+
} catch (error) {
|
|
24494
|
+
const strikes = (await this.ctx.storage.get(KEY_UNREACHABLE) ?? 0) + 1;
|
|
24495
|
+
await this.ctx.storage.put(KEY_UNREACHABLE, strikes);
|
|
24496
|
+
checkin = {
|
|
24497
|
+
at: startedAt,
|
|
24498
|
+
verdict: "unreachable",
|
|
24499
|
+
action: `probe failed (${strikes} consecutive)`,
|
|
24500
|
+
executing: null,
|
|
24501
|
+
incompleteLogs: null,
|
|
24502
|
+
silenceMs: null,
|
|
24503
|
+
probeMs: Date.now() - startedAt,
|
|
24504
|
+
error: error instanceof Error ? error.message : String(error)
|
|
24505
|
+
};
|
|
24506
|
+
if (strikes >= MAX_UNREACHABLE_STRIKES) {
|
|
24507
|
+
console.error(
|
|
24508
|
+
`[ThreadWatchdog] CRITICAL: thread ${armed.threadId} unreachable for ${strikes} consecutive probes \u2014 standing down. Manual intervention required.`
|
|
24509
|
+
);
|
|
24510
|
+
standDown = true;
|
|
24511
|
+
}
|
|
24512
|
+
}
|
|
24513
|
+
await this.recordCheckin(checkin);
|
|
24514
|
+
if (requestRestart) {
|
|
24515
|
+
try {
|
|
24516
|
+
const stub = this.getThreadStub(armed.threadId);
|
|
24517
|
+
await stub.forceRestart(
|
|
24518
|
+
"watchdog takeover: wedged execution aborted; restarting so the recovery alarm can fire"
|
|
24519
|
+
);
|
|
24520
|
+
} catch {
|
|
24521
|
+
}
|
|
24522
|
+
}
|
|
24523
|
+
if (standDown) {
|
|
24524
|
+
await this.ctx.storage.delete(KEY_ARMED);
|
|
24525
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
24526
|
+
await this.ctx.storage.deleteAlarm();
|
|
24527
|
+
}
|
|
24528
|
+
}
|
|
24529
|
+
getThreadStub(threadId) {
|
|
24530
|
+
const namespace = this.env.AGENT_BUILDER_THREAD;
|
|
24531
|
+
const id = namespace.idFromName(threadId);
|
|
24532
|
+
return namespace.get(id);
|
|
24533
|
+
}
|
|
24534
|
+
async probeThread(threadId) {
|
|
24535
|
+
const stub = this.getThreadStub(threadId);
|
|
24536
|
+
let timer;
|
|
24537
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
24538
|
+
timer = setTimeout(
|
|
24539
|
+
() => reject(new Error(`probe timed out after ${PROBE_TIMEOUT_MS}ms`)),
|
|
24540
|
+
PROBE_TIMEOUT_MS
|
|
24541
|
+
);
|
|
24542
|
+
});
|
|
24543
|
+
try {
|
|
24544
|
+
return await Promise.race([stub.watchdogProbe(threadId), timeout]);
|
|
24545
|
+
} finally {
|
|
24546
|
+
if (timer) clearTimeout(timer);
|
|
24547
|
+
}
|
|
24548
|
+
}
|
|
24549
|
+
async recordCheckin(checkin) {
|
|
24550
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
24551
|
+
history.unshift(checkin);
|
|
24552
|
+
await this.ctx.storage.put(KEY_HISTORY, history.slice(0, HISTORY_CAP));
|
|
24553
|
+
}
|
|
24554
|
+
};
|
|
24555
|
+
function describeAction(result) {
|
|
24556
|
+
switch (result.action) {
|
|
24557
|
+
case "finalized_resumed":
|
|
24558
|
+
return `finalized ${result.incompleteLogs} lost request(s), resumed thread`;
|
|
24559
|
+
case "finalized_parked":
|
|
24560
|
+
return `finalized ${result.incompleteLogs} lost request(s), left thread stopped`;
|
|
24561
|
+
case "finalized_deferred":
|
|
24562
|
+
return `finalized ${result.incompleteLogs} lost request(s); queued turn will continue`;
|
|
24563
|
+
case "aborted_takeover_resumed":
|
|
24564
|
+
return `took over wedged execution (silent ${formatMs(result.silenceMs)}), finalized and resumed`;
|
|
24565
|
+
case "none":
|
|
24566
|
+
default:
|
|
24567
|
+
if (result.verdict === "observing") {
|
|
24568
|
+
return `execution silent ${formatMs(result.silenceMs)} (ceiling ${formatMs(result.ceilingMs)}) \u2014 observing`;
|
|
24569
|
+
}
|
|
24570
|
+
if (result.verdict === "idle") {
|
|
24571
|
+
return "thread idle and clean \u2014 standing down";
|
|
24572
|
+
}
|
|
24573
|
+
if (result.executing) {
|
|
24574
|
+
return result.incompleteLogs > 0 ? `streaming, last progress ${formatMs(result.silenceMs)} ago` : "executing (tool run, no open request)";
|
|
24575
|
+
}
|
|
24576
|
+
return "no action needed";
|
|
24577
|
+
}
|
|
24578
|
+
}
|
|
24579
|
+
function formatMs(ms) {
|
|
24580
|
+
if (ms === null || !Number.isFinite(ms)) return "\u221E";
|
|
24581
|
+
if (ms < 1e3) return `${ms}ms`;
|
|
24582
|
+
const seconds = Math.round(ms / 1e3);
|
|
24583
|
+
if (seconds < 60) return `${seconds}s`;
|
|
24584
|
+
const minutes = Math.floor(seconds / 60);
|
|
24585
|
+
return `${minutes}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
24586
|
+
}
|
|
24162
24587
|
|
|
24163
24588
|
// src/runtime.ts
|
|
24164
24589
|
init_utilities();
|
|
@@ -24521,6 +24946,6 @@ init_context();
|
|
|
24521
24946
|
init_ProviderRegistry();
|
|
24522
24947
|
init_types2();
|
|
24523
24948
|
|
|
24524
|
-
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, ProviderRegistry, authenticate, buildImageDescription, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateImageDescription, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
24949
|
+
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, ProviderRegistry, ThreadWatchdog, authenticate, buildImageDescription, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateImageDescription, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
24525
24950
|
//# sourceMappingURL=runtime.js.map
|
|
24526
24951
|
//# sourceMappingURL=runtime.js.map
|