@standardagents/builder 0.25.6 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/built-in-routes.js +269 -17
- package/dist/built-in-routes.js.map +1 -1
- package/dist/client/{ApiKeysView-B9XLopSb.js → ApiKeysView-jhfg43UH.js} +1 -1
- package/dist/client/CenteredContentView-CS0mV2EJ.js +1 -0
- package/dist/client/{CompositionView-DJOtNN7f.js → CompositionView-DhkLWmtE.js} +1 -1
- package/dist/client/{ConfirmDialog.vue_vue_type_script_setup_true_lang-55dXt19T.js → ConfirmDialog.vue_vue_type_script_setup_true_lang-D-WToRMy.js} +1 -1
- package/dist/client/{CopyButton.vue_vue_type_script_setup_true_lang-B6NqQmW9.js → CopyButton.vue_vue_type_script_setup_true_lang-Dhq0t0dd.js} +1 -1
- package/dist/client/{DataTable.vue_vue_type_script_setup_true_lang-0cbPVIIz.js → DataTable.vue_vue_type_script_setup_true_lang-D1MDCns-.js} +1 -1
- package/dist/client/{JsonViewer-B7yHhxqP.js → JsonViewer-BoHrE76S.js} +1 -1
- package/dist/client/{LoginView-mY9OytK6.js → LoginView-BpR2nMNf.js} +1 -1
- package/dist/client/{MarketplaceView-B9Z9YMuB.js → MarketplaceView-uUf1JYla.js} +1 -1
- package/dist/client/{Modal.vue_vue_type_script_setup_true_lang-DEVCRbXl.js → Modal.vue_vue_type_script_setup_true_lang-D-SDVFQI.js} +1 -1
- package/dist/client/{ModelModal.vue_vue_type_script_setup_true_lang-FxEBZ3A8.js → ModelModal.vue_vue_type_script_setup_true_lang-DE57Zr0r.js} +1 -1
- package/dist/client/{ModelsView-DTU8pk-T.js → ModelsView-CuM_OieH.js} +1 -1
- package/dist/client/{PromptEditView-BAKoULxX.js → PromptEditView-BWjrA2zr.js} +1 -1
- package/dist/client/{PromptModal-CoinSKIM.js → PromptModal-BFxhuG0Z.js} +1 -1
- package/dist/client/{PromptsView-DIkj3q-o.js → PromptsView-AlGsRB6k.js} +1 -1
- package/dist/client/{ProvidersView-B8A6ezDR.js → ProvidersView-CbXBs3SH.js} +1 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-Dm3d37hz.js +22 -0
- package/dist/client/{ThreadsView-DGCVdDej.js → ThreadsView-rN9rzJi1.js} +1 -1
- package/dist/client/{ToolsView-CC-WsZet.js → ToolsView-0Gpo_RFg.js} +1 -1
- package/dist/client/{UiCodeBlock.vue_vue_type_script_setup_true_lang-BH3WC9G_.js → UiCodeBlock.vue_vue_type_script_setup_true_lang-CNncALQo.js} +1 -1
- package/dist/client/{UsersView-U3Stwxui.js → UsersView-iXPbcguY.js} +1 -1
- package/dist/client/{VariablesView-0-a1F4cn.js → VariablesView-C9BB9h34.js} +1 -1
- package/dist/client/assets/{ThreadInspectorPane-ouC_RyKA.css → ThreadInspectorPane-CfpdDnUQ.css} +1 -1
- package/dist/client/assets/index-BTPsPULO.css +1 -0
- package/dist/client/{check-Dkzs1C3V.js → check-BdJVlUmI.js} +1 -1
- package/dist/client/{chevron-right-BGirddPg.js → chevron-right-D49O3XLU.js} +1 -1
- package/dist/client/{copy-C08OR8rY.js → copy-40uDyTgY.js} +1 -1
- package/dist/client/{formkit-YJ1Z7D5r.js → formkit-Bl-sERnF.js} +1 -1
- package/dist/client/index-BCoE48be.js +6 -0
- package/dist/client/index.html +4 -4
- package/dist/client/{package-guyzMTDG.js → package-1UgoERpx.js} +1 -1
- package/dist/client/{plus-DG3ZN5wg.js → plus-BzCjfYs0.js} +1 -1
- package/dist/client/{timeAgo-D7V247YG.js → timeAgo-cFwldMu-.js} +1 -1
- package/dist/client/{useActionFeedback-byARNQjQ.js → useActionFeedback-BVifffYQ.js} +1 -1
- package/dist/client/{vue-BqC6IZDY.js → vue-CmmJuziT.js} +1 -1
- package/dist/{index-CX9Ti98s.d.ts → index-DJ7B8CTp.d.ts} +63 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +555 -84
- package/dist/index.js.map +1 -1
- package/dist/plugin.js +27 -3
- package/dist/plugin.js.map +1 -1
- package/dist/runtime.d.ts +184 -6
- package/dist/runtime.js +528 -81
- package/dist/runtime.js.map +1 -1
- package/dist/test.d.ts +1 -1
- package/package.json +5 -5
- package/dist/client/CenteredContentView-Bwv4MVSC.js +0 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-CvssvzLz.js +0 -22
- package/dist/client/assets/index-Ca3he3fq.css +0 -1
- package/dist/client/index-4Zwz6zXD.js +0 -6
package/dist/runtime.js
CHANGED
|
@@ -1070,6 +1070,24 @@ function resolveWatchdogStaleCeilingMs(env) {
|
|
|
1070
1070
|
);
|
|
1071
1071
|
return Math.max(firstChunk, stall) + 6e4;
|
|
1072
1072
|
}
|
|
1073
|
+
async function awaitBounded(work, budgetMs, label) {
|
|
1074
|
+
let timer;
|
|
1075
|
+
work.catch(() => {
|
|
1076
|
+
});
|
|
1077
|
+
const timeout = new Promise((resolve) => {
|
|
1078
|
+
timer = setTimeout(() => {
|
|
1079
|
+
console.error(
|
|
1080
|
+
`[LLMRequest] ${label} did not settle within ${budgetMs}ms \u2014 continuing without it`
|
|
1081
|
+
);
|
|
1082
|
+
resolve(void 0);
|
|
1083
|
+
}, budgetMs);
|
|
1084
|
+
});
|
|
1085
|
+
try {
|
|
1086
|
+
return await Promise.race([work, timeout]);
|
|
1087
|
+
} finally {
|
|
1088
|
+
if (timer) clearTimeout(timer);
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1073
1091
|
function modelSupportsImageInput(modelDef) {
|
|
1074
1092
|
const supportsImages = modelDef.capabilities?.supportsImages;
|
|
1075
1093
|
if (typeof supportsImages === "boolean") {
|
|
@@ -1283,13 +1301,42 @@ function transformMessages(messages) {
|
|
|
1283
1301
|
arguments: typeof tc.function.arguments === "string" ? JSON.parse(tc.function.arguments) : tc.function.arguments,
|
|
1284
1302
|
extraContent: tc.extra_content && typeof tc.extra_content === "object" ? tc.extra_content : void 0
|
|
1285
1303
|
}));
|
|
1304
|
+
const storedDetails = msg.reasoning_details;
|
|
1305
|
+
const reasoningDetails = Array.isArray(
|
|
1306
|
+
storedDetails
|
|
1307
|
+
) ? storedDetails.flatMap((entry) => {
|
|
1308
|
+
if (!entry || typeof entry !== "object") return [];
|
|
1309
|
+
const raw = entry;
|
|
1310
|
+
const type = raw.type === "summary" || raw.type === "encrypted" ? raw.type : "text";
|
|
1311
|
+
return [
|
|
1312
|
+
{
|
|
1313
|
+
type,
|
|
1314
|
+
id: typeof raw.id === "string" ? raw.id : void 0,
|
|
1315
|
+
text: typeof raw.text === "string" ? raw.text : void 0,
|
|
1316
|
+
data: typeof raw.data === "string" ? raw.data : void 0,
|
|
1317
|
+
format: typeof raw.format === "string" ? raw.format : void 0
|
|
1318
|
+
}
|
|
1319
|
+
];
|
|
1320
|
+
}) : void 0;
|
|
1321
|
+
const storedReasoning = msg.reasoning_content;
|
|
1322
|
+
let reasoning = typeof storedReasoning === "string" && storedReasoning.length > 0 ? storedReasoning : void 0;
|
|
1323
|
+
if (!reasoning && Array.isArray(reasoningDetails)) {
|
|
1324
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1325
|
+
const texts = [];
|
|
1326
|
+
for (const d of reasoningDetails) {
|
|
1327
|
+
if (typeof d?.text === "string" && d.text.length > 0 && !seen.has(d.text)) {
|
|
1328
|
+
seen.add(d.text);
|
|
1329
|
+
texts.push(d.text);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
if (texts.length) reasoning = texts.join("\n");
|
|
1333
|
+
}
|
|
1286
1334
|
return {
|
|
1287
1335
|
role: "assistant",
|
|
1288
1336
|
content: typeof msg.content === "string" ? msg.content : null,
|
|
1289
1337
|
toolCalls: toolCalls?.length ? toolCalls : void 0,
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
reasoningDetails: msg.reasoning_details || void 0
|
|
1338
|
+
reasoning,
|
|
1339
|
+
reasoningDetails: reasoningDetails?.length ? reasoningDetails : void 0
|
|
1293
1340
|
};
|
|
1294
1341
|
}
|
|
1295
1342
|
case "tool":
|
|
@@ -1464,7 +1511,7 @@ function normalizeProviderMetadataValue(value) {
|
|
|
1464
1511
|
if (!trimmed || trimmed.toLowerCase() === "unknown") return void 0;
|
|
1465
1512
|
return trimmed;
|
|
1466
1513
|
}
|
|
1467
|
-
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, LLMRequest;
|
|
1514
|
+
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, HOOK_AWAIT_BUDGET_MS, STREAM_CLEANUP_BUDGET_MS, LLMRequest;
|
|
1468
1515
|
var init_LLMRequest = __esm({
|
|
1469
1516
|
"src/agents/LLMRequest.ts"() {
|
|
1470
1517
|
init_pricing();
|
|
@@ -1475,6 +1522,8 @@ var init_LLMRequest = __esm({
|
|
|
1475
1522
|
NON_VISION_PLACEHOLDER_TEXT = "[Image omitted: model does not support vision]";
|
|
1476
1523
|
STREAM_FIRST_CHUNK_TIMEOUT_MS = 12e4;
|
|
1477
1524
|
STREAM_STALL_TIMEOUT_MS = 45e3;
|
|
1525
|
+
HOOK_AWAIT_BUDGET_MS = 3e4;
|
|
1526
|
+
STREAM_CLEANUP_BUDGET_MS = 5e3;
|
|
1478
1527
|
LLMRequest = class {
|
|
1479
1528
|
/**
|
|
1480
1529
|
* Execute an LLM request with full retry/fallback logic
|
|
@@ -1958,7 +2007,14 @@ var init_LLMRequest = __esm({
|
|
|
1958
2007
|
parentSignal.removeEventListener("abort", onParentAbort);
|
|
1959
2008
|
}
|
|
1960
2009
|
try {
|
|
1961
|
-
|
|
2010
|
+
const cleanup = iterator.return?.();
|
|
2011
|
+
if (cleanup) {
|
|
2012
|
+
await awaitBounded(
|
|
2013
|
+
Promise.resolve(cleanup),
|
|
2014
|
+
STREAM_CLEANUP_BUDGET_MS,
|
|
2015
|
+
"stream iterator cleanup"
|
|
2016
|
+
);
|
|
2017
|
+
}
|
|
1962
2018
|
} catch {
|
|
1963
2019
|
}
|
|
1964
2020
|
}
|
|
@@ -2091,7 +2147,11 @@ var init_LLMRequest = __esm({
|
|
|
2091
2147
|
*/
|
|
2092
2148
|
static async resolveProgressArgument(state, toolName) {
|
|
2093
2149
|
try {
|
|
2094
|
-
const mod = await
|
|
2150
|
+
const mod = await awaitBounded(
|
|
2151
|
+
state.thread.instance.loadTool(toolName),
|
|
2152
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2153
|
+
`loadTool(${toolName}) for progress argument`
|
|
2154
|
+
);
|
|
2095
2155
|
const pa = mod?.progressArgument ?? mod?.default?.progressArgument;
|
|
2096
2156
|
return typeof pa === "string" && pa ? pa : void 0;
|
|
2097
2157
|
} catch {
|
|
@@ -2112,7 +2172,11 @@ var init_LLMRequest = __esm({
|
|
|
2112
2172
|
function: { name: entry.name, arguments: entry.argumentsDelta || "" }
|
|
2113
2173
|
};
|
|
2114
2174
|
try {
|
|
2115
|
-
await
|
|
2175
|
+
await awaitBounded(
|
|
2176
|
+
FlowEngine.runToolCallStartedHook(state, toolCall, progress),
|
|
2177
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2178
|
+
`tool_call_started hook for ${entry.name}`
|
|
2179
|
+
);
|
|
2116
2180
|
} catch (error) {
|
|
2117
2181
|
console.error("[LLMRequest] Error firing tool_call_started:", error);
|
|
2118
2182
|
}
|
|
@@ -7807,6 +7871,25 @@ This usually points to something I can't fix from here \u2014 for example the wo
|
|
|
7807
7871
|
return null;
|
|
7808
7872
|
}
|
|
7809
7873
|
}
|
|
7874
|
+
/**
|
|
7875
|
+
* Registry entries the model may see (and therefore manage). Hidden
|
|
7876
|
+
* subagent relationships (`SubagentToolConfig.hidden` — infrastructure
|
|
7877
|
+
* children like background compaction) are excluded from the injected
|
|
7878
|
+
* "Active subagent instances" message: a listed instance reads as the
|
|
7879
|
+
* model's to clean up, and terminate_subagent on an infrastructure child
|
|
7880
|
+
* kills work the runtime depends on. Static and pure for direct testing.
|
|
7881
|
+
*/
|
|
7882
|
+
static filterModelVisibleSubagents(registry, promptTools) {
|
|
7883
|
+
if (registry.length === 0) return registry;
|
|
7884
|
+
const hidden = /* @__PURE__ */ new Set();
|
|
7885
|
+
for (const config of promptTools ?? []) {
|
|
7886
|
+
if (typeof config === "object" && config !== null && "name" in config && config.hidden === true) {
|
|
7887
|
+
hidden.add(config.name);
|
|
7888
|
+
}
|
|
7889
|
+
}
|
|
7890
|
+
if (hidden.size === 0) return registry;
|
|
7891
|
+
return registry.filter((entry) => !hidden.has(entry.name));
|
|
7892
|
+
}
|
|
7810
7893
|
static async getSubagentRegistry(state) {
|
|
7811
7894
|
if (typeof state.thread.instance.getChildrenRegistry === "function") {
|
|
7812
7895
|
try {
|
|
@@ -8065,7 +8148,10 @@ ${msg.content}` : `${imageDescriptions}${nonImageList}`;
|
|
|
8065
8148
|
});
|
|
8066
8149
|
}
|
|
8067
8150
|
}
|
|
8068
|
-
const subagentRegistry =
|
|
8151
|
+
const subagentRegistry = _FlowEngine.filterModelVisibleSubagents(
|
|
8152
|
+
await this.getSubagentRegistry(state),
|
|
8153
|
+
state.prompt?._tools
|
|
8154
|
+
);
|
|
8069
8155
|
if (subagentRegistry.length > 0) {
|
|
8070
8156
|
const fallbackNameMap = await this.resolveRegistryThreadNameMap(state, subagentRegistry);
|
|
8071
8157
|
const lines = subagentRegistry.map((entry) => {
|
|
@@ -12675,8 +12761,12 @@ var init_ThreadStateImpl = __esm({
|
|
|
12675
12761
|
options?.includeSilent,
|
|
12676
12762
|
options?.maxDepth
|
|
12677
12763
|
);
|
|
12764
|
+
const messages = result.messages.map(this._mapMessage);
|
|
12765
|
+
if (options?.order === "desc") {
|
|
12766
|
+
messages.reverse();
|
|
12767
|
+
}
|
|
12678
12768
|
return {
|
|
12679
|
-
messages
|
|
12769
|
+
messages,
|
|
12680
12770
|
total: result.total,
|
|
12681
12771
|
hasMore: result.hasMore
|
|
12682
12772
|
};
|
|
@@ -16884,6 +16974,11 @@ var DurableThread = class _DurableThread extends DurableObject {
|
|
|
16884
16974
|
if (this.currentAbortController) {
|
|
16885
16975
|
this.currentAbortController.abort();
|
|
16886
16976
|
}
|
|
16977
|
+
try {
|
|
16978
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
16979
|
+
} catch (error) {
|
|
16980
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
16981
|
+
}
|
|
16887
16982
|
const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
|
|
16888
16983
|
const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
|
|
16889
16984
|
await agentBuilder.updateThread(threadId, { terminated: terminatedAt });
|
|
@@ -18043,6 +18138,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
18043
18138
|
*/
|
|
18044
18139
|
async getMessages(limit = 100, offset = 0, order = "DESC", includeSilent = false, maxDepth = 0) {
|
|
18045
18140
|
await this.ensureMigrated();
|
|
18141
|
+
const normalizedOrder = String(order).toUpperCase() === "ASC" ? "ASC" : "DESC";
|
|
18046
18142
|
try {
|
|
18047
18143
|
const whereClauses = [];
|
|
18048
18144
|
if (!includeSilent) {
|
|
@@ -18050,7 +18146,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
18050
18146
|
}
|
|
18051
18147
|
whereClauses.push(`(depth <= ${maxDepth} OR depth IS NULL)`);
|
|
18052
18148
|
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
|
|
18053
|
-
const orderDirection =
|
|
18149
|
+
const orderDirection = normalizedOrder;
|
|
18054
18150
|
const combinedMessagesSql = `
|
|
18055
18151
|
WITH combined_messages AS (
|
|
18056
18152
|
SELECT
|
|
@@ -18176,7 +18272,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
18176
18272
|
return projection ? { ...message, ...projection } : message;
|
|
18177
18273
|
});
|
|
18178
18274
|
}
|
|
18179
|
-
if (
|
|
18275
|
+
if (normalizedOrder === "DESC") {
|
|
18180
18276
|
messages = messages.reverse();
|
|
18181
18277
|
}
|
|
18182
18278
|
const hasMore = offset + messages.length < total;
|
|
@@ -19009,6 +19105,14 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19009
19105
|
}
|
|
19010
19106
|
this.logSockets.clear();
|
|
19011
19107
|
this.messageSockets.clear();
|
|
19108
|
+
try {
|
|
19109
|
+
const threadId = this.ctx.id.name;
|
|
19110
|
+
if (threadId) {
|
|
19111
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
19112
|
+
}
|
|
19113
|
+
} catch (error) {
|
|
19114
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
19115
|
+
}
|
|
19012
19116
|
await this.ctx.storage.deleteAlarm();
|
|
19013
19117
|
await this.ctx.storage.deleteAll();
|
|
19014
19118
|
this.migratedToVersion = null;
|
|
@@ -19664,6 +19768,18 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19664
19768
|
noteExecutionProgress() {
|
|
19665
19769
|
this.lastProgressAt = Date.now();
|
|
19666
19770
|
}
|
|
19771
|
+
/**
|
|
19772
|
+
* The per-thread companion watchdog DO (ThreadWatchdog), derived from the
|
|
19773
|
+
* thread id. It exists because this thread's OWN alarm cannot fire while a
|
|
19774
|
+
* queue-dispatched turn runs inside alarm() — the companion delivers the
|
|
19775
|
+
* same check via RPC, which interleaves at await points even mid-turn.
|
|
19776
|
+
* Returns null on deployments whose generated config predates the binding.
|
|
19777
|
+
*/
|
|
19778
|
+
watchdogCompanion(threadId) {
|
|
19779
|
+
const namespace = this.env.AGENT_BUILDER_WATCHDOG;
|
|
19780
|
+
if (!namespace || typeof namespace.idFromName !== "function") return null;
|
|
19781
|
+
return namespace.get(namespace.idFromName(threadId));
|
|
19782
|
+
}
|
|
19667
19783
|
/** Arm the standing watchdog for an execution start. One pending row max. */
|
|
19668
19784
|
async armExecutionWatchdog(threadId) {
|
|
19669
19785
|
try {
|
|
@@ -19672,14 +19788,20 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19672
19788
|
const pending = (await this.ctx.storage.sql.exec(
|
|
19673
19789
|
`SELECT id FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'pending' LIMIT 1`
|
|
19674
19790
|
)).toArray();
|
|
19675
|
-
if (pending.length
|
|
19676
|
-
|
|
19791
|
+
if (pending.length === 0) {
|
|
19792
|
+
await this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
19793
|
+
}
|
|
19677
19794
|
} catch (error) {
|
|
19678
19795
|
console.error("[DurableThread] Failed to arm execution watchdog (non-fatal):", error);
|
|
19679
19796
|
}
|
|
19797
|
+
try {
|
|
19798
|
+
await this.watchdogCompanion(threadId)?.arm(threadId);
|
|
19799
|
+
} catch (error) {
|
|
19800
|
+
console.error("[DurableThread] Failed to arm watchdog companion (non-fatal):", error);
|
|
19801
|
+
}
|
|
19680
19802
|
}
|
|
19681
19803
|
/** Disarm on clean execution end. Completed check rows are noise, not history. */
|
|
19682
|
-
async disarmExecutionWatchdog() {
|
|
19804
|
+
async disarmExecutionWatchdog(threadId) {
|
|
19683
19805
|
try {
|
|
19684
19806
|
await this.ctx.storage.sql.exec(
|
|
19685
19807
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status IN ('pending', 'completed')`
|
|
@@ -19688,10 +19810,21 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19688
19810
|
} catch (error) {
|
|
19689
19811
|
console.error("[DurableThread] Failed to disarm execution watchdog (non-fatal):", error);
|
|
19690
19812
|
}
|
|
19813
|
+
if (threadId) {
|
|
19814
|
+
try {
|
|
19815
|
+
await this.watchdogCompanion(threadId)?.disarmHint();
|
|
19816
|
+
} catch (error) {
|
|
19817
|
+
console.error("[DurableThread] Failed to hint watchdog companion (non-fatal):", error);
|
|
19818
|
+
}
|
|
19819
|
+
}
|
|
19691
19820
|
}
|
|
19692
19821
|
/**
|
|
19693
|
-
* The standing check. Evaluates the thread's state AT
|
|
19694
|
-
* nothing about whichever request was in flight when it
|
|
19822
|
+
* The standing check, alarm-queue delivery. Evaluates the thread's state AT
|
|
19823
|
+
* FIRE TIME — it knows nothing about whichever request was in flight when it
|
|
19824
|
+
* was armed. NOTE this delivery path cannot fire while a queue-dispatched
|
|
19825
|
+
* turn runs (the turn owns this DO's single alarm invocation); the
|
|
19826
|
+
* ThreadWatchdog companion delivers the same assessment via watchdogProbe()
|
|
19827
|
+
* in exactly that window.
|
|
19695
19828
|
*/
|
|
19696
19829
|
async watchdogCheck(threadId) {
|
|
19697
19830
|
const rearm = () => this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
@@ -19699,80 +19832,190 @@ ${resultContent}${attachmentSummary}`;
|
|
|
19699
19832
|
await this.ctx.storage.sql.exec(
|
|
19700
19833
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'completed'`
|
|
19701
19834
|
);
|
|
19702
|
-
const
|
|
19703
|
-
|
|
19704
|
-
)).toArray();
|
|
19705
|
-
let incomplete = await readIncomplete();
|
|
19706
|
-
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
19707
|
-
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
19708
|
-
if (this.isExecuting) {
|
|
19709
|
-
if (incomplete.length === 0) {
|
|
19710
|
-
this.watchdogWedgeStrikes = 0;
|
|
19711
|
-
await rearm();
|
|
19712
|
-
return;
|
|
19713
|
-
}
|
|
19714
|
-
if (silenceMs <= ceiling) {
|
|
19715
|
-
this.watchdogWedgeStrikes = 0;
|
|
19716
|
-
await rearm();
|
|
19717
|
-
return;
|
|
19718
|
-
}
|
|
19719
|
-
this.watchdogWedgeStrikes++;
|
|
19720
|
-
if (this.watchdogWedgeStrikes < 2) {
|
|
19721
|
-
console.warn(
|
|
19722
|
-
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
19723
|
-
);
|
|
19724
|
-
await rearm();
|
|
19725
|
-
return;
|
|
19726
|
-
}
|
|
19727
|
-
console.error(
|
|
19728
|
-
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
19729
|
-
);
|
|
19730
|
-
try {
|
|
19731
|
-
this.currentAbortController?.abort();
|
|
19732
|
-
} catch {
|
|
19733
|
-
}
|
|
19734
|
-
this.isExecuting = false;
|
|
19735
|
-
this.currentAbortController = null;
|
|
19736
|
-
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
19835
|
+
const result = await this.performWatchdogAssessment(threadId);
|
|
19836
|
+
if (result.verdict === "healthy" || result.verdict === "observing") {
|
|
19737
19837
|
await rearm();
|
|
19738
19838
|
return;
|
|
19739
19839
|
}
|
|
19740
|
-
|
|
19741
|
-
|
|
19840
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
19841
|
+
} catch (error) {
|
|
19842
|
+
console.error("[DurableThread] Watchdog check failed (non-fatal):", error);
|
|
19843
|
+
try {
|
|
19742
19844
|
await rearm();
|
|
19743
|
-
|
|
19744
|
-
}
|
|
19745
|
-
if (incomplete.length > 0) {
|
|
19746
|
-
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
19845
|
+
} catch {
|
|
19747
19846
|
}
|
|
19748
|
-
|
|
19749
|
-
|
|
19750
|
-
|
|
19751
|
-
|
|
19752
|
-
|
|
19753
|
-
|
|
19754
|
-
|
|
19847
|
+
}
|
|
19848
|
+
}
|
|
19849
|
+
/**
|
|
19850
|
+
* RPC delivery of the standing check — called by the ThreadWatchdog
|
|
19851
|
+
* companion (and the diagnostics revive endpoint). Runs in a fresh request
|
|
19852
|
+
* context, so it interleaves with a running turn at await points and its
|
|
19853
|
+
* own awaits/timers are alive even when a wedged turn's context is frozen.
|
|
19854
|
+
* Does NOT touch the internal watchdogCheck alarm rows — those belong to
|
|
19855
|
+
* the alarm delivery path.
|
|
19856
|
+
*/
|
|
19857
|
+
async watchdogProbe(threadId) {
|
|
19858
|
+
await this.ensureMigrated();
|
|
19859
|
+
return await this.performWatchdogAssessment(threadId);
|
|
19860
|
+
}
|
|
19861
|
+
/**
|
|
19862
|
+
* Reset this Durable Object instance on request. Used by the watchdog
|
|
19863
|
+
* companion after a wedged-executor takeover: the zombie turn may still own
|
|
19864
|
+
* this DO's single alarm invocation, which would block the just-enqueued
|
|
19865
|
+
* recovery alarm forever. Aborting guarantees a fresh instance whose alarm
|
|
19866
|
+
* chain re-arms from persisted queue state on first wake.
|
|
19867
|
+
*/
|
|
19868
|
+
async forceRestart(reason) {
|
|
19869
|
+
console.error(`[DurableThread] Force restart requested: ${reason}`);
|
|
19870
|
+
this.ctx.abort(reason);
|
|
19871
|
+
}
|
|
19872
|
+
/**
|
|
19873
|
+
* Execution-health snapshot for the diagnostics endpoint and `agents diag`.
|
|
19874
|
+
* Exposes the state the 2026-07-08 post-mortem could not see from outside:
|
|
19875
|
+
* the alarm queue (was a turn item pending/processing/failed? was the
|
|
19876
|
+
* watchdog row gone?), the volatile execution flags, and open request logs.
|
|
19877
|
+
* Read-only; safe to call at any time.
|
|
19878
|
+
*/
|
|
19879
|
+
async getDiagnostics() {
|
|
19880
|
+
await this.ensureMigrated();
|
|
19881
|
+
const incompleteLogs = (await this.ctx.storage.sql.exec(
|
|
19882
|
+
`SELECT id, model, provider, created_at FROM logs WHERE is_complete = 0 ORDER BY created_at DESC LIMIT 20`
|
|
19883
|
+
)).toArray();
|
|
19884
|
+
const alarmQueue = (await this.ctx.storage.sql.exec(
|
|
19885
|
+
`SELECT id, method, status, scheduled_at, created_at, attempts, error
|
|
19886
|
+
FROM alarm_queue
|
|
19887
|
+
ORDER BY CASE WHEN status IN ('pending', 'processing') THEN 0 ELSE 1 END ASC,
|
|
19888
|
+
scheduled_at DESC
|
|
19889
|
+
LIMIT 50`
|
|
19890
|
+
)).toArray();
|
|
19891
|
+
const messageQueueCount = (await this.ctx.storage.sql.exec(
|
|
19892
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
19893
|
+
)).toArray()[0]?.c ?? 0;
|
|
19894
|
+
let contextUsage = null;
|
|
19895
|
+
try {
|
|
19896
|
+
const row = (await this.ctx.storage.sql.exec(
|
|
19897
|
+
`SELECT value FROM thread_values WHERE key = 'context_usage' LIMIT 1`
|
|
19898
|
+
)).toArray()[0];
|
|
19899
|
+
if (row?.value) {
|
|
19900
|
+
const parsed = JSON.parse(row.value);
|
|
19901
|
+
contextUsage = this.normalizePlainRecord(parsed);
|
|
19755
19902
|
}
|
|
19756
|
-
|
|
19757
|
-
|
|
19758
|
-
|
|
19903
|
+
} catch {
|
|
19904
|
+
}
|
|
19905
|
+
return {
|
|
19906
|
+
executing: this.isExecuting,
|
|
19907
|
+
lastProgressAt: this.lastProgressAt > 0 ? this.lastProgressAt : null,
|
|
19908
|
+
silenceMs: this.lastProgressAt > 0 ? Date.now() - this.lastProgressAt : null,
|
|
19909
|
+
watchdogStrikes: this.watchdogWedgeStrikes,
|
|
19910
|
+
watchdogCeilingMs: resolveWatchdogStaleCeilingMs(this.env),
|
|
19911
|
+
stopped: await this.shouldStop(),
|
|
19912
|
+
incompleteLogs,
|
|
19913
|
+
alarmQueue,
|
|
19914
|
+
messageQueueCount,
|
|
19915
|
+
contextUsage
|
|
19916
|
+
};
|
|
19917
|
+
}
|
|
19918
|
+
/**
|
|
19919
|
+
* The shared watchdog decision core. Both deliveries (alarm + companion
|
|
19920
|
+
* probe) evaluate and ACT through this method; the caller only decides what
|
|
19921
|
+
* to do with its own scheduling based on the verdict.
|
|
19922
|
+
*/
|
|
19923
|
+
async performWatchdogAssessment(threadId) {
|
|
19924
|
+
const readIncomplete = async () => (await this.ctx.storage.sql.exec(
|
|
19925
|
+
`SELECT id FROM logs WHERE is_complete = 0`
|
|
19926
|
+
)).toArray();
|
|
19927
|
+
let incomplete = await readIncomplete();
|
|
19928
|
+
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
19929
|
+
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
19930
|
+
const reportedSilence = Number.isFinite(silenceMs) ? silenceMs : null;
|
|
19931
|
+
const report = (verdict, action, extra = {}) => ({
|
|
19932
|
+
verdict,
|
|
19933
|
+
action,
|
|
19934
|
+
executing: this.isExecuting,
|
|
19935
|
+
incompleteLogs: incomplete.length,
|
|
19936
|
+
queuedMessages: 0,
|
|
19937
|
+
silenceMs: reportedSilence,
|
|
19938
|
+
ceilingMs: ceiling,
|
|
19939
|
+
strikes: this.watchdogWedgeStrikes,
|
|
19940
|
+
requestRestart: false,
|
|
19941
|
+
...extra
|
|
19942
|
+
});
|
|
19943
|
+
let tookOver = false;
|
|
19944
|
+
if (this.isExecuting) {
|
|
19945
|
+
if (incomplete.length === 0) {
|
|
19946
|
+
this.watchdogWedgeStrikes = 0;
|
|
19947
|
+
return report("healthy", "none");
|
|
19948
|
+
}
|
|
19949
|
+
if (silenceMs <= ceiling) {
|
|
19950
|
+
this.watchdogWedgeStrikes = 0;
|
|
19951
|
+
return report("healthy", "none");
|
|
19952
|
+
}
|
|
19953
|
+
this.watchdogWedgeStrikes++;
|
|
19954
|
+
if (this.watchdogWedgeStrikes < 2) {
|
|
19759
19955
|
console.warn(
|
|
19760
|
-
`[DurableThread] Watchdog:
|
|
19761
|
-
);
|
|
19762
|
-
} else {
|
|
19763
|
-
console.error(
|
|
19764
|
-
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
19956
|
+
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
19765
19957
|
);
|
|
19766
|
-
|
|
19958
|
+
return report("observing", "none");
|
|
19767
19959
|
}
|
|
19768
|
-
|
|
19769
|
-
|
|
19770
|
-
|
|
19960
|
+
console.error(
|
|
19961
|
+
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
19962
|
+
);
|
|
19771
19963
|
try {
|
|
19772
|
-
|
|
19964
|
+
this.currentAbortController?.abort();
|
|
19773
19965
|
} catch {
|
|
19774
19966
|
}
|
|
19967
|
+
this.isExecuting = false;
|
|
19968
|
+
this.currentAbortController = null;
|
|
19969
|
+
tookOver = true;
|
|
19970
|
+
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
19971
|
+
return report("observing", "none");
|
|
19972
|
+
}
|
|
19973
|
+
incomplete = await readIncomplete();
|
|
19974
|
+
if (this.isExecuting) {
|
|
19975
|
+
return report("healthy", "none");
|
|
19976
|
+
}
|
|
19977
|
+
const finalizedCount = incomplete.length;
|
|
19978
|
+
if (incomplete.length > 0) {
|
|
19979
|
+
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
19980
|
+
}
|
|
19981
|
+
const stopped = await this.shouldStop();
|
|
19982
|
+
const queued = (await this.ctx.storage.sql.exec(
|
|
19983
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
19984
|
+
)).toArray()[0]?.c ?? 0;
|
|
19985
|
+
if (stopped || finalizedCount === 0 && queued === 0) {
|
|
19986
|
+
if (finalizedCount > 0) {
|
|
19987
|
+
return report("parked", "finalized_parked", {
|
|
19988
|
+
queuedMessages: queued,
|
|
19989
|
+
requestRestart: tookOver
|
|
19990
|
+
});
|
|
19991
|
+
}
|
|
19992
|
+
return report("idle", "none", { queuedMessages: queued });
|
|
19993
|
+
}
|
|
19994
|
+
if (await this.hasPendingTurnAlarm()) {
|
|
19995
|
+
return report("recovered", "finalized_deferred", {
|
|
19996
|
+
queuedMessages: queued,
|
|
19997
|
+
requestRestart: tookOver
|
|
19998
|
+
});
|
|
19775
19999
|
}
|
|
20000
|
+
if (await this.consumeWatchdogResumeBudget()) {
|
|
20001
|
+
const side = await this.getLatestQueuedMessageSide() ?? "a";
|
|
20002
|
+
await this.alarmQueue.enqueue("continueExecution", { threadId, side }, 0);
|
|
20003
|
+
console.warn(
|
|
20004
|
+
`[DurableThread] Watchdog: lost execution recovered \u2014 resuming thread ${threadId}`
|
|
20005
|
+
);
|
|
20006
|
+
return report("recovered", tookOver ? "aborted_takeover_resumed" : "finalized_resumed", {
|
|
20007
|
+
queuedMessages: queued,
|
|
20008
|
+
requestRestart: tookOver
|
|
20009
|
+
});
|
|
20010
|
+
}
|
|
20011
|
+
console.error(
|
|
20012
|
+
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
20013
|
+
);
|
|
20014
|
+
await this.notifyParentOfLostExecution(threadId);
|
|
20015
|
+
return report("parked", "finalized_parked", {
|
|
20016
|
+
queuedMessages: queued,
|
|
20017
|
+
requestRestart: tookOver
|
|
20018
|
+
});
|
|
19776
20019
|
}
|
|
19777
20020
|
/** Mark logs the dead executor abandoned, and tell connected clients. */
|
|
19778
20021
|
async finalizeLostExecutionLogs(logIds) {
|
|
@@ -19881,6 +20124,18 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
19881
20124
|
*/
|
|
19882
20125
|
async executeFlow(threadId, agentName, initial_messages, data, awaitCompletion = true) {
|
|
19883
20126
|
await this.assertNotTerminated();
|
|
20127
|
+
if (this.isExecuting) {
|
|
20128
|
+
const busyWaitBudgetMs = 3e4;
|
|
20129
|
+
const waitStartedAt = Date.now();
|
|
20130
|
+
while (this.isExecuting && Date.now() - waitStartedAt < busyWaitBudgetMs) {
|
|
20131
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
20132
|
+
}
|
|
20133
|
+
if (this.isExecuting) {
|
|
20134
|
+
throw new Error(
|
|
20135
|
+
`Thread ${threadId} is already executing; refusing to start a second concurrent flow`
|
|
20136
|
+
);
|
|
20137
|
+
}
|
|
20138
|
+
}
|
|
19884
20139
|
this.isExecuting = true;
|
|
19885
20140
|
this.currentAbortController = new AbortController();
|
|
19886
20141
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -20014,7 +20269,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20014
20269
|
} finally {
|
|
20015
20270
|
this.isExecuting = false;
|
|
20016
20271
|
this.currentAbortController = null;
|
|
20017
|
-
await this.disarmExecutionWatchdog();
|
|
20272
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
20018
20273
|
}
|
|
20019
20274
|
}
|
|
20020
20275
|
/**
|
|
@@ -20054,6 +20309,19 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20054
20309
|
*/
|
|
20055
20310
|
async processMessage(threadId, content, role = "user", attachments, queuedMessageId, queuedAt) {
|
|
20056
20311
|
await this.assertNotTerminated();
|
|
20312
|
+
if (this.isExecuting) {
|
|
20313
|
+
console.warn(
|
|
20314
|
+
`[DurableThread] processMessage while executing \u2014 diverting message to queue for thread ${threadId}`
|
|
20315
|
+
);
|
|
20316
|
+
await this.queueMessage(threadId, {
|
|
20317
|
+
role: role === "assistant" ? "assistant" : "user",
|
|
20318
|
+
content,
|
|
20319
|
+
attachments: this.parseAttachmentRefsJson(attachments ?? null) ?? [],
|
|
20320
|
+
silent: false,
|
|
20321
|
+
metadata: { diverted_from: "processMessage" }
|
|
20322
|
+
});
|
|
20323
|
+
return;
|
|
20324
|
+
}
|
|
20057
20325
|
if (role === "user") {
|
|
20058
20326
|
await this.ctx.storage.sql.exec(
|
|
20059
20327
|
`UPDATE execution_state SET value = 'false' WHERE key = 'stopped'`
|
|
@@ -20200,7 +20468,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20200
20468
|
} finally {
|
|
20201
20469
|
this.isExecuting = false;
|
|
20202
20470
|
this.currentAbortController = null;
|
|
20203
|
-
await this.disarmExecutionWatchdog();
|
|
20471
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
20204
20472
|
}
|
|
20205
20473
|
}
|
|
20206
20474
|
/**
|
|
@@ -20209,6 +20477,12 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20209
20477
|
*/
|
|
20210
20478
|
async doContinueExecution(threadId, side = "a") {
|
|
20211
20479
|
await this.assertNotTerminated();
|
|
20480
|
+
if (this.isExecuting) {
|
|
20481
|
+
console.warn(
|
|
20482
|
+
`[DurableThread] continueExecution while executing \u2014 skipping (running flow owns the queue) for thread ${threadId}`
|
|
20483
|
+
);
|
|
20484
|
+
return;
|
|
20485
|
+
}
|
|
20212
20486
|
this.isExecuting = true;
|
|
20213
20487
|
this.currentAbortController = new AbortController();
|
|
20214
20488
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -20320,7 +20594,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
20320
20594
|
} finally {
|
|
20321
20595
|
this.isExecuting = false;
|
|
20322
20596
|
this.currentAbortController = null;
|
|
20323
|
-
await this.disarmExecutionWatchdog();
|
|
20597
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
20324
20598
|
const latestQueuedSide = await this.getLatestQueuedMessageSide();
|
|
20325
20599
|
if (latestQueuedSide) {
|
|
20326
20600
|
await this.ensureQueuedMessageExecution(threadId, latestQueuedSide);
|
|
@@ -24137,6 +24411,179 @@ var CodeExecutionBridge = class extends WorkerEntrypoint {
|
|
|
24137
24411
|
return thread.callCodeExecutionBridge(runId, functionId, encodedArgs);
|
|
24138
24412
|
}
|
|
24139
24413
|
};
|
|
24414
|
+
var CHECK_CADENCE_MS = 3e4;
|
|
24415
|
+
var DISARM_CONFIRM_DELAY_MS = 3e3;
|
|
24416
|
+
var PROBE_TIMEOUT_MS = 15e3;
|
|
24417
|
+
var MAX_UNREACHABLE_STRIKES = 40;
|
|
24418
|
+
var HISTORY_CAP = 20;
|
|
24419
|
+
var KEY_ARMED = "armed";
|
|
24420
|
+
var KEY_HISTORY = "history";
|
|
24421
|
+
var KEY_UNREACHABLE = "unreachable_strikes";
|
|
24422
|
+
var ThreadWatchdog = class extends DurableObject {
|
|
24423
|
+
/**
|
|
24424
|
+
* Arm (or re-arm) the standing check for a thread execution. Idempotent —
|
|
24425
|
+
* called at every execution start. Resets the unreachable counter because a
|
|
24426
|
+
* fresh arm proves the thread is answering RPCs right now.
|
|
24427
|
+
*/
|
|
24428
|
+
async arm(threadId) {
|
|
24429
|
+
await this.ctx.storage.put(KEY_ARMED, {
|
|
24430
|
+
threadId,
|
|
24431
|
+
armedAt: Date.now()
|
|
24432
|
+
});
|
|
24433
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
24434
|
+
const existing = await this.ctx.storage.getAlarm();
|
|
24435
|
+
if (existing === null) {
|
|
24436
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
24437
|
+
}
|
|
24438
|
+
}
|
|
24439
|
+
/**
|
|
24440
|
+
* Hint that the thread believes its execution ended cleanly. The companion
|
|
24441
|
+
* does NOT trust this (a bogus "clean end" was one of the incident's failure
|
|
24442
|
+
* modes) — it just moves the next check close so a genuinely clean thread
|
|
24443
|
+
* stands down in seconds instead of a full cadence.
|
|
24444
|
+
*/
|
|
24445
|
+
async disarmHint() {
|
|
24446
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
24447
|
+
if (!armed) return;
|
|
24448
|
+
await this.ctx.storage.setAlarm(Date.now() + DISARM_CONFIRM_DELAY_MS);
|
|
24449
|
+
}
|
|
24450
|
+
/** Diagnostics view for the thread inspector UI and the diag CLI. */
|
|
24451
|
+
async getState() {
|
|
24452
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
24453
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
24454
|
+
return {
|
|
24455
|
+
armed: Boolean(armed),
|
|
24456
|
+
threadId: armed?.threadId ?? null,
|
|
24457
|
+
armedAt: armed?.armedAt ?? null,
|
|
24458
|
+
history
|
|
24459
|
+
};
|
|
24460
|
+
}
|
|
24461
|
+
/**
|
|
24462
|
+
* Explicit cascade from the parent thread (delete/terminate). Cloudflare
|
|
24463
|
+
* has no automatic parent-child cleanup — this IS the cleanup.
|
|
24464
|
+
*/
|
|
24465
|
+
async destroy() {
|
|
24466
|
+
await this.ctx.storage.deleteAlarm();
|
|
24467
|
+
await this.ctx.storage.deleteAll();
|
|
24468
|
+
}
|
|
24469
|
+
async alarm() {
|
|
24470
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
24471
|
+
if (!armed) {
|
|
24472
|
+
return;
|
|
24473
|
+
}
|
|
24474
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
24475
|
+
const startedAt = Date.now();
|
|
24476
|
+
let checkin;
|
|
24477
|
+
let requestRestart = false;
|
|
24478
|
+
let standDown = false;
|
|
24479
|
+
try {
|
|
24480
|
+
const result = await this.probeThread(armed.threadId);
|
|
24481
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
24482
|
+
requestRestart = result.requestRestart;
|
|
24483
|
+
standDown = result.verdict === "idle" || result.verdict === "parked";
|
|
24484
|
+
checkin = {
|
|
24485
|
+
at: startedAt,
|
|
24486
|
+
verdict: result.verdict,
|
|
24487
|
+
action: describeAction(result),
|
|
24488
|
+
executing: result.executing,
|
|
24489
|
+
incompleteLogs: result.incompleteLogs,
|
|
24490
|
+
silenceMs: result.silenceMs,
|
|
24491
|
+
probeMs: Date.now() - startedAt
|
|
24492
|
+
};
|
|
24493
|
+
} catch (error) {
|
|
24494
|
+
const strikes = (await this.ctx.storage.get(KEY_UNREACHABLE) ?? 0) + 1;
|
|
24495
|
+
await this.ctx.storage.put(KEY_UNREACHABLE, strikes);
|
|
24496
|
+
checkin = {
|
|
24497
|
+
at: startedAt,
|
|
24498
|
+
verdict: "unreachable",
|
|
24499
|
+
action: `probe failed (${strikes} consecutive)`,
|
|
24500
|
+
executing: null,
|
|
24501
|
+
incompleteLogs: null,
|
|
24502
|
+
silenceMs: null,
|
|
24503
|
+
probeMs: Date.now() - startedAt,
|
|
24504
|
+
error: error instanceof Error ? error.message : String(error)
|
|
24505
|
+
};
|
|
24506
|
+
if (strikes >= MAX_UNREACHABLE_STRIKES) {
|
|
24507
|
+
console.error(
|
|
24508
|
+
`[ThreadWatchdog] CRITICAL: thread ${armed.threadId} unreachable for ${strikes} consecutive probes \u2014 standing down. Manual intervention required.`
|
|
24509
|
+
);
|
|
24510
|
+
standDown = true;
|
|
24511
|
+
}
|
|
24512
|
+
}
|
|
24513
|
+
await this.recordCheckin(checkin);
|
|
24514
|
+
if (requestRestart) {
|
|
24515
|
+
try {
|
|
24516
|
+
const stub = this.getThreadStub(armed.threadId);
|
|
24517
|
+
await stub.forceRestart(
|
|
24518
|
+
"watchdog takeover: wedged execution aborted; restarting so the recovery alarm can fire"
|
|
24519
|
+
);
|
|
24520
|
+
} catch {
|
|
24521
|
+
}
|
|
24522
|
+
}
|
|
24523
|
+
if (standDown) {
|
|
24524
|
+
await this.ctx.storage.delete(KEY_ARMED);
|
|
24525
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
24526
|
+
await this.ctx.storage.deleteAlarm();
|
|
24527
|
+
}
|
|
24528
|
+
}
|
|
24529
|
+
getThreadStub(threadId) {
|
|
24530
|
+
const namespace = this.env.AGENT_BUILDER_THREAD;
|
|
24531
|
+
const id = namespace.idFromName(threadId);
|
|
24532
|
+
return namespace.get(id);
|
|
24533
|
+
}
|
|
24534
|
+
async probeThread(threadId) {
|
|
24535
|
+
const stub = this.getThreadStub(threadId);
|
|
24536
|
+
let timer;
|
|
24537
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
24538
|
+
timer = setTimeout(
|
|
24539
|
+
() => reject(new Error(`probe timed out after ${PROBE_TIMEOUT_MS}ms`)),
|
|
24540
|
+
PROBE_TIMEOUT_MS
|
|
24541
|
+
);
|
|
24542
|
+
});
|
|
24543
|
+
try {
|
|
24544
|
+
return await Promise.race([stub.watchdogProbe(threadId), timeout]);
|
|
24545
|
+
} finally {
|
|
24546
|
+
if (timer) clearTimeout(timer);
|
|
24547
|
+
}
|
|
24548
|
+
}
|
|
24549
|
+
async recordCheckin(checkin) {
|
|
24550
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
24551
|
+
history.unshift(checkin);
|
|
24552
|
+
await this.ctx.storage.put(KEY_HISTORY, history.slice(0, HISTORY_CAP));
|
|
24553
|
+
}
|
|
24554
|
+
};
|
|
24555
|
+
function describeAction(result) {
|
|
24556
|
+
switch (result.action) {
|
|
24557
|
+
case "finalized_resumed":
|
|
24558
|
+
return `finalized ${result.incompleteLogs} lost request(s), resumed thread`;
|
|
24559
|
+
case "finalized_parked":
|
|
24560
|
+
return `finalized ${result.incompleteLogs} lost request(s), left thread stopped`;
|
|
24561
|
+
case "finalized_deferred":
|
|
24562
|
+
return `finalized ${result.incompleteLogs} lost request(s); queued turn will continue`;
|
|
24563
|
+
case "aborted_takeover_resumed":
|
|
24564
|
+
return `took over wedged execution (silent ${formatMs(result.silenceMs)}), finalized and resumed`;
|
|
24565
|
+
case "none":
|
|
24566
|
+
default:
|
|
24567
|
+
if (result.verdict === "observing") {
|
|
24568
|
+
return `execution silent ${formatMs(result.silenceMs)} (ceiling ${formatMs(result.ceilingMs)}) \u2014 observing`;
|
|
24569
|
+
}
|
|
24570
|
+
if (result.verdict === "idle") {
|
|
24571
|
+
return "thread idle and clean \u2014 standing down";
|
|
24572
|
+
}
|
|
24573
|
+
if (result.executing) {
|
|
24574
|
+
return result.incompleteLogs > 0 ? `streaming, last progress ${formatMs(result.silenceMs)} ago` : "executing (tool run, no open request)";
|
|
24575
|
+
}
|
|
24576
|
+
return "no action needed";
|
|
24577
|
+
}
|
|
24578
|
+
}
|
|
24579
|
+
function formatMs(ms) {
|
|
24580
|
+
if (ms === null || !Number.isFinite(ms)) return "\u221E";
|
|
24581
|
+
if (ms < 1e3) return `${ms}ms`;
|
|
24582
|
+
const seconds = Math.round(ms / 1e3);
|
|
24583
|
+
if (seconds < 60) return `${seconds}s`;
|
|
24584
|
+
const minutes = Math.floor(seconds / 60);
|
|
24585
|
+
return `${minutes}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
24586
|
+
}
|
|
24140
24587
|
|
|
24141
24588
|
// src/runtime.ts
|
|
24142
24589
|
init_utilities();
|
|
@@ -24499,6 +24946,6 @@ init_context();
|
|
|
24499
24946
|
init_ProviderRegistry();
|
|
24500
24947
|
init_types2();
|
|
24501
24948
|
|
|
24502
|
-
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, ProviderRegistry, authenticate, buildImageDescription, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateImageDescription, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
24949
|
+
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, ProviderRegistry, ThreadWatchdog, authenticate, buildImageDescription, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateImageDescription, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
24503
24950
|
//# sourceMappingURL=runtime.js.map
|
|
24504
24951
|
//# sourceMappingURL=runtime.js.map
|