@standardagents/builder 0.25.7 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/built-in-routes.js +246 -16
- package/dist/built-in-routes.js.map +1 -1
- package/dist/client/{ApiKeysView-B9XLopSb.js → ApiKeysView-jhfg43UH.js} +1 -1
- package/dist/client/CenteredContentView-CS0mV2EJ.js +1 -0
- package/dist/client/{CompositionView-DJOtNN7f.js → CompositionView-DhkLWmtE.js} +1 -1
- package/dist/client/{ConfirmDialog.vue_vue_type_script_setup_true_lang-55dXt19T.js → ConfirmDialog.vue_vue_type_script_setup_true_lang-D-WToRMy.js} +1 -1
- package/dist/client/{CopyButton.vue_vue_type_script_setup_true_lang-B6NqQmW9.js → CopyButton.vue_vue_type_script_setup_true_lang-Dhq0t0dd.js} +1 -1
- package/dist/client/{DataTable.vue_vue_type_script_setup_true_lang-0cbPVIIz.js → DataTable.vue_vue_type_script_setup_true_lang-D1MDCns-.js} +1 -1
- package/dist/client/{JsonViewer-B7yHhxqP.js → JsonViewer-BoHrE76S.js} +1 -1
- package/dist/client/{LoginView-mY9OytK6.js → LoginView-BpR2nMNf.js} +1 -1
- package/dist/client/{MarketplaceView-B9Z9YMuB.js → MarketplaceView-uUf1JYla.js} +1 -1
- package/dist/client/{Modal.vue_vue_type_script_setup_true_lang-DEVCRbXl.js → Modal.vue_vue_type_script_setup_true_lang-D-SDVFQI.js} +1 -1
- package/dist/client/{ModelModal.vue_vue_type_script_setup_true_lang-FxEBZ3A8.js → ModelModal.vue_vue_type_script_setup_true_lang-DE57Zr0r.js} +1 -1
- package/dist/client/{ModelsView-DTU8pk-T.js → ModelsView-CuM_OieH.js} +1 -1
- package/dist/client/{PromptEditView-BAKoULxX.js → PromptEditView-BWjrA2zr.js} +1 -1
- package/dist/client/{PromptModal-CoinSKIM.js → PromptModal-BFxhuG0Z.js} +1 -1
- package/dist/client/{PromptsView-DIkj3q-o.js → PromptsView-AlGsRB6k.js} +1 -1
- package/dist/client/{ProvidersView-B8A6ezDR.js → ProvidersView-CbXBs3SH.js} +1 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-Dm3d37hz.js +22 -0
- package/dist/client/{ThreadsView-DGCVdDej.js → ThreadsView-rN9rzJi1.js} +1 -1
- package/dist/client/{ToolsView-CC-WsZet.js → ToolsView-0Gpo_RFg.js} +1 -1
- package/dist/client/{UiCodeBlock.vue_vue_type_script_setup_true_lang-BH3WC9G_.js → UiCodeBlock.vue_vue_type_script_setup_true_lang-CNncALQo.js} +1 -1
- package/dist/client/{UsersView-U3Stwxui.js → UsersView-iXPbcguY.js} +1 -1
- package/dist/client/{VariablesView-0-a1F4cn.js → VariablesView-C9BB9h34.js} +1 -1
- package/dist/client/assets/{ThreadInspectorPane-ouC_RyKA.css → ThreadInspectorPane-CfpdDnUQ.css} +1 -1
- package/dist/client/assets/index-BTPsPULO.css +1 -0
- package/dist/client/{check-Dkzs1C3V.js → check-BdJVlUmI.js} +1 -1
- package/dist/client/{chevron-right-BGirddPg.js → chevron-right-D49O3XLU.js} +1 -1
- package/dist/client/{copy-C08OR8rY.js → copy-40uDyTgY.js} +1 -1
- package/dist/client/{formkit-YJ1Z7D5r.js → formkit-Bl-sERnF.js} +1 -1
- package/dist/client/index-BCoE48be.js +6 -0
- package/dist/client/index.html +4 -4
- package/dist/client/{package-guyzMTDG.js → package-1UgoERpx.js} +1 -1
- package/dist/client/{plus-DG3ZN5wg.js → plus-BzCjfYs0.js} +1 -1
- package/dist/client/{timeAgo-D7V247YG.js → timeAgo-cFwldMu-.js} +1 -1
- package/dist/client/{useActionFeedback-byARNQjQ.js → useActionFeedback-BVifffYQ.js} +1 -1
- package/dist/client/{vue-BqC6IZDY.js → vue-CmmJuziT.js} +1 -1
- package/dist/{index-CX9Ti98s.d.ts → index-DJ7B8CTp.d.ts} +63 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +532 -83
- package/dist/index.js.map +1 -1
- package/dist/plugin.js +27 -3
- package/dist/plugin.js.map +1 -1
- package/dist/runtime.d.ts +184 -6
- package/dist/runtime.js +505 -80
- package/dist/runtime.js.map +1 -1
- package/dist/test.d.ts +1 -1
- package/package.json +5 -5
- package/dist/client/CenteredContentView-Bwv4MVSC.js +0 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-CvssvzLz.js +0 -22
- package/dist/client/assets/index-Ca3he3fq.css +0 -1
- package/dist/client/index-4Zwz6zXD.js +0 -6
package/dist/index.js
CHANGED
|
@@ -1079,6 +1079,24 @@ function resolveWatchdogStaleCeilingMs(env) {
|
|
|
1079
1079
|
);
|
|
1080
1080
|
return Math.max(firstChunk, stall) + 6e4;
|
|
1081
1081
|
}
|
|
1082
|
+
async function awaitBounded(work, budgetMs, label) {
|
|
1083
|
+
let timer;
|
|
1084
|
+
work.catch(() => {
|
|
1085
|
+
});
|
|
1086
|
+
const timeout = new Promise((resolve2) => {
|
|
1087
|
+
timer = setTimeout(() => {
|
|
1088
|
+
console.error(
|
|
1089
|
+
`[LLMRequest] ${label} did not settle within ${budgetMs}ms \u2014 continuing without it`
|
|
1090
|
+
);
|
|
1091
|
+
resolve2(void 0);
|
|
1092
|
+
}, budgetMs);
|
|
1093
|
+
});
|
|
1094
|
+
try {
|
|
1095
|
+
return await Promise.race([work, timeout]);
|
|
1096
|
+
} finally {
|
|
1097
|
+
if (timer) clearTimeout(timer);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1082
1100
|
function modelSupportsImageInput(modelDef) {
|
|
1083
1101
|
const supportsImages = modelDef.capabilities?.supportsImages;
|
|
1084
1102
|
if (typeof supportsImages === "boolean") {
|
|
@@ -1292,13 +1310,42 @@ function transformMessages(messages) {
|
|
|
1292
1310
|
arguments: typeof tc.function.arguments === "string" ? JSON.parse(tc.function.arguments) : tc.function.arguments,
|
|
1293
1311
|
extraContent: tc.extra_content && typeof tc.extra_content === "object" ? tc.extra_content : void 0
|
|
1294
1312
|
}));
|
|
1313
|
+
const storedDetails = msg.reasoning_details;
|
|
1314
|
+
const reasoningDetails = Array.isArray(
|
|
1315
|
+
storedDetails
|
|
1316
|
+
) ? storedDetails.flatMap((entry) => {
|
|
1317
|
+
if (!entry || typeof entry !== "object") return [];
|
|
1318
|
+
const raw = entry;
|
|
1319
|
+
const type = raw.type === "summary" || raw.type === "encrypted" ? raw.type : "text";
|
|
1320
|
+
return [
|
|
1321
|
+
{
|
|
1322
|
+
type,
|
|
1323
|
+
id: typeof raw.id === "string" ? raw.id : void 0,
|
|
1324
|
+
text: typeof raw.text === "string" ? raw.text : void 0,
|
|
1325
|
+
data: typeof raw.data === "string" ? raw.data : void 0,
|
|
1326
|
+
format: typeof raw.format === "string" ? raw.format : void 0
|
|
1327
|
+
}
|
|
1328
|
+
];
|
|
1329
|
+
}) : void 0;
|
|
1330
|
+
const storedReasoning = msg.reasoning_content;
|
|
1331
|
+
let reasoning = typeof storedReasoning === "string" && storedReasoning.length > 0 ? storedReasoning : void 0;
|
|
1332
|
+
if (!reasoning && Array.isArray(reasoningDetails)) {
|
|
1333
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1334
|
+
const texts = [];
|
|
1335
|
+
for (const d of reasoningDetails) {
|
|
1336
|
+
if (typeof d?.text === "string" && d.text.length > 0 && !seen.has(d.text)) {
|
|
1337
|
+
seen.add(d.text);
|
|
1338
|
+
texts.push(d.text);
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
if (texts.length) reasoning = texts.join("\n");
|
|
1342
|
+
}
|
|
1295
1343
|
return {
|
|
1296
1344
|
role: "assistant",
|
|
1297
1345
|
content: typeof msg.content === "string" ? msg.content : null,
|
|
1298
1346
|
toolCalls: toolCalls?.length ? toolCalls : void 0,
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
reasoningDetails: msg.reasoning_details || void 0
|
|
1347
|
+
reasoning,
|
|
1348
|
+
reasoningDetails: reasoningDetails?.length ? reasoningDetails : void 0
|
|
1302
1349
|
};
|
|
1303
1350
|
}
|
|
1304
1351
|
case "tool":
|
|
@@ -1473,7 +1520,7 @@ function normalizeProviderMetadataValue(value) {
|
|
|
1473
1520
|
if (!trimmed || trimmed.toLowerCase() === "unknown") return void 0;
|
|
1474
1521
|
return trimmed;
|
|
1475
1522
|
}
|
|
1476
|
-
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, LLMRequest;
|
|
1523
|
+
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, HOOK_AWAIT_BUDGET_MS, STREAM_CLEANUP_BUDGET_MS, LLMRequest;
|
|
1477
1524
|
var init_LLMRequest = __esm({
|
|
1478
1525
|
"src/agents/LLMRequest.ts"() {
|
|
1479
1526
|
init_pricing();
|
|
@@ -1484,6 +1531,8 @@ var init_LLMRequest = __esm({
|
|
|
1484
1531
|
NON_VISION_PLACEHOLDER_TEXT = "[Image omitted: model does not support vision]";
|
|
1485
1532
|
STREAM_FIRST_CHUNK_TIMEOUT_MS = 12e4;
|
|
1486
1533
|
STREAM_STALL_TIMEOUT_MS = 45e3;
|
|
1534
|
+
HOOK_AWAIT_BUDGET_MS = 3e4;
|
|
1535
|
+
STREAM_CLEANUP_BUDGET_MS = 5e3;
|
|
1487
1536
|
LLMRequest = class {
|
|
1488
1537
|
/**
|
|
1489
1538
|
* Execute an LLM request with full retry/fallback logic
|
|
@@ -1967,7 +2016,14 @@ var init_LLMRequest = __esm({
|
|
|
1967
2016
|
parentSignal.removeEventListener("abort", onParentAbort);
|
|
1968
2017
|
}
|
|
1969
2018
|
try {
|
|
1970
|
-
|
|
2019
|
+
const cleanup = iterator.return?.();
|
|
2020
|
+
if (cleanup) {
|
|
2021
|
+
await awaitBounded(
|
|
2022
|
+
Promise.resolve(cleanup),
|
|
2023
|
+
STREAM_CLEANUP_BUDGET_MS,
|
|
2024
|
+
"stream iterator cleanup"
|
|
2025
|
+
);
|
|
2026
|
+
}
|
|
1971
2027
|
} catch {
|
|
1972
2028
|
}
|
|
1973
2029
|
}
|
|
@@ -2100,7 +2156,11 @@ var init_LLMRequest = __esm({
|
|
|
2100
2156
|
*/
|
|
2101
2157
|
static async resolveProgressArgument(state, toolName) {
|
|
2102
2158
|
try {
|
|
2103
|
-
const mod = await
|
|
2159
|
+
const mod = await awaitBounded(
|
|
2160
|
+
state.thread.instance.loadTool(toolName),
|
|
2161
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2162
|
+
`loadTool(${toolName}) for progress argument`
|
|
2163
|
+
);
|
|
2104
2164
|
const pa = mod?.progressArgument ?? mod?.default?.progressArgument;
|
|
2105
2165
|
return typeof pa === "string" && pa ? pa : void 0;
|
|
2106
2166
|
} catch {
|
|
@@ -2121,7 +2181,11 @@ var init_LLMRequest = __esm({
|
|
|
2121
2181
|
function: { name: entry.name, arguments: entry.argumentsDelta || "" }
|
|
2122
2182
|
};
|
|
2123
2183
|
try {
|
|
2124
|
-
await
|
|
2184
|
+
await awaitBounded(
|
|
2185
|
+
FlowEngine.runToolCallStartedHook(state, toolCall, progress),
|
|
2186
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2187
|
+
`tool_call_started hook for ${entry.name}`
|
|
2188
|
+
);
|
|
2125
2189
|
} catch (error) {
|
|
2126
2190
|
console.error("[LLMRequest] Error firing tool_call_started:", error);
|
|
2127
2191
|
}
|
|
@@ -12706,8 +12770,12 @@ var init_ThreadStateImpl = __esm({
|
|
|
12706
12770
|
options?.includeSilent,
|
|
12707
12771
|
options?.maxDepth
|
|
12708
12772
|
);
|
|
12773
|
+
const messages = result.messages.map(this._mapMessage);
|
|
12774
|
+
if (options?.order === "desc") {
|
|
12775
|
+
messages.reverse();
|
|
12776
|
+
}
|
|
12709
12777
|
return {
|
|
12710
|
-
messages
|
|
12778
|
+
messages,
|
|
12711
12779
|
total: result.total,
|
|
12712
12780
|
hasMore: result.hasMore
|
|
12713
12781
|
};
|
|
@@ -14016,6 +14084,7 @@ declare module 'virtual:@standardagents/builder' {
|
|
|
14016
14084
|
import type { DurableThread as BaseDurableThread } from '@standardagents/builder/runtime';
|
|
14017
14085
|
import type { DurableAgentBuilder as BaseDurableAgentBuilder } from '@standardagents/builder/runtime';
|
|
14018
14086
|
import type { CodeExecutionBridge as BaseCodeExecutionBridge } from '@standardagents/builder/runtime';
|
|
14087
|
+
import type { ThreadWatchdog as BaseThreadWatchdog } from '@standardagents/builder/runtime';
|
|
14019
14088
|
import type { ThreadEnv } from '@standardagents/builder/runtime';
|
|
14020
14089
|
|
|
14021
14090
|
// ============================================================
|
|
@@ -14494,6 +14563,12 @@ declare module 'virtual:@standardagents/builder' {
|
|
|
14494
14563
|
*/
|
|
14495
14564
|
export class CodeExecutionBridge extends BaseCodeExecutionBridge {}
|
|
14496
14565
|
|
|
14566
|
+
/**
|
|
14567
|
+
* Per-thread watchdog companion Durable Object (AGENT_BUILDER_WATCHDOG
|
|
14568
|
+
* binding). Delivers execution liveness checks a running turn cannot starve.
|
|
14569
|
+
*/
|
|
14570
|
+
export class ThreadWatchdog extends BaseThreadWatchdog {}
|
|
14571
|
+
|
|
14497
14572
|
// ============================================================
|
|
14498
14573
|
// Router Function
|
|
14499
14574
|
// ============================================================
|
|
@@ -20660,10 +20735,11 @@ async function loadBuilderVirtualModule(context) {
|
|
|
20660
20735
|
}
|
|
20661
20736
|
}).join("\n");
|
|
20662
20737
|
return `// Consolidated virtual module: virtual:@standardagents/builder /* v${hmr.versions.builder} */
|
|
20663
|
-
// Provides DurableThread, DurableAgentBuilder, and router
|
|
20738
|
+
// Provides DurableThread, DurableAgentBuilder, ThreadWatchdog, and router
|
|
20664
20739
|
import { DurableThread as _BaseDurableThread } from '@standardagents/builder/runtime';
|
|
20665
20740
|
import { DurableAgentBuilder as _BaseDurableAgentBuilder } from '@standardagents/builder/runtime';
|
|
20666
20741
|
import { CodeExecutionBridge as _BaseCodeExecutionBridge } from '@standardagents/builder/runtime';
|
|
20742
|
+
import { ThreadWatchdog as _BaseThreadWatchdog } from '@standardagents/builder/runtime';
|
|
20667
20743
|
|
|
20668
20744
|
// Import sip WASM module and readiness helper
|
|
20669
20745
|
// Static import allows workerd to pre-compile the WASM at bundle time
|
|
@@ -20783,6 +20859,12 @@ export class DurableAgentBuilder extends _BaseDurableAgentBuilder {
|
|
|
20783
20859
|
}
|
|
20784
20860
|
|
|
20785
20861
|
export class CodeExecutionBridge extends _BaseCodeExecutionBridge {}
|
|
20862
|
+
|
|
20863
|
+
/**
|
|
20864
|
+
* Per-thread watchdog companion. Needs no virtual-module registries \u2014 it only
|
|
20865
|
+
* probes its DurableThread over RPC.
|
|
20866
|
+
*/
|
|
20867
|
+
export class ThreadWatchdog extends _BaseThreadWatchdog {}
|
|
20786
20868
|
`;
|
|
20787
20869
|
}
|
|
20788
20870
|
async function loadRegistryVirtualModule(context) {
|
|
@@ -21750,6 +21832,10 @@ function createAgentBuilderWorkerConfig(options) {
|
|
|
21750
21832
|
{
|
|
21751
21833
|
name: "AGENT_BUILDER",
|
|
21752
21834
|
class_name: "DurableAgentBuilder"
|
|
21835
|
+
},
|
|
21836
|
+
{
|
|
21837
|
+
name: "AGENT_BUILDER_WATCHDOG",
|
|
21838
|
+
class_name: "ThreadWatchdog"
|
|
21753
21839
|
}
|
|
21754
21840
|
]
|
|
21755
21841
|
},
|
|
@@ -21761,6 +21847,12 @@ function createAgentBuilderWorkerConfig(options) {
|
|
|
21761
21847
|
{
|
|
21762
21848
|
tag: "v2",
|
|
21763
21849
|
new_sqlite_classes: ["DurableAgentBuilder"]
|
|
21850
|
+
},
|
|
21851
|
+
// Migration tags are append-only: existing deployments have already
|
|
21852
|
+
// recorded v1/v2, so a new class MUST arrive as a new tag.
|
|
21853
|
+
{
|
|
21854
|
+
tag: "v3",
|
|
21855
|
+
new_sqlite_classes: ["ThreadWatchdog"]
|
|
21764
21856
|
}
|
|
21765
21857
|
]
|
|
21766
21858
|
};
|
|
@@ -21848,7 +21940,7 @@ function agentbuilder(options = {}) {
|
|
|
21848
21940
|
function writeGeneratedWorkerEntry() {
|
|
21849
21941
|
const content = `// Generated by @standardagents/builder.
|
|
21850
21942
|
// Future agents: update builder/src/plugin.ts when changing this entrypoint.
|
|
21851
|
-
import { CodeExecutionBridge, DurableAgentBuilder, DurableThread, router } from "virtual:@standardagents/builder";
|
|
21943
|
+
import { CodeExecutionBridge, DurableAgentBuilder, DurableThread, ThreadWatchdog, router } from "virtual:@standardagents/builder";
|
|
21852
21944
|
|
|
21853
21945
|
export default {
|
|
21854
21946
|
async fetch(request: Request, env: Parameters<typeof router>[1]) {
|
|
@@ -21857,7 +21949,7 @@ export default {
|
|
|
21857
21949
|
},
|
|
21858
21950
|
};
|
|
21859
21951
|
|
|
21860
|
-
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread };
|
|
21952
|
+
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, ThreadWatchdog };
|
|
21861
21953
|
`;
|
|
21862
21954
|
fs8__default.mkdirSync(outputDir, { recursive: true });
|
|
21863
21955
|
if (!fs8__default.existsSync(generatedWorkerEntry) || fs8__default.readFileSync(generatedWorkerEntry, "utf-8") !== content) {
|
|
@@ -25197,6 +25289,11 @@ var DurableThread = class _DurableThread extends DurableObject {
|
|
|
25197
25289
|
if (this.currentAbortController) {
|
|
25198
25290
|
this.currentAbortController.abort();
|
|
25199
25291
|
}
|
|
25292
|
+
try {
|
|
25293
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
25294
|
+
} catch (error) {
|
|
25295
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
25296
|
+
}
|
|
25200
25297
|
const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
|
|
25201
25298
|
const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
|
|
25202
25299
|
await agentBuilder.updateThread(threadId, { terminated: terminatedAt });
|
|
@@ -26356,6 +26453,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
26356
26453
|
*/
|
|
26357
26454
|
async getMessages(limit = 100, offset = 0, order = "DESC", includeSilent = false, maxDepth = 0) {
|
|
26358
26455
|
await this.ensureMigrated();
|
|
26456
|
+
const normalizedOrder = String(order).toUpperCase() === "ASC" ? "ASC" : "DESC";
|
|
26359
26457
|
try {
|
|
26360
26458
|
const whereClauses = [];
|
|
26361
26459
|
if (!includeSilent) {
|
|
@@ -26363,7 +26461,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
26363
26461
|
}
|
|
26364
26462
|
whereClauses.push(`(depth <= ${maxDepth} OR depth IS NULL)`);
|
|
26365
26463
|
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
|
|
26366
|
-
const orderDirection =
|
|
26464
|
+
const orderDirection = normalizedOrder;
|
|
26367
26465
|
const combinedMessagesSql = `
|
|
26368
26466
|
WITH combined_messages AS (
|
|
26369
26467
|
SELECT
|
|
@@ -26489,7 +26587,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
26489
26587
|
return projection ? { ...message, ...projection } : message;
|
|
26490
26588
|
});
|
|
26491
26589
|
}
|
|
26492
|
-
if (
|
|
26590
|
+
if (normalizedOrder === "DESC") {
|
|
26493
26591
|
messages = messages.reverse();
|
|
26494
26592
|
}
|
|
26495
26593
|
const hasMore = offset + messages.length < total;
|
|
@@ -27322,6 +27420,14 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27322
27420
|
}
|
|
27323
27421
|
this.logSockets.clear();
|
|
27324
27422
|
this.messageSockets.clear();
|
|
27423
|
+
try {
|
|
27424
|
+
const threadId = this.ctx.id.name;
|
|
27425
|
+
if (threadId) {
|
|
27426
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
27427
|
+
}
|
|
27428
|
+
} catch (error) {
|
|
27429
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
27430
|
+
}
|
|
27325
27431
|
await this.ctx.storage.deleteAlarm();
|
|
27326
27432
|
await this.ctx.storage.deleteAll();
|
|
27327
27433
|
this.migratedToVersion = null;
|
|
@@ -27977,6 +28083,18 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27977
28083
|
noteExecutionProgress() {
|
|
27978
28084
|
this.lastProgressAt = Date.now();
|
|
27979
28085
|
}
|
|
28086
|
+
/**
|
|
28087
|
+
* The per-thread companion watchdog DO (ThreadWatchdog), derived from the
|
|
28088
|
+
* thread id. It exists because this thread's OWN alarm cannot fire while a
|
|
28089
|
+
* queue-dispatched turn runs inside alarm() — the companion delivers the
|
|
28090
|
+
* same check via RPC, which interleaves at await points even mid-turn.
|
|
28091
|
+
* Returns null on deployments whose generated config predates the binding.
|
|
28092
|
+
*/
|
|
28093
|
+
watchdogCompanion(threadId) {
|
|
28094
|
+
const namespace = this.env.AGENT_BUILDER_WATCHDOG;
|
|
28095
|
+
if (!namespace || typeof namespace.idFromName !== "function") return null;
|
|
28096
|
+
return namespace.get(namespace.idFromName(threadId));
|
|
28097
|
+
}
|
|
27980
28098
|
/** Arm the standing watchdog for an execution start. One pending row max. */
|
|
27981
28099
|
async armExecutionWatchdog(threadId) {
|
|
27982
28100
|
try {
|
|
@@ -27985,14 +28103,20 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27985
28103
|
const pending = (await this.ctx.storage.sql.exec(
|
|
27986
28104
|
`SELECT id FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'pending' LIMIT 1`
|
|
27987
28105
|
)).toArray();
|
|
27988
|
-
if (pending.length
|
|
27989
|
-
|
|
28106
|
+
if (pending.length === 0) {
|
|
28107
|
+
await this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
28108
|
+
}
|
|
27990
28109
|
} catch (error) {
|
|
27991
28110
|
console.error("[DurableThread] Failed to arm execution watchdog (non-fatal):", error);
|
|
27992
28111
|
}
|
|
28112
|
+
try {
|
|
28113
|
+
await this.watchdogCompanion(threadId)?.arm(threadId);
|
|
28114
|
+
} catch (error) {
|
|
28115
|
+
console.error("[DurableThread] Failed to arm watchdog companion (non-fatal):", error);
|
|
28116
|
+
}
|
|
27993
28117
|
}
|
|
27994
28118
|
/** Disarm on clean execution end. Completed check rows are noise, not history. */
|
|
27995
|
-
async disarmExecutionWatchdog() {
|
|
28119
|
+
async disarmExecutionWatchdog(threadId) {
|
|
27996
28120
|
try {
|
|
27997
28121
|
await this.ctx.storage.sql.exec(
|
|
27998
28122
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status IN ('pending', 'completed')`
|
|
@@ -28001,10 +28125,21 @@ ${resultContent}${attachmentSummary}`;
|
|
|
28001
28125
|
} catch (error) {
|
|
28002
28126
|
console.error("[DurableThread] Failed to disarm execution watchdog (non-fatal):", error);
|
|
28003
28127
|
}
|
|
28128
|
+
if (threadId) {
|
|
28129
|
+
try {
|
|
28130
|
+
await this.watchdogCompanion(threadId)?.disarmHint();
|
|
28131
|
+
} catch (error) {
|
|
28132
|
+
console.error("[DurableThread] Failed to hint watchdog companion (non-fatal):", error);
|
|
28133
|
+
}
|
|
28134
|
+
}
|
|
28004
28135
|
}
|
|
28005
28136
|
/**
|
|
28006
|
-
* The standing check. Evaluates the thread's state AT
|
|
28007
|
-
* nothing about whichever request was in flight when it
|
|
28137
|
+
* The standing check, alarm-queue delivery. Evaluates the thread's state AT
|
|
28138
|
+
* FIRE TIME — it knows nothing about whichever request was in flight when it
|
|
28139
|
+
* was armed. NOTE this delivery path cannot fire while a queue-dispatched
|
|
28140
|
+
* turn runs (the turn owns this DO's single alarm invocation); the
|
|
28141
|
+
* ThreadWatchdog companion delivers the same assessment via watchdogProbe()
|
|
28142
|
+
* in exactly that window.
|
|
28008
28143
|
*/
|
|
28009
28144
|
async watchdogCheck(threadId) {
|
|
28010
28145
|
const rearm = () => this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
@@ -28012,80 +28147,190 @@ ${resultContent}${attachmentSummary}`;
|
|
|
28012
28147
|
await this.ctx.storage.sql.exec(
|
|
28013
28148
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'completed'`
|
|
28014
28149
|
);
|
|
28015
|
-
const
|
|
28016
|
-
|
|
28017
|
-
)).toArray();
|
|
28018
|
-
let incomplete = await readIncomplete();
|
|
28019
|
-
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
28020
|
-
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
28021
|
-
if (this.isExecuting) {
|
|
28022
|
-
if (incomplete.length === 0) {
|
|
28023
|
-
this.watchdogWedgeStrikes = 0;
|
|
28024
|
-
await rearm();
|
|
28025
|
-
return;
|
|
28026
|
-
}
|
|
28027
|
-
if (silenceMs <= ceiling) {
|
|
28028
|
-
this.watchdogWedgeStrikes = 0;
|
|
28029
|
-
await rearm();
|
|
28030
|
-
return;
|
|
28031
|
-
}
|
|
28032
|
-
this.watchdogWedgeStrikes++;
|
|
28033
|
-
if (this.watchdogWedgeStrikes < 2) {
|
|
28034
|
-
console.warn(
|
|
28035
|
-
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
28036
|
-
);
|
|
28037
|
-
await rearm();
|
|
28038
|
-
return;
|
|
28039
|
-
}
|
|
28040
|
-
console.error(
|
|
28041
|
-
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
28042
|
-
);
|
|
28043
|
-
try {
|
|
28044
|
-
this.currentAbortController?.abort();
|
|
28045
|
-
} catch {
|
|
28046
|
-
}
|
|
28047
|
-
this.isExecuting = false;
|
|
28048
|
-
this.currentAbortController = null;
|
|
28049
|
-
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
28150
|
+
const result = await this.performWatchdogAssessment(threadId);
|
|
28151
|
+
if (result.verdict === "healthy" || result.verdict === "observing") {
|
|
28050
28152
|
await rearm();
|
|
28051
28153
|
return;
|
|
28052
28154
|
}
|
|
28053
|
-
|
|
28054
|
-
|
|
28155
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28156
|
+
} catch (error) {
|
|
28157
|
+
console.error("[DurableThread] Watchdog check failed (non-fatal):", error);
|
|
28158
|
+
try {
|
|
28055
28159
|
await rearm();
|
|
28056
|
-
|
|
28057
|
-
}
|
|
28058
|
-
if (incomplete.length > 0) {
|
|
28059
|
-
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
28160
|
+
} catch {
|
|
28060
28161
|
}
|
|
28061
|
-
|
|
28062
|
-
|
|
28063
|
-
|
|
28064
|
-
|
|
28065
|
-
|
|
28066
|
-
|
|
28067
|
-
|
|
28162
|
+
}
|
|
28163
|
+
}
|
|
28164
|
+
/**
|
|
28165
|
+
* RPC delivery of the standing check — called by the ThreadWatchdog
|
|
28166
|
+
* companion (and the diagnostics revive endpoint). Runs in a fresh request
|
|
28167
|
+
* context, so it interleaves with a running turn at await points and its
|
|
28168
|
+
* own awaits/timers are alive even when a wedged turn's context is frozen.
|
|
28169
|
+
* Does NOT touch the internal watchdogCheck alarm rows — those belong to
|
|
28170
|
+
* the alarm delivery path.
|
|
28171
|
+
*/
|
|
28172
|
+
async watchdogProbe(threadId) {
|
|
28173
|
+
await this.ensureMigrated();
|
|
28174
|
+
return await this.performWatchdogAssessment(threadId);
|
|
28175
|
+
}
|
|
28176
|
+
/**
|
|
28177
|
+
* Reset this Durable Object instance on request. Used by the watchdog
|
|
28178
|
+
* companion after a wedged-executor takeover: the zombie turn may still own
|
|
28179
|
+
* this DO's single alarm invocation, which would block the just-enqueued
|
|
28180
|
+
* recovery alarm forever. Aborting guarantees a fresh instance whose alarm
|
|
28181
|
+
* chain re-arms from persisted queue state on first wake.
|
|
28182
|
+
*/
|
|
28183
|
+
async forceRestart(reason) {
|
|
28184
|
+
console.error(`[DurableThread] Force restart requested: ${reason}`);
|
|
28185
|
+
this.ctx.abort(reason);
|
|
28186
|
+
}
|
|
28187
|
+
/**
|
|
28188
|
+
* Execution-health snapshot for the diagnostics endpoint and `agents diag`.
|
|
28189
|
+
* Exposes the state the 2026-07-08 post-mortem could not see from outside:
|
|
28190
|
+
* the alarm queue (was a turn item pending/processing/failed? was the
|
|
28191
|
+
* watchdog row gone?), the volatile execution flags, and open request logs.
|
|
28192
|
+
* Read-only; safe to call at any time.
|
|
28193
|
+
*/
|
|
28194
|
+
async getDiagnostics() {
|
|
28195
|
+
await this.ensureMigrated();
|
|
28196
|
+
const incompleteLogs = (await this.ctx.storage.sql.exec(
|
|
28197
|
+
`SELECT id, model, provider, created_at FROM logs WHERE is_complete = 0 ORDER BY created_at DESC LIMIT 20`
|
|
28198
|
+
)).toArray();
|
|
28199
|
+
const alarmQueue = (await this.ctx.storage.sql.exec(
|
|
28200
|
+
`SELECT id, method, status, scheduled_at, created_at, attempts, error
|
|
28201
|
+
FROM alarm_queue
|
|
28202
|
+
ORDER BY CASE WHEN status IN ('pending', 'processing') THEN 0 ELSE 1 END ASC,
|
|
28203
|
+
scheduled_at DESC
|
|
28204
|
+
LIMIT 50`
|
|
28205
|
+
)).toArray();
|
|
28206
|
+
const messageQueueCount = (await this.ctx.storage.sql.exec(
|
|
28207
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
28208
|
+
)).toArray()[0]?.c ?? 0;
|
|
28209
|
+
let contextUsage = null;
|
|
28210
|
+
try {
|
|
28211
|
+
const row = (await this.ctx.storage.sql.exec(
|
|
28212
|
+
`SELECT value FROM thread_values WHERE key = 'context_usage' LIMIT 1`
|
|
28213
|
+
)).toArray()[0];
|
|
28214
|
+
if (row?.value) {
|
|
28215
|
+
const parsed = JSON.parse(row.value);
|
|
28216
|
+
contextUsage = this.normalizePlainRecord(parsed);
|
|
28068
28217
|
}
|
|
28069
|
-
|
|
28070
|
-
|
|
28071
|
-
|
|
28218
|
+
} catch {
|
|
28219
|
+
}
|
|
28220
|
+
return {
|
|
28221
|
+
executing: this.isExecuting,
|
|
28222
|
+
lastProgressAt: this.lastProgressAt > 0 ? this.lastProgressAt : null,
|
|
28223
|
+
silenceMs: this.lastProgressAt > 0 ? Date.now() - this.lastProgressAt : null,
|
|
28224
|
+
watchdogStrikes: this.watchdogWedgeStrikes,
|
|
28225
|
+
watchdogCeilingMs: resolveWatchdogStaleCeilingMs(this.env),
|
|
28226
|
+
stopped: await this.shouldStop(),
|
|
28227
|
+
incompleteLogs,
|
|
28228
|
+
alarmQueue,
|
|
28229
|
+
messageQueueCount,
|
|
28230
|
+
contextUsage
|
|
28231
|
+
};
|
|
28232
|
+
}
|
|
28233
|
+
/**
|
|
28234
|
+
* The shared watchdog decision core. Both deliveries (alarm + companion
|
|
28235
|
+
* probe) evaluate and ACT through this method; the caller only decides what
|
|
28236
|
+
* to do with its own scheduling based on the verdict.
|
|
28237
|
+
*/
|
|
28238
|
+
async performWatchdogAssessment(threadId) {
|
|
28239
|
+
const readIncomplete = async () => (await this.ctx.storage.sql.exec(
|
|
28240
|
+
`SELECT id FROM logs WHERE is_complete = 0`
|
|
28241
|
+
)).toArray();
|
|
28242
|
+
let incomplete = await readIncomplete();
|
|
28243
|
+
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
28244
|
+
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
28245
|
+
const reportedSilence = Number.isFinite(silenceMs) ? silenceMs : null;
|
|
28246
|
+
const report = (verdict, action, extra = {}) => ({
|
|
28247
|
+
verdict,
|
|
28248
|
+
action,
|
|
28249
|
+
executing: this.isExecuting,
|
|
28250
|
+
incompleteLogs: incomplete.length,
|
|
28251
|
+
queuedMessages: 0,
|
|
28252
|
+
silenceMs: reportedSilence,
|
|
28253
|
+
ceilingMs: ceiling,
|
|
28254
|
+
strikes: this.watchdogWedgeStrikes,
|
|
28255
|
+
requestRestart: false,
|
|
28256
|
+
...extra
|
|
28257
|
+
});
|
|
28258
|
+
let tookOver = false;
|
|
28259
|
+
if (this.isExecuting) {
|
|
28260
|
+
if (incomplete.length === 0) {
|
|
28261
|
+
this.watchdogWedgeStrikes = 0;
|
|
28262
|
+
return report("healthy", "none");
|
|
28263
|
+
}
|
|
28264
|
+
if (silenceMs <= ceiling) {
|
|
28265
|
+
this.watchdogWedgeStrikes = 0;
|
|
28266
|
+
return report("healthy", "none");
|
|
28267
|
+
}
|
|
28268
|
+
this.watchdogWedgeStrikes++;
|
|
28269
|
+
if (this.watchdogWedgeStrikes < 2) {
|
|
28072
28270
|
console.warn(
|
|
28073
|
-
`[DurableThread] Watchdog:
|
|
28074
|
-
);
|
|
28075
|
-
} else {
|
|
28076
|
-
console.error(
|
|
28077
|
-
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
28271
|
+
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
28078
28272
|
);
|
|
28079
|
-
|
|
28273
|
+
return report("observing", "none");
|
|
28080
28274
|
}
|
|
28081
|
-
|
|
28082
|
-
|
|
28083
|
-
|
|
28275
|
+
console.error(
|
|
28276
|
+
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
28277
|
+
);
|
|
28084
28278
|
try {
|
|
28085
|
-
|
|
28279
|
+
this.currentAbortController?.abort();
|
|
28086
28280
|
} catch {
|
|
28087
28281
|
}
|
|
28282
|
+
this.isExecuting = false;
|
|
28283
|
+
this.currentAbortController = null;
|
|
28284
|
+
tookOver = true;
|
|
28285
|
+
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
28286
|
+
return report("observing", "none");
|
|
28287
|
+
}
|
|
28288
|
+
incomplete = await readIncomplete();
|
|
28289
|
+
if (this.isExecuting) {
|
|
28290
|
+
return report("healthy", "none");
|
|
28291
|
+
}
|
|
28292
|
+
const finalizedCount = incomplete.length;
|
|
28293
|
+
if (incomplete.length > 0) {
|
|
28294
|
+
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
28295
|
+
}
|
|
28296
|
+
const stopped = await this.shouldStop();
|
|
28297
|
+
const queued = (await this.ctx.storage.sql.exec(
|
|
28298
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
28299
|
+
)).toArray()[0]?.c ?? 0;
|
|
28300
|
+
if (stopped || finalizedCount === 0 && queued === 0) {
|
|
28301
|
+
if (finalizedCount > 0) {
|
|
28302
|
+
return report("parked", "finalized_parked", {
|
|
28303
|
+
queuedMessages: queued,
|
|
28304
|
+
requestRestart: tookOver
|
|
28305
|
+
});
|
|
28306
|
+
}
|
|
28307
|
+
return report("idle", "none", { queuedMessages: queued });
|
|
28308
|
+
}
|
|
28309
|
+
if (await this.hasPendingTurnAlarm()) {
|
|
28310
|
+
return report("recovered", "finalized_deferred", {
|
|
28311
|
+
queuedMessages: queued,
|
|
28312
|
+
requestRestart: tookOver
|
|
28313
|
+
});
|
|
28088
28314
|
}
|
|
28315
|
+
if (await this.consumeWatchdogResumeBudget()) {
|
|
28316
|
+
const side = await this.getLatestQueuedMessageSide() ?? "a";
|
|
28317
|
+
await this.alarmQueue.enqueue("continueExecution", { threadId, side }, 0);
|
|
28318
|
+
console.warn(
|
|
28319
|
+
`[DurableThread] Watchdog: lost execution recovered \u2014 resuming thread ${threadId}`
|
|
28320
|
+
);
|
|
28321
|
+
return report("recovered", tookOver ? "aborted_takeover_resumed" : "finalized_resumed", {
|
|
28322
|
+
queuedMessages: queued,
|
|
28323
|
+
requestRestart: tookOver
|
|
28324
|
+
});
|
|
28325
|
+
}
|
|
28326
|
+
console.error(
|
|
28327
|
+
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
28328
|
+
);
|
|
28329
|
+
await this.notifyParentOfLostExecution(threadId);
|
|
28330
|
+
return report("parked", "finalized_parked", {
|
|
28331
|
+
queuedMessages: queued,
|
|
28332
|
+
requestRestart: tookOver
|
|
28333
|
+
});
|
|
28089
28334
|
}
|
|
28090
28335
|
/** Mark logs the dead executor abandoned, and tell connected clients. */
|
|
28091
28336
|
async finalizeLostExecutionLogs(logIds) {
|
|
@@ -28194,6 +28439,18 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28194
28439
|
*/
|
|
28195
28440
|
async executeFlow(threadId, agentName, initial_messages, data, awaitCompletion = true) {
|
|
28196
28441
|
await this.assertNotTerminated();
|
|
28442
|
+
if (this.isExecuting) {
|
|
28443
|
+
const busyWaitBudgetMs = 3e4;
|
|
28444
|
+
const waitStartedAt = Date.now();
|
|
28445
|
+
while (this.isExecuting && Date.now() - waitStartedAt < busyWaitBudgetMs) {
|
|
28446
|
+
await new Promise((resolve2) => setTimeout(resolve2, 500));
|
|
28447
|
+
}
|
|
28448
|
+
if (this.isExecuting) {
|
|
28449
|
+
throw new Error(
|
|
28450
|
+
`Thread ${threadId} is already executing; refusing to start a second concurrent flow`
|
|
28451
|
+
);
|
|
28452
|
+
}
|
|
28453
|
+
}
|
|
28197
28454
|
this.isExecuting = true;
|
|
28198
28455
|
this.currentAbortController = new AbortController();
|
|
28199
28456
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -28327,7 +28584,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28327
28584
|
} finally {
|
|
28328
28585
|
this.isExecuting = false;
|
|
28329
28586
|
this.currentAbortController = null;
|
|
28330
|
-
await this.disarmExecutionWatchdog();
|
|
28587
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28331
28588
|
}
|
|
28332
28589
|
}
|
|
28333
28590
|
/**
|
|
@@ -28367,6 +28624,19 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28367
28624
|
*/
|
|
28368
28625
|
async processMessage(threadId, content, role = "user", attachments, queuedMessageId, queuedAt) {
|
|
28369
28626
|
await this.assertNotTerminated();
|
|
28627
|
+
if (this.isExecuting) {
|
|
28628
|
+
console.warn(
|
|
28629
|
+
`[DurableThread] processMessage while executing \u2014 diverting message to queue for thread ${threadId}`
|
|
28630
|
+
);
|
|
28631
|
+
await this.queueMessage(threadId, {
|
|
28632
|
+
role: role === "assistant" ? "assistant" : "user",
|
|
28633
|
+
content,
|
|
28634
|
+
attachments: this.parseAttachmentRefsJson(attachments ?? null) ?? [],
|
|
28635
|
+
silent: false,
|
|
28636
|
+
metadata: { diverted_from: "processMessage" }
|
|
28637
|
+
});
|
|
28638
|
+
return;
|
|
28639
|
+
}
|
|
28370
28640
|
if (role === "user") {
|
|
28371
28641
|
await this.ctx.storage.sql.exec(
|
|
28372
28642
|
`UPDATE execution_state SET value = 'false' WHERE key = 'stopped'`
|
|
@@ -28513,7 +28783,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28513
28783
|
} finally {
|
|
28514
28784
|
this.isExecuting = false;
|
|
28515
28785
|
this.currentAbortController = null;
|
|
28516
|
-
await this.disarmExecutionWatchdog();
|
|
28786
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28517
28787
|
}
|
|
28518
28788
|
}
|
|
28519
28789
|
/**
|
|
@@ -28522,6 +28792,12 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28522
28792
|
*/
|
|
28523
28793
|
async doContinueExecution(threadId, side = "a") {
|
|
28524
28794
|
await this.assertNotTerminated();
|
|
28795
|
+
if (this.isExecuting) {
|
|
28796
|
+
console.warn(
|
|
28797
|
+
`[DurableThread] continueExecution while executing \u2014 skipping (running flow owns the queue) for thread ${threadId}`
|
|
28798
|
+
);
|
|
28799
|
+
return;
|
|
28800
|
+
}
|
|
28525
28801
|
this.isExecuting = true;
|
|
28526
28802
|
this.currentAbortController = new AbortController();
|
|
28527
28803
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -28633,7 +28909,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28633
28909
|
} finally {
|
|
28634
28910
|
this.isExecuting = false;
|
|
28635
28911
|
this.currentAbortController = null;
|
|
28636
|
-
await this.disarmExecutionWatchdog();
|
|
28912
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28637
28913
|
const latestQueuedSide = await this.getLatestQueuedMessageSide();
|
|
28638
28914
|
if (latestQueuedSide) {
|
|
28639
28915
|
await this.ensureQueuedMessageExecution(threadId, latestQueuedSide);
|
|
@@ -32450,6 +32726,179 @@ var CodeExecutionBridge = class extends WorkerEntrypoint {
|
|
|
32450
32726
|
return thread.callCodeExecutionBridge(runId, functionId, encodedArgs);
|
|
32451
32727
|
}
|
|
32452
32728
|
};
|
|
32729
|
+
var CHECK_CADENCE_MS = 3e4;
|
|
32730
|
+
var DISARM_CONFIRM_DELAY_MS = 3e3;
|
|
32731
|
+
var PROBE_TIMEOUT_MS = 15e3;
|
|
32732
|
+
var MAX_UNREACHABLE_STRIKES = 40;
|
|
32733
|
+
var HISTORY_CAP = 20;
|
|
32734
|
+
var KEY_ARMED = "armed";
|
|
32735
|
+
var KEY_HISTORY = "history";
|
|
32736
|
+
var KEY_UNREACHABLE = "unreachable_strikes";
|
|
32737
|
+
var ThreadWatchdog = class extends DurableObject {
|
|
32738
|
+
/**
|
|
32739
|
+
* Arm (or re-arm) the standing check for a thread execution. Idempotent —
|
|
32740
|
+
* called at every execution start. Resets the unreachable counter because a
|
|
32741
|
+
* fresh arm proves the thread is answering RPCs right now.
|
|
32742
|
+
*/
|
|
32743
|
+
async arm(threadId) {
|
|
32744
|
+
await this.ctx.storage.put(KEY_ARMED, {
|
|
32745
|
+
threadId,
|
|
32746
|
+
armedAt: Date.now()
|
|
32747
|
+
});
|
|
32748
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
32749
|
+
const existing = await this.ctx.storage.getAlarm();
|
|
32750
|
+
if (existing === null) {
|
|
32751
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
32752
|
+
}
|
|
32753
|
+
}
|
|
32754
|
+
/**
|
|
32755
|
+
* Hint that the thread believes its execution ended cleanly. The companion
|
|
32756
|
+
* does NOT trust this (a bogus "clean end" was one of the incident's failure
|
|
32757
|
+
* modes) — it just moves the next check close so a genuinely clean thread
|
|
32758
|
+
* stands down in seconds instead of a full cadence.
|
|
32759
|
+
*/
|
|
32760
|
+
async disarmHint() {
|
|
32761
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
32762
|
+
if (!armed) return;
|
|
32763
|
+
await this.ctx.storage.setAlarm(Date.now() + DISARM_CONFIRM_DELAY_MS);
|
|
32764
|
+
}
|
|
32765
|
+
/** Diagnostics view for the thread inspector UI and the diag CLI. */
|
|
32766
|
+
async getState() {
|
|
32767
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
32768
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
32769
|
+
return {
|
|
32770
|
+
armed: Boolean(armed),
|
|
32771
|
+
threadId: armed?.threadId ?? null,
|
|
32772
|
+
armedAt: armed?.armedAt ?? null,
|
|
32773
|
+
history
|
|
32774
|
+
};
|
|
32775
|
+
}
|
|
32776
|
+
/**
|
|
32777
|
+
* Explicit cascade from the parent thread (delete/terminate). Cloudflare
|
|
32778
|
+
* has no automatic parent-child cleanup — this IS the cleanup.
|
|
32779
|
+
*/
|
|
32780
|
+
async destroy() {
|
|
32781
|
+
await this.ctx.storage.deleteAlarm();
|
|
32782
|
+
await this.ctx.storage.deleteAll();
|
|
32783
|
+
}
|
|
32784
|
+
async alarm() {
|
|
32785
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
32786
|
+
if (!armed) {
|
|
32787
|
+
return;
|
|
32788
|
+
}
|
|
32789
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
32790
|
+
const startedAt = Date.now();
|
|
32791
|
+
let checkin;
|
|
32792
|
+
let requestRestart = false;
|
|
32793
|
+
let standDown = false;
|
|
32794
|
+
try {
|
|
32795
|
+
const result = await this.probeThread(armed.threadId);
|
|
32796
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
32797
|
+
requestRestart = result.requestRestart;
|
|
32798
|
+
standDown = result.verdict === "idle" || result.verdict === "parked";
|
|
32799
|
+
checkin = {
|
|
32800
|
+
at: startedAt,
|
|
32801
|
+
verdict: result.verdict,
|
|
32802
|
+
action: describeAction(result),
|
|
32803
|
+
executing: result.executing,
|
|
32804
|
+
incompleteLogs: result.incompleteLogs,
|
|
32805
|
+
silenceMs: result.silenceMs,
|
|
32806
|
+
probeMs: Date.now() - startedAt
|
|
32807
|
+
};
|
|
32808
|
+
} catch (error) {
|
|
32809
|
+
const strikes = (await this.ctx.storage.get(KEY_UNREACHABLE) ?? 0) + 1;
|
|
32810
|
+
await this.ctx.storage.put(KEY_UNREACHABLE, strikes);
|
|
32811
|
+
checkin = {
|
|
32812
|
+
at: startedAt,
|
|
32813
|
+
verdict: "unreachable",
|
|
32814
|
+
action: `probe failed (${strikes} consecutive)`,
|
|
32815
|
+
executing: null,
|
|
32816
|
+
incompleteLogs: null,
|
|
32817
|
+
silenceMs: null,
|
|
32818
|
+
probeMs: Date.now() - startedAt,
|
|
32819
|
+
error: error instanceof Error ? error.message : String(error)
|
|
32820
|
+
};
|
|
32821
|
+
if (strikes >= MAX_UNREACHABLE_STRIKES) {
|
|
32822
|
+
console.error(
|
|
32823
|
+
`[ThreadWatchdog] CRITICAL: thread ${armed.threadId} unreachable for ${strikes} consecutive probes \u2014 standing down. Manual intervention required.`
|
|
32824
|
+
);
|
|
32825
|
+
standDown = true;
|
|
32826
|
+
}
|
|
32827
|
+
}
|
|
32828
|
+
await this.recordCheckin(checkin);
|
|
32829
|
+
if (requestRestart) {
|
|
32830
|
+
try {
|
|
32831
|
+
const stub = this.getThreadStub(armed.threadId);
|
|
32832
|
+
await stub.forceRestart(
|
|
32833
|
+
"watchdog takeover: wedged execution aborted; restarting so the recovery alarm can fire"
|
|
32834
|
+
);
|
|
32835
|
+
} catch {
|
|
32836
|
+
}
|
|
32837
|
+
}
|
|
32838
|
+
if (standDown) {
|
|
32839
|
+
await this.ctx.storage.delete(KEY_ARMED);
|
|
32840
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
32841
|
+
await this.ctx.storage.deleteAlarm();
|
|
32842
|
+
}
|
|
32843
|
+
}
|
|
32844
|
+
getThreadStub(threadId) {
|
|
32845
|
+
const namespace = this.env.AGENT_BUILDER_THREAD;
|
|
32846
|
+
const id = namespace.idFromName(threadId);
|
|
32847
|
+
return namespace.get(id);
|
|
32848
|
+
}
|
|
32849
|
+
async probeThread(threadId) {
|
|
32850
|
+
const stub = this.getThreadStub(threadId);
|
|
32851
|
+
let timer;
|
|
32852
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
32853
|
+
timer = setTimeout(
|
|
32854
|
+
() => reject(new Error(`probe timed out after ${PROBE_TIMEOUT_MS}ms`)),
|
|
32855
|
+
PROBE_TIMEOUT_MS
|
|
32856
|
+
);
|
|
32857
|
+
});
|
|
32858
|
+
try {
|
|
32859
|
+
return await Promise.race([stub.watchdogProbe(threadId), timeout]);
|
|
32860
|
+
} finally {
|
|
32861
|
+
if (timer) clearTimeout(timer);
|
|
32862
|
+
}
|
|
32863
|
+
}
|
|
32864
|
+
async recordCheckin(checkin) {
|
|
32865
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
32866
|
+
history.unshift(checkin);
|
|
32867
|
+
await this.ctx.storage.put(KEY_HISTORY, history.slice(0, HISTORY_CAP));
|
|
32868
|
+
}
|
|
32869
|
+
};
|
|
32870
|
+
function describeAction(result) {
|
|
32871
|
+
switch (result.action) {
|
|
32872
|
+
case "finalized_resumed":
|
|
32873
|
+
return `finalized ${result.incompleteLogs} lost request(s), resumed thread`;
|
|
32874
|
+
case "finalized_parked":
|
|
32875
|
+
return `finalized ${result.incompleteLogs} lost request(s), left thread stopped`;
|
|
32876
|
+
case "finalized_deferred":
|
|
32877
|
+
return `finalized ${result.incompleteLogs} lost request(s); queued turn will continue`;
|
|
32878
|
+
case "aborted_takeover_resumed":
|
|
32879
|
+
return `took over wedged execution (silent ${formatMs(result.silenceMs)}), finalized and resumed`;
|
|
32880
|
+
case "none":
|
|
32881
|
+
default:
|
|
32882
|
+
if (result.verdict === "observing") {
|
|
32883
|
+
return `execution silent ${formatMs(result.silenceMs)} (ceiling ${formatMs(result.ceilingMs)}) \u2014 observing`;
|
|
32884
|
+
}
|
|
32885
|
+
if (result.verdict === "idle") {
|
|
32886
|
+
return "thread idle and clean \u2014 standing down";
|
|
32887
|
+
}
|
|
32888
|
+
if (result.executing) {
|
|
32889
|
+
return result.incompleteLogs > 0 ? `streaming, last progress ${formatMs(result.silenceMs)} ago` : "executing (tool run, no open request)";
|
|
32890
|
+
}
|
|
32891
|
+
return "no action needed";
|
|
32892
|
+
}
|
|
32893
|
+
}
|
|
32894
|
+
function formatMs(ms) {
|
|
32895
|
+
if (ms === null || !Number.isFinite(ms)) return "\u221E";
|
|
32896
|
+
if (ms < 1e3) return `${ms}ms`;
|
|
32897
|
+
const seconds = Math.round(ms / 1e3);
|
|
32898
|
+
if (seconds < 60) return `${seconds}s`;
|
|
32899
|
+
const minutes = Math.floor(seconds / 60);
|
|
32900
|
+
return `${minutes}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
32901
|
+
}
|
|
32453
32902
|
|
|
32454
32903
|
// src/index.ts
|
|
32455
32904
|
init_utilities();
|
|
@@ -32812,6 +33261,6 @@ init_context();
|
|
|
32812
33261
|
init_ProviderRegistry();
|
|
32813
33262
|
init_types2();
|
|
32814
33263
|
|
|
32815
|
-
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, PackageDiscoveryService, ProviderRegistry, agentbuilder, authenticate, buildImageDescription, builder, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, discoverPackages, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateAgentFile, generateImageDescription, generateModelFile, generatePromptFile, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
33264
|
+
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, PackageDiscoveryService, ProviderRegistry, ThreadWatchdog, agentbuilder, authenticate, buildImageDescription, builder, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, discoverPackages, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateAgentFile, generateImageDescription, generateModelFile, generatePromptFile, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
32816
33265
|
//# sourceMappingURL=index.js.map
|
|
32817
33266
|
//# sourceMappingURL=index.js.map
|