@standardagents/builder 0.25.6 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/built-in-routes.js +269 -17
- package/dist/built-in-routes.js.map +1 -1
- package/dist/client/{ApiKeysView-B9XLopSb.js → ApiKeysView-jhfg43UH.js} +1 -1
- package/dist/client/CenteredContentView-CS0mV2EJ.js +1 -0
- package/dist/client/{CompositionView-DJOtNN7f.js → CompositionView-DhkLWmtE.js} +1 -1
- package/dist/client/{ConfirmDialog.vue_vue_type_script_setup_true_lang-55dXt19T.js → ConfirmDialog.vue_vue_type_script_setup_true_lang-D-WToRMy.js} +1 -1
- package/dist/client/{CopyButton.vue_vue_type_script_setup_true_lang-B6NqQmW9.js → CopyButton.vue_vue_type_script_setup_true_lang-Dhq0t0dd.js} +1 -1
- package/dist/client/{DataTable.vue_vue_type_script_setup_true_lang-0cbPVIIz.js → DataTable.vue_vue_type_script_setup_true_lang-D1MDCns-.js} +1 -1
- package/dist/client/{JsonViewer-B7yHhxqP.js → JsonViewer-BoHrE76S.js} +1 -1
- package/dist/client/{LoginView-mY9OytK6.js → LoginView-BpR2nMNf.js} +1 -1
- package/dist/client/{MarketplaceView-B9Z9YMuB.js → MarketplaceView-uUf1JYla.js} +1 -1
- package/dist/client/{Modal.vue_vue_type_script_setup_true_lang-DEVCRbXl.js → Modal.vue_vue_type_script_setup_true_lang-D-SDVFQI.js} +1 -1
- package/dist/client/{ModelModal.vue_vue_type_script_setup_true_lang-FxEBZ3A8.js → ModelModal.vue_vue_type_script_setup_true_lang-DE57Zr0r.js} +1 -1
- package/dist/client/{ModelsView-DTU8pk-T.js → ModelsView-CuM_OieH.js} +1 -1
- package/dist/client/{PromptEditView-BAKoULxX.js → PromptEditView-BWjrA2zr.js} +1 -1
- package/dist/client/{PromptModal-CoinSKIM.js → PromptModal-BFxhuG0Z.js} +1 -1
- package/dist/client/{PromptsView-DIkj3q-o.js → PromptsView-AlGsRB6k.js} +1 -1
- package/dist/client/{ProvidersView-B8A6ezDR.js → ProvidersView-CbXBs3SH.js} +1 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-Dm3d37hz.js +22 -0
- package/dist/client/{ThreadsView-DGCVdDej.js → ThreadsView-rN9rzJi1.js} +1 -1
- package/dist/client/{ToolsView-CC-WsZet.js → ToolsView-0Gpo_RFg.js} +1 -1
- package/dist/client/{UiCodeBlock.vue_vue_type_script_setup_true_lang-BH3WC9G_.js → UiCodeBlock.vue_vue_type_script_setup_true_lang-CNncALQo.js} +1 -1
- package/dist/client/{UsersView-U3Stwxui.js → UsersView-iXPbcguY.js} +1 -1
- package/dist/client/{VariablesView-0-a1F4cn.js → VariablesView-C9BB9h34.js} +1 -1
- package/dist/client/assets/{ThreadInspectorPane-ouC_RyKA.css → ThreadInspectorPane-CfpdDnUQ.css} +1 -1
- package/dist/client/assets/index-BTPsPULO.css +1 -0
- package/dist/client/{check-Dkzs1C3V.js → check-BdJVlUmI.js} +1 -1
- package/dist/client/{chevron-right-BGirddPg.js → chevron-right-D49O3XLU.js} +1 -1
- package/dist/client/{copy-C08OR8rY.js → copy-40uDyTgY.js} +1 -1
- package/dist/client/{formkit-YJ1Z7D5r.js → formkit-Bl-sERnF.js} +1 -1
- package/dist/client/index-BCoE48be.js +6 -0
- package/dist/client/index.html +4 -4
- package/dist/client/{package-guyzMTDG.js → package-1UgoERpx.js} +1 -1
- package/dist/client/{plus-DG3ZN5wg.js → plus-BzCjfYs0.js} +1 -1
- package/dist/client/{timeAgo-D7V247YG.js → timeAgo-cFwldMu-.js} +1 -1
- package/dist/client/{useActionFeedback-byARNQjQ.js → useActionFeedback-BVifffYQ.js} +1 -1
- package/dist/client/{vue-BqC6IZDY.js → vue-CmmJuziT.js} +1 -1
- package/dist/{index-CX9Ti98s.d.ts → index-DJ7B8CTp.d.ts} +63 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +555 -84
- package/dist/index.js.map +1 -1
- package/dist/plugin.js +27 -3
- package/dist/plugin.js.map +1 -1
- package/dist/runtime.d.ts +184 -6
- package/dist/runtime.js +528 -81
- package/dist/runtime.js.map +1 -1
- package/dist/test.d.ts +1 -1
- package/package.json +5 -5
- package/dist/client/CenteredContentView-Bwv4MVSC.js +0 -1
- package/dist/client/ThreadInspectorPane.vue_vue_type_script_setup_true_lang-CvssvzLz.js +0 -22
- package/dist/client/assets/index-Ca3he3fq.css +0 -1
- package/dist/client/index-4Zwz6zXD.js +0 -6
package/dist/index.js
CHANGED
|
@@ -1079,6 +1079,24 @@ function resolveWatchdogStaleCeilingMs(env) {
|
|
|
1079
1079
|
);
|
|
1080
1080
|
return Math.max(firstChunk, stall) + 6e4;
|
|
1081
1081
|
}
|
|
1082
|
+
async function awaitBounded(work, budgetMs, label) {
|
|
1083
|
+
let timer;
|
|
1084
|
+
work.catch(() => {
|
|
1085
|
+
});
|
|
1086
|
+
const timeout = new Promise((resolve2) => {
|
|
1087
|
+
timer = setTimeout(() => {
|
|
1088
|
+
console.error(
|
|
1089
|
+
`[LLMRequest] ${label} did not settle within ${budgetMs}ms \u2014 continuing without it`
|
|
1090
|
+
);
|
|
1091
|
+
resolve2(void 0);
|
|
1092
|
+
}, budgetMs);
|
|
1093
|
+
});
|
|
1094
|
+
try {
|
|
1095
|
+
return await Promise.race([work, timeout]);
|
|
1096
|
+
} finally {
|
|
1097
|
+
if (timer) clearTimeout(timer);
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1082
1100
|
function modelSupportsImageInput(modelDef) {
|
|
1083
1101
|
const supportsImages = modelDef.capabilities?.supportsImages;
|
|
1084
1102
|
if (typeof supportsImages === "boolean") {
|
|
@@ -1292,13 +1310,42 @@ function transformMessages(messages) {
|
|
|
1292
1310
|
arguments: typeof tc.function.arguments === "string" ? JSON.parse(tc.function.arguments) : tc.function.arguments,
|
|
1293
1311
|
extraContent: tc.extra_content && typeof tc.extra_content === "object" ? tc.extra_content : void 0
|
|
1294
1312
|
}));
|
|
1313
|
+
const storedDetails = msg.reasoning_details;
|
|
1314
|
+
const reasoningDetails = Array.isArray(
|
|
1315
|
+
storedDetails
|
|
1316
|
+
) ? storedDetails.flatMap((entry) => {
|
|
1317
|
+
if (!entry || typeof entry !== "object") return [];
|
|
1318
|
+
const raw = entry;
|
|
1319
|
+
const type = raw.type === "summary" || raw.type === "encrypted" ? raw.type : "text";
|
|
1320
|
+
return [
|
|
1321
|
+
{
|
|
1322
|
+
type,
|
|
1323
|
+
id: typeof raw.id === "string" ? raw.id : void 0,
|
|
1324
|
+
text: typeof raw.text === "string" ? raw.text : void 0,
|
|
1325
|
+
data: typeof raw.data === "string" ? raw.data : void 0,
|
|
1326
|
+
format: typeof raw.format === "string" ? raw.format : void 0
|
|
1327
|
+
}
|
|
1328
|
+
];
|
|
1329
|
+
}) : void 0;
|
|
1330
|
+
const storedReasoning = msg.reasoning_content;
|
|
1331
|
+
let reasoning = typeof storedReasoning === "string" && storedReasoning.length > 0 ? storedReasoning : void 0;
|
|
1332
|
+
if (!reasoning && Array.isArray(reasoningDetails)) {
|
|
1333
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1334
|
+
const texts = [];
|
|
1335
|
+
for (const d of reasoningDetails) {
|
|
1336
|
+
if (typeof d?.text === "string" && d.text.length > 0 && !seen.has(d.text)) {
|
|
1337
|
+
seen.add(d.text);
|
|
1338
|
+
texts.push(d.text);
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
if (texts.length) reasoning = texts.join("\n");
|
|
1342
|
+
}
|
|
1295
1343
|
return {
|
|
1296
1344
|
role: "assistant",
|
|
1297
1345
|
content: typeof msg.content === "string" ? msg.content : null,
|
|
1298
1346
|
toolCalls: toolCalls?.length ? toolCalls : void 0,
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
reasoningDetails: msg.reasoning_details || void 0
|
|
1347
|
+
reasoning,
|
|
1348
|
+
reasoningDetails: reasoningDetails?.length ? reasoningDetails : void 0
|
|
1302
1349
|
};
|
|
1303
1350
|
}
|
|
1304
1351
|
case "tool":
|
|
@@ -1473,7 +1520,7 @@ function normalizeProviderMetadataValue(value) {
|
|
|
1473
1520
|
if (!trimmed || trimmed.toLowerCase() === "unknown") return void 0;
|
|
1474
1521
|
return trimmed;
|
|
1475
1522
|
}
|
|
1476
|
-
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, LLMRequest;
|
|
1523
|
+
var NON_VISION_PLACEHOLDER_TEXT, STREAM_FIRST_CHUNK_TIMEOUT_MS, STREAM_STALL_TIMEOUT_MS, HOOK_AWAIT_BUDGET_MS, STREAM_CLEANUP_BUDGET_MS, LLMRequest;
|
|
1477
1524
|
var init_LLMRequest = __esm({
|
|
1478
1525
|
"src/agents/LLMRequest.ts"() {
|
|
1479
1526
|
init_pricing();
|
|
@@ -1484,6 +1531,8 @@ var init_LLMRequest = __esm({
|
|
|
1484
1531
|
NON_VISION_PLACEHOLDER_TEXT = "[Image omitted: model does not support vision]";
|
|
1485
1532
|
STREAM_FIRST_CHUNK_TIMEOUT_MS = 12e4;
|
|
1486
1533
|
STREAM_STALL_TIMEOUT_MS = 45e3;
|
|
1534
|
+
HOOK_AWAIT_BUDGET_MS = 3e4;
|
|
1535
|
+
STREAM_CLEANUP_BUDGET_MS = 5e3;
|
|
1487
1536
|
LLMRequest = class {
|
|
1488
1537
|
/**
|
|
1489
1538
|
* Execute an LLM request with full retry/fallback logic
|
|
@@ -1967,7 +2016,14 @@ var init_LLMRequest = __esm({
|
|
|
1967
2016
|
parentSignal.removeEventListener("abort", onParentAbort);
|
|
1968
2017
|
}
|
|
1969
2018
|
try {
|
|
1970
|
-
|
|
2019
|
+
const cleanup = iterator.return?.();
|
|
2020
|
+
if (cleanup) {
|
|
2021
|
+
await awaitBounded(
|
|
2022
|
+
Promise.resolve(cleanup),
|
|
2023
|
+
STREAM_CLEANUP_BUDGET_MS,
|
|
2024
|
+
"stream iterator cleanup"
|
|
2025
|
+
);
|
|
2026
|
+
}
|
|
1971
2027
|
} catch {
|
|
1972
2028
|
}
|
|
1973
2029
|
}
|
|
@@ -2100,7 +2156,11 @@ var init_LLMRequest = __esm({
|
|
|
2100
2156
|
*/
|
|
2101
2157
|
static async resolveProgressArgument(state, toolName) {
|
|
2102
2158
|
try {
|
|
2103
|
-
const mod = await
|
|
2159
|
+
const mod = await awaitBounded(
|
|
2160
|
+
state.thread.instance.loadTool(toolName),
|
|
2161
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2162
|
+
`loadTool(${toolName}) for progress argument`
|
|
2163
|
+
);
|
|
2104
2164
|
const pa = mod?.progressArgument ?? mod?.default?.progressArgument;
|
|
2105
2165
|
return typeof pa === "string" && pa ? pa : void 0;
|
|
2106
2166
|
} catch {
|
|
@@ -2121,7 +2181,11 @@ var init_LLMRequest = __esm({
|
|
|
2121
2181
|
function: { name: entry.name, arguments: entry.argumentsDelta || "" }
|
|
2122
2182
|
};
|
|
2123
2183
|
try {
|
|
2124
|
-
await
|
|
2184
|
+
await awaitBounded(
|
|
2185
|
+
FlowEngine.runToolCallStartedHook(state, toolCall, progress),
|
|
2186
|
+
HOOK_AWAIT_BUDGET_MS,
|
|
2187
|
+
`tool_call_started hook for ${entry.name}`
|
|
2188
|
+
);
|
|
2125
2189
|
} catch (error) {
|
|
2126
2190
|
console.error("[LLMRequest] Error firing tool_call_started:", error);
|
|
2127
2191
|
}
|
|
@@ -7816,6 +7880,25 @@ This usually points to something I can't fix from here \u2014 for example the wo
|
|
|
7816
7880
|
return null;
|
|
7817
7881
|
}
|
|
7818
7882
|
}
|
|
7883
|
+
/**
|
|
7884
|
+
* Registry entries the model may see (and therefore manage). Hidden
|
|
7885
|
+
* subagent relationships (`SubagentToolConfig.hidden` — infrastructure
|
|
7886
|
+
* children like background compaction) are excluded from the injected
|
|
7887
|
+
* "Active subagent instances" message: a listed instance reads as the
|
|
7888
|
+
* model's to clean up, and terminate_subagent on an infrastructure child
|
|
7889
|
+
* kills work the runtime depends on. Static and pure for direct testing.
|
|
7890
|
+
*/
|
|
7891
|
+
static filterModelVisibleSubagents(registry, promptTools) {
|
|
7892
|
+
if (registry.length === 0) return registry;
|
|
7893
|
+
const hidden = /* @__PURE__ */ new Set();
|
|
7894
|
+
for (const config of promptTools ?? []) {
|
|
7895
|
+
if (typeof config === "object" && config !== null && "name" in config && config.hidden === true) {
|
|
7896
|
+
hidden.add(config.name);
|
|
7897
|
+
}
|
|
7898
|
+
}
|
|
7899
|
+
if (hidden.size === 0) return registry;
|
|
7900
|
+
return registry.filter((entry) => !hidden.has(entry.name));
|
|
7901
|
+
}
|
|
7819
7902
|
static async getSubagentRegistry(state) {
|
|
7820
7903
|
if (typeof state.thread.instance.getChildrenRegistry === "function") {
|
|
7821
7904
|
try {
|
|
@@ -8074,7 +8157,10 @@ ${msg.content}` : `${imageDescriptions}${nonImageList}`;
|
|
|
8074
8157
|
});
|
|
8075
8158
|
}
|
|
8076
8159
|
}
|
|
8077
|
-
const subagentRegistry =
|
|
8160
|
+
const subagentRegistry = _FlowEngine.filterModelVisibleSubagents(
|
|
8161
|
+
await this.getSubagentRegistry(state),
|
|
8162
|
+
state.prompt?._tools
|
|
8163
|
+
);
|
|
8078
8164
|
if (subagentRegistry.length > 0) {
|
|
8079
8165
|
const fallbackNameMap = await this.resolveRegistryThreadNameMap(state, subagentRegistry);
|
|
8080
8166
|
const lines = subagentRegistry.map((entry) => {
|
|
@@ -12684,8 +12770,12 @@ var init_ThreadStateImpl = __esm({
|
|
|
12684
12770
|
options?.includeSilent,
|
|
12685
12771
|
options?.maxDepth
|
|
12686
12772
|
);
|
|
12773
|
+
const messages = result.messages.map(this._mapMessage);
|
|
12774
|
+
if (options?.order === "desc") {
|
|
12775
|
+
messages.reverse();
|
|
12776
|
+
}
|
|
12687
12777
|
return {
|
|
12688
|
-
messages
|
|
12778
|
+
messages,
|
|
12689
12779
|
total: result.total,
|
|
12690
12780
|
hasMore: result.hasMore
|
|
12691
12781
|
};
|
|
@@ -13994,6 +14084,7 @@ declare module 'virtual:@standardagents/builder' {
|
|
|
13994
14084
|
import type { DurableThread as BaseDurableThread } from '@standardagents/builder/runtime';
|
|
13995
14085
|
import type { DurableAgentBuilder as BaseDurableAgentBuilder } from '@standardagents/builder/runtime';
|
|
13996
14086
|
import type { CodeExecutionBridge as BaseCodeExecutionBridge } from '@standardagents/builder/runtime';
|
|
14087
|
+
import type { ThreadWatchdog as BaseThreadWatchdog } from '@standardagents/builder/runtime';
|
|
13997
14088
|
import type { ThreadEnv } from '@standardagents/builder/runtime';
|
|
13998
14089
|
|
|
13999
14090
|
// ============================================================
|
|
@@ -14472,6 +14563,12 @@ declare module 'virtual:@standardagents/builder' {
|
|
|
14472
14563
|
*/
|
|
14473
14564
|
export class CodeExecutionBridge extends BaseCodeExecutionBridge {}
|
|
14474
14565
|
|
|
14566
|
+
/**
|
|
14567
|
+
* Per-thread watchdog companion Durable Object (AGENT_BUILDER_WATCHDOG
|
|
14568
|
+
* binding). Delivers execution liveness checks a running turn cannot starve.
|
|
14569
|
+
*/
|
|
14570
|
+
export class ThreadWatchdog extends BaseThreadWatchdog {}
|
|
14571
|
+
|
|
14475
14572
|
// ============================================================
|
|
14476
14573
|
// Router Function
|
|
14477
14574
|
// ============================================================
|
|
@@ -20638,10 +20735,11 @@ async function loadBuilderVirtualModule(context) {
|
|
|
20638
20735
|
}
|
|
20639
20736
|
}).join("\n");
|
|
20640
20737
|
return `// Consolidated virtual module: virtual:@standardagents/builder /* v${hmr.versions.builder} */
|
|
20641
|
-
// Provides DurableThread, DurableAgentBuilder, and router
|
|
20738
|
+
// Provides DurableThread, DurableAgentBuilder, ThreadWatchdog, and router
|
|
20642
20739
|
import { DurableThread as _BaseDurableThread } from '@standardagents/builder/runtime';
|
|
20643
20740
|
import { DurableAgentBuilder as _BaseDurableAgentBuilder } from '@standardagents/builder/runtime';
|
|
20644
20741
|
import { CodeExecutionBridge as _BaseCodeExecutionBridge } from '@standardagents/builder/runtime';
|
|
20742
|
+
import { ThreadWatchdog as _BaseThreadWatchdog } from '@standardagents/builder/runtime';
|
|
20645
20743
|
|
|
20646
20744
|
// Import sip WASM module and readiness helper
|
|
20647
20745
|
// Static import allows workerd to pre-compile the WASM at bundle time
|
|
@@ -20761,6 +20859,12 @@ export class DurableAgentBuilder extends _BaseDurableAgentBuilder {
|
|
|
20761
20859
|
}
|
|
20762
20860
|
|
|
20763
20861
|
export class CodeExecutionBridge extends _BaseCodeExecutionBridge {}
|
|
20862
|
+
|
|
20863
|
+
/**
|
|
20864
|
+
* Per-thread watchdog companion. Needs no virtual-module registries \u2014 it only
|
|
20865
|
+
* probes its DurableThread over RPC.
|
|
20866
|
+
*/
|
|
20867
|
+
export class ThreadWatchdog extends _BaseThreadWatchdog {}
|
|
20764
20868
|
`;
|
|
20765
20869
|
}
|
|
20766
20870
|
async function loadRegistryVirtualModule(context) {
|
|
@@ -21728,6 +21832,10 @@ function createAgentBuilderWorkerConfig(options) {
|
|
|
21728
21832
|
{
|
|
21729
21833
|
name: "AGENT_BUILDER",
|
|
21730
21834
|
class_name: "DurableAgentBuilder"
|
|
21835
|
+
},
|
|
21836
|
+
{
|
|
21837
|
+
name: "AGENT_BUILDER_WATCHDOG",
|
|
21838
|
+
class_name: "ThreadWatchdog"
|
|
21731
21839
|
}
|
|
21732
21840
|
]
|
|
21733
21841
|
},
|
|
@@ -21739,6 +21847,12 @@ function createAgentBuilderWorkerConfig(options) {
|
|
|
21739
21847
|
{
|
|
21740
21848
|
tag: "v2",
|
|
21741
21849
|
new_sqlite_classes: ["DurableAgentBuilder"]
|
|
21850
|
+
},
|
|
21851
|
+
// Migration tags are append-only: existing deployments have already
|
|
21852
|
+
// recorded v1/v2, so a new class MUST arrive as a new tag.
|
|
21853
|
+
{
|
|
21854
|
+
tag: "v3",
|
|
21855
|
+
new_sqlite_classes: ["ThreadWatchdog"]
|
|
21742
21856
|
}
|
|
21743
21857
|
]
|
|
21744
21858
|
};
|
|
@@ -21826,7 +21940,7 @@ function agentbuilder(options = {}) {
|
|
|
21826
21940
|
function writeGeneratedWorkerEntry() {
|
|
21827
21941
|
const content = `// Generated by @standardagents/builder.
|
|
21828
21942
|
// Future agents: update builder/src/plugin.ts when changing this entrypoint.
|
|
21829
|
-
import { CodeExecutionBridge, DurableAgentBuilder, DurableThread, router } from "virtual:@standardagents/builder";
|
|
21943
|
+
import { CodeExecutionBridge, DurableAgentBuilder, DurableThread, ThreadWatchdog, router } from "virtual:@standardagents/builder";
|
|
21830
21944
|
|
|
21831
21945
|
export default {
|
|
21832
21946
|
async fetch(request: Request, env: Parameters<typeof router>[1]) {
|
|
@@ -21835,7 +21949,7 @@ export default {
|
|
|
21835
21949
|
},
|
|
21836
21950
|
};
|
|
21837
21951
|
|
|
21838
|
-
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread };
|
|
21952
|
+
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, ThreadWatchdog };
|
|
21839
21953
|
`;
|
|
21840
21954
|
fs8__default.mkdirSync(outputDir, { recursive: true });
|
|
21841
21955
|
if (!fs8__default.existsSync(generatedWorkerEntry) || fs8__default.readFileSync(generatedWorkerEntry, "utf-8") !== content) {
|
|
@@ -25175,6 +25289,11 @@ var DurableThread = class _DurableThread extends DurableObject {
|
|
|
25175
25289
|
if (this.currentAbortController) {
|
|
25176
25290
|
this.currentAbortController.abort();
|
|
25177
25291
|
}
|
|
25292
|
+
try {
|
|
25293
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
25294
|
+
} catch (error) {
|
|
25295
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
25296
|
+
}
|
|
25178
25297
|
const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
|
|
25179
25298
|
const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
|
|
25180
25299
|
await agentBuilder.updateThread(threadId, { terminated: terminatedAt });
|
|
@@ -26334,6 +26453,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
26334
26453
|
*/
|
|
26335
26454
|
async getMessages(limit = 100, offset = 0, order = "DESC", includeSilent = false, maxDepth = 0) {
|
|
26336
26455
|
await this.ensureMigrated();
|
|
26456
|
+
const normalizedOrder = String(order).toUpperCase() === "ASC" ? "ASC" : "DESC";
|
|
26337
26457
|
try {
|
|
26338
26458
|
const whereClauses = [];
|
|
26339
26459
|
if (!includeSilent) {
|
|
@@ -26341,7 +26461,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
26341
26461
|
}
|
|
26342
26462
|
whereClauses.push(`(depth <= ${maxDepth} OR depth IS NULL)`);
|
|
26343
26463
|
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
|
|
26344
|
-
const orderDirection =
|
|
26464
|
+
const orderDirection = normalizedOrder;
|
|
26345
26465
|
const combinedMessagesSql = `
|
|
26346
26466
|
WITH combined_messages AS (
|
|
26347
26467
|
SELECT
|
|
@@ -26467,7 +26587,7 @@ ${resultContent}${attachmentSummary}`;
|
|
|
26467
26587
|
return projection ? { ...message, ...projection } : message;
|
|
26468
26588
|
});
|
|
26469
26589
|
}
|
|
26470
|
-
if (
|
|
26590
|
+
if (normalizedOrder === "DESC") {
|
|
26471
26591
|
messages = messages.reverse();
|
|
26472
26592
|
}
|
|
26473
26593
|
const hasMore = offset + messages.length < total;
|
|
@@ -27300,6 +27420,14 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27300
27420
|
}
|
|
27301
27421
|
this.logSockets.clear();
|
|
27302
27422
|
this.messageSockets.clear();
|
|
27423
|
+
try {
|
|
27424
|
+
const threadId = this.ctx.id.name;
|
|
27425
|
+
if (threadId) {
|
|
27426
|
+
await this.watchdogCompanion(threadId)?.destroy();
|
|
27427
|
+
}
|
|
27428
|
+
} catch (error) {
|
|
27429
|
+
console.error("[DurableThread] Failed to destroy watchdog companion (non-fatal):", error);
|
|
27430
|
+
}
|
|
27303
27431
|
await this.ctx.storage.deleteAlarm();
|
|
27304
27432
|
await this.ctx.storage.deleteAll();
|
|
27305
27433
|
this.migratedToVersion = null;
|
|
@@ -27955,6 +28083,18 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27955
28083
|
noteExecutionProgress() {
|
|
27956
28084
|
this.lastProgressAt = Date.now();
|
|
27957
28085
|
}
|
|
28086
|
+
/**
|
|
28087
|
+
* The per-thread companion watchdog DO (ThreadWatchdog), derived from the
|
|
28088
|
+
* thread id. It exists because this thread's OWN alarm cannot fire while a
|
|
28089
|
+
* queue-dispatched turn runs inside alarm() — the companion delivers the
|
|
28090
|
+
* same check via RPC, which interleaves at await points even mid-turn.
|
|
28091
|
+
* Returns null on deployments whose generated config predates the binding.
|
|
28092
|
+
*/
|
|
28093
|
+
watchdogCompanion(threadId) {
|
|
28094
|
+
const namespace = this.env.AGENT_BUILDER_WATCHDOG;
|
|
28095
|
+
if (!namespace || typeof namespace.idFromName !== "function") return null;
|
|
28096
|
+
return namespace.get(namespace.idFromName(threadId));
|
|
28097
|
+
}
|
|
27958
28098
|
/** Arm the standing watchdog for an execution start. One pending row max. */
|
|
27959
28099
|
async armExecutionWatchdog(threadId) {
|
|
27960
28100
|
try {
|
|
@@ -27963,14 +28103,20 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27963
28103
|
const pending = (await this.ctx.storage.sql.exec(
|
|
27964
28104
|
`SELECT id FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'pending' LIMIT 1`
|
|
27965
28105
|
)).toArray();
|
|
27966
|
-
if (pending.length
|
|
27967
|
-
|
|
28106
|
+
if (pending.length === 0) {
|
|
28107
|
+
await this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
28108
|
+
}
|
|
27968
28109
|
} catch (error) {
|
|
27969
28110
|
console.error("[DurableThread] Failed to arm execution watchdog (non-fatal):", error);
|
|
27970
28111
|
}
|
|
28112
|
+
try {
|
|
28113
|
+
await this.watchdogCompanion(threadId)?.arm(threadId);
|
|
28114
|
+
} catch (error) {
|
|
28115
|
+
console.error("[DurableThread] Failed to arm watchdog companion (non-fatal):", error);
|
|
28116
|
+
}
|
|
27971
28117
|
}
|
|
27972
28118
|
/** Disarm on clean execution end. Completed check rows are noise, not history. */
|
|
27973
|
-
async disarmExecutionWatchdog() {
|
|
28119
|
+
async disarmExecutionWatchdog(threadId) {
|
|
27974
28120
|
try {
|
|
27975
28121
|
await this.ctx.storage.sql.exec(
|
|
27976
28122
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status IN ('pending', 'completed')`
|
|
@@ -27979,10 +28125,21 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27979
28125
|
} catch (error) {
|
|
27980
28126
|
console.error("[DurableThread] Failed to disarm execution watchdog (non-fatal):", error);
|
|
27981
28127
|
}
|
|
28128
|
+
if (threadId) {
|
|
28129
|
+
try {
|
|
28130
|
+
await this.watchdogCompanion(threadId)?.disarmHint();
|
|
28131
|
+
} catch (error) {
|
|
28132
|
+
console.error("[DurableThread] Failed to hint watchdog companion (non-fatal):", error);
|
|
28133
|
+
}
|
|
28134
|
+
}
|
|
27982
28135
|
}
|
|
27983
28136
|
/**
|
|
27984
|
-
* The standing check. Evaluates the thread's state AT
|
|
27985
|
-
* nothing about whichever request was in flight when it
|
|
28137
|
+
* The standing check, alarm-queue delivery. Evaluates the thread's state AT
|
|
28138
|
+
* FIRE TIME — it knows nothing about whichever request was in flight when it
|
|
28139
|
+
* was armed. NOTE this delivery path cannot fire while a queue-dispatched
|
|
28140
|
+
* turn runs (the turn owns this DO's single alarm invocation); the
|
|
28141
|
+
* ThreadWatchdog companion delivers the same assessment via watchdogProbe()
|
|
28142
|
+
* in exactly that window.
|
|
27986
28143
|
*/
|
|
27987
28144
|
async watchdogCheck(threadId) {
|
|
27988
28145
|
const rearm = () => this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
|
|
@@ -27990,80 +28147,190 @@ ${resultContent}${attachmentSummary}`;
|
|
|
27990
28147
|
await this.ctx.storage.sql.exec(
|
|
27991
28148
|
`DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'completed'`
|
|
27992
28149
|
);
|
|
27993
|
-
const
|
|
27994
|
-
|
|
27995
|
-
)).toArray();
|
|
27996
|
-
let incomplete = await readIncomplete();
|
|
27997
|
-
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
27998
|
-
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
27999
|
-
if (this.isExecuting) {
|
|
28000
|
-
if (incomplete.length === 0) {
|
|
28001
|
-
this.watchdogWedgeStrikes = 0;
|
|
28002
|
-
await rearm();
|
|
28003
|
-
return;
|
|
28004
|
-
}
|
|
28005
|
-
if (silenceMs <= ceiling) {
|
|
28006
|
-
this.watchdogWedgeStrikes = 0;
|
|
28007
|
-
await rearm();
|
|
28008
|
-
return;
|
|
28009
|
-
}
|
|
28010
|
-
this.watchdogWedgeStrikes++;
|
|
28011
|
-
if (this.watchdogWedgeStrikes < 2) {
|
|
28012
|
-
console.warn(
|
|
28013
|
-
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
28014
|
-
);
|
|
28015
|
-
await rearm();
|
|
28016
|
-
return;
|
|
28017
|
-
}
|
|
28018
|
-
console.error(
|
|
28019
|
-
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
28020
|
-
);
|
|
28021
|
-
try {
|
|
28022
|
-
this.currentAbortController?.abort();
|
|
28023
|
-
} catch {
|
|
28024
|
-
}
|
|
28025
|
-
this.isExecuting = false;
|
|
28026
|
-
this.currentAbortController = null;
|
|
28027
|
-
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
28150
|
+
const result = await this.performWatchdogAssessment(threadId);
|
|
28151
|
+
if (result.verdict === "healthy" || result.verdict === "observing") {
|
|
28028
28152
|
await rearm();
|
|
28029
28153
|
return;
|
|
28030
28154
|
}
|
|
28031
|
-
|
|
28032
|
-
|
|
28155
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28156
|
+
} catch (error) {
|
|
28157
|
+
console.error("[DurableThread] Watchdog check failed (non-fatal):", error);
|
|
28158
|
+
try {
|
|
28033
28159
|
await rearm();
|
|
28034
|
-
|
|
28035
|
-
}
|
|
28036
|
-
if (incomplete.length > 0) {
|
|
28037
|
-
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
28160
|
+
} catch {
|
|
28038
28161
|
}
|
|
28039
|
-
|
|
28040
|
-
|
|
28041
|
-
|
|
28042
|
-
|
|
28043
|
-
|
|
28044
|
-
|
|
28045
|
-
|
|
28162
|
+
}
|
|
28163
|
+
}
|
|
28164
|
+
/**
|
|
28165
|
+
* RPC delivery of the standing check — called by the ThreadWatchdog
|
|
28166
|
+
* companion (and the diagnostics revive endpoint). Runs in a fresh request
|
|
28167
|
+
* context, so it interleaves with a running turn at await points and its
|
|
28168
|
+
* own awaits/timers are alive even when a wedged turn's context is frozen.
|
|
28169
|
+
* Does NOT touch the internal watchdogCheck alarm rows — those belong to
|
|
28170
|
+
* the alarm delivery path.
|
|
28171
|
+
*/
|
|
28172
|
+
async watchdogProbe(threadId) {
|
|
28173
|
+
await this.ensureMigrated();
|
|
28174
|
+
return await this.performWatchdogAssessment(threadId);
|
|
28175
|
+
}
|
|
28176
|
+
/**
|
|
28177
|
+
* Reset this Durable Object instance on request. Used by the watchdog
|
|
28178
|
+
* companion after a wedged-executor takeover: the zombie turn may still own
|
|
28179
|
+
* this DO's single alarm invocation, which would block the just-enqueued
|
|
28180
|
+
* recovery alarm forever. Aborting guarantees a fresh instance whose alarm
|
|
28181
|
+
* chain re-arms from persisted queue state on first wake.
|
|
28182
|
+
*/
|
|
28183
|
+
async forceRestart(reason) {
|
|
28184
|
+
console.error(`[DurableThread] Force restart requested: ${reason}`);
|
|
28185
|
+
this.ctx.abort(reason);
|
|
28186
|
+
}
|
|
28187
|
+
/**
|
|
28188
|
+
* Execution-health snapshot for the diagnostics endpoint and `agents diag`.
|
|
28189
|
+
* Exposes the state the 2026-07-08 post-mortem could not see from outside:
|
|
28190
|
+
* the alarm queue (was a turn item pending/processing/failed? was the
|
|
28191
|
+
* watchdog row gone?), the volatile execution flags, and open request logs.
|
|
28192
|
+
* Read-only; safe to call at any time.
|
|
28193
|
+
*/
|
|
28194
|
+
async getDiagnostics() {
|
|
28195
|
+
await this.ensureMigrated();
|
|
28196
|
+
const incompleteLogs = (await this.ctx.storage.sql.exec(
|
|
28197
|
+
`SELECT id, model, provider, created_at FROM logs WHERE is_complete = 0 ORDER BY created_at DESC LIMIT 20`
|
|
28198
|
+
)).toArray();
|
|
28199
|
+
const alarmQueue = (await this.ctx.storage.sql.exec(
|
|
28200
|
+
`SELECT id, method, status, scheduled_at, created_at, attempts, error
|
|
28201
|
+
FROM alarm_queue
|
|
28202
|
+
ORDER BY CASE WHEN status IN ('pending', 'processing') THEN 0 ELSE 1 END ASC,
|
|
28203
|
+
scheduled_at DESC
|
|
28204
|
+
LIMIT 50`
|
|
28205
|
+
)).toArray();
|
|
28206
|
+
const messageQueueCount = (await this.ctx.storage.sql.exec(
|
|
28207
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
28208
|
+
)).toArray()[0]?.c ?? 0;
|
|
28209
|
+
let contextUsage = null;
|
|
28210
|
+
try {
|
|
28211
|
+
const row = (await this.ctx.storage.sql.exec(
|
|
28212
|
+
`SELECT value FROM thread_values WHERE key = 'context_usage' LIMIT 1`
|
|
28213
|
+
)).toArray()[0];
|
|
28214
|
+
if (row?.value) {
|
|
28215
|
+
const parsed = JSON.parse(row.value);
|
|
28216
|
+
contextUsage = this.normalizePlainRecord(parsed);
|
|
28046
28217
|
}
|
|
28047
|
-
|
|
28048
|
-
|
|
28049
|
-
|
|
28218
|
+
} catch {
|
|
28219
|
+
}
|
|
28220
|
+
return {
|
|
28221
|
+
executing: this.isExecuting,
|
|
28222
|
+
lastProgressAt: this.lastProgressAt > 0 ? this.lastProgressAt : null,
|
|
28223
|
+
silenceMs: this.lastProgressAt > 0 ? Date.now() - this.lastProgressAt : null,
|
|
28224
|
+
watchdogStrikes: this.watchdogWedgeStrikes,
|
|
28225
|
+
watchdogCeilingMs: resolveWatchdogStaleCeilingMs(this.env),
|
|
28226
|
+
stopped: await this.shouldStop(),
|
|
28227
|
+
incompleteLogs,
|
|
28228
|
+
alarmQueue,
|
|
28229
|
+
messageQueueCount,
|
|
28230
|
+
contextUsage
|
|
28231
|
+
};
|
|
28232
|
+
}
|
|
28233
|
+
/**
|
|
28234
|
+
* The shared watchdog decision core. Both deliveries (alarm + companion
|
|
28235
|
+
* probe) evaluate and ACT through this method; the caller only decides what
|
|
28236
|
+
* to do with its own scheduling based on the verdict.
|
|
28237
|
+
*/
|
|
28238
|
+
async performWatchdogAssessment(threadId) {
|
|
28239
|
+
const readIncomplete = async () => (await this.ctx.storage.sql.exec(
|
|
28240
|
+
`SELECT id FROM logs WHERE is_complete = 0`
|
|
28241
|
+
)).toArray();
|
|
28242
|
+
let incomplete = await readIncomplete();
|
|
28243
|
+
const ceiling = resolveWatchdogStaleCeilingMs(this.env);
|
|
28244
|
+
const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
|
|
28245
|
+
const reportedSilence = Number.isFinite(silenceMs) ? silenceMs : null;
|
|
28246
|
+
const report = (verdict, action, extra = {}) => ({
|
|
28247
|
+
verdict,
|
|
28248
|
+
action,
|
|
28249
|
+
executing: this.isExecuting,
|
|
28250
|
+
incompleteLogs: incomplete.length,
|
|
28251
|
+
queuedMessages: 0,
|
|
28252
|
+
silenceMs: reportedSilence,
|
|
28253
|
+
ceilingMs: ceiling,
|
|
28254
|
+
strikes: this.watchdogWedgeStrikes,
|
|
28255
|
+
requestRestart: false,
|
|
28256
|
+
...extra
|
|
28257
|
+
});
|
|
28258
|
+
let tookOver = false;
|
|
28259
|
+
if (this.isExecuting) {
|
|
28260
|
+
if (incomplete.length === 0) {
|
|
28261
|
+
this.watchdogWedgeStrikes = 0;
|
|
28262
|
+
return report("healthy", "none");
|
|
28263
|
+
}
|
|
28264
|
+
if (silenceMs <= ceiling) {
|
|
28265
|
+
this.watchdogWedgeStrikes = 0;
|
|
28266
|
+
return report("healthy", "none");
|
|
28267
|
+
}
|
|
28268
|
+
this.watchdogWedgeStrikes++;
|
|
28269
|
+
if (this.watchdogWedgeStrikes < 2) {
|
|
28050
28270
|
console.warn(
|
|
28051
|
-
`[DurableThread] Watchdog:
|
|
28271
|
+
`[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
|
|
28052
28272
|
);
|
|
28053
|
-
|
|
28054
|
-
console.error(
|
|
28055
|
-
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
28056
|
-
);
|
|
28057
|
-
await this.notifyParentOfLostExecution(threadId);
|
|
28273
|
+
return report("observing", "none");
|
|
28058
28274
|
}
|
|
28059
|
-
|
|
28060
|
-
|
|
28061
|
-
|
|
28275
|
+
console.error(
|
|
28276
|
+
`[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
|
|
28277
|
+
);
|
|
28062
28278
|
try {
|
|
28063
|
-
|
|
28279
|
+
this.currentAbortController?.abort();
|
|
28064
28280
|
} catch {
|
|
28065
28281
|
}
|
|
28282
|
+
this.isExecuting = false;
|
|
28283
|
+
this.currentAbortController = null;
|
|
28284
|
+
tookOver = true;
|
|
28285
|
+
} else if (incomplete.length > 0 && silenceMs <= ceiling) {
|
|
28286
|
+
return report("observing", "none");
|
|
28287
|
+
}
|
|
28288
|
+
incomplete = await readIncomplete();
|
|
28289
|
+
if (this.isExecuting) {
|
|
28290
|
+
return report("healthy", "none");
|
|
28291
|
+
}
|
|
28292
|
+
const finalizedCount = incomplete.length;
|
|
28293
|
+
if (incomplete.length > 0) {
|
|
28294
|
+
await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
|
|
28295
|
+
}
|
|
28296
|
+
const stopped = await this.shouldStop();
|
|
28297
|
+
const queued = (await this.ctx.storage.sql.exec(
|
|
28298
|
+
`SELECT COUNT(*) AS c FROM message_queue`
|
|
28299
|
+
)).toArray()[0]?.c ?? 0;
|
|
28300
|
+
if (stopped || finalizedCount === 0 && queued === 0) {
|
|
28301
|
+
if (finalizedCount > 0) {
|
|
28302
|
+
return report("parked", "finalized_parked", {
|
|
28303
|
+
queuedMessages: queued,
|
|
28304
|
+
requestRestart: tookOver
|
|
28305
|
+
});
|
|
28306
|
+
}
|
|
28307
|
+
return report("idle", "none", { queuedMessages: queued });
|
|
28308
|
+
}
|
|
28309
|
+
if (await this.hasPendingTurnAlarm()) {
|
|
28310
|
+
return report("recovered", "finalized_deferred", {
|
|
28311
|
+
queuedMessages: queued,
|
|
28312
|
+
requestRestart: tookOver
|
|
28313
|
+
});
|
|
28314
|
+
}
|
|
28315
|
+
if (await this.consumeWatchdogResumeBudget()) {
|
|
28316
|
+
const side = await this.getLatestQueuedMessageSide() ?? "a";
|
|
28317
|
+
await this.alarmQueue.enqueue("continueExecution", { threadId, side }, 0);
|
|
28318
|
+
console.warn(
|
|
28319
|
+
`[DurableThread] Watchdog: lost execution recovered \u2014 resuming thread ${threadId}`
|
|
28320
|
+
);
|
|
28321
|
+
return report("recovered", tookOver ? "aborted_takeover_resumed" : "finalized_resumed", {
|
|
28322
|
+
queuedMessages: queued,
|
|
28323
|
+
requestRestart: tookOver
|
|
28324
|
+
});
|
|
28066
28325
|
}
|
|
28326
|
+
console.error(
|
|
28327
|
+
`[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
|
|
28328
|
+
);
|
|
28329
|
+
await this.notifyParentOfLostExecution(threadId);
|
|
28330
|
+
return report("parked", "finalized_parked", {
|
|
28331
|
+
queuedMessages: queued,
|
|
28332
|
+
requestRestart: tookOver
|
|
28333
|
+
});
|
|
28067
28334
|
}
|
|
28068
28335
|
/** Mark logs the dead executor abandoned, and tell connected clients. */
|
|
28069
28336
|
async finalizeLostExecutionLogs(logIds) {
|
|
@@ -28172,6 +28439,18 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28172
28439
|
*/
|
|
28173
28440
|
async executeFlow(threadId, agentName, initial_messages, data, awaitCompletion = true) {
|
|
28174
28441
|
await this.assertNotTerminated();
|
|
28442
|
+
if (this.isExecuting) {
|
|
28443
|
+
const busyWaitBudgetMs = 3e4;
|
|
28444
|
+
const waitStartedAt = Date.now();
|
|
28445
|
+
while (this.isExecuting && Date.now() - waitStartedAt < busyWaitBudgetMs) {
|
|
28446
|
+
await new Promise((resolve2) => setTimeout(resolve2, 500));
|
|
28447
|
+
}
|
|
28448
|
+
if (this.isExecuting) {
|
|
28449
|
+
throw new Error(
|
|
28450
|
+
`Thread ${threadId} is already executing; refusing to start a second concurrent flow`
|
|
28451
|
+
);
|
|
28452
|
+
}
|
|
28453
|
+
}
|
|
28175
28454
|
this.isExecuting = true;
|
|
28176
28455
|
this.currentAbortController = new AbortController();
|
|
28177
28456
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -28305,7 +28584,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28305
28584
|
} finally {
|
|
28306
28585
|
this.isExecuting = false;
|
|
28307
28586
|
this.currentAbortController = null;
|
|
28308
|
-
await this.disarmExecutionWatchdog();
|
|
28587
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28309
28588
|
}
|
|
28310
28589
|
}
|
|
28311
28590
|
/**
|
|
@@ -28345,6 +28624,19 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28345
28624
|
*/
|
|
28346
28625
|
async processMessage(threadId, content, role = "user", attachments, queuedMessageId, queuedAt) {
|
|
28347
28626
|
await this.assertNotTerminated();
|
|
28627
|
+
if (this.isExecuting) {
|
|
28628
|
+
console.warn(
|
|
28629
|
+
`[DurableThread] processMessage while executing \u2014 diverting message to queue for thread ${threadId}`
|
|
28630
|
+
);
|
|
28631
|
+
await this.queueMessage(threadId, {
|
|
28632
|
+
role: role === "assistant" ? "assistant" : "user",
|
|
28633
|
+
content,
|
|
28634
|
+
attachments: this.parseAttachmentRefsJson(attachments ?? null) ?? [],
|
|
28635
|
+
silent: false,
|
|
28636
|
+
metadata: { diverted_from: "processMessage" }
|
|
28637
|
+
});
|
|
28638
|
+
return;
|
|
28639
|
+
}
|
|
28348
28640
|
if (role === "user") {
|
|
28349
28641
|
await this.ctx.storage.sql.exec(
|
|
28350
28642
|
`UPDATE execution_state SET value = 'false' WHERE key = 'stopped'`
|
|
@@ -28491,7 +28783,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28491
28783
|
} finally {
|
|
28492
28784
|
this.isExecuting = false;
|
|
28493
28785
|
this.currentAbortController = null;
|
|
28494
|
-
await this.disarmExecutionWatchdog();
|
|
28786
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28495
28787
|
}
|
|
28496
28788
|
}
|
|
28497
28789
|
/**
|
|
@@ -28500,6 +28792,12 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28500
28792
|
*/
|
|
28501
28793
|
async doContinueExecution(threadId, side = "a") {
|
|
28502
28794
|
await this.assertNotTerminated();
|
|
28795
|
+
if (this.isExecuting) {
|
|
28796
|
+
console.warn(
|
|
28797
|
+
`[DurableThread] continueExecution while executing \u2014 skipping (running flow owns the queue) for thread ${threadId}`
|
|
28798
|
+
);
|
|
28799
|
+
return;
|
|
28800
|
+
}
|
|
28503
28801
|
this.isExecuting = true;
|
|
28504
28802
|
this.currentAbortController = new AbortController();
|
|
28505
28803
|
await this.armExecutionWatchdog(threadId);
|
|
@@ -28611,7 +28909,7 @@ Its execution was lost (the executor restarted or hung) and automatic recovery w
|
|
|
28611
28909
|
} finally {
|
|
28612
28910
|
this.isExecuting = false;
|
|
28613
28911
|
this.currentAbortController = null;
|
|
28614
|
-
await this.disarmExecutionWatchdog();
|
|
28912
|
+
await this.disarmExecutionWatchdog(threadId);
|
|
28615
28913
|
const latestQueuedSide = await this.getLatestQueuedMessageSide();
|
|
28616
28914
|
if (latestQueuedSide) {
|
|
28617
28915
|
await this.ensureQueuedMessageExecution(threadId, latestQueuedSide);
|
|
@@ -32428,6 +32726,179 @@ var CodeExecutionBridge = class extends WorkerEntrypoint {
|
|
|
32428
32726
|
return thread.callCodeExecutionBridge(runId, functionId, encodedArgs);
|
|
32429
32727
|
}
|
|
32430
32728
|
};
|
|
32729
|
+
var CHECK_CADENCE_MS = 3e4;
|
|
32730
|
+
var DISARM_CONFIRM_DELAY_MS = 3e3;
|
|
32731
|
+
var PROBE_TIMEOUT_MS = 15e3;
|
|
32732
|
+
var MAX_UNREACHABLE_STRIKES = 40;
|
|
32733
|
+
var HISTORY_CAP = 20;
|
|
32734
|
+
var KEY_ARMED = "armed";
|
|
32735
|
+
var KEY_HISTORY = "history";
|
|
32736
|
+
var KEY_UNREACHABLE = "unreachable_strikes";
|
|
32737
|
+
var ThreadWatchdog = class extends DurableObject {
|
|
32738
|
+
/**
|
|
32739
|
+
* Arm (or re-arm) the standing check for a thread execution. Idempotent —
|
|
32740
|
+
* called at every execution start. Resets the unreachable counter because a
|
|
32741
|
+
* fresh arm proves the thread is answering RPCs right now.
|
|
32742
|
+
*/
|
|
32743
|
+
async arm(threadId) {
|
|
32744
|
+
await this.ctx.storage.put(KEY_ARMED, {
|
|
32745
|
+
threadId,
|
|
32746
|
+
armedAt: Date.now()
|
|
32747
|
+
});
|
|
32748
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
32749
|
+
const existing = await this.ctx.storage.getAlarm();
|
|
32750
|
+
if (existing === null) {
|
|
32751
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
32752
|
+
}
|
|
32753
|
+
}
|
|
32754
|
+
/**
|
|
32755
|
+
* Hint that the thread believes its execution ended cleanly. The companion
|
|
32756
|
+
* does NOT trust this (a bogus "clean end" was one of the incident's failure
|
|
32757
|
+
* modes) — it just moves the next check close so a genuinely clean thread
|
|
32758
|
+
* stands down in seconds instead of a full cadence.
|
|
32759
|
+
*/
|
|
32760
|
+
async disarmHint() {
|
|
32761
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
32762
|
+
if (!armed) return;
|
|
32763
|
+
await this.ctx.storage.setAlarm(Date.now() + DISARM_CONFIRM_DELAY_MS);
|
|
32764
|
+
}
|
|
32765
|
+
/** Diagnostics view for the thread inspector UI and the diag CLI. */
|
|
32766
|
+
async getState() {
|
|
32767
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
32768
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
32769
|
+
return {
|
|
32770
|
+
armed: Boolean(armed),
|
|
32771
|
+
threadId: armed?.threadId ?? null,
|
|
32772
|
+
armedAt: armed?.armedAt ?? null,
|
|
32773
|
+
history
|
|
32774
|
+
};
|
|
32775
|
+
}
|
|
32776
|
+
/**
|
|
32777
|
+
* Explicit cascade from the parent thread (delete/terminate). Cloudflare
|
|
32778
|
+
* has no automatic parent-child cleanup — this IS the cleanup.
|
|
32779
|
+
*/
|
|
32780
|
+
async destroy() {
|
|
32781
|
+
await this.ctx.storage.deleteAlarm();
|
|
32782
|
+
await this.ctx.storage.deleteAll();
|
|
32783
|
+
}
|
|
32784
|
+
async alarm() {
|
|
32785
|
+
const armed = await this.ctx.storage.get(KEY_ARMED);
|
|
32786
|
+
if (!armed) {
|
|
32787
|
+
return;
|
|
32788
|
+
}
|
|
32789
|
+
await this.ctx.storage.setAlarm(Date.now() + CHECK_CADENCE_MS);
|
|
32790
|
+
const startedAt = Date.now();
|
|
32791
|
+
let checkin;
|
|
32792
|
+
let requestRestart = false;
|
|
32793
|
+
let standDown = false;
|
|
32794
|
+
try {
|
|
32795
|
+
const result = await this.probeThread(armed.threadId);
|
|
32796
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
32797
|
+
requestRestart = result.requestRestart;
|
|
32798
|
+
standDown = result.verdict === "idle" || result.verdict === "parked";
|
|
32799
|
+
checkin = {
|
|
32800
|
+
at: startedAt,
|
|
32801
|
+
verdict: result.verdict,
|
|
32802
|
+
action: describeAction(result),
|
|
32803
|
+
executing: result.executing,
|
|
32804
|
+
incompleteLogs: result.incompleteLogs,
|
|
32805
|
+
silenceMs: result.silenceMs,
|
|
32806
|
+
probeMs: Date.now() - startedAt
|
|
32807
|
+
};
|
|
32808
|
+
} catch (error) {
|
|
32809
|
+
const strikes = (await this.ctx.storage.get(KEY_UNREACHABLE) ?? 0) + 1;
|
|
32810
|
+
await this.ctx.storage.put(KEY_UNREACHABLE, strikes);
|
|
32811
|
+
checkin = {
|
|
32812
|
+
at: startedAt,
|
|
32813
|
+
verdict: "unreachable",
|
|
32814
|
+
action: `probe failed (${strikes} consecutive)`,
|
|
32815
|
+
executing: null,
|
|
32816
|
+
incompleteLogs: null,
|
|
32817
|
+
silenceMs: null,
|
|
32818
|
+
probeMs: Date.now() - startedAt,
|
|
32819
|
+
error: error instanceof Error ? error.message : String(error)
|
|
32820
|
+
};
|
|
32821
|
+
if (strikes >= MAX_UNREACHABLE_STRIKES) {
|
|
32822
|
+
console.error(
|
|
32823
|
+
`[ThreadWatchdog] CRITICAL: thread ${armed.threadId} unreachable for ${strikes} consecutive probes \u2014 standing down. Manual intervention required.`
|
|
32824
|
+
);
|
|
32825
|
+
standDown = true;
|
|
32826
|
+
}
|
|
32827
|
+
}
|
|
32828
|
+
await this.recordCheckin(checkin);
|
|
32829
|
+
if (requestRestart) {
|
|
32830
|
+
try {
|
|
32831
|
+
const stub = this.getThreadStub(armed.threadId);
|
|
32832
|
+
await stub.forceRestart(
|
|
32833
|
+
"watchdog takeover: wedged execution aborted; restarting so the recovery alarm can fire"
|
|
32834
|
+
);
|
|
32835
|
+
} catch {
|
|
32836
|
+
}
|
|
32837
|
+
}
|
|
32838
|
+
if (standDown) {
|
|
32839
|
+
await this.ctx.storage.delete(KEY_ARMED);
|
|
32840
|
+
await this.ctx.storage.delete(KEY_UNREACHABLE);
|
|
32841
|
+
await this.ctx.storage.deleteAlarm();
|
|
32842
|
+
}
|
|
32843
|
+
}
|
|
32844
|
+
getThreadStub(threadId) {
|
|
32845
|
+
const namespace = this.env.AGENT_BUILDER_THREAD;
|
|
32846
|
+
const id = namespace.idFromName(threadId);
|
|
32847
|
+
return namespace.get(id);
|
|
32848
|
+
}
|
|
32849
|
+
async probeThread(threadId) {
|
|
32850
|
+
const stub = this.getThreadStub(threadId);
|
|
32851
|
+
let timer;
|
|
32852
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
32853
|
+
timer = setTimeout(
|
|
32854
|
+
() => reject(new Error(`probe timed out after ${PROBE_TIMEOUT_MS}ms`)),
|
|
32855
|
+
PROBE_TIMEOUT_MS
|
|
32856
|
+
);
|
|
32857
|
+
});
|
|
32858
|
+
try {
|
|
32859
|
+
return await Promise.race([stub.watchdogProbe(threadId), timeout]);
|
|
32860
|
+
} finally {
|
|
32861
|
+
if (timer) clearTimeout(timer);
|
|
32862
|
+
}
|
|
32863
|
+
}
|
|
32864
|
+
async recordCheckin(checkin) {
|
|
32865
|
+
const history = await this.ctx.storage.get(KEY_HISTORY) ?? [];
|
|
32866
|
+
history.unshift(checkin);
|
|
32867
|
+
await this.ctx.storage.put(KEY_HISTORY, history.slice(0, HISTORY_CAP));
|
|
32868
|
+
}
|
|
32869
|
+
};
|
|
32870
|
+
function describeAction(result) {
|
|
32871
|
+
switch (result.action) {
|
|
32872
|
+
case "finalized_resumed":
|
|
32873
|
+
return `finalized ${result.incompleteLogs} lost request(s), resumed thread`;
|
|
32874
|
+
case "finalized_parked":
|
|
32875
|
+
return `finalized ${result.incompleteLogs} lost request(s), left thread stopped`;
|
|
32876
|
+
case "finalized_deferred":
|
|
32877
|
+
return `finalized ${result.incompleteLogs} lost request(s); queued turn will continue`;
|
|
32878
|
+
case "aborted_takeover_resumed":
|
|
32879
|
+
return `took over wedged execution (silent ${formatMs(result.silenceMs)}), finalized and resumed`;
|
|
32880
|
+
case "none":
|
|
32881
|
+
default:
|
|
32882
|
+
if (result.verdict === "observing") {
|
|
32883
|
+
return `execution silent ${formatMs(result.silenceMs)} (ceiling ${formatMs(result.ceilingMs)}) \u2014 observing`;
|
|
32884
|
+
}
|
|
32885
|
+
if (result.verdict === "idle") {
|
|
32886
|
+
return "thread idle and clean \u2014 standing down";
|
|
32887
|
+
}
|
|
32888
|
+
if (result.executing) {
|
|
32889
|
+
return result.incompleteLogs > 0 ? `streaming, last progress ${formatMs(result.silenceMs)} ago` : "executing (tool run, no open request)";
|
|
32890
|
+
}
|
|
32891
|
+
return "no action needed";
|
|
32892
|
+
}
|
|
32893
|
+
}
|
|
32894
|
+
function formatMs(ms) {
|
|
32895
|
+
if (ms === null || !Number.isFinite(ms)) return "\u221E";
|
|
32896
|
+
if (ms < 1e3) return `${ms}ms`;
|
|
32897
|
+
const seconds = Math.round(ms / 1e3);
|
|
32898
|
+
if (seconds < 60) return `${seconds}s`;
|
|
32899
|
+
const minutes = Math.floor(seconds / 60);
|
|
32900
|
+
return `${minutes}m${String(seconds % 60).padStart(2, "0")}s`;
|
|
32901
|
+
}
|
|
32431
32902
|
|
|
32432
32903
|
// src/index.ts
|
|
32433
32904
|
init_utilities();
|
|
@@ -32790,6 +33261,6 @@ init_context();
|
|
|
32790
33261
|
init_ProviderRegistry();
|
|
32791
33262
|
init_types2();
|
|
32792
33263
|
|
|
32793
|
-
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, PackageDiscoveryService, ProviderRegistry, agentbuilder, authenticate, buildImageDescription, builder, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, discoverPackages, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateAgentFile, generateImageDescription, generateModelFile, generatePromptFile, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
33264
|
+
export { CodeExecutionBridge, DurableAgentBuilder, DurableThread, FlowStateSdk, NamespaceResolutionError, PackageDiscoveryService, ProviderRegistry, ThreadWatchdog, agentbuilder, authenticate, buildImageDescription, builder, cat, createNamespaceContext, createThreadEndpointHandler, defineController2 as defineController, discoverPackages, emitThreadEvent, enhanceFlowState, exists, find, forceTurn, generateAgentFile, generateImageDescription, generateModelFile, generatePromptFile, getFileStats, getMessages, getMessagesToSummarize, getShortName, getThumbnail, getUnsummarizedImageAttachments, getVisibleAgentNames, getVisibleModelNames, getVisiblePromptNames, getVisibleToolNames, grep, hasImageAttachments, head, injectMessage, isQualifiedName, linkFile, mkdir, move, optimizeImageContext, parseQualifiedName, qualifyName, queueTool, readFile, readdir, reloadHistory, rename, replaceImagesWithDescriptions, requireAdmin, requireAuth, resolveAgent, resolveHook, resolveModel, resolvePrompt, resolveTool, rmdir, stat, tail, unlink, updateThread, writeFile, writeImage };
|
|
32794
33265
|
//# sourceMappingURL=index.js.map
|
|
32795
33266
|
//# sourceMappingURL=index.js.map
|