@op1/threads 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1049 @@
1
+ import { createHash } from "node:crypto";
2
+ import type { Plugin } from "@opencode/plugin";
3
+ import { Session } from "@opencode/schema/session";
4
+ import { SessionMessage } from "@opencode/schema/session-message";
5
+ import Ajv from "ajv";
6
+ import { parseWorkflow, executeWorkflow, type WorkflowHost } from "./workflow-runtime";
7
+ import { requireDelegation } from "./permissions";
8
+ import {
9
+ serialized,
10
+ type threads,
11
+ WorkflowWorker,
12
+ workerIdentity,
13
+ workerLink,
14
+ workflowExecutionAuthorized,
15
+ watchWorkflowExecution,
16
+ } from "./threads";
17
+ import { workflowHash, workflowStore } from "./workflow-store";
18
+ import {
19
+ Json,
20
+ WORKFLOW_DIAGNOSTIC_JSON_BYTES,
21
+ WORKFLOW_SETTLEMENT_DIAGNOSTIC_JSON_BYTES,
22
+ WorkflowAgentInput,
23
+ WorkflowResult,
24
+ WorkflowRun,
25
+ WorkflowSettlement,
26
+ WorkflowStart,
27
+ type WorkflowStep,
28
+ type WorkflowSettlement as Settlement,
29
+ } from "./workflow-types";
30
+ import {
31
+ interruptWorkflowWorkers,
32
+ withWorkflowSlot,
33
+ workflowDirectory,
34
+ workflowDirectoryPlan,
35
+ workflowSourceDirectory,
36
+ workflowTask,
37
+ } from "./workflow-worker";
38
+
39
+ type Threads = ReturnType<typeof threads>;
40
+ type Runtime = { agent: string; model: { providerID: string; id: string; variant?: string } };
41
+ type Control = { runID: string; action: "pause" | "resume" | "stop"; checkpointKey?: string; response?: Json };
42
+
43
+ const processState = globalThis as typeof globalThis & {
44
+ __opWorkflowLeases?: Map<string, { controller: AbortController; promise: Promise<void>; owner: symbol }>;
45
+ __opWorkflowSignals?: Map<string, Set<() => void>>;
46
+ };
47
+ const leases = processState.__opWorkflowLeases ??= new Map();
48
+ const workflowSignals = processState.__opWorkflowSignals ??= new Map();
49
+ const digest = (...parts: string[]) => createHash("sha256").update(JSON.stringify(parts)).digest("hex");
50
+ const runIdentity = (ownerID: string, key: string) => `wfr_${digest(ownerID, key).slice(0, 32)}`;
51
+ const deliveryIdentity = (runID: string) => SessionMessage.ID.make(`msg_${digest(runID, "delivery").slice(0, 32)}`);
52
+ const resultKey = (workerID: string) => `workflows/results/${workerID}`;
53
+ const nestedKey = (runID: string, identity: string) => `workflows/nested/${runID}/${digest(identity)}`;
54
+ const completionKey = (runID: string) => `workflows/completions/${runID}`;
55
+ const terminal = new Set(["stopped", "failed", "completed"]);
56
+ const activeStatus = new Set(["running", "pausing", "stopping", "waiting"]);
57
+ const settlementLimit = 16 * 1024 * 1024;
58
+ const bounded = (text: string, length = 20_000) => text.length <= length ? text : `${text.slice(0, length)}\n…[truncated]`;
59
+ const boundedBytes = (text: string, limit: number) => {
60
+ if (Buffer.byteLength(JSON.stringify(text), "utf8") - 2 <= limit) return text;
61
+ const suffix = "\n…[truncated]";
62
+ let low = 0;
63
+ let high = text.length;
64
+ while (low < high) {
65
+ const middle = Math.ceil((low + high) / 2);
66
+ if (Buffer.byteLength(JSON.stringify(`${text.slice(0, middle)}${suffix}`), "utf8") - 2 <= limit) low = middle;
67
+ else high = middle - 1;
68
+ }
69
+ return `${text.slice(0, low)}${suffix}`;
70
+ };
71
+ const isCompleted = (step: WorkflowStep): step is Extract<WorkflowStep, { status: "completed" }> => step.status === "completed";
72
+ const notify = (runID: string) => {
73
+ const listeners = workflowSignals.get(runID);
74
+ if (!listeners) return;
75
+ workflowSignals.delete(runID);
76
+ for (const listener of listeners) listener();
77
+ };
78
+ const waitForChange = (runID: string, signal: AbortSignal, timeoutMs = 1_000) => new Promise<void>((resolve, reject) => {
79
+ const listeners = workflowSignals.get(runID) ?? new Set<() => void>();
80
+ workflowSignals.set(runID, listeners);
81
+ let timer: ReturnType<typeof setTimeout>;
82
+ const done = () => {
83
+ clearTimeout(timer);
84
+ signal.removeEventListener("abort", aborted);
85
+ listeners.delete(done);
86
+ if (listeners.size === 0 && workflowSignals.get(runID) === listeners) workflowSignals.delete(runID);
87
+ resolve();
88
+ };
89
+ const aborted = () => {
90
+ clearTimeout(timer);
91
+ listeners.delete(done);
92
+ if (listeners.size === 0 && workflowSignals.get(runID) === listeners) workflowSignals.delete(runID);
93
+ reject(signal.reason ?? new Error("Workflow cancelled"));
94
+ };
95
+ listeners.add(done);
96
+ timer = setTimeout(done, timeoutMs);
97
+ signal.addEventListener("abort", aborted, { once: true });
98
+ if (signal.aborted) aborted();
99
+ });
100
+ function boundedJson(value: unknown, label: string) {
101
+ const parsed = Json.parse(value);
102
+ if (new TextEncoder().encode(JSON.stringify(parsed)).byteLength > 1_048_576) {
103
+ throw new Error(`${label} exceeds the 1 MiB durable output limit`);
104
+ }
105
+ return parsed;
106
+ }
107
+
108
+ function errorText(error: unknown) {
109
+ return boundedBytes(error instanceof Error ? error.message : String(error), WORKFLOW_DIAGNOSTIC_JSON_BYTES);
110
+ }
111
+
112
+ const settlementError = (error: unknown) => boundedBytes(errorText(error), WORKFLOW_SETTLEMENT_DIAGNOSTIC_JSON_BYTES);
113
+ const assertSettlementLimit = (settlements: Settlement[]) => {
114
+ if (Buffer.byteLength(JSON.stringify(settlements), "utf8") > settlementLimit) {
115
+ throw new Error("Workflow settlement journal exceeds the 16 MiB durable limit");
116
+ }
117
+ };
118
+
119
+ function readSettlements(run: WorkflowRun, stored: unknown): Settlement[] {
120
+ const raw = run.settlements !== undefined && (run.settlements.length > 0 || stored === undefined)
121
+ ? run.settlements
122
+ : stored ?? [];
123
+ if (!Array.isArray(raw)) throw new Error("Workflow settlement journal must be an array");
124
+ const legacy = raw.every((entry) => typeof entry === "string");
125
+ if (legacy && raw.length > 0 && run.checkpoints.some((checkpoint) => checkpoint.response !== undefined)) {
126
+ throw new Error("Legacy completion journal cannot deterministically replay answered checkpoints; start a new workflow run key");
127
+ }
128
+ const settlements: Settlement[] = legacy
129
+ ? raw.map((key: string) => ({ kind: "agent", key, outcome: "success" }))
130
+ : raw.map((entry: unknown) => WorkflowSettlement.parse(entry));
131
+ assertSettlementLimit(settlements);
132
+ return settlements;
133
+ }
134
+
135
+ class UncertainWriteError extends Error {}
136
+ class SchedulingDeferred extends Error {}
137
+ class AdmissionDenied extends Error {}
138
+
139
+ function usage(session: Awaited<ReturnType<Plugin.Context["session"]["get"]>>) {
140
+ const tokens = session.tokens;
141
+ const measured = tokens !== undefined && session.cost !== undefined;
142
+ return {
143
+ tokens: measured
144
+ ? tokens.input + tokens.output
145
+ : 0,
146
+ cost: measured ? session.cost : 0,
147
+ measured,
148
+ };
149
+ }
150
+
151
+ export function workflowEngine(
152
+ ctx: Plugin.Context,
153
+ workers: Threads,
154
+ options: { maxWorkers?: number; loadSaved?: (name: string) => Promise<string>; warmWorker?: (workerID: string) => Promise<void> } = {},
155
+ ) {
156
+ const diagnosed = new Set<string>();
157
+ async function diagnose(issue: { id: string; ownerID: string; message: string }) {
158
+ const message = errorText(issue.message);
159
+ const identity = digest(issue.ownerID, issue.id, message);
160
+ await serialized(`workflow-diagnostic:${identity}`, async () => {
161
+ if (diagnosed.has(identity)) return;
162
+ try {
163
+ await ctx.session.synthetic({
164
+ sessionID: issue.ownerID,
165
+ id: SessionMessage.ID.make(`msg_${identity.slice(0, 32)}`),
166
+ text: `Workflow ${issue.id} requires journal recovery: ${message}\nRaw evidence remains at workflows/runs/${issue.id}. Inspect the retained record and repair it or start a new workflow run key. Other runs remain available.`,
167
+ metadata: { opWorkflowJournalDiagnostic: { runID: issue.id, error: message } },
168
+ delivery: "queue", resume: true,
169
+ });
170
+ diagnosed.add(identity);
171
+ } catch {
172
+ // Diagnostic delivery must not make a damaged sibling block healthy runs.
173
+ }
174
+ });
175
+ }
176
+ const store = workflowStore(ctx.storage, { onDiagnostic: diagnose });
177
+ const maxWorkers = options.maxWorkers ?? 4;
178
+ if (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1) throw new Error("maxWorkers must be a positive integer");
179
+ const ajv = new Ajv({ allErrors: true, strict: false });
180
+ const recoveredOwners = new Set<string>();
181
+ const engineOwner = Symbol("workflow-engine");
182
+ let disposed = false;
183
+
184
+ async function nativeWorker(workerID: string) {
185
+ try {
186
+ return await ctx.session.get({ sessionID: workerID });
187
+ } catch (error) {
188
+ if (typeof error === "object" && error !== null && "_tag" in error && error._tag === "Session.NotFoundError" && "sessionID" in error && error.sessionID === workerID) return undefined;
189
+ throw error;
190
+ }
191
+ }
192
+
193
+ async function owned(ownerID: string, runID: string) {
194
+ const run = await store.get(runID);
195
+ if (run.ownerID !== ownerID) throw new Error("Only the owning session may access this workflow");
196
+ return run;
197
+ }
198
+
199
+ async function deliver(runID: string) {
200
+ return serialized(`workflow-delivery:${runID}`, async () => {
201
+ const run = await store.get(runID);
202
+ if (!terminal.has(run.status) || run.delivered) return run;
203
+ const completed = run.steps.filter(isCompleted);
204
+ const report = {
205
+ runID: run.id,
206
+ status: run.status,
207
+ ...(run.error === undefined ? {} : { error: bounded(run.error, 600) }),
208
+ counts: {
209
+ total: run.steps.length,
210
+ completed: completed.length,
211
+ failed: run.steps.filter((step) => step.status === "failed").length,
212
+ },
213
+ evidence: completed.slice(0, 6).map((step) => ({
214
+ key: step.key,
215
+ verdict: step.report.verdict,
216
+ summary: bounded(step.report.summary, 160),
217
+ evidence: step.report.evidence.slice(0, 2).map((item) => bounded(item, 160)),
218
+ })),
219
+ };
220
+ const compact = bounded(JSON.stringify(report), 7_000);
221
+ await ctx.session.synthetic({
222
+ sessionID: run.ownerID,
223
+ id: SessionMessage.ID.make(run.deliveryID),
224
+ text: `Workflow ${run.name} (${run.id}) finished:\n${compact}\nUse workflows_inspect with runID ${run.id} for the full durable result and step details.`,
225
+ metadata: { opWorkflowDelivery: Json.parse(report), runID: run.id },
226
+ delivery: "queue",
227
+ resume: true,
228
+ });
229
+ return store.update(run.id, (current) => { current.delivered = true; }, "control");
230
+ });
231
+ }
232
+
233
+ async function recover(ownerID: string) {
234
+ const runs = await store.list(ownerID);
235
+ if (!recoveredOwners.has(ownerID)) {
236
+ recoveredOwners.add(ownerID);
237
+ for (const run of runs) {
238
+ if (activeStatus.has(run.status) && !leases.has(run.id)) {
239
+ try {
240
+ if (run.status === "stopping") {
241
+ await interruptWorkflowWorkers(workers, ownerID, run);
242
+ await store.update(run.id, (current) => {
243
+ current.status = "stopped";
244
+ current.error = "Workflow stop was recovered after server restart.";
245
+ }, "control");
246
+ continue;
247
+ }
248
+ await store.update(run.id, (current) => {
249
+ current.status = "interrupted";
250
+ current.error = "OpenCode stopped while this workflow was active. Inspect its recorded steps, then resume explicitly.";
251
+ }, "control");
252
+ } catch (error) {
253
+ await diagnose({ id: run.id, ownerID, message: errorText(error) });
254
+ }
255
+ }
256
+ }
257
+ }
258
+ for (const run of await store.list(ownerID)) {
259
+ if (terminal.has(run.status) && !run.delivered) {
260
+ await deliver(run.id).catch((error) => diagnose({ id: run.id, ownerID, message: errorText(error) }));
261
+ }
262
+ }
263
+ }
264
+
265
+ async function waitUntilRunnable(runID: string, signal: AbortSignal) {
266
+ for (;;) {
267
+ if (signal.aborted) throw signal.reason ?? new Error("Workflow cancelled");
268
+ const run = await store.get(runID);
269
+ if (run.status === "running") return;
270
+ if (run.status === "stopping" || run.status === "stopped") throw new Error("Workflow stopped");
271
+ if (run.status === "pausing" && !run.steps.some((step) => step.status === "running")) {
272
+ await store.update(runID, (current) => { if (current.status === "pausing") current.status = "paused"; }, "control");
273
+ }
274
+ await waitForChange(runID, signal);
275
+ }
276
+ }
277
+
278
+ async function resolveWorktree(run: WorkflowRun, input: WorkflowAgentInput, key: string) {
279
+ return serialized(`workflow-worktree:${run.id}:${key}`, () => workflowDirectory(ctx, run, input, key));
280
+ }
281
+
282
+ function launch(runID: string, runtime: Runtime, recovering = false) {
283
+ if (disposed || leases.has(runID)) return leases.get(runID)?.promise;
284
+ const controller = new AbortController();
285
+ const task = executeRun(runID, runtime, controller, recovering).finally(() => {
286
+ if (leases.get(runID)?.promise === task) leases.delete(runID);
287
+ });
288
+ leases.set(runID, { controller, promise: task, owner: engineOwner });
289
+ void task.catch(() => {});
290
+ return task;
291
+ }
292
+
293
+ async function executeRun(runID: string, runtime: Runtime, controller: AbortController, recovering: boolean) {
294
+ let run = await store.get(runID);
295
+ const remaining = run.created + run.limits.timeoutMs - Date.now();
296
+ if (remaining <= 0) {
297
+ await store.update(runID, (current) => { current.status = "failed"; current.error = "Workflow timeout exceeded"; }, "control");
298
+ await deliver(runID);
299
+ return;
300
+ }
301
+ const timer = setTimeout(() => controller.abort(new Error("Workflow timeout exceeded")), remaining);
302
+ const seen = new Set<string>();
303
+ const pendingAgents = new Set<Promise<unknown>>();
304
+ const maxCalls = run.limits.maxAgents * 8 + 100;
305
+ let hostCalls = 0;
306
+ const countCall = () => {
307
+ hostCalls++;
308
+ if (hostCalls > maxCalls) throw new Error(`Workflow cumulative call limit exceeded (${maxCalls})`);
309
+ };
310
+ let completionOrder: Settlement[];
311
+ try {
312
+ const storedCompletionOrder = await ctx.storage.get(completionKey(runID));
313
+ if (run.settlements === undefined || (run.settlements.length === 0 && storedCompletionOrder !== undefined)) {
314
+ run = await store.update(runID, (current) => { current.settlements = readSettlements(current, storedCompletionOrder); });
315
+ }
316
+ completionOrder = readSettlements(run, undefined);
317
+ } catch (error) {
318
+ try {
319
+ await store.update(runID, (current) => { current.status = "failed"; current.error = errorText(error); }, "control")
320
+ .finally(() => clearTimeout(timer));
321
+ } catch (persistError) {
322
+ await diagnose({
323
+ id: runID, ownerID: run.ownerID,
324
+ message: `${errorText(error)}; recording the workflow failure also failed: ${errorText(persistError)}`,
325
+ });
326
+ return;
327
+ }
328
+ await deliver(runID);
329
+ return;
330
+ }
331
+ let completionCursor = 0;
332
+ const refreshSettlements = async () => {
333
+ const refreshed = readSettlements(await store.get(runID), undefined);
334
+ if (refreshed.length >= completionOrder.length) completionOrder = refreshed;
335
+ };
336
+ const appendSettlement = async (entry: Settlement) => {
337
+ const updated = await store.update(runID, (current) => {
338
+ current.settlements ??= [];
339
+ if (!current.settlements.some((item) => item.kind === entry.kind && item.key === entry.key)) {
340
+ const settlements = [...current.settlements, entry];
341
+ assertSettlementLimit(settlements);
342
+ current.settlements = settlements;
343
+ }
344
+ }, entry.kind === "agent" && entry.outcome === "failure" ? "control" : "payload");
345
+ completionOrder = updated.settlements ?? [];
346
+ notify(runID);
347
+ };
348
+ const recordedSettlement = (kind: Settlement["kind"], key: string) => completionOrder.find((item) => item.kind === kind && item.key === key);
349
+ const consumeSettlement = async (kind: Settlement["kind"], key: string) => {
350
+ for (;;) {
351
+ if (controller.signal.aborted) throw controller.signal.reason;
352
+ const next = completionOrder[completionCursor];
353
+ if (next?.kind === kind && next.key === key) {
354
+ completionCursor++;
355
+ notify(runID);
356
+ return next;
357
+ }
358
+ await waitForChange(runID, controller.signal);
359
+ await refreshSettlements();
360
+ }
361
+ };
362
+ const nestedIndexes = new Map<string, number>();
363
+ let active = 0;
364
+ const hostFor = (prefix: string, depth: number): WorkflowHost => ({
365
+ agent: async (raw) => {
366
+ countCall();
367
+ const input = WorkflowAgentInput.parse(raw);
368
+ if (input.schema !== undefined) ajv.compile(input.schema);
369
+ const key = `${prefix}${input.key}`;
370
+ if (seen.has(key)) throw new Error(`Duplicate workflow agent key in this execution: ${key}`);
371
+ seen.add(key);
372
+ await waitUntilRunnable(runID, controller.signal);
373
+ const requestFingerprint = workflowHash(input);
374
+ const beforeAdmission = await store.get(runID);
375
+ const source = await workflowSourceDirectory(ctx, beforeAdmission, input);
376
+ const directoryInput = { ...input, directory: source };
377
+ const owner = await ctx.session.get({ sessionID: beforeAdmission.ownerID });
378
+ const caller = await ctx.agent.get({ agentID: beforeAdmission.callerAgent, location: owner.location });
379
+ requireDelegation([...caller.data.permissions, ...(owner.permissions ?? [])], input.agent);
380
+ const sourceWorker = beforeAdmission.steps.find((step) => step.directory === source);
381
+ if (sourceWorker && options.warmWorker) {
382
+ const existing = await nativeWorker(sourceWorker.workerID);
383
+ if (existing) await options.warmWorker(existing.id);
384
+ }
385
+ const sourceProfile = await ctx.agent.get({ agentID: input.agent, location: { directory: source } });
386
+ const sourceModel = sourceProfile.data.model ?? beforeAdmission.model;
387
+ const sourceProfileFingerprint = workflowHash({
388
+ model: sourceModel,
389
+ permissions: sourceProfile.data.permissions,
390
+ system: sourceProfile.data.system ?? null,
391
+ });
392
+ run = await store.update(runID, (current) => {
393
+ const existing = current.steps.find((step) => step.key === key);
394
+ if (existing) {
395
+ if (existing.fingerprint !== requestFingerprint) throw new Error(`Workflow key ${key} was resumed with different input; use a new run key`);
396
+ return;
397
+ }
398
+ if (current.steps.length >= current.limits.maxAgents) throw new Error(`Workflow agent limit reached (${current.limits.maxAgents})`);
399
+ const plan = workflowDirectoryPlan(current, directoryInput, key);
400
+ const spawnKey = `workflow:${current.id}:${key}`;
401
+ current.steps.push({
402
+ status: "prepared",
403
+ key,
404
+ fingerprint: requestFingerprint,
405
+ index: current.steps.reduce((maximum, step) => Math.max(maximum, step.index), -1) + 1,
406
+ input,
407
+ workerID: workerIdentity(current.ownerID, spawnKey),
408
+ spawnKey,
409
+ created: Date.now(),
410
+ phase: input.phase ?? current.phase,
411
+ directory: plan.directory,
412
+ model: sourceModel,
413
+ profileFingerprint: `pending:${sourceProfileFingerprint}`,
414
+ });
415
+ });
416
+ let previous = run.steps.find((step) => step.key === key)!;
417
+ if (previous.profileFingerprint.startsWith("pending:") && previous.profileFingerprint !== `pending:${sourceProfileFingerprint}`) {
418
+ throw new Error(`Source agent profile for workflow key ${key} changed before worktree allocation; start a new workflow run key`);
419
+ }
420
+ const spawnKey = previous.spawnKey;
421
+ const priorSettlement = recordedSettlement("agent", key);
422
+ if (priorSettlement?.kind === "agent" && priorSettlement.outcome === "failure") {
423
+ let expectedFingerprint = `pending:${sourceProfileFingerprint}`;
424
+ if (!previous.profileFingerprint.startsWith("pending:")) {
425
+ const replayProfile = await ctx.agent.get({ agentID: input.agent, location: { directory: previous.directory } });
426
+ const replayModel = replayProfile.data.model ?? run.model;
427
+ expectedFingerprint = workflowHash({ model: replayModel, permissions: replayProfile.data.permissions, system: replayProfile.data.system ?? null });
428
+ }
429
+ if (previous.profileFingerprint !== expectedFingerprint) {
430
+ throw new Error(`Agent profile for workflow key ${key} changed; start a new workflow run key`);
431
+ }
432
+ const settlement = await consumeSettlement("agent", key);
433
+ throw new Error(settlement.kind === "agent" ? settlement.error ?? `Workflow step ${key} previously failed` : `Workflow step ${key} previously failed`);
434
+ }
435
+ const directory = previous.profileFingerprint.startsWith("pending:")
436
+ ? await resolveWorktree(run, directoryInput, key)
437
+ : previous.directory;
438
+ const dispatch = async () => {
439
+ let admitted = await store.get(run.id);
440
+ if (admitted.status !== "running") throw new SchedulingDeferred();
441
+ if (admitted.limits.tokenBudget !== undefined && admitted.steps.find((step) => step.key === key)?.status === "prepared") {
442
+ const tokenBudget = admitted.limits.tokenBudget;
443
+ const discovered = new Map<string, ReturnType<typeof usage>>();
444
+ for (const step of admitted.steps) {
445
+ if (step.key === key || step.status === "prepared") continue;
446
+ if ("usage" in step && step.usage !== undefined) continue;
447
+ if (step.status === "failed") throw new AdmissionDenied("Workflow token budget cannot continue because prior worker usage is unmeasured");
448
+ if (step.status === "running") {
449
+ const native = await nativeWorker(step.workerID);
450
+ if (native?.outcome !== undefined) discovered.set(step.key, usage(native));
451
+ }
452
+ }
453
+ if (discovered.size > 0) {
454
+ admitted = await store.update(run.id, (value) => {
455
+ for (const step of value.steps) {
456
+ const measured = discovered.get(step.key);
457
+ if (measured && step.status === "running") step.usage = measured;
458
+ }
459
+ });
460
+ }
461
+ const spent = admitted.steps.flatMap((step) => step.key !== key && "usage" in step && step.usage !== undefined ? [step.usage] : []);
462
+ if (spent.some((item) => !item.measured)) {
463
+ throw new AdmissionDenied("Workflow token budget cannot continue because prior worker usage is unmeasured");
464
+ }
465
+ const consumed = spent.reduce((sum, item) => sum + item.tokens, 0);
466
+ if (consumed >= tokenBudget) {
467
+ throw new AdmissionDenied(`Workflow token budget reached (${tokenBudget}); in-flight usage may overshoot the soft budget`);
468
+ }
469
+ }
470
+ let current = admitted.steps.find((step) => step.key === key)!;
471
+ const ensureWorker = () => workers.spawnWorkflow(run.ownerID, {
472
+ key: spawnKey,
473
+ title: input.label ?? `Workflow: ${key}`,
474
+ directory,
475
+ task: workflowTask(input),
476
+ agent: input.agent,
477
+ }, runtime as Parameters<Threads["spawnWorkflow"]>[2], {
478
+ ownerID: Session.ID.make(run.ownerID), runID: run.id, stepKey: key,
479
+ callerAgent: run.callerAgent, access: input.access,
480
+ });
481
+ const finalizeProfile = async () => {
482
+ if (options.warmWorker && await nativeWorker(current.workerID)) await options.warmWorker(current.workerID);
483
+ const profile = await ctx.agent.get({ agentID: input.agent, location: { directory } });
484
+ const model = profile.data.model ?? run.model;
485
+ const profileFingerprint = workflowHash({ model, permissions: profile.data.permissions, system: profile.data.system ?? null });
486
+ run = await store.update(run.id, (value) => {
487
+ const step = value.steps.find((item) => item.key === key)!;
488
+ if (!step.profileFingerprint.startsWith("pending:") && step.profileFingerprint !== profileFingerprint) {
489
+ throw new Error(`Agent profile for workflow key ${key} changed; start a new workflow run key`);
490
+ }
491
+ step.directory = directory;
492
+ step.model = model;
493
+ step.profileFingerprint = profileFingerprint;
494
+ });
495
+ return run.steps.find((step) => step.key === key)!;
496
+ };
497
+ if (current.status === "completed") {
498
+ const result = current.report.result;
499
+ await finalizeProfile();
500
+ return result;
501
+ }
502
+ const fresh = current.status === "prepared";
503
+ if (fresh) {
504
+ run = await store.update(run.id, (value) => {
505
+ const index = value.steps.findIndex((step) => step.key === key);
506
+ if (value.steps[index].status === "prepared") {
507
+ value.steps[index] = { ...value.steps[index], status: "running" } as WorkflowStep;
508
+ }
509
+ }, "control");
510
+ current = run.steps.find((step) => step.key === key)!;
511
+ } else {
512
+ try {
513
+ await ctx.session.get({ sessionID: current.workerID });
514
+ } catch (error) {
515
+ const missing = typeof error === "object" && error !== null &&
516
+ "_tag" in error && error._tag === "Session.NotFoundError" &&
517
+ "sessionID" in error && error.sessionID === current.workerID;
518
+ if (!missing) throw error;
519
+ const message = input.access === "write"
520
+ ? "Previously dispatched write worker is missing. Its external state is ambiguous, so it will not be recreated or replayed."
521
+ : "Previously dispatched worker is missing and cannot be safely recreated under the same durable identity.";
522
+ await store.update(run.id, (value) => {
523
+ if (input.access === "write") {
524
+ value.status = "interrupted";
525
+ value.error = message;
526
+ return;
527
+ }
528
+ const index = value.steps.findIndex((step) => step.key === key);
529
+ value.steps[index] = { ...value.steps[index], status: "failed", error: message, retryable: false } as WorkflowStep;
530
+ }, "control");
531
+ throw new Error(message);
532
+ }
533
+ }
534
+ await ensureWorker();
535
+ current = await finalizeProfile();
536
+ const finishRecorded = async (recorded: unknown) => {
537
+ const report = WorkflowResult.parse(recorded);
538
+ const native = await ctx.session.get({ sessionID: current.workerID });
539
+ if (native.outcome !== "succeeded") {
540
+ const message = `Worker reported ${report.verdict} but its native execution ended ${native.outcome ?? "without a successful outcome"}`;
541
+ if (recovering) {
542
+ const resolution = "A durable report exists, but the native execution was interrupted. Ask the same worker to inspect and explicitly resolve the crash window, then resume this workflow.";
543
+ await store.update(run.id, (value) => { value.status = "interrupted"; value.error = resolution; }, "control");
544
+ notify(run.id);
545
+ throw new UncertainWriteError(resolution);
546
+ }
547
+ await store.update(run.id, (value) => {
548
+ const index = value.steps.findIndex((step) => step.key === key);
549
+ value.steps[index] = { ...value.steps[index], status: "failed", error: message, retryable: false, usage: usage(native) } as WorkflowStep;
550
+ }, "control");
551
+ throw new Error(message);
552
+ }
553
+ await store.update(run.id, (value) => {
554
+ const index = value.steps.findIndex((step) => step.key === key);
555
+ value.steps[index] = { ...value.steps[index], status: "completed", completed: Date.now(), report, usage: usage(native) } as WorkflowStep;
556
+ });
557
+ return report.result;
558
+ };
559
+ let recorded = await ctx.storage.get(resultKey(current.workerID));
560
+ if (recorded === undefined) {
561
+ if (current.status === "failed") {
562
+ if (!current.retryable) throw new Error(current.error);
563
+ await workers.send(run.ownerID, {
564
+ workerID: Session.ID.make(current.workerID),
565
+ key: `workflow-retry:${run.id}:${key}`,
566
+ text: "The previous native execution failed before a valid workflows_result report was recorded. Inspect the existing session work, repair the issue, and call workflows_result. Do not redo already-completed external effects.",
567
+ });
568
+ } else {
569
+ const native = await ctx.session.get({ sessionID: current.workerID }).catch(() => undefined);
570
+ if (current.status === "running" && native?.outcome !== undefined) {
571
+ if (input.access === "write") {
572
+ const message = "Interrupted write has uncertain external state. Inspect the retained worker/worktree and send that same worker an explicit resolution request; after it reports, resume this run. The write will not be replayed automatically.";
573
+ await store.update(run.id, (value) => { value.status = "interrupted"; value.error = message; }, "control");
574
+ throw new UncertainWriteError(message);
575
+ }
576
+ await workers.send(run.ownerID, {
577
+ workerID: Session.ID.make(current.workerID),
578
+ key: `workflow-reconcile:${run.id}:${key}`,
579
+ text: "The service resumed this read-only step after its prior execution ended without a validated report. Inspect the existing context, finish the task, and call workflows_result.",
580
+ });
581
+ }
582
+ }
583
+ }
584
+ await store.update(run.id, (value) => {
585
+ const index = value.steps.findIndex((step) => step.key === key);
586
+ if (index >= 0 && value.steps[index].status !== "completed") value.steps[index] = { ...value.steps[index], status: "running" } as WorkflowStep;
587
+ }, "control");
588
+ const agentTimeout = input.timeoutMs ?? run.limits.agentTimeoutMs;
589
+ const deadline = Date.now() + agentTimeout;
590
+ const deadlineReason = () => controller.signal.aborted
591
+ ? controller.signal.reason ?? new Error("Workflow cancelled")
592
+ : Date.now() >= deadline ? new Error(`Workflow worker timed out after ${agentTimeout}ms`) : undefined;
593
+ const waitWorker = async () => {
594
+ let timeout: ReturnType<typeof setTimeout> | undefined;
595
+ let abort = () => {};
596
+ let forcedReason: unknown;
597
+ let interrupt: Promise<void> | undefined;
598
+ const force = (reason: unknown) => {
599
+ if (forcedReason === undefined) {
600
+ forcedReason = reason;
601
+ interrupt = Promise.resolve().then(() => workers.interrupt(run.ownerID, { workerID: Session.ID.make(current.workerID) }))
602
+ .then(() => undefined, () => undefined);
603
+ }
604
+ };
605
+ try {
606
+ await Promise.race([
607
+ ctx.session.wait({ sessionID: current.workerID }),
608
+ new Promise<never>((_, reject) => {
609
+ timeout = setTimeout(() => {
610
+ const reason = new Error(`Workflow worker timed out after ${agentTimeout}ms`);
611
+ force(reason);
612
+ reject(reason);
613
+ }, Math.max(0, deadline - Date.now()));
614
+ }),
615
+ new Promise<never>((_, reject) => {
616
+ abort = () => {
617
+ const reason = controller.signal.reason ?? new Error("Workflow cancelled");
618
+ force(reason);
619
+ reject(reason);
620
+ };
621
+ controller.signal.addEventListener("abort", abort, { once: true });
622
+ if (controller.signal.aborted) abort();
623
+ }),
624
+ ]);
625
+ const reason = forcedReason ?? deadlineReason();
626
+ if (reason !== undefined) { force(reason); throw reason; }
627
+ } catch (error) {
628
+ const reason = forcedReason ?? deadlineReason();
629
+ if (reason !== undefined) { force(reason); throw reason; }
630
+ throw error;
631
+ } finally {
632
+ if (timeout !== undefined) clearTimeout(timeout);
633
+ controller.signal.removeEventListener("abort", abort);
634
+ await interrupt;
635
+ }
636
+ };
637
+ await waitWorker();
638
+ recorded ??= await ctx.storage.get(resultKey(current.workerID));
639
+ if (recorded === undefined) {
640
+ const native = await ctx.session.get({ sessionID: current.workerID });
641
+ if (native.outcome === "succeeded" || (input.access === "read" && (native.outcome === "failed" || native.outcome === "interrupted"))) {
642
+ const reason = deadlineReason();
643
+ if (reason !== undefined) {
644
+ await workers.interrupt(run.ownerID, { workerID: Session.ID.make(current.workerID) }).catch(() => undefined);
645
+ throw reason;
646
+ }
647
+ await workers.send(run.ownerID, {
648
+ workerID: Session.ID.make(current.workerID),
649
+ key: `workflow-report-repair:${run.id}:${key}`,
650
+ text: native.outcome === "succeeded"
651
+ ? "Your native execution completed without a validated workflows_result. Do not redo the task. Review the existing work and submit the required structured report now."
652
+ : "The read-only execution ended before a validated report was recorded. Inspect the existing context, repair or finish the read, and submit workflows_result without creating a new worker.",
653
+ });
654
+ await waitWorker();
655
+ recorded = await ctx.storage.get(resultKey(current.workerID));
656
+ }
657
+ }
658
+ if (recorded === undefined) {
659
+ const native = await ctx.session.get({ sessionID: current.workerID });
660
+ const message = native.outcome === "succeeded"
661
+ ? "Worker completed without a workflows_result report; native success is not workflow success"
662
+ : `Worker ended ${native.outcome ?? "without a terminal outcome"} before a valid workflows_result report`;
663
+ await store.update(run.id, (value) => {
664
+ const index = value.steps.findIndex((step) => step.key === key);
665
+ value.steps[index] = { ...value.steps[index], status: "failed", error: message, retryable: input.access === "read", usage: usage(native) } as WorkflowStep;
666
+ }, "control");
667
+ throw new Error(message);
668
+ }
669
+ return finishRecorded(recorded);
670
+ };
671
+ active++;
672
+ try {
673
+ const attempt = () => withWorkflowSlot(
674
+ run.ownerID,
675
+ maxWorkers,
676
+ run.id,
677
+ run.limits.concurrency,
678
+ controller.signal,
679
+ dispatch,
680
+ );
681
+ const perform = async (): Promise<Json> => {
682
+ for (;;) {
683
+ await waitUntilRunnable(run.id, controller.signal);
684
+ try {
685
+ return await (input.access === "write" && input.isolation === "shared"
686
+ ? serialized(`workflow-write:${directory}`, attempt)
687
+ : attempt()) as Json;
688
+ } catch (error) {
689
+ if (!(error instanceof SchedulingDeferred)) throw error;
690
+ }
691
+ }
692
+ };
693
+ const pending = perform();
694
+ pendingAgents.add(pending);
695
+ try {
696
+ const existing = recordedSettlement("agent", key);
697
+ let value: Json | undefined;
698
+ let failure: unknown;
699
+ try {
700
+ value = await pending;
701
+ } catch (error) {
702
+ failure = error;
703
+ }
704
+ if (failure !== undefined) {
705
+ if (failure instanceof UncertainWriteError) throw failure;
706
+ const state = await store.get(run.id);
707
+ if (state.status === "interrupted" || state.status === "stopping" || state.status === "stopped") throw failure;
708
+ if (existing?.kind === "agent" && existing.outcome === "success") throw failure;
709
+ if (!(failure instanceof AdmissionDenied)) {
710
+ await store.update(run.id, (value) => {
711
+ if (value.status === "interrupted") return;
712
+ const index = value.steps.findIndex((step) => step.key === key);
713
+ const step = value.steps[index];
714
+ if (step && step.status !== "completed" && step.status !== "failed") {
715
+ value.steps[index] = { ...step, status: "failed", error: errorText(failure), retryable: input.access === "read" } as WorkflowStep;
716
+ }
717
+ }, "control");
718
+ }
719
+ if (!existing) await appendSettlement({ kind: "agent", key, outcome: "failure", error: settlementError(failure) });
720
+ const settlement = await consumeSettlement("agent", key);
721
+ throw new Error(settlement.kind === "agent" && settlement.outcome === "failure" ? settlement.error ?? errorText(failure) : errorText(failure));
722
+ }
723
+ if (!existing) await appendSettlement({ kind: "agent", key, outcome: "success" });
724
+ const settlement = await consumeSettlement("agent", key);
725
+ if (settlement.kind === "agent" && settlement.outcome === "failure") throw new Error(settlement.error ?? `Workflow step ${key} previously failed`);
726
+ return value!;
727
+ } finally {
728
+ pendingAgents.delete(pending);
729
+ }
730
+ } catch (error) {
731
+ throw error;
732
+ } finally {
733
+ active--;
734
+ const latest = await store.get(run.id);
735
+ if (active === 0 && latest.status === "pausing") await store.update(run.id, (value) => { if (value.status === "pausing") value.status = "paused"; }, "control");
736
+ }
737
+ },
738
+ phase: async (title) => {
739
+ countCall();
740
+ const phase = bounded(String(title), 160);
741
+ await store.update(runID, (current) => { current.phase = phase; });
742
+ },
743
+ log: async (message) => {
744
+ countCall();
745
+ await store.update(runID, (current) => {
746
+ current.logs.push({ time: Date.now(), text: bounded(String(message), 2_000) });
747
+ if (current.logs.length > 200) current.logs.splice(0, current.logs.length - 200);
748
+ });
749
+ },
750
+ checkpoint: async (raw) => {
751
+ countCall();
752
+ const request = raw as { key?: unknown; prompt?: unknown };
753
+ const localKey = String(request.key ?? "");
754
+ const key = `${prefix}${localKey}`;
755
+ const prompt = bounded(String(request.prompt ?? ""), 10_000);
756
+ if (!localKey || !prompt) throw new Error("Checkpoint requires non-empty key and prompt");
757
+ const checkpoint = await store.update(runID, (current) => {
758
+ const checkpoint = current.checkpoints.find((item) => item.key === key);
759
+ if (checkpoint && checkpoint.prompt !== prompt) throw new Error(`Checkpoint ${key} changed on resume`);
760
+ if (!checkpoint) current.checkpoints.push({ key, prompt });
761
+ if (checkpoint?.response === undefined && current.status === "running") current.status = "waiting";
762
+ });
763
+ const recorded = checkpoint.checkpoints.find((item) => item.key === key)!;
764
+ const reconcileSettlement = async () => {
765
+ await refreshSettlements();
766
+ const settlement = recordedSettlement("checkpoint", key);
767
+ if (settlement?.kind !== "checkpoint") return undefined;
768
+ const reconciled = await store.update(runID, (current) => {
769
+ const item = current.checkpoints.find((candidate) => candidate.key === key);
770
+ if (!item) throw new Error(`Checkpoint ${key} settlement has no matching checkpoint`);
771
+ if (item.response !== undefined && JSON.stringify(item.response) !== JSON.stringify(settlement.response)) {
772
+ throw new Error(`Checkpoint ${key} response conflicts with its durable settlement`);
773
+ }
774
+ item.response = settlement.response;
775
+ if (current.status === "running" || current.status === "waiting") {
776
+ current.status = current.checkpoints.some((candidate) => candidate.response === undefined) ? "waiting" : "running";
777
+ }
778
+ });
779
+ notify(runID);
780
+ const ordered = await consumeSettlement("checkpoint", key);
781
+ return ordered.kind === "checkpoint"
782
+ ? ordered.response
783
+ : reconciled.checkpoints.find((item) => item.key === key)!.response;
784
+ };
785
+ if (recorded.response !== undefined) {
786
+ const response = await reconcileSettlement();
787
+ if (response === undefined) throw new Error(`Checkpoint ${key} has a response but no deterministic settlement record`);
788
+ return response;
789
+ }
790
+ for (;;) {
791
+ if (controller.signal.aborted) throw controller.signal.reason;
792
+ const reconciled = await reconcileSettlement();
793
+ if (reconciled !== undefined) return reconciled;
794
+ const current = await store.get(runID);
795
+ const checkpoint = current.checkpoints.find((item) => item.key === key)!;
796
+ if (checkpoint.response !== undefined) {
797
+ const raced = await reconcileSettlement();
798
+ if (raced !== undefined) return raced;
799
+ throw new Error(`Checkpoint ${key} has a response but no deterministic settlement record`);
800
+ }
801
+ await waitForChange(runID, controller.signal);
802
+ }
803
+ },
804
+ workflow: async (input) => {
805
+ countCall();
806
+ if (depth >= 4) throw new Error("Nested workflow depth limit reached (4)");
807
+ if (!options.loadSaved) throw new Error("Saved workflow loading is not configured");
808
+ const scope = `${prefix}workflow:${input.name}`;
809
+ const index = nestedIndexes.get(scope) ?? 0;
810
+ nestedIndexes.set(scope, index + 1);
811
+ const identity = `${scope}:${index}`;
812
+ const key = nestedKey(run.id, identity);
813
+ const existing = await ctx.storage.get(key);
814
+ let script: string;
815
+ if (existing === undefined) {
816
+ const loaded = await options.loadSaved(input.name);
817
+ const pinned = { name: input.name, identity, script: loaded, fingerprint: digest(loaded) };
818
+ await serialized(key, async () => {
819
+ const raced = await ctx.storage.get(key);
820
+ if (raced === undefined) await ctx.storage.set(key, Json.parse(pinned));
821
+ });
822
+ const durable = await ctx.storage.get(key) as Partial<typeof pinned> | undefined;
823
+ if (durable?.name !== input.name || durable.identity !== identity || typeof durable.script !== "string" || durable.fingerprint !== digest(durable.script)) {
824
+ throw new Error(`Nested workflow pin ${identity} is invalid`);
825
+ }
826
+ script = durable.script;
827
+ } else {
828
+ const durable = existing as { name?: unknown; identity?: unknown; script?: unknown; fingerprint?: unknown };
829
+ if (durable.name !== input.name || durable.identity !== identity || typeof durable.script !== "string" || durable.fingerprint !== digest(durable.script)) {
830
+ throw new Error(`Nested workflow pin ${identity} is invalid`);
831
+ }
832
+ script = durable.script;
833
+ }
834
+ parseWorkflow(script);
835
+ return executeWorkflow({
836
+ script,
837
+ args: input.args ?? null,
838
+ signal: controller.signal,
839
+ host: hostFor(`${prefix}workflow:${input.name}:${index}/`, depth + 1),
840
+ maxCalls,
841
+ timeoutMs: remaining,
842
+ });
843
+ },
844
+ });
845
+ try {
846
+ const output = await executeWorkflow({
847
+ script: run.script, args: run.args, signal: controller.signal,
848
+ host: hostFor("", 0), maxCalls, timeoutMs: remaining,
849
+ });
850
+ const latest = await store.get(runID);
851
+ if (latest.status === "stopping" || latest.status === "stopped") {
852
+ await store.update(runID, (current) => { current.status = "stopped"; }, "control");
853
+ } else if (latest.status === "pausing" || latest.status === "paused") {
854
+ await store.update(runID, (current) => { current.status = "paused"; }, "control");
855
+ } else {
856
+ const unresolved = latest.steps.find((step) => step.status !== "completed");
857
+ if (unresolved) throw new Error(`Workflow cannot complete with unresolved step ${unresolved.key} (${unresolved.status})`);
858
+ const adverse = latest.steps.filter(isCompleted).find((step) => step.report.verdict === "FAIL" || step.report.verdict === "INCONCLUSIVE");
859
+ if (adverse) throw new Error(`Step ${adverse.key} reported ${adverse.report.verdict}: ${adverse.report.summary}`);
860
+ await store.update(runID, (current) => {
861
+ current.status = "completed";
862
+ if (output === undefined) delete current.result;
863
+ else current.result = boundedJson(output, "Workflow result");
864
+ delete current.error;
865
+ });
866
+ }
867
+ } catch (error) {
868
+ if (!controller.signal.aborted) controller.abort(error);
869
+ const latest = await store.get(runID);
870
+ await interruptWorkflowWorkers(workers, run.ownerID, latest);
871
+ await Promise.allSettled([...pendingAgents]);
872
+ await store.update(runID, (current) => {
873
+ if (latest.status === "stopping" || latest.status === "stopped") current.status = "stopped";
874
+ else if (latest.status === "paused" || latest.status === "interrupted") return;
875
+ else { current.status = "failed"; current.error = errorText(error); }
876
+ }, "control");
877
+ } finally {
878
+ clearTimeout(timer);
879
+ const latest = await store.get(runID);
880
+ if (terminal.has(latest.status)) await deliver(runID);
881
+ }
882
+ }
883
+
884
+ return {
885
+ async start(ownerID: string, raw: WorkflowStart, runtime: Runtime) {
886
+ await recover(ownerID);
887
+ const input = WorkflowStart.parse(raw);
888
+ const owner = await ctx.session.get({ sessionID: ownerID });
889
+ if (owner.parentID !== undefined || owner.metadata?.opThreads !== undefined) throw new Error("Only a root session may start workflows");
890
+ const script = input.script ?? await options.loadSaved?.(input.name!);
891
+ if (script === undefined) throw new Error("Saved workflow loading is not configured");
892
+ const parsed = parseWorkflow(script);
893
+ const id = runIdentity(ownerID, input.key);
894
+ const fingerprint = workflowHash({ script, args: input.args, limits: {
895
+ concurrency: input.concurrency, maxAgents: input.maxAgents,
896
+ agentTimeoutMs: input.agentTimeoutMs, timeoutMs: input.timeoutMs,
897
+ ...(input.tokenBudget === undefined ? {} : { tokenBudget: input.tokenBudget }),
898
+ }, runtime });
899
+ const now = Date.now();
900
+ const run = WorkflowRun.parse({
901
+ version: 1, id, key: input.key, ownerID, callerAgent: runtime.agent,
902
+ model: runtime.model, projectID: owner.projectID, directory: owner.location.directory,
903
+ name: parsed.meta.name, description: parsed.meta.description, script, args: input.args,
904
+ fingerprint, limits: input, status: "running", created: now, updated: now,
905
+ phase: parsed.meta.phases?.[0]?.title ?? "Starting", steps: [], logs: [], checkpoints: [],
906
+ settlements: [],
907
+ deliveryID: deliveryIdentity(id), delivered: false,
908
+ });
909
+ const admitted = await store.create(run);
910
+ if (admitted.fingerprint !== fingerprint) throw new Error("This workflow key belongs to a different request");
911
+ if (admitted === run) launch(id, runtime);
912
+ return store.get(id);
913
+ },
914
+ async get(ownerID: string, runID: string) {
915
+ await recover(ownerID);
916
+ return owned(ownerID, runID);
917
+ },
918
+ async list(ownerID: string) {
919
+ await recover(ownerID);
920
+ return store.list(ownerID);
921
+ },
922
+ async control(ownerID: string, input: Control) {
923
+ let checkpointReply: { key: string; response: Json } | undefined;
924
+ if (input.action === "resume" && (input.checkpointKey !== undefined || input.response !== undefined)) {
925
+ if (!input.checkpointKey || input.response === undefined) throw new Error("Checkpoint resume requires checkpointKey and response");
926
+ checkpointReply = { key: input.checkpointKey, response: boundedJson(input.response, "Checkpoint response") };
927
+ }
928
+ return serialized(`workflow-control:${input.runID}`, async () => {
929
+ await recover(ownerID);
930
+ let run = await owned(ownerID, input.runID);
931
+ if (input.action === "pause") {
932
+ if (run.status !== "running") throw new Error(`Cannot pause workflow in ${run.status}`);
933
+ run = await store.update(run.id, (current) => { current.status = "pausing"; }, "control");
934
+ if (!run.steps.some((step) => step.status === "running")) run = await store.update(run.id, (current) => { current.status = "paused"; }, "control");
935
+ } else if (input.action === "stop") {
936
+ if (!terminal.has(run.status)) {
937
+ run = await store.update(run.id, (current) => { current.status = "stopping"; }, "control");
938
+ leases.get(run.id)?.controller.abort(new Error("Workflow stopped"));
939
+ await interruptWorkflowWorkers(workers, ownerID, run);
940
+ await leases.get(run.id)?.promise.catch(() => {});
941
+ run = await store.update(run.id, (current) => { current.status = "stopped"; }, "control");
942
+ await deliver(run.id);
943
+ }
944
+ } else {
945
+ if (run.status === "interrupted") {
946
+ await leases.get(run.id)?.promise.catch(() => {});
947
+ run = await owned(ownerID, run.id);
948
+ }
949
+ const recovering = run.status === "interrupted";
950
+ if (checkpointReply !== undefined) {
951
+ const { key, response } = checkpointReply;
952
+ if (terminal.has(run.status) || run.status === "stopping") throw new Error(`Cannot answer a checkpoint in ${run.status}`);
953
+ const stored = await ctx.storage.get(completionKey(run.id));
954
+ run = await store.update(run.id, (current) => {
955
+ if (terminal.has(current.status) || current.status === "stopping") throw new Error(`Cannot answer a checkpoint in ${current.status}`);
956
+ const checkpoint = current.checkpoints.find((item) => item.key === key);
957
+ if (!checkpoint) throw new Error(`Checkpoint ${key} not found`);
958
+ if (checkpoint.response !== undefined && JSON.stringify(checkpoint.response) !== JSON.stringify(response)) throw new Error("Checkpoint already has a different response");
959
+ const settlements = readSettlements(current, stored);
960
+ const existing = settlements.find((item) => item.kind === "checkpoint" && item.key === key);
961
+ if (existing?.kind === "checkpoint" && JSON.stringify(existing.response) !== JSON.stringify(response)) {
962
+ throw new Error("Checkpoint already has a different response");
963
+ }
964
+ current.settlements = existing ? settlements : [...settlements, { kind: "checkpoint", key, response }];
965
+ assertSettlementLimit(current.settlements);
966
+ checkpoint.response = response;
967
+ current.status = current.checkpoints.some((item) => item.response === undefined) ? "waiting" : "running";
968
+ delete current.error;
969
+ });
970
+ } else {
971
+ if (!["paused", "interrupted"].includes(run.status)) {
972
+ if (run.status === "waiting") throw new Error("Waiting workflow requires checkpointKey and response");
973
+ throw new Error(`Cannot resume workflow in ${run.status}`);
974
+ }
975
+ run = await store.update(run.id, (current) => { current.status = "running"; delete current.error; }, "control");
976
+ }
977
+ notify(run.id);
978
+ launch(run.id, { agent: run.callerAgent, model: run.model }, recovering);
979
+ }
980
+ notify(run.id);
981
+ return run;
982
+ });
983
+ },
984
+ async result(workerID: string, raw: WorkflowResult) {
985
+ const session = await ctx.session.get({ sessionID: workerID });
986
+ workerLink(session);
987
+ const metadata = WorkflowWorker.parse(session.metadata?.opWorkflow);
988
+ if (metadata.ownerID === workerID) throw new Error("Invalid workflow worker ownership");
989
+ const run = await owned(metadata.ownerID, metadata.runID);
990
+ if (run.status === "stopping" || terminal.has(run.status)) throw new Error(`Workflow ${run.id} is no longer accepting results`);
991
+ const step = run.steps.find((item) => item.workerID === workerID && item.key === metadata.stepKey);
992
+ if (!step) throw new Error("Workflow result does not match its server-recorded step");
993
+ const input = WorkflowResult.parse(raw);
994
+ if (input.summary.length > 20_000 || input.evidence.length > 100 || input.evidence.some((item) => item.length > 20_000)) {
995
+ throw new Error("Workflow report exceeds its bounded summary or evidence limits");
996
+ }
997
+ boundedJson(input, "Workflow worker report");
998
+ if (step.input.schema !== undefined) {
999
+ const validate = ajv.compile(step.input.schema);
1000
+ if (!validate(input.result)) throw new Error(`Workflow result schema validation failed: ${ajv.errorsText(validate.errors)}`);
1001
+ }
1002
+ await serialized(resultKey(workerID), async () => {
1003
+ const existing = await ctx.storage.get(resultKey(workerID));
1004
+ if (existing !== undefined && JSON.stringify(WorkflowResult.parse(existing)) !== JSON.stringify(input)) throw new Error("This worker already reported a different workflow result");
1005
+ await ctx.storage.set(resultKey(workerID), Json.parse(input));
1006
+ });
1007
+ await workers.reportWorkflow(workerID, input);
1008
+ return { accepted: true as const };
1009
+ },
1010
+ async preparePrompt(sessionID: string, messageID: string) {
1011
+ await workers.prepareWorkflowPrompt(sessionID, messageID);
1012
+ },
1013
+ async prepareContext(sessionID: string) {
1014
+ const session = await ctx.session.get({ sessionID });
1015
+ const metadata = WorkflowWorker.safeParse(session.metadata?.opWorkflow);
1016
+ if (!metadata.success) return;
1017
+ workerLink(session);
1018
+ const run = await owned(metadata.data.ownerID, metadata.data.runID);
1019
+ const step = run.steps.find((item) =>
1020
+ item.workerID === sessionID && item.key === metadata.data.stepKey
1021
+ );
1022
+ if (!step) throw new Error("Workflow worker context does not match its durable step");
1023
+ const lease = leases.get(run.id);
1024
+ const leased = lease !== undefined && !lease.controller.signal.aborted && !terminal.has(run.status);
1025
+ watchWorkflowExecution(sessionID, () => ctx.session.wait({ sessionID }));
1026
+ if (!leased && !workflowExecutionAuthorized(sessionID)) {
1027
+ throw new Error(
1028
+ "Workflow worker generation is blocked until its owning workflow is explicitly resumed or the owner sends an authorized follow-up.",
1029
+ );
1030
+ }
1031
+ },
1032
+ async dispose() {
1033
+ disposed = true;
1034
+ const pending: Promise<void>[] = [];
1035
+ for (const [runID, lease] of leases) {
1036
+ if (lease.owner !== engineOwner) continue;
1037
+ await store.update(runID, (run) => {
1038
+ if (!terminal.has(run.status)) {
1039
+ run.status = "interrupted";
1040
+ run.error = "Workflow engine disposed while the run was active; resume explicitly.";
1041
+ }
1042
+ }, "control");
1043
+ lease.controller.abort(new Error("Workflow engine disposed"));
1044
+ pending.push(lease.promise.catch(() => {}));
1045
+ }
1046
+ await Promise.all(pending);
1047
+ },
1048
+ };
1049
+ }