@abelxiaoxing/cadence 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +172 -13
  2. package/agents/design-explorer.md +17 -11
  3. package/agents/diagnosis-worker.md +9 -1
  4. package/agents/implementation-worker.md +3 -1
  5. package/config/plan-draft.example.json +122 -0
  6. package/config/plan-draft.multiple-tasks.example.json +345 -0
  7. package/package.json +9 -5
  8. package/prompts/abel-design.md +141 -38
  9. package/prompts/abel-diagnose.md +10 -1
  10. package/prompts/abel-implement.md +118 -20
  11. package/prompts/abel-init.md +9 -2
  12. package/src/activity-contracts.ts +82 -0
  13. package/src/apply-transaction.ts +16 -14
  14. package/src/artifact-store.ts +17 -0
  15. package/src/candidate-artifact.ts +143 -0
  16. package/src/candidate-patch.ts +33 -12
  17. package/src/candidate-workspace.ts +74 -0
  18. package/src/canonical.ts +16 -0
  19. package/src/change-contract-schema.ts +191 -0
  20. package/src/change-contract.ts +273 -0
  21. package/src/change-verification.ts +896 -0
  22. package/src/child-model.ts +129 -0
  23. package/src/child-session.ts +296 -328
  24. package/src/contracts.ts +325 -109
  25. package/src/control-contracts.ts +51 -1
  26. package/src/delivery-compiler.ts +241 -141
  27. package/src/delivery-revision.ts +42 -0
  28. package/src/design-control.ts +182 -15
  29. package/src/design-diagnostics.ts +150 -0
  30. package/src/design-journal.ts +216 -72
  31. package/src/durable-contracts.ts +210 -0
  32. package/src/durable-workflow.ts +963 -0
  33. package/src/evidence-draft.ts +122 -0
  34. package/src/implement-graph.ts +14 -16
  35. package/src/implement-plan.ts +70 -0
  36. package/src/index.ts +341 -1405
  37. package/src/isolation-backend.ts +84 -23
  38. package/src/model-source.ts +22 -0
  39. package/src/package-candidate.ts +224 -0
  40. package/src/package-delivery.ts +486 -0
  41. package/src/package-verification.ts +509 -0
  42. package/src/package-workflow.ts +253 -0
  43. package/src/packet-runtime.ts +27 -48
  44. package/src/parent-provider.ts +86 -198
  45. package/src/phase-execution.ts +2248 -0
  46. package/src/pi-adapter.ts +19 -0
  47. package/src/plan-draft-summary.ts +146 -0
  48. package/src/plan-draft.ts +493 -0
  49. package/src/route-policy.ts +1 -11
  50. package/src/run-state.ts +4 -16
  51. package/src/run-store.ts +5 -144
  52. package/src/scoped-grep-worker.mjs +23 -0
  53. package/src/scoped-tools.ts +312 -116
  54. package/src/sqlite-schema.ts +171 -0
  55. package/src/storage-schema.ts +352 -0
  56. package/src/subagent-activity.ts +46 -67
  57. package/src/submit-tool.ts +40 -61
  58. package/src/task-ledger.ts +14 -55
  59. package/src/transport-budget.ts +32 -0
  60. package/src/verification-capability.ts +298 -102
  61. package/src/verification-environment.ts +213 -0
  62. package/src/verification-identity.ts +77 -0
  63. package/src/worker-broker.ts +134 -91
  64. package/src/workflow-engine.ts +28 -8285
  65. package/src/workflow-policy.ts +1388 -0
  66. package/src/workflow-recovery-policy.ts +95 -0
  67. package/src/workflow-scheduling.ts +105 -0
  68. package/src/workflow-state-machine.ts +4086 -0
  69. package/src/workflow-status.ts +384 -0
  70. package/src/workspace-io-entry.mjs +58 -0
  71. package/src/workspace-io-worker.mjs +991 -0
  72. package/src/workspace-io.ts +84 -0
  73. package/src/workspace-store.ts +125 -19
  74. package/skills/abel-workflow/SKILL.md +0 -205
  75. package/src/empty-resource-loader.ts +0 -38
  76. package/src/parent-payload-bridge.ts +0 -523
@@ -0,0 +1,4086 @@
1
+ import { readFileSync } from "node:fs";
2
+ import path from "node:path";
3
+ import { DatabaseSync } from "node:sqlite";
4
+ import type { WorkflowActivityUpdate } from "./activity-contracts.ts";
5
+ import {
6
+ assertPlanWithinChangeContract,
7
+ normalizeChangeContract,
8
+ retainedChangeContract,
9
+ } from "./change-contract.ts";
10
+ import { LIMITS } from "./contracts.ts";
11
+ import {
12
+ assertControlCommand,
13
+ type ControlCommand,
14
+ type ControlStage,
15
+ } from "./control-contracts.ts";
16
+ import {
17
+ compileImplementPlan,
18
+ DeliveryValidationError,
19
+ type ImplementPlan,
20
+ type PlanTaskDraft,
21
+ } from "./delivery-compiler.ts";
22
+ import { compareDeliveryRevision } from "./delivery-revision.ts";
23
+ import { snapshotFiles } from "./file-snapshot.ts";
24
+ import type { RoutePolicy } from "./route-policy.ts";
25
+ import {
26
+ canonicalJson,
27
+ type RunProjection,
28
+ type RunState,
29
+ } from "./run-state.ts";
30
+ import { type OperationLease, RunStore } from "./run-store.ts";
31
+ import { isSafeRegularFile } from "./safe-path.ts";
32
+ import { configureSqlite, ensureSqliteSchema } from "./sqlite-schema.ts";
33
+ import type { ResolvedStateRoot } from "./state-root.ts";
34
+ import {
35
+ AMENDMENT_SCHEMA,
36
+ ENGINE_ADDITIONS,
37
+ ENGINE_SCHEMA,
38
+ RECOVERY_ADDITIONS,
39
+ RECOVERY_GRANT_SCHEMA,
40
+ RECOVERY_SCHEMA,
41
+ REJECTED_DELIVERY_SCHEMA,
42
+ } from "./storage-schema.ts";
43
+ import { permitsContextRead } from "./submit-tool.ts";
44
+ import {
45
+ type ActiveOperation,
46
+ assertDelivery,
47
+ type BootstrapRow,
48
+ bootstrapAcceptanceHash,
49
+ CHANGE_NAME,
50
+ cancellableRead,
51
+ classifyPersistedContextRequest,
52
+ decideRecoveryAction,
53
+ type EngineOperationRow,
54
+ type EngineRunRow,
55
+ type EngineTaskRow,
56
+ emitWorkflowActivity,
57
+ expectedClassification,
58
+ type HeldOperationLease,
59
+ hash,
60
+ IDENTIFIER,
61
+ isCancellationException,
62
+ isRecord,
63
+ isWorkflowApprovalCode,
64
+ normalizeBootstrapAcceptanceFacts,
65
+ normalizeWorkflowContextRequest,
66
+ normalizeWorkflowVerificationStatus,
67
+ parseJsonRecord,
68
+ parsePlan,
69
+ phases,
70
+ recoveryKey,
71
+ type SafeAttemptDiagnostic,
72
+ SHA256,
73
+ type WorkflowApplication,
74
+ type WorkflowApplicationContext,
75
+ type WorkflowAttemptOutcome,
76
+ type WorkflowAvailableDelivery,
77
+ type WorkflowChangeVerifier,
78
+ type WorkflowContextRequest,
79
+ type WorkflowDelivery,
80
+ type WorkflowDeliverySource,
81
+ type WorkflowEngineOptions,
82
+ type WorkflowRecoveryFact,
83
+ type WorkflowRunLifecycle,
84
+ type WorkflowVerificationStatus,
85
+ type WorkflowWorker,
86
+ } from "./workflow-policy.ts";
87
+ import {
88
+ assessRecoveryGrant,
89
+ recoveryExhausted,
90
+ } from "./workflow-recovery-policy.ts";
91
+ import { selectRunnableTasks } from "./workflow-scheduling.ts";
92
+ import {
93
+ firstPausedTask,
94
+ projectWorkflowStatus,
95
+ type WorkflowResourceBudget,
96
+ } from "./workflow-status.ts";
97
+ export class WorkflowEngine {
98
+ readonly #consumerRoot: string;
99
+ readonly #stateRoot: ResolvedStateRoot;
100
+ readonly #runStore: RunStore;
101
+ readonly #database: DatabaseSync;
102
+ readonly #deliverySource: WorkflowDeliverySource;
103
+ readonly #worker: WorkflowWorker;
104
+ readonly #changeVerifier: WorkflowChangeVerifier;
105
+ readonly #application?: WorkflowApplication;
106
+ readonly #lifecycle?: WorkflowRunLifecycle;
107
+ readonly #now: () => number;
108
+ readonly #leaseTtlMs: number;
109
+ readonly #workHardLimit: number;
110
+ #activeTasks = 0;
111
+ readonly #capacityWaiters = new Set<() => void>();
112
+ readonly #active = new Map<string, ActiveOperation>();
113
+ readonly #operationLeases = new Map<string, HeldOperationLease>();
114
+ #orphanRecoveryTimer?: ReturnType<typeof setTimeout>;
115
+ readonly #commands = new Map<
116
+ AbortController,
117
+ { command: ControlCommand; settled: Promise<Record<string, unknown>> }
118
+ >();
119
+ #closing?: Promise<void>;
120
+ readonly #amendments = new Map<string, Promise<Record<string, unknown>>>();
121
+ #closed = false;
122
+
123
+ private constructor(options: WorkflowEngineOptions) {
124
+ this.#consumerRoot = path.resolve(options.consumerRoot);
125
+ this.#stateRoot = options.stateRoot;
126
+ this.#deliverySource = options.deliverySource;
127
+ this.#worker = options.worker;
128
+ this.#changeVerifier = options.changeVerifier;
129
+ this.#application = options.application;
130
+ this.#lifecycle = options.lifecycle;
131
+ this.#now = options.now ?? Date.now;
132
+ this.#leaseTtlMs = options.leaseTtlMs ?? 60_000;
133
+ this.#workHardLimit = options.workHardLimit ?? 512;
134
+ if (!Number.isSafeInteger(this.#workHardLimit) || this.#workHardLimit < 1)
135
+ throw new Error("workflow-work-limit-invalid");
136
+ this.#runStore = RunStore.open(options.stateRoot, { now: this.#now });
137
+ this.#database = new DatabaseSync(options.stateRoot.databasePath);
138
+ try {
139
+ configureSqlite(this.#database);
140
+ ensureSqliteSchema(this.#database, ENGINE_SCHEMA, ENGINE_ADDITIONS);
141
+ ensureSqliteSchema(this.#database, RECOVERY_SCHEMA, RECOVERY_ADDITIONS);
142
+ ensureSqliteSchema(this.#database, RECOVERY_GRANT_SCHEMA);
143
+ ensureSqliteSchema(this.#database, AMENDMENT_SCHEMA);
144
+ ensureSqliteSchema(this.#database, REJECTED_DELIVERY_SCHEMA);
145
+ } catch (error) {
146
+ this.#database.close();
147
+ this.#runStore.close();
148
+ throw error;
149
+ }
150
+ this.#transaction(() => {
151
+ const budgets = this.#database
152
+ .prepare(
153
+ "SELECT run_id FROM workflow_work_budget WHERE recovery_policy = 0",
154
+ )
155
+ .all() as { run_id: string }[];
156
+ for (const { run_id } of budgets) {
157
+ const plans = this.#database
158
+ .prepare(
159
+ "SELECT plan_json FROM workflow_engine_deliveries WHERE run_id = ?",
160
+ )
161
+ .all(run_id) as { plan_json: string }[];
162
+ const high = Math.max(
163
+ 0,
164
+ ...plans.map((row) => this.#phaseCount(JSON.parse(row.plan_json))),
165
+ );
166
+ this.#database
167
+ .prepare(
168
+ "UPDATE workflow_work_budget SET phase_high_water = ?, hard_limit = MAX(hard_limit, max_work, used), recovery_policy = 1 WHERE run_id = ?",
169
+ )
170
+ .run(high, run_id);
171
+ }
172
+ });
173
+ this.#recoverInterruptedOperations();
174
+ this.#recoverTerminalCleanup();
175
+ }
176
+
177
+ static open(options: WorkflowEngineOptions): WorkflowEngine {
178
+ if (
179
+ path.resolve(options.consumerRoot) !== options.stateRoot.consumerRoot ||
180
+ options.stateRoot.consumerRootHash.length !== 64
181
+ ) {
182
+ throw new Error("workflow-engine-root-mismatch");
183
+ }
184
+ if (
185
+ !options.deliverySource ||
186
+ typeof options.deliverySource.load !== "function" ||
187
+ !options.worker ||
188
+ typeof options.worker.runAttempt !== "function" ||
189
+ typeof options.worker.rebind !== "function" ||
190
+ !options.changeVerifier ||
191
+ typeof options.changeVerifier.verify !== "function" ||
192
+ (options.application !== undefined &&
193
+ (typeof options.application.prepare === "function") !==
194
+ (typeof options.application.applyPrepared === "function"))
195
+ ) {
196
+ throw new Error("workflow-engine-service-invalid");
197
+ }
198
+ if (
199
+ options.leaseTtlMs !== undefined &&
200
+ (!Number.isSafeInteger(options.leaseTtlMs) || options.leaseTtlMs < 1)
201
+ ) {
202
+ throw new Error("workflow-engine-lease-invalid");
203
+ }
204
+ return new WorkflowEngine(options);
205
+ }
206
+
207
+ updateRoutePolicy(policy: RoutePolicy): void {
208
+ this.#assertOpen();
209
+ if (!this.#worker.updateRoutePolicy) {
210
+ throw new Error("route-policy-update-unavailable");
211
+ }
212
+ this.#worker.updateRoutePolicy(policy);
213
+ }
214
+
215
+ routePolicyStatus(): Record<string, unknown> | undefined {
216
+ this.#assertOpen();
217
+ return this.#worker.routePolicyStatus?.();
218
+ }
219
+
220
+ #assertOpen(): void {
221
+ if (this.#closed) throw new Error("workflow-engine-closed");
222
+ }
223
+
224
+ #transaction<T>(work: () => T): T {
225
+ this.#database.exec("BEGIN IMMEDIATE");
226
+ try {
227
+ const result = work();
228
+ this.#database.exec("COMMIT");
229
+ return result;
230
+ } catch (error) {
231
+ this.#database.exec("ROLLBACK");
232
+ throw error;
233
+ }
234
+ }
235
+
236
+ #operationKey(runId: string, operationId: string): string {
237
+ return `${runId}\0${operationId}`;
238
+ }
239
+
240
+ #leaseOperationId(runId: string, _operationId: string): string {
241
+ return `engine-run-${hash(runId).slice(0, 40)}`;
242
+ }
243
+
244
+ #holdOperationLease(lease: OperationLease, commandOperationId: string): void {
245
+ const key = this.#operationKey(lease.runId, commandOperationId);
246
+ const existing = this.#operationLeases.get(key);
247
+ if (existing) clearInterval(existing.timer);
248
+ const intervalMs = Math.max(1, Math.floor(this.#leaseTtlMs / 3));
249
+ const held: HeldOperationLease = {
250
+ lease,
251
+ timer: setInterval(() => {
252
+ if (this.#operationLeases.get(key) !== held) return;
253
+ try {
254
+ held.lease = this.#runStore.renewLease(held.lease, this.#leaseTtlMs);
255
+ this.#database
256
+ .prepare(
257
+ `UPDATE workflow_engine_operations
258
+ SET lease_expires_at = ?
259
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'
260
+ AND lease_token = ?`,
261
+ )
262
+ .run(
263
+ held.lease.expiresAt,
264
+ lease.runId,
265
+ commandOperationId,
266
+ lease.token,
267
+ );
268
+ } catch {
269
+ held.error = new Error("lease-fenced");
270
+ clearInterval(held.timer);
271
+ const active = this.#active.get(lease.runId);
272
+ if (active?.operationId === commandOperationId) {
273
+ active.controller.abort(held.error);
274
+ }
275
+ }
276
+ }, intervalMs),
277
+ };
278
+ held.timer.unref?.();
279
+ this.#operationLeases.set(key, held);
280
+ }
281
+
282
+ #operationLease(runId: string, operationId: string): OperationLease {
283
+ const held = this.#operationLeases.get(
284
+ this.#operationKey(runId, operationId),
285
+ );
286
+ if (!held || held.error) throw held?.error ?? new Error("lease-fenced");
287
+ this.#runStore.assertLease(held.lease);
288
+ return held.lease;
289
+ }
290
+
291
+ #releaseOperationLease(runId: string, operationId: string): void {
292
+ const key = this.#operationKey(runId, operationId);
293
+ const held = this.#operationLeases.get(key);
294
+ if (!held) return;
295
+ clearInterval(held.timer);
296
+ this.#operationLeases.delete(key);
297
+ }
298
+
299
+ #assertLeaseInTransaction(lease: OperationLease): void {
300
+ const row = this.#database
301
+ .prepare(
302
+ `SELECT state, lease_token, lease_expires_at
303
+ FROM operations WHERE run_id = ? AND operation_id = ?`,
304
+ )
305
+ .get(lease.runId, lease.operationId) as
306
+ | {
307
+ state: string;
308
+ lease_token: string | null;
309
+ lease_expires_at: number | null;
310
+ }
311
+ | undefined;
312
+ if (
313
+ row?.state !== "running" ||
314
+ row.lease_token !== lease.token ||
315
+ row.lease_expires_at === null ||
316
+ row.lease_expires_at < this.#now()
317
+ ) {
318
+ throw new Error("lease-fenced");
319
+ }
320
+ }
321
+
322
+ #leasedTransaction<T>(lease: OperationLease, work: () => T): T {
323
+ return this.#transaction(() => {
324
+ this.#assertLeaseInTransaction(lease);
325
+ return work();
326
+ });
327
+ }
328
+
329
+ amend(
330
+ change: string,
331
+ batchId: string,
332
+ request: unknown,
333
+ execute: (assertAuthority: () => void) => Promise<Record<string, unknown>>,
334
+ ): Promise<Record<string, unknown>> {
335
+ this.#assertOpen();
336
+ if (this.#closing) throw new Error("workflow-engine-closed");
337
+ const runId = this.#lookupRun("abel-implement", change);
338
+ if (!runId) throw new Error("run-not-found");
339
+ if (this.#amendments.has(runId)) throw new Error("amendment-busy");
340
+ const operationId = `amend-${hash(batchId, canonicalJson(request)).slice(0, 48)}`;
341
+ const pending = Promise.resolve()
342
+ .then(async () => {
343
+ const replay = this.#beginOperation(runId, operationId, "amend");
344
+ if (replay) return replay;
345
+ const lease = this.#operationLease(runId, operationId);
346
+ const assertAuthority = () => {
347
+ this.#assertOpen();
348
+ if (this.#closing) throw new Error("workflow-engine-closed");
349
+ this.#runStore.assertLease(lease);
350
+ const status = this.#statusByRun(runId);
351
+ if (
352
+ !["approval-needed", "paused"].includes(String(status.state)) ||
353
+ (status.decisionBatch as { id?: string } | undefined)?.id !==
354
+ batchId
355
+ )
356
+ throw new Error("amendment-batch-stale");
357
+ };
358
+ try {
359
+ assertAuthority();
360
+ if (
361
+ !isRecord(request) ||
362
+ !["status", "validate-plan-draft"].includes(
363
+ String(request.operation),
364
+ )
365
+ ) {
366
+ this.#leasedTransaction(lease, () => {
367
+ this.#database
368
+ .prepare(
369
+ "INSERT INTO workflow_amendment_budget(run_id, used) VALUES (?, 0) ON CONFLICT DO NOTHING",
370
+ )
371
+ .run(runId);
372
+ const reserved = this.#database
373
+ .prepare(
374
+ "UPDATE workflow_amendment_budget SET used = used + 1 WHERE run_id = ? AND used < 64",
375
+ )
376
+ .run(runId);
377
+ if (reserved.changes !== 1)
378
+ throw new Error("amendment-budget-exhausted");
379
+ });
380
+ }
381
+ this.#assertAmendmentContract(runId, change, request);
382
+ const outcome = await execute(assertAuthority);
383
+ assertAuthority();
384
+ return this.#commitOperation(runId, operationId, outcome);
385
+ } catch (error) {
386
+ this.#interruptOperation(runId, operationId);
387
+ throw error;
388
+ }
389
+ })
390
+ .finally(() => {
391
+ if (this.#amendments.get(runId) === pending)
392
+ this.#amendments.delete(runId);
393
+ });
394
+ this.#amendments.set(runId, pending);
395
+ return pending;
396
+ }
397
+
398
+ #assertAmendmentContract(
399
+ runId: string,
400
+ change: string,
401
+ request: unknown,
402
+ ): void {
403
+ if (!isRecord(request)) throw new Error("amendment-contract-invalid");
404
+ if (
405
+ (request.operation === "approve-gate" && request.gate === "gate-a") ||
406
+ (request.operation === "record-decision" &&
407
+ request.category === "behavior")
408
+ )
409
+ throw new Error("amendment-changes-accepted-behavior");
410
+ if (
411
+ ![
412
+ "write-artifact",
413
+ "delete-artifact",
414
+ "compile-plan",
415
+ "validate-plan-draft",
416
+ "finalize-delivery",
417
+ ].includes(String(request.operation))
418
+ )
419
+ return;
420
+ const relative = String(request.path);
421
+ if (relative === "proposal.md" || relative.startsWith("specs/")) {
422
+ const file = `openspec/changes/${change}/${relative}`;
423
+ if (
424
+ request.operation !== "write-artifact" ||
425
+ !isSafeRegularFile(this.#consumerRoot, file) ||
426
+ readFileSync(path.join(this.#consumerRoot, file), "utf8") !==
427
+ request.content
428
+ )
429
+ throw new Error("amendment-changes-accepted-behavior");
430
+ }
431
+ const compile = [
432
+ "compile-plan",
433
+ "validate-plan-draft",
434
+ "finalize-delivery",
435
+ ].includes(String(request.operation));
436
+ if (!compile) return;
437
+ const draftPath = `openspec/changes/${change}/plan-draft.json`;
438
+ if (!isSafeRegularFile(this.#consumerRoot, draftPath)) return;
439
+ const run = this.#engineRun(runId);
440
+ if (run.current_revision === null) return;
441
+ const prior = this.#planForRevision(runId, run.current_revision);
442
+ const contract = retainedChangeContract(prior);
443
+ const draft = JSON.parse(
444
+ readFileSync(path.join(this.#consumerRoot, draftPath), "utf8"),
445
+ );
446
+ if (
447
+ draft.changeContract !== undefined &&
448
+ canonicalJson(normalizeChangeContract(draft.changeContract)) !==
449
+ canonicalJson(contract)
450
+ )
451
+ throw new Error("change-contract-mismatch");
452
+ // Match DesignController's inheritance before validating authorized modes.
453
+ if (draft.changeContract === undefined && prior.changeContract)
454
+ draft.changeContract = contract;
455
+ const next = compileImplementPlan(draft, {
456
+ consumerRoot: this.#consumerRoot,
457
+ bindExecutionInputs: true,
458
+ }).plan;
459
+ assertPlanWithinChangeContract(contract, next.tasks, [
460
+ next.verification.change.fullSuite,
461
+ next.verification.change.postApply,
462
+ ]);
463
+ }
464
+
465
+ #lookupRun(stage: ControlStage, change: string): string | undefined {
466
+ return (
467
+ this.#database
468
+ .prepare(
469
+ `SELECT run_id FROM runs
470
+ WHERE root_hash = ? AND stage = ? AND lookup_key = ?`,
471
+ )
472
+ .get(this.#stateRoot.consumerRootHash, stage, `change:${change}`) as
473
+ | { run_id: string }
474
+ | undefined
475
+ )?.run_id;
476
+ }
477
+
478
+ #ensureEngineRun(runId: string): void {
479
+ this.#database
480
+ .prepare(
481
+ `INSERT INTO workflow_engine_runs(
482
+ run_id, current_revision, baseline_workspace_revision,
483
+ current_workspace_revision, cleanup_state, next_queue_position
484
+ ) VALUES (?, NULL, NULL, NULL, 'none', 1)
485
+ ON CONFLICT(run_id) DO NOTHING`,
486
+ )
487
+ .run(runId);
488
+ }
489
+
490
+ #engineRun(runId: string): EngineRunRow {
491
+ const row = this.#database
492
+ .prepare(
493
+ `SELECT current_revision, baseline_workspace_revision,
494
+ current_workspace_revision, cleanup_state, route_id,
495
+ route_fingerprint,
496
+ transaction_id, verification_json, delivery_diagnostics_json,
497
+ next_queue_position
498
+ FROM workflow_engine_runs WHERE run_id = ?`,
499
+ )
500
+ .get(runId) as EngineRunRow | undefined;
501
+ if (!row) throw new Error("workflow-engine-run-unavailable");
502
+ return row;
503
+ }
504
+
505
+ #operation(
506
+ runId: string,
507
+ operationId: string,
508
+ ): EngineOperationRow | undefined {
509
+ return this.#database
510
+ .prepare(
511
+ `SELECT command, state, outcome_json FROM workflow_engine_operations
512
+ WHERE run_id = ? AND operation_id = ?`,
513
+ )
514
+ .get(runId, operationId) as EngineOperationRow | undefined;
515
+ }
516
+
517
+ #operationReplay(
518
+ runId: string,
519
+ operationId: string,
520
+ command: string,
521
+ ): Record<string, unknown> | undefined {
522
+ const row = this.#operation(runId, operationId);
523
+ if (row && row.command !== command) {
524
+ throw new Error("operation-command-mismatch");
525
+ }
526
+ return row?.state === "committed" && row.outcome_json
527
+ ? parseJsonRecord(row.outcome_json)
528
+ : undefined;
529
+ }
530
+
531
+ #beginOperation(
532
+ runId: string,
533
+ operationId: string,
534
+ command: string,
535
+ ): Record<string, unknown> | undefined {
536
+ if (!IDENTIFIER.test(operationId)) throw new Error("invalid-operation-id");
537
+ const replay = this.#operationReplay(runId, operationId, command);
538
+ if (replay) return replay;
539
+ const existing = this.#operation(runId, operationId);
540
+ if (existing?.state === "running") {
541
+ const leaseStatus = this.#runStore.inspectOperation(
542
+ runId,
543
+ this.#leaseOperationId(runId, operationId),
544
+ );
545
+ if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
546
+ this.#database
547
+ .prepare(
548
+ `UPDATE workflow_engine_operations
549
+ SET state = 'committed', lease_token = NULL,
550
+ lease_expires_at = NULL, outcome_json = ?
551
+ WHERE run_id = ? AND operation_id = ?`,
552
+ )
553
+ .run(JSON.stringify(leaseStatus.outcome), runId, operationId);
554
+ return structuredClone(leaseStatus.outcome);
555
+ }
556
+ if (leaseStatus?.state === "running") {
557
+ throw new Error("operation-already-running");
558
+ }
559
+ this.#database
560
+ .prepare(
561
+ `UPDATE workflow_engine_operations
562
+ SET state = 'interrupted', lease_token = NULL,
563
+ lease_expires_at = NULL
564
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
565
+ )
566
+ .run(runId, operationId);
567
+ }
568
+ const leaseOperationId = this.#leaseOperationId(runId, operationId);
569
+ const priorLease = this.#runStore.inspectOperation(runId, leaseOperationId);
570
+ const preempt = command === "cancel" || command === "discard";
571
+ if (priorLease?.state === "running" && !preempt) {
572
+ throw new Error("operation-already-running");
573
+ }
574
+ const lease = this.#runStore.acquireLease({
575
+ runId,
576
+ operationId: leaseOperationId,
577
+ ttlMs: this.#leaseTtlMs,
578
+ exclusive: true,
579
+ ...(preempt ? { preempt: true } : {}),
580
+ });
581
+ if (preempt) {
582
+ this.#database
583
+ .prepare(
584
+ `UPDATE workflow_engine_operations
585
+ SET state = 'interrupted', lease_token = NULL,
586
+ lease_expires_at = NULL
587
+ WHERE run_id = ? AND operation_id <> ? AND state = 'running'`,
588
+ )
589
+ .run(runId, operationId);
590
+ }
591
+ this.#database
592
+ .prepare(
593
+ `INSERT INTO workflow_engine_operations(
594
+ run_id, operation_id, command, state, lease_token,
595
+ lease_expires_at, outcome_json
596
+ ) VALUES (?, ?, ?, 'running', ?, ?, NULL)
597
+ ON CONFLICT(run_id, operation_id) DO UPDATE SET
598
+ command = excluded.command, state = 'running',
599
+ lease_token = excluded.lease_token,
600
+ lease_expires_at = excluded.lease_expires_at,
601
+ outcome_json = NULL`,
602
+ )
603
+ .run(runId, operationId, command, lease.token, lease.expiresAt);
604
+ this.#holdOperationLease(lease, operationId);
605
+ return undefined;
606
+ }
607
+
608
+ #commitOperation(
609
+ runId: string,
610
+ operationId: string,
611
+ outcome: Record<string, unknown>,
612
+ ): Record<string, unknown> {
613
+ const lease = this.#operationLease(runId, operationId);
614
+ try {
615
+ const committed = this.#runStore.commitLease(lease, outcome);
616
+ const result = this.#database
617
+ .prepare(
618
+ `UPDATE workflow_engine_operations
619
+ SET state = 'committed', lease_token = NULL,
620
+ lease_expires_at = NULL, outcome_json = ?
621
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'
622
+ AND lease_token = ?`,
623
+ )
624
+ .run(JSON.stringify(committed), runId, operationId, lease.token);
625
+ if (Number(result.changes) !== 1) {
626
+ throw new Error("operation-journal-fenced");
627
+ }
628
+ return structuredClone(committed);
629
+ } finally {
630
+ this.#releaseOperationLease(runId, operationId);
631
+ }
632
+ }
633
+
634
+ #interruptOperation(runId: string, operationId: string): void {
635
+ let lease: OperationLease;
636
+ try {
637
+ lease = this.#operationLease(runId, operationId);
638
+ this.#runStore.interruptLease(lease);
639
+ this.#database
640
+ .prepare(
641
+ `UPDATE workflow_engine_operations
642
+ SET state = 'interrupted', lease_token = NULL,
643
+ lease_expires_at = NULL
644
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'
645
+ AND lease_token = ?`,
646
+ )
647
+ .run(runId, operationId, lease.token);
648
+ } catch (error) {
649
+ if (!(error instanceof Error) || error.message !== "lease-fenced") {
650
+ throw error;
651
+ }
652
+ } finally {
653
+ this.#releaseOperationLease(runId, operationId);
654
+ }
655
+ }
656
+
657
+ #transition(
658
+ runId: string,
659
+ to: RunState,
660
+ identity: string,
661
+ code?: string,
662
+ lease?: OperationLease,
663
+ ): RunProjection {
664
+ if (lease) this.#runStore.assertLease(lease);
665
+ const current = this.#runStore.status(runId);
666
+ if (
667
+ current.state === to &&
668
+ (to !== "paused" || code === undefined || current.pauseCode === code)
669
+ ) {
670
+ return current;
671
+ }
672
+ return this.#runStore.transition({
673
+ runId,
674
+ to,
675
+ operationId: `eng-${hash(runId, identity, String(current.sequence), to).slice(0, 40)}`,
676
+ ...(code ? { code } : {}),
677
+ ...(lease ? { lease } : {}),
678
+ });
679
+ }
680
+
681
+ #recoverInterruptedOperations(): void {
682
+ if (this.#orphanRecoveryTimer) {
683
+ clearTimeout(this.#orphanRecoveryTimer);
684
+ this.#orphanRecoveryTimer = undefined;
685
+ }
686
+ let nextExpiry: number | undefined;
687
+ const interrupted = this.#database
688
+ .prepare(
689
+ `SELECT run_id, operation_id FROM workflow_engine_operations
690
+ WHERE state = 'running'`,
691
+ )
692
+ .all() as unknown as Array<{ run_id: string; operation_id: string }>;
693
+ for (const row of interrupted) {
694
+ const leaseStatus = this.#runStore.inspectOperation(
695
+ row.run_id,
696
+ this.#leaseOperationId(row.run_id, row.operation_id),
697
+ );
698
+ if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
699
+ this.#database
700
+ .prepare(
701
+ `UPDATE workflow_engine_operations
702
+ SET state = 'committed', lease_token = NULL,
703
+ lease_expires_at = NULL, outcome_json = ?
704
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
705
+ )
706
+ .run(
707
+ JSON.stringify(leaseStatus.outcome),
708
+ row.run_id,
709
+ row.operation_id,
710
+ );
711
+ continue;
712
+ }
713
+ if (leaseStatus?.state === "running") {
714
+ if (leaseStatus.expiresAt !== undefined) {
715
+ nextExpiry = Math.min(nextExpiry ?? Infinity, leaseStatus.expiresAt);
716
+ }
717
+ continue;
718
+ }
719
+ this.#database
720
+ .prepare(
721
+ `UPDATE workflow_engine_operations
722
+ SET state = 'interrupted', lease_token = NULL,
723
+ lease_expires_at = NULL
724
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
725
+ )
726
+ .run(row.run_id, row.operation_id);
727
+ const projection = this.#runStore.status(row.run_id);
728
+ if (["completed", "discarded", "rejected"].includes(projection.state)) {
729
+ continue;
730
+ }
731
+ this.#database
732
+ .prepare(
733
+ `UPDATE workflow_engine_tasks
734
+ SET state = 'paused',
735
+ pause_code = CASE
736
+ WHEN pause_code = 'red-artifact-constraint'
737
+ THEN pause_code
738
+ ELSE 'operation-interrupted'
739
+ END,
740
+ context_request_json = CASE
741
+ WHEN pause_code = 'red-artifact-constraint'
742
+ THEN context_request_json
743
+ ELSE NULL
744
+ END,
745
+ queue_position = NULL
746
+ WHERE run_id = ? AND state IN ('phase-running', 'validating')`,
747
+ )
748
+ .run(row.run_id);
749
+ if (projection.state === "applying") {
750
+ this.#transition(
751
+ row.run_id,
752
+ "recovering",
753
+ "restart-apply-recovery",
754
+ "operation-interrupted",
755
+ );
756
+ } else if (
757
+ [
758
+ "created",
759
+ "validating-delivery",
760
+ "ready",
761
+ "queued",
762
+ "connecting",
763
+ "running",
764
+ "validating",
765
+ "verifying",
766
+ "retryable",
767
+ "approval-needed",
768
+ "change-verifying",
769
+ "ready-to-apply",
770
+ ].includes(projection.state)
771
+ ) {
772
+ this.#transition(
773
+ row.run_id,
774
+ "paused",
775
+ "restart-operation-interrupted",
776
+ "operation-interrupted",
777
+ );
778
+ }
779
+ }
780
+ if (nextExpiry !== undefined && !this.#closed) {
781
+ const delay = Math.max(1, nextExpiry - this.#now() + 1);
782
+ this.#orphanRecoveryTimer = setTimeout(() => {
783
+ this.#orphanRecoveryTimer = undefined;
784
+ if (this.#closed) return;
785
+ try {
786
+ this.#recoverInterruptedOperations();
787
+ } catch {
788
+ // A later status/restart will surface durable journal corruption.
789
+ }
790
+ }, delay);
791
+ this.#orphanRecoveryTimer.unref?.();
792
+ }
793
+ }
794
+
795
+ #recoverTerminalCleanup(): void {
796
+ const rows = this.#database
797
+ .prepare(
798
+ `SELECT workflow_engine_runs.run_id
799
+ FROM workflow_engine_runs
800
+ JOIN runs ON runs.run_id = workflow_engine_runs.run_id
801
+ WHERE runs.state IN ('completed', 'discarded')
802
+ AND workflow_engine_runs.cleanup_state = 'retained'`,
803
+ )
804
+ .all() as unknown as Array<{ run_id: string }>;
805
+ for (const row of rows) {
806
+ try {
807
+ this.#cleanupTerminal(row.run_id, "restart-cleanup");
808
+ } catch {
809
+ // Terminal status remains truthful and a later reload retries cleanup.
810
+ }
811
+ }
812
+ }
813
+
814
+ #tasks(runId: string): EngineTaskRow[] {
815
+ return this.#database
816
+ .prepare(
817
+ `SELECT task_id, task_order, delivery_revision, plan_json, state,
818
+ phase, pause_code, context_request_json, attempt_diagnostic_json,
819
+ route_id, route_fingerprint, queue_position
820
+ FROM workflow_engine_tasks WHERE run_id = ? ORDER BY task_order`,
821
+ )
822
+ .all(runId) as unknown as EngineTaskRow[];
823
+ }
824
+
825
+ #taskPlan(row: EngineTaskRow): PlanTaskDraft {
826
+ return JSON.parse(row.plan_json) as PlanTaskDraft;
827
+ }
828
+
829
+ #currentPlan(runId: string): ImplementPlan {
830
+ const revision = this.#engineRun(runId).current_revision;
831
+ if (revision === null) throw new Error("delivery-not-bound");
832
+ return this.#planForRevision(runId, revision);
833
+ }
834
+
835
+ #planForRevision(runId: string, revision: number): ImplementPlan {
836
+ const row = this.#database
837
+ .prepare(
838
+ `SELECT plan_json FROM workflow_engine_deliveries
839
+ WHERE run_id = ? AND revision = ?`,
840
+ )
841
+ .get(runId, revision) as { plan_json: string } | undefined;
842
+ if (!row) throw new Error("delivery-not-bound");
843
+ return parsePlan(row.plan_json);
844
+ }
845
+
846
+ async #loadDelivery(
847
+ runId: string,
848
+ lease: OperationLease,
849
+ stage: ControlStage,
850
+ change: string,
851
+ deliveryRevision?: number,
852
+ receiptHash?: string,
853
+ signal?: AbortSignal,
854
+ ): Promise<WorkflowDelivery> {
855
+ if (deliveryRevision === undefined && this.#deliverySource.discoverLatest) {
856
+ try {
857
+ const discover = this.#deliverySource.discoverLatest;
858
+ const available = await cancellableRead(
859
+ () => discover.call(this.#deliverySource, { stage, change, signal }),
860
+ signal,
861
+ );
862
+ if (
863
+ available &&
864
+ Number.isSafeInteger(available.deliveryRevision) &&
865
+ available.deliveryRevision > 0 &&
866
+ SHA256.test(available.receiptHash)
867
+ ) {
868
+ deliveryRevision = available.deliveryRevision;
869
+ receiptHash = available.receiptHash;
870
+ }
871
+ } catch {
872
+ // Discovery is optional; authoritative loading still reports diagnostics.
873
+ signal?.throwIfAborted();
874
+ }
875
+ }
876
+ try {
877
+ const delivery = await cancellableRead(
878
+ () =>
879
+ this.#deliverySource.load({
880
+ stage,
881
+ change,
882
+ ...(deliveryRevision === undefined ? {} : { deliveryRevision }),
883
+ ...(receiptHash === undefined ? {} : { receiptHash }),
884
+ ...(signal ? { signal } : {}),
885
+ }),
886
+ signal,
887
+ );
888
+ signal?.throwIfAborted();
889
+ assertDelivery(delivery, stage, change, deliveryRevision, receiptHash);
890
+ return structuredClone(delivery);
891
+ } catch (error) {
892
+ if (
893
+ error instanceof DeliveryValidationError &&
894
+ deliveryRevision !== undefined &&
895
+ receiptHash !== undefined
896
+ ) {
897
+ this.#leasedTransaction(lease, () => {
898
+ this.#database
899
+ .prepare(
900
+ "INSERT INTO workflow_rejected_deliveries(run_id, delivery_revision, receipt_hash) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
901
+ )
902
+ .run(runId, deliveryRevision, receiptHash);
903
+ });
904
+ }
905
+ throw error;
906
+ }
907
+ }
908
+
909
+ #admitDelivery(
910
+ runId: string,
911
+ delivery: WorkflowDelivery,
912
+ lease: OperationLease,
913
+ ): void {
914
+ this.#runStore.assertLease(lease);
915
+ const existing = this.#database
916
+ .prepare(
917
+ `SELECT receipt_hash, plan_json FROM workflow_engine_deliveries
918
+ WHERE run_id = ? AND revision = ?`,
919
+ )
920
+ .get(runId, delivery.revision) as
921
+ | { receipt_hash: string; plan_json: string }
922
+ | undefined;
923
+ const capturedBudget = this.#database
924
+ .prepare("SELECT hard_limit FROM workflow_work_budget WHERE run_id = ?")
925
+ .get(runId) as { hard_limit: number } | undefined;
926
+ if (
927
+ this.#phaseCount(delivery.plan) >
928
+ (capturedBudget?.hard_limit ?? this.#workHardLimit) &&
929
+ !existing
930
+ )
931
+ throw new DeliveryValidationError([
932
+ "verification-work-budget-insufficient",
933
+ ]);
934
+ const serializedPlan = JSON.stringify(delivery.plan);
935
+ if (
936
+ existing &&
937
+ (existing.receipt_hash !== delivery.receiptHash ||
938
+ existing.plan_json !== serializedPlan)
939
+ ) {
940
+ throw new Error("delivery-revision-conflict");
941
+ }
942
+ const engineRun = this.#engineRun(runId);
943
+ if (
944
+ engineRun.current_revision !== null &&
945
+ delivery.revision < engineRun.current_revision
946
+ ) {
947
+ throw new Error("delivery-revision-stale");
948
+ }
949
+ const priorRows = this.#tasks(runId);
950
+ const priorById = new Map(priorRows.map((row) => [row.task_id, row]));
951
+ const priorPlan =
952
+ engineRun.current_revision === null
953
+ ? undefined
954
+ : this.#planForRevision(runId, engineRun.current_revision);
955
+ const comparison = compareDeliveryRevision(
956
+ priorPlan,
957
+ delivery.plan,
958
+ priorRows,
959
+ );
960
+ const { compatible, boundaryExpanded } = comparison;
961
+ let invalidated = comparison.invalidated;
962
+ let revalidatedWorkspace:
963
+ | {
964
+ baselineRevisionId: string;
965
+ currentWorkspaceRevisionId: string;
966
+ }
967
+ | undefined;
968
+ if (
969
+ priorPlan &&
970
+ (invalidated.size > 0 || boundaryExpanded) &&
971
+ engineRun.baseline_workspace_revision &&
972
+ engineRun.current_workspace_revision
973
+ ) {
974
+ if (!this.#worker.revalidateDelivery) {
975
+ if (boundaryExpanded) {
976
+ throw new Error("workflow-delivery-revalidation-unavailable");
977
+ }
978
+ if (
979
+ engineRun.current_workspace_revision !==
980
+ engineRun.baseline_workspace_revision
981
+ ) {
982
+ throw new Error("workflow-delivery-revalidation-unavailable");
983
+ }
984
+ } else {
985
+ const result = this.#worker.revalidateDelivery({
986
+ runId,
987
+ deliveryRevision: delivery.revision,
988
+ plan: structuredClone(delivery.plan),
989
+ tasks: priorRows
990
+ .filter((row) => compatible.has(row.task_id))
991
+ .map((row) => ({
992
+ taskId: row.task_id,
993
+ deliveryRevision: row.delivery_revision,
994
+ state: row.state,
995
+ phase: row.phase,
996
+ contextReadPaths: this.#contextReads(runId, row.task_id),
997
+ })),
998
+ invalidatedTaskIds: [...invalidated].sort(),
999
+ baselineRevisionId: engineRun.baseline_workspace_revision,
1000
+ currentWorkspaceRevisionId: engineRun.current_workspace_revision,
1001
+ });
1002
+ const resultIds = new Set(result.invalidatedTaskIds);
1003
+ if (
1004
+ !Array.isArray(result.invalidatedTaskIds) ||
1005
+ resultIds.size !== result.invalidatedTaskIds.length ||
1006
+ [...invalidated].some((taskId) => !resultIds.has(taskId)) ||
1007
+ [...resultIds].some((taskId) => !priorById.has(taskId)) ||
1008
+ !SHA256.test(result.baselineRevisionId) ||
1009
+ !SHA256.test(result.currentWorkspaceRevisionId)
1010
+ ) {
1011
+ throw new Error("workflow-delivery-revalidation-invalid");
1012
+ }
1013
+ invalidated = resultIds;
1014
+ revalidatedWorkspace = {
1015
+ baselineRevisionId: result.baselineRevisionId,
1016
+ currentWorkspaceRevisionId: result.currentWorkspaceRevisionId,
1017
+ };
1018
+ }
1019
+ }
1020
+ const deliveryBindings = delivery.approvalProofs
1021
+ ? ([
1022
+ {
1023
+ runId,
1024
+ gate: "gate-a" as const,
1025
+ revision: delivery.revision,
1026
+ receiptHash: delivery.receiptHash,
1027
+ approvalProof: structuredClone(delivery.approvalProofs.gateA),
1028
+ operationId: `bind-a-${hash(
1029
+ runId,
1030
+ String(delivery.revision),
1031
+ delivery.approvalProofs.gateA.recordHash,
1032
+ ).slice(0, 40)}`,
1033
+ lease,
1034
+ },
1035
+ {
1036
+ runId,
1037
+ gate: "gate-b" as const,
1038
+ revision: delivery.revision,
1039
+ receiptHash: delivery.receiptHash,
1040
+ approvalProof: structuredClone(delivery.approvalProofs.gateB),
1041
+ operationId: `bind-b-${hash(
1042
+ runId,
1043
+ String(delivery.revision),
1044
+ delivery.approvalProofs.gateB.recordHash,
1045
+ ).slice(0, 40)}`,
1046
+ lease,
1047
+ },
1048
+ ] as const)
1049
+ : ([
1050
+ {
1051
+ runId,
1052
+ gate: delivery.gate,
1053
+ revision: delivery.revision,
1054
+ receiptHash: delivery.receiptHash,
1055
+ operationId: `bind-${hash(
1056
+ runId,
1057
+ String(delivery.revision),
1058
+ delivery.receiptHash,
1059
+ ).slice(0, 40)}`,
1060
+ lease,
1061
+ },
1062
+ ] as const);
1063
+ this.#runStore.bindDeliveriesAtomically(deliveryBindings, (database) => {
1064
+ const engineLease = database
1065
+ .prepare(
1066
+ `SELECT state, lease_expires_at
1067
+ FROM workflow_engine_operations
1068
+ WHERE run_id = ? AND lease_token = ?`,
1069
+ )
1070
+ .get(runId, lease.token) as
1071
+ | { state: string; lease_expires_at: number | null }
1072
+ | undefined;
1073
+ if (
1074
+ engineLease?.state !== "running" ||
1075
+ engineLease.lease_expires_at === null ||
1076
+ engineLease.lease_expires_at < this.#now()
1077
+ ) {
1078
+ throw new Error("lease-fenced");
1079
+ }
1080
+ database
1081
+ .prepare(
1082
+ `INSERT INTO workflow_engine_runs(
1083
+ run_id, current_revision, delivery_diagnostics_json,
1084
+ next_queue_position
1085
+ ) VALUES (?, ?, NULL, 1)
1086
+ ON CONFLICT(run_id) DO UPDATE SET
1087
+ current_revision = excluded.current_revision,
1088
+ verification_json = NULL,
1089
+ delivery_diagnostics_json = NULL`,
1090
+ )
1091
+ .run(runId, delivery.revision);
1092
+ if (revalidatedWorkspace) {
1093
+ database
1094
+ .prepare(
1095
+ `UPDATE workflow_engine_runs
1096
+ SET baseline_workspace_revision = ?,
1097
+ current_workspace_revision = ?, cleanup_state = 'retained'
1098
+ WHERE run_id = ?`,
1099
+ )
1100
+ .run(
1101
+ revalidatedWorkspace.baselineRevisionId,
1102
+ revalidatedWorkspace.currentWorkspaceRevisionId,
1103
+ runId,
1104
+ );
1105
+ }
1106
+ database
1107
+ .prepare(
1108
+ `INSERT INTO workflow_engine_deliveries(
1109
+ run_id, revision, gate, receipt_hash, plan_json
1110
+ ) VALUES (?, ?, ?, ?, ?)
1111
+ ON CONFLICT(run_id, revision) DO NOTHING`,
1112
+ )
1113
+ .run(
1114
+ runId,
1115
+ delivery.revision,
1116
+ delivery.gate,
1117
+ delivery.receiptHash,
1118
+ serializedPlan,
1119
+ );
1120
+
1121
+ if (
1122
+ database
1123
+ .prepare("SELECT 1 FROM workflow_work_budget WHERE run_id = ?")
1124
+ .get(runId)
1125
+ )
1126
+ this.#growWorkBudget(database, runId, delivery.plan);
1127
+ const admitted = new Set<string>();
1128
+ delivery.plan.tasks.forEach((task, taskOrder) => {
1129
+ admitted.add(task.taskId);
1130
+ const taskJson = JSON.stringify(task);
1131
+ const prior = priorById.get(task.taskId);
1132
+ if (!prior) {
1133
+ database
1134
+ .prepare(
1135
+ `INSERT INTO workflow_engine_tasks(
1136
+ run_id, task_id, task_order, delivery_revision, plan_json,
1137
+ state, phase
1138
+ ) VALUES (?, ?, ?, ?, ?, 'pending', ?)`,
1139
+ )
1140
+ .run(
1141
+ runId,
1142
+ task.taskId,
1143
+ taskOrder,
1144
+ delivery.revision,
1145
+ taskJson,
1146
+ phases(task)[0],
1147
+ );
1148
+ return;
1149
+ }
1150
+ if (compatible.has(task.taskId) && !invalidated.has(task.taskId)) {
1151
+ database
1152
+ .prepare(
1153
+ `UPDATE workflow_engine_tasks
1154
+ SET task_order = ?, plan_json = ?
1155
+ WHERE run_id = ? AND task_id = ?`,
1156
+ )
1157
+ .run(taskOrder, taskJson, runId, task.taskId);
1158
+ return;
1159
+ }
1160
+ database
1161
+ .prepare(
1162
+ "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1163
+ )
1164
+ .run(runId, task.taskId);
1165
+ database
1166
+ .prepare(
1167
+ `UPDATE workflow_engine_tasks
1168
+ SET task_order = ?, delivery_revision = ?, plan_json = ?,
1169
+ state = 'pending', phase = ?, pause_code = NULL,
1170
+ context_request_json = NULL, attempt_diagnostic_json = NULL,
1171
+ queue_position = NULL
1172
+ WHERE run_id = ? AND task_id = ?`,
1173
+ )
1174
+ .run(
1175
+ taskOrder,
1176
+ delivery.revision,
1177
+ taskJson,
1178
+ phases(task)[0],
1179
+ runId,
1180
+ task.taskId,
1181
+ );
1182
+ });
1183
+ for (const row of priorRows) {
1184
+ if (!admitted.has(row.task_id)) {
1185
+ database
1186
+ .prepare(
1187
+ "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1188
+ )
1189
+ .run(runId, row.task_id);
1190
+ database
1191
+ .prepare(
1192
+ `DELETE FROM workflow_engine_tasks
1193
+ WHERE run_id = ? AND task_id = ?`,
1194
+ )
1195
+ .run(runId, row.task_id);
1196
+ }
1197
+ }
1198
+ });
1199
+ }
1200
+
1201
+ #cancelledDelivery(
1202
+ runId: string,
1203
+ operationId: string,
1204
+ error: unknown,
1205
+ signal: AbortSignal | undefined,
1206
+ lease: OperationLease,
1207
+ ): Record<string, unknown> | undefined {
1208
+ if (!signal || !isCancellationException(error, signal)) return undefined;
1209
+ try {
1210
+ this.#runStore.assertLease(lease);
1211
+ if (
1212
+ ["validating-delivery", "ready"].includes(
1213
+ this.#runStore.status(runId).state,
1214
+ )
1215
+ ) {
1216
+ this.#transition(
1217
+ runId,
1218
+ "paused",
1219
+ `${operationId}:delivery-cancelled`,
1220
+ "operation-cancelled",
1221
+ lease,
1222
+ );
1223
+ }
1224
+ } catch (failure) {
1225
+ if (!(failure instanceof Error) || failure.message !== "lease-fenced")
1226
+ throw failure;
1227
+ } finally {
1228
+ this.#interruptOperation(runId, operationId);
1229
+ }
1230
+ return this.#statusByRun(runId);
1231
+ }
1232
+
1233
+ #deliveryFailure(
1234
+ runId: string,
1235
+ operationId: string,
1236
+ error: unknown,
1237
+ lease: OperationLease,
1238
+ ): Record<string, unknown> | undefined {
1239
+ if (!(error instanceof DeliveryValidationError)) return undefined;
1240
+ this.#leasedTransaction(lease, () => {
1241
+ this.#database
1242
+ .prepare(
1243
+ `UPDATE workflow_engine_runs
1244
+ SET delivery_diagnostics_json = ? WHERE run_id = ?`,
1245
+ )
1246
+ .run(JSON.stringify(error.diagnostics), runId);
1247
+ });
1248
+ this.#transition(
1249
+ runId,
1250
+ "paused",
1251
+ `${operationId}:delivery-invalid`,
1252
+ "delivery-invalid",
1253
+ lease,
1254
+ );
1255
+ return {
1256
+ ...this.#statusByRun(runId),
1257
+ delivery: {
1258
+ code: "delivery-invalid",
1259
+ diagnostics: [...error.diagnostics],
1260
+ },
1261
+ };
1262
+ }
1263
+
1264
+ #setTask(
1265
+ runId: string,
1266
+ taskId: string,
1267
+ values: {
1268
+ state?: string;
1269
+ phase?: "red" | "green" | "refactor";
1270
+ pauseCode?: string | null;
1271
+ contextRequest?: WorkflowContextRequest | null;
1272
+ attemptDiagnostic?: SafeAttemptDiagnostic | null;
1273
+ routeId?: string | null;
1274
+ routeFingerprint?: string | null;
1275
+ queuePosition?: number | null;
1276
+ },
1277
+ lease?: OperationLease,
1278
+ ): void {
1279
+ if (values.attemptDiagnostic?.recovery && lease) {
1280
+ const fact = values.attemptDiagnostic.recovery;
1281
+ const prior = this.#recovery(runId, fact.key);
1282
+ if (
1283
+ !prior ||
1284
+ fact.failures > prior.failures ||
1285
+ fact.feedback.maxAttempts < prior.feedback.maxAttempts
1286
+ )
1287
+ this.#recordRecovery(runId, fact, lease);
1288
+ }
1289
+ const assignments: string[] = [];
1290
+ const parameters: Array<string | number | null> = [];
1291
+ if (values.state !== undefined) {
1292
+ assignments.push("state = ?");
1293
+ parameters.push(values.state);
1294
+ }
1295
+ if (values.phase !== undefined) {
1296
+ assignments.push("phase = ?");
1297
+ parameters.push(values.phase);
1298
+ }
1299
+ if (values.pauseCode !== undefined) {
1300
+ assignments.push("pause_code = ?");
1301
+ parameters.push(values.pauseCode);
1302
+ }
1303
+ if (values.contextRequest !== undefined) {
1304
+ assignments.push("context_request_json = ?");
1305
+ parameters.push(
1306
+ values.contextRequest === null
1307
+ ? null
1308
+ : JSON.stringify(
1309
+ normalizeWorkflowContextRequest(values.contextRequest),
1310
+ ),
1311
+ );
1312
+ }
1313
+ if (values.attemptDiagnostic !== undefined) {
1314
+ assignments.push("attempt_diagnostic_json = ?");
1315
+ parameters.push(
1316
+ values.attemptDiagnostic === null
1317
+ ? null
1318
+ : JSON.stringify(
1319
+ Object.fromEntries(
1320
+ Object.entries(values.attemptDiagnostic).filter(
1321
+ ([key]) => key !== "recovery",
1322
+ ),
1323
+ ),
1324
+ ),
1325
+ );
1326
+ }
1327
+ if (values.routeId !== undefined) {
1328
+ assignments.push("route_id = ?");
1329
+ parameters.push(values.routeId);
1330
+ }
1331
+ if (values.routeFingerprint !== undefined) {
1332
+ assignments.push("route_fingerprint = ?");
1333
+ parameters.push(values.routeFingerprint);
1334
+ }
1335
+ if (values.queuePosition !== undefined) {
1336
+ assignments.push("queue_position = ?");
1337
+ parameters.push(values.queuePosition);
1338
+ }
1339
+ if (assignments.length === 0) return;
1340
+ const write = () => {
1341
+ this.#database
1342
+ .prepare(
1343
+ `UPDATE workflow_engine_tasks SET ${assignments.join(", ")}
1344
+ WHERE run_id = ? AND task_id = ?`,
1345
+ )
1346
+ .run(...parameters, runId, taskId);
1347
+ };
1348
+ if (lease) this.#leasedTransaction(lease, write);
1349
+ else write();
1350
+ }
1351
+
1352
+ #setVerificationStatus(
1353
+ runId: string,
1354
+ status: WorkflowVerificationStatus | null,
1355
+ lease: OperationLease,
1356
+ ): void {
1357
+ const serialized =
1358
+ status === null
1359
+ ? null
1360
+ : JSON.stringify(normalizeWorkflowVerificationStatus(status));
1361
+ this.#leasedTransaction(lease, () => {
1362
+ this.#database
1363
+ .prepare(
1364
+ `UPDATE workflow_engine_runs SET verification_json = ?
1365
+ WHERE run_id = ?`,
1366
+ )
1367
+ .run(serialized, runId);
1368
+ });
1369
+ }
1370
+
1371
+ #queueTask(
1372
+ runId: string,
1373
+ row: EngineTaskRow,
1374
+ lease: OperationLease,
1375
+ reason: "conflict" | "capacity" = "conflict",
1376
+ ): void {
1377
+ const pauseCode = reason === "capacity" ? "worker-capacity" : null;
1378
+ if (row.state === "queued" && row.queue_position !== null) {
1379
+ if (row.pause_code !== pauseCode)
1380
+ this.#setTask(runId, row.task_id, { pauseCode }, lease);
1381
+ return;
1382
+ }
1383
+ const engineRun = this.#engineRun(runId);
1384
+ this.#leasedTransaction(lease, () => {
1385
+ this.#setTask(runId, row.task_id, {
1386
+ state: "queued",
1387
+ pauseCode,
1388
+ queuePosition: engineRun.next_queue_position,
1389
+ });
1390
+ this.#database
1391
+ .prepare(
1392
+ `UPDATE workflow_engine_runs
1393
+ SET next_queue_position = next_queue_position + 1
1394
+ WHERE run_id = ?`,
1395
+ )
1396
+ .run(runId);
1397
+ });
1398
+ }
1399
+
1400
+ #recordWorkspaceFacts(
1401
+ runId: string,
1402
+ outcome: WorkflowAttemptOutcome,
1403
+ lease: OperationLease,
1404
+ ): void {
1405
+ const baselineRevisionId = outcome.baselineRevisionId;
1406
+ const currentWorkspaceRevisionId =
1407
+ outcome.kind === "phase-committed"
1408
+ ? outcome.isolatedRevisionId
1409
+ : outcome.currentWorkspaceRevisionId;
1410
+ if (
1411
+ (baselineRevisionId !== undefined && !SHA256.test(baselineRevisionId)) ||
1412
+ (currentWorkspaceRevisionId !== undefined &&
1413
+ !SHA256.test(currentWorkspaceRevisionId)) ||
1414
+ (outcome.kind !== "phase-committed" &&
1415
+ (baselineRevisionId === undefined) !==
1416
+ (currentWorkspaceRevisionId === undefined))
1417
+ ) {
1418
+ throw new Error("workflow-attempt-workspace-fact-invalid");
1419
+ }
1420
+ if (currentWorkspaceRevisionId === undefined) return;
1421
+ this.#leasedTransaction(lease, () => {
1422
+ this.#database
1423
+ .prepare(
1424
+ `UPDATE workflow_engine_runs
1425
+ SET baseline_workspace_revision = COALESCE(
1426
+ baseline_workspace_revision, ?
1427
+ ),
1428
+ current_workspace_revision = ?,
1429
+ cleanup_state = CASE
1430
+ WHEN ? IS NULL THEN cleanup_state ELSE 'retained'
1431
+ END
1432
+ WHERE run_id = ?`,
1433
+ )
1434
+ .run(
1435
+ baselineRevisionId ?? null,
1436
+ currentWorkspaceRevisionId,
1437
+ baselineRevisionId ?? null,
1438
+ runId,
1439
+ );
1440
+ });
1441
+ }
1442
+
1443
+ #recordRouteFacts(
1444
+ runId: string,
1445
+ taskId: string,
1446
+ outcome: WorkflowAttemptOutcome,
1447
+ lease: OperationLease,
1448
+ ): void {
1449
+ if (
1450
+ (outcome.routeId === undefined) !==
1451
+ (outcome.routeFingerprint === undefined) ||
1452
+ (outcome.routeId !== undefined && !IDENTIFIER.test(outcome.routeId)) ||
1453
+ (outcome.routeFingerprint !== undefined &&
1454
+ !SHA256.test(outcome.routeFingerprint))
1455
+ ) {
1456
+ throw new Error("workflow-attempt-route-fact-invalid");
1457
+ }
1458
+ if (!outcome.routeId || !outcome.routeFingerprint) return;
1459
+ const routeId = outcome.routeId;
1460
+ const routeFingerprint = outcome.routeFingerprint;
1461
+ this.#leasedTransaction(lease, () => {
1462
+ this.#setTask(runId, taskId, {
1463
+ routeId,
1464
+ routeFingerprint,
1465
+ });
1466
+ this.#database
1467
+ .prepare(
1468
+ `UPDATE workflow_engine_runs
1469
+ SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
1470
+ )
1471
+ .run(routeId, routeFingerprint, runId);
1472
+ });
1473
+ }
1474
+
1475
+ async #runTask(
1476
+ runId: string,
1477
+ row: EngineTaskRow,
1478
+ operationId: string,
1479
+ signal: AbortSignal,
1480
+ lease: OperationLease,
1481
+ onActivity?: (event: WorkflowActivityUpdate) => void,
1482
+ ): Promise<void> {
1483
+ const task = this.#taskPlan(row);
1484
+ const orderedPhases = phases(task);
1485
+ let phaseIndex = orderedPhases.indexOf(row.phase);
1486
+ let artifactAttempts = 0;
1487
+ const persistedContextRequest = row.context_request_json
1488
+ ? normalizeWorkflowContextRequest(
1489
+ JSON.parse(row.context_request_json) as unknown,
1490
+ )
1491
+ : undefined;
1492
+ const persistedRedArtifactCorrection =
1493
+ row.phase === "green" &&
1494
+ persistedContextRequest !== undefined &&
1495
+ classifyPersistedContextRequest(persistedContextRequest, task, "green")
1496
+ .code === "red-artifact-constraint";
1497
+ let redArtifactCorrection =
1498
+ row.phase === "green" &&
1499
+ (row.pause_code === "red-artifact-constraint" ||
1500
+ persistedRedArtifactCorrection);
1501
+ let requestedContext = persistedContextRequest;
1502
+ let correctionContextRequest = redArtifactCorrection
1503
+ ? persistedContextRequest
1504
+ : undefined;
1505
+ if (phaseIndex < 0) throw new Error("workflow-task-phase-invalid");
1506
+ while (phaseIndex < orderedPhases.length) {
1507
+ const phase = orderedPhases[phaseIndex];
1508
+ row.phase = phase;
1509
+ if (signal.aborted) {
1510
+ this.#setTask(
1511
+ runId,
1512
+ row.task_id,
1513
+ {
1514
+ state: "paused",
1515
+ phase,
1516
+ pauseCode: "operation-cancelled",
1517
+ contextRequest: null,
1518
+ queuePosition: null,
1519
+ },
1520
+ lease,
1521
+ );
1522
+ return;
1523
+ }
1524
+ const current = this.#engineRun(runId);
1525
+ if (current.current_revision === null)
1526
+ throw new Error("delivery-not-bound");
1527
+ const deliveryRevision = current.current_revision;
1528
+ const plan = this.#planForRevision(runId, deliveryRevision);
1529
+ this.#leasedTransaction(lease, () => {
1530
+ for (const ref of requestedContext?.refs ?? []) {
1531
+ if (
1532
+ ref.kind === "requested-path" &&
1533
+ ref.access === "read" &&
1534
+ permitsContextRead(ref.path, task.roots) &&
1535
+ (current.current_workspace_revision
1536
+ ? current.baseline_workspace_revision &&
1537
+ this.#worker.isContextReadAvailable?.({
1538
+ runId,
1539
+ deliveryRevision,
1540
+ plan,
1541
+ baselineRevisionId: current.baseline_workspace_revision,
1542
+ currentWorkspaceRevisionId:
1543
+ current.current_workspace_revision,
1544
+ path: ref.path,
1545
+ })
1546
+ : isSafeRegularFile(this.#consumerRoot, ref.path))
1547
+ ) {
1548
+ this.#database
1549
+ .prepare(
1550
+ "INSERT INTO workflow_context_reads(run_id, task_id, path) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
1551
+ )
1552
+ .run(runId, row.task_id, ref.path);
1553
+ }
1554
+ }
1555
+ });
1556
+ const verificationStatus = current.verification_json
1557
+ ? normalizeWorkflowVerificationStatus(
1558
+ JSON.parse(current.verification_json) as unknown,
1559
+ )
1560
+ : undefined;
1561
+ const priorDiagnostic = this.#safeAttemptDiagnostic(runId, row);
1562
+ const priorRecovery = priorDiagnostic?.recovery;
1563
+ const activeRecovery =
1564
+ priorRecovery?.key === recoveryKey(task, phase, row, current)
1565
+ ? priorRecovery
1566
+ : undefined;
1567
+ const maxRecoveryAttempts = Math.min(
1568
+ plan.verification.artifactCorrection.maxAttempts,
1569
+ activeRecovery?.feedback.maxAttempts ?? Infinity,
1570
+ );
1571
+ if (
1572
+ activeRecovery &&
1573
+ activeRecovery.feedback.maxAttempts !== maxRecoveryAttempts
1574
+ ) {
1575
+ this.#leasedTransaction(lease, () => {
1576
+ this.#database
1577
+ .prepare(
1578
+ "UPDATE workflow_recovery_incidents SET max_attempts = MIN(max_attempts, ?) WHERE run_id = ? AND incident_key = ?",
1579
+ )
1580
+ .run(maxRecoveryAttempts, runId, activeRecovery.key);
1581
+ });
1582
+ activeRecovery.feedback.maxAttempts = maxRecoveryAttempts;
1583
+ }
1584
+ const granted = this.#hasRecoveryGrant(
1585
+ runId,
1586
+ operationId,
1587
+ recoveryKey(task, phase, row, current),
1588
+ );
1589
+ const verificationOnly =
1590
+ recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted) &&
1591
+ (await this.#worker.hasPendingVerification?.({
1592
+ runId,
1593
+ operationId,
1594
+ deliveryRevision: row.delivery_revision,
1595
+ taskId: row.task_id,
1596
+ phase,
1597
+ task,
1598
+ plan,
1599
+ signal,
1600
+ ...(current.baseline_workspace_revision
1601
+ ? { baselineRevisionId: current.baseline_workspace_revision }
1602
+ : {}),
1603
+ ...(current.current_workspace_revision
1604
+ ? { currentWorkspaceRevisionId: current.current_workspace_revision }
1605
+ : {}),
1606
+ })) === true;
1607
+ if (
1608
+ !verificationOnly &&
1609
+ activeRecovery &&
1610
+ recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted)
1611
+ ) {
1612
+ this.#setTask(
1613
+ runId,
1614
+ row.task_id,
1615
+ {
1616
+ state: "paused",
1617
+ phase,
1618
+ pauseCode: activeRecovery.feedback.code,
1619
+ attemptDiagnostic: priorDiagnostic ?? null,
1620
+ queuePosition: null,
1621
+ },
1622
+ lease,
1623
+ );
1624
+ return;
1625
+ }
1626
+ if (activeRecovery)
1627
+ artifactAttempts = Math.max(artifactAttempts, activeRecovery.failures);
1628
+ const repair =
1629
+ verificationStatus?.scope === "change-task-affected" &&
1630
+ verificationStatus.attribution === "introduced" &&
1631
+ verificationStatus.taskId === row.task_id &&
1632
+ verificationStatus.failureIdentities &&
1633
+ verificationStatus.failureIdentities.length > 0
1634
+ ? {
1635
+ attribution: "introduced" as const,
1636
+ failureIdentities: [...verificationStatus.failureIdentities],
1637
+ }
1638
+ : undefined;
1639
+ const artifactCorrection =
1640
+ redArtifactCorrection && phase === "green"
1641
+ ? {
1642
+ code: "red-artifact-constraint" as const,
1643
+ attempt: artifactAttempts + 1,
1644
+ maxAttempts: maxRecoveryAttempts,
1645
+ ...(correctionContextRequest
1646
+ ? {
1647
+ contextRequest: structuredClone(correctionContextRequest),
1648
+ }
1649
+ : {}),
1650
+ }
1651
+ : undefined;
1652
+ this.#setTask(
1653
+ runId,
1654
+ row.task_id,
1655
+ {
1656
+ state: "phase-running",
1657
+ phase,
1658
+ pauseCode: artifactCorrection?.code ?? null,
1659
+ contextRequest: requestedContext ?? null,
1660
+ attemptDiagnostic: priorDiagnostic ?? null,
1661
+ queuePosition: null,
1662
+ },
1663
+ lease,
1664
+ );
1665
+ if (
1666
+ !this.#reserveWork(
1667
+ runId,
1668
+ plan,
1669
+ recoveryKey(task, phase, row, current),
1670
+ lease,
1671
+ granted ? operationId : undefined,
1672
+ )
1673
+ ) {
1674
+ this.#setTask(
1675
+ runId,
1676
+ row.task_id,
1677
+ {
1678
+ state: "paused",
1679
+ phase,
1680
+ pauseCode: "change-work-budget-exhausted",
1681
+ queuePosition: null,
1682
+ },
1683
+ lease,
1684
+ );
1685
+ return;
1686
+ }
1687
+ let initialReservation = true;
1688
+ let outcome: WorkflowAttemptOutcome;
1689
+ try {
1690
+ outcome = await this.#worker.runAttempt({
1691
+ runId,
1692
+ operationId,
1693
+ deliveryRevision: row.delivery_revision,
1694
+ taskId: row.task_id,
1695
+ phase,
1696
+ task: structuredClone(task),
1697
+ contextReadPaths: this.#contextReads(runId, row.task_id),
1698
+ plan: structuredClone(plan),
1699
+ ...(current.baseline_workspace_revision
1700
+ ? { baselineRevisionId: current.baseline_workspace_revision }
1701
+ : {}),
1702
+ ...(current.current_workspace_revision
1703
+ ? {
1704
+ currentWorkspaceRevisionId: current.current_workspace_revision,
1705
+ }
1706
+ : {}),
1707
+ ...(row.route_id || current.route_id
1708
+ ? { routeId: row.route_id ?? current.route_id ?? undefined }
1709
+ : {}),
1710
+ ...(row.route_fingerprint || current.route_fingerprint
1711
+ ? {
1712
+ routeFingerprint:
1713
+ row.route_fingerprint ??
1714
+ current.route_fingerprint ??
1715
+ undefined,
1716
+ }
1717
+ : {}),
1718
+ ...(repair ? { repair } : {}),
1719
+ ...(artifactCorrection ? { artifactCorrection } : {}),
1720
+ ...(requestedContext ? { contextRequest: requestedContext } : {}),
1721
+ ...(granted ? { additionalAttempt: true } : {}),
1722
+ ...(verificationOnly ? { verificationOnly: true } : {}),
1723
+ ...(activeRecovery
1724
+ ? { recoveryFeedback: activeRecovery.feedback }
1725
+ : {}),
1726
+ signal,
1727
+ recoveryDecision: (request) => {
1728
+ signal.throwIfAborted();
1729
+ this.#runStore.assertLease(lease);
1730
+ return decideRecoveryAction(plan, request, {
1731
+ additionalAttempt: granted,
1732
+ verificationOnly,
1733
+ });
1734
+ },
1735
+ reserveCandidate: () => {
1736
+ signal.throwIfAborted();
1737
+ if (verificationOnly) return false;
1738
+ if (initialReservation) {
1739
+ initialReservation = false;
1740
+ return true;
1741
+ }
1742
+ if (granted) return false;
1743
+ return this.#reserveWork(
1744
+ runId,
1745
+ plan,
1746
+ recoveryKey(task, phase, row, current),
1747
+ lease,
1748
+ );
1749
+ },
1750
+ onActivity: (event) => {
1751
+ const projection = this.#runStore.status(runId);
1752
+ emitWorkflowActivity(onActivity, {
1753
+ ...event,
1754
+ stage: projection.stage,
1755
+ runId,
1756
+ ...(projection.change ? { change: projection.change } : {}),
1757
+ taskId: row.task_id,
1758
+ phase,
1759
+ objective: task.objective,
1760
+ legalCommands: ["status", "cancel", "discard"],
1761
+ });
1762
+ },
1763
+ });
1764
+ } catch (error) {
1765
+ if (isCancellationException(error, signal)) {
1766
+ outcome = { kind: "operation-cancelled", code: "cancelled" };
1767
+ } else {
1768
+ throw error;
1769
+ }
1770
+ }
1771
+ if (!isRecord(outcome) || typeof outcome.kind !== "string") {
1772
+ throw new Error("workflow-attempt-outcome-invalid");
1773
+ }
1774
+ this.#recordWorkspaceFacts(runId, outcome, lease);
1775
+ this.#recordRouteFacts(runId, row.task_id, outcome, lease);
1776
+ if (outcome.routeId && outcome.routeFingerprint) {
1777
+ row.route_id = outcome.routeId;
1778
+ row.route_fingerprint = outcome.routeFingerprint;
1779
+ }
1780
+ if (
1781
+ outcome.kind !== "phase-committed" &&
1782
+ outcome.kind !== "operation-cancelled" &&
1783
+ outcome.verification
1784
+ ) {
1785
+ this.#setVerificationStatus(runId, outcome.verification, lease);
1786
+ }
1787
+ if (
1788
+ outcome.kind !== "phase-committed" &&
1789
+ outcome.kind !== "operation-cancelled" &&
1790
+ outcome.contextRequest
1791
+ ) {
1792
+ requestedContext = normalizeWorkflowContextRequest(
1793
+ outcome.contextRequest,
1794
+ );
1795
+ }
1796
+ if (
1797
+ outcome.kind !== "phase-committed" &&
1798
+ outcome.kind !== "operation-cancelled" &&
1799
+ outcome.attemptDiagnostic
1800
+ ) {
1801
+ const previous = row.attempt_diagnostic_json
1802
+ ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
1803
+ : undefined;
1804
+ const fingerprint = hash(
1805
+ outcome.code,
1806
+ outcome.attemptDiagnostic.finalCategory ?? "unknown",
1807
+ outcome.attemptDiagnostic.schema ?? "unknown",
1808
+ JSON.stringify(outcome.attemptDiagnostic.identityMismatch ?? []),
1809
+ ).slice(0, 16);
1810
+ const sameFailure = previous?.fingerprint === fingerprint;
1811
+ const sameFailureCount = sameFailure
1812
+ ? (previous?.sameFailureCount ?? 1) + 1
1813
+ : 1;
1814
+ const attemptDiagnostic = {
1815
+ ...outcome.attemptDiagnostic,
1816
+ ...(priorDiagnostic?.recovery
1817
+ ? { recovery: priorDiagnostic.recovery }
1818
+ : {}),
1819
+ fingerprint,
1820
+ sameFailureCount,
1821
+ ...(sameFailureCount > 1
1822
+ ? { action: "rebind-or-revise-delivery" as const }
1823
+ : {}),
1824
+ };
1825
+ this.#setTask(runId, row.task_id, { attemptDiagnostic }, lease);
1826
+ row.attempt_diagnostic_json = JSON.stringify(attemptDiagnostic);
1827
+ }
1828
+ if (
1829
+ outcome.kind === "paused" &&
1830
+ outcome.code === "repair-attempts-exhausted"
1831
+ ) {
1832
+ const maxAttempts = plan.verification.repair.maxAttempts;
1833
+ const failures = Math.max(
1834
+ maxAttempts,
1835
+ (activeRecovery?.failures ?? 0) +
1836
+ (granted || verificationOnly ? 1 : 0),
1837
+ );
1838
+ const diagnostic: SafeAttemptDiagnostic = {
1839
+ recovery: {
1840
+ key: recoveryKey(task, phase, row, this.#engineRun(runId)),
1841
+ failures,
1842
+ feedback: {
1843
+ code: outcome.code,
1844
+ attempt: failures + 1,
1845
+ maxAttempts,
1846
+ strategy: "repair-verification",
1847
+ },
1848
+ },
1849
+ };
1850
+ this.#setTask(
1851
+ runId,
1852
+ row.task_id,
1853
+ { attemptDiagnostic: diagnostic },
1854
+ lease,
1855
+ );
1856
+ row.attempt_diagnostic_json = JSON.stringify(diagnostic);
1857
+ }
1858
+ if (
1859
+ outcome.kind === "retryable" &&
1860
+ ["artifact", "stale", "verification"].includes(
1861
+ outcome.retryPolicy ?? "",
1862
+ )
1863
+ ) {
1864
+ const key = recoveryKey(task, phase, row, this.#engineRun(runId));
1865
+ const failures =
1866
+ (activeRecovery?.key === key ? activeRecovery.failures : 0) + 1;
1867
+ const strategy =
1868
+ outcome.code === "needs-task-split" ||
1869
+ outcome.code === "task-split-needed"
1870
+ ? ("compact-patch" as const)
1871
+ : outcome.retryPolicy === "stale"
1872
+ ? ("refresh-candidate" as const)
1873
+ : outcome.retryPolicy === "verification"
1874
+ ? ("repair-verification" as const)
1875
+ : ("revise-candidate" as const);
1876
+ const diagnostic: SafeAttemptDiagnostic = {
1877
+ ...this.#safeAttemptDiagnostic(runId, row),
1878
+ recovery: {
1879
+ key,
1880
+ failures,
1881
+ feedback: {
1882
+ code: outcome.code,
1883
+ attempt: failures + 1,
1884
+ maxAttempts: maxRecoveryAttempts,
1885
+ strategy,
1886
+ ...(outcome.verification?.failureIdentities
1887
+ ? { failureIdentities: outcome.verification.failureIdentities }
1888
+ : {}),
1889
+ },
1890
+ },
1891
+ };
1892
+ this.#setTask(
1893
+ runId,
1894
+ row.task_id,
1895
+ { attemptDiagnostic: diagnostic },
1896
+ lease,
1897
+ );
1898
+ row.attempt_diagnostic_json = JSON.stringify(diagnostic);
1899
+ if (outcome.code === "red-artifact-constraint") {
1900
+ redArtifactCorrection = true;
1901
+ correctionContextRequest = outcome.contextRequest
1902
+ ? normalizeWorkflowContextRequest(outcome.contextRequest)
1903
+ : correctionContextRequest;
1904
+ requestedContext = correctionContextRequest ?? requestedContext;
1905
+ }
1906
+ artifactAttempts = Math.max(artifactAttempts + 1, failures);
1907
+ const maxAttempts = maxRecoveryAttempts;
1908
+ if (artifactAttempts < maxAttempts) {
1909
+ this.#setTask(
1910
+ runId,
1911
+ row.task_id,
1912
+ {
1913
+ state: "retryable",
1914
+ phase,
1915
+ pauseCode: redArtifactCorrection
1916
+ ? "red-artifact-constraint"
1917
+ : outcome.code,
1918
+ contextRequest: requestedContext ?? null,
1919
+ queuePosition: null,
1920
+ },
1921
+ lease,
1922
+ );
1923
+ emitWorkflowActivity(onActivity, {
1924
+ state: "retrying",
1925
+ stage: this.#runStore.status(runId).stage,
1926
+ runId,
1927
+ taskId: row.task_id,
1928
+ phase,
1929
+ objective: task.objective,
1930
+ legalCommands: ["status", "cancel", "discard"],
1931
+ code: outcome.code,
1932
+ attempt: artifactAttempts,
1933
+ maxAttempts,
1934
+ wait: "artifact-correction",
1935
+ });
1936
+ continue;
1937
+ }
1938
+ }
1939
+ if (outcome.kind === "phase-committed") {
1940
+ if (
1941
+ !SHA256.test(outcome.artifactHash) ||
1942
+ !SHA256.test(outcome.isolatedRevisionId) ||
1943
+ (outcome.baselineRevisionId !== undefined &&
1944
+ !SHA256.test(outcome.baselineRevisionId)) ||
1945
+ outcome.classification !== expectedClassification(phase)
1946
+ ) {
1947
+ throw new Error("workflow-attempt-fact-invalid");
1948
+ }
1949
+ this.#leasedTransaction(lease, () => {
1950
+ const resolvedKey = recoveryKey(task, phase, row, current);
1951
+ if (this.#recovery(runId, resolvedKey)) {
1952
+ this.#database
1953
+ .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
1954
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
1955
+ .run(runId, resolvedKey, runId);
1956
+ }
1957
+ this.#database
1958
+ .prepare(
1959
+ "DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
1960
+ )
1961
+ .run(runId, recoveryKey(task, phase, row, current));
1962
+ });
1963
+ if (repair) {
1964
+ this.#setVerificationStatus(runId, null, lease);
1965
+ this.#setTask(
1966
+ runId,
1967
+ row.task_id,
1968
+ {
1969
+ state: "verified",
1970
+ phase,
1971
+ pauseCode: null,
1972
+ contextRequest: null,
1973
+ attemptDiagnostic: null,
1974
+ queuePosition: null,
1975
+ },
1976
+ lease,
1977
+ );
1978
+ return;
1979
+ }
1980
+ artifactAttempts = 0;
1981
+ row.attempt_diagnostic_json = null;
1982
+ redArtifactCorrection = false;
1983
+ requestedContext = undefined;
1984
+ correctionContextRequest = undefined;
1985
+ phaseIndex += 1;
1986
+ if (phaseIndex >= orderedPhases.length) {
1987
+ this.#setTask(
1988
+ runId,
1989
+ row.task_id,
1990
+ {
1991
+ state: "verified",
1992
+ phase,
1993
+ pauseCode: null,
1994
+ contextRequest: null,
1995
+ attemptDiagnostic: null,
1996
+ queuePosition: null,
1997
+ },
1998
+ lease,
1999
+ );
2000
+ return;
2001
+ }
2002
+ this.#setTask(
2003
+ runId,
2004
+ row.task_id,
2005
+ {
2006
+ state: "phase-ready",
2007
+ phase: orderedPhases[phaseIndex],
2008
+ pauseCode: null,
2009
+ contextRequest: null,
2010
+ attemptDiagnostic: null,
2011
+ queuePosition: null,
2012
+ },
2013
+ lease,
2014
+ );
2015
+ continue;
2016
+ }
2017
+ if (outcome.kind === "operation-cancelled") {
2018
+ this.#setTask(
2019
+ runId,
2020
+ row.task_id,
2021
+ {
2022
+ state: "paused",
2023
+ phase,
2024
+ pauseCode: "operation-cancelled",
2025
+ contextRequest: null,
2026
+ queuePosition: null,
2027
+ },
2028
+ lease,
2029
+ );
2030
+ return;
2031
+ }
2032
+ if (
2033
+ outcome.kind === "paused" ||
2034
+ outcome.kind === "retryable" ||
2035
+ outcome.kind === "approval-needed"
2036
+ ) {
2037
+ if (typeof outcome.code !== "string" || outcome.code.length === 0) {
2038
+ throw new Error("workflow-attempt-outcome-invalid");
2039
+ }
2040
+ const approvalCodeInvalid =
2041
+ outcome.kind === "approval-needed" &&
2042
+ !isWorkflowApprovalCode(outcome.code);
2043
+ this.#setTask(
2044
+ runId,
2045
+ row.task_id,
2046
+ {
2047
+ state: approvalCodeInvalid ? "paused" : outcome.kind,
2048
+ phase,
2049
+ pauseCode: approvalCodeInvalid
2050
+ ? "approval-code-invalid"
2051
+ : outcome.code,
2052
+ contextRequest: approvalCodeInvalid
2053
+ ? null
2054
+ : (requestedContext ?? null),
2055
+ queuePosition: null,
2056
+ },
2057
+ lease,
2058
+ );
2059
+ return;
2060
+ }
2061
+ throw new Error("workflow-attempt-outcome-invalid");
2062
+ }
2063
+ }
2064
+
2065
+ #firstPaused(rows: EngineTaskRow[]): EngineTaskRow | undefined {
2066
+ return firstPausedTask(rows);
2067
+ }
2068
+
2069
+ #recoverLegacyContextApproval(runId: string): boolean {
2070
+ const projection = this.#runStore.status(runId);
2071
+ if (!["approval-needed", "paused"].includes(projection.state)) return false;
2072
+ const rows = this.#tasks(runId);
2073
+ const approvalRows = rows.filter((row) => row.state === "approval-needed");
2074
+ if (approvalRows.length === 0) return false;
2075
+ const reclassified = rows.flatMap((row) => {
2076
+ if (
2077
+ row.state !== "approval-needed" ||
2078
+ row.pause_code !== "boundary-review-needed" ||
2079
+ row.context_request_json === null
2080
+ ) {
2081
+ return [];
2082
+ }
2083
+ try {
2084
+ const request = normalizeWorkflowContextRequest(
2085
+ JSON.parse(row.context_request_json) as unknown,
2086
+ );
2087
+ const classified = classifyPersistedContextRequest(
2088
+ request,
2089
+ this.#taskPlan(row),
2090
+ row.phase,
2091
+ );
2092
+ if (classified.kind === "approval-needed") return [];
2093
+ return [
2094
+ {
2095
+ row,
2096
+ state: classified.kind === "retryable" ? "retryable" : "paused",
2097
+ code: classified.code,
2098
+ contextRequest: JSON.stringify(
2099
+ normalizeWorkflowContextRequest(classified.contextRequest),
2100
+ ),
2101
+ },
2102
+ ];
2103
+ } catch {
2104
+ return [];
2105
+ }
2106
+ });
2107
+ const reclassifiedTaskIds = new Set(
2108
+ reclassified.map((entry) => entry.row.task_id),
2109
+ );
2110
+ const retainedApprovals = approvalRows.filter(
2111
+ (row) => !reclassifiedTaskIds.has(row.task_id),
2112
+ );
2113
+ const updateTasks = (database: DatabaseSync) => {
2114
+ for (const entry of reclassified) {
2115
+ const updated = database
2116
+ .prepare(
2117
+ `UPDATE workflow_engine_tasks
2118
+ SET state = ?, phase = ?, pause_code = ?, context_request_json = ?,
2119
+ queue_position = NULL
2120
+ WHERE run_id = ? AND task_id = ?
2121
+ AND state = 'approval-needed'
2122
+ AND pause_code = 'boundary-review-needed'`,
2123
+ )
2124
+ .run(
2125
+ entry.state,
2126
+ entry.row.phase,
2127
+ entry.code,
2128
+ entry.contextRequest,
2129
+ runId,
2130
+ entry.row.task_id,
2131
+ );
2132
+ if (Number(updated.changes) !== 1) {
2133
+ throw new Error("workflow-legacy-context-race");
2134
+ }
2135
+ }
2136
+ };
2137
+ const transitionWithUpdates = (
2138
+ to: "paused" | "approval-needed",
2139
+ code: string,
2140
+ ) =>
2141
+ this.#runStore.transitionAtomically(
2142
+ {
2143
+ runId,
2144
+ to,
2145
+ operationId: `legacy-context-${hash(
2146
+ runId,
2147
+ String(projection.sequence),
2148
+ to,
2149
+ code,
2150
+ JSON.stringify(
2151
+ reclassified.map((entry) => [entry.row.task_id, entry.code]),
2152
+ ),
2153
+ ).slice(0, 40)}`,
2154
+ code,
2155
+ },
2156
+ updateTasks,
2157
+ );
2158
+ if (retainedApprovals.length > 0) {
2159
+ if (projection.state === "approval-needed") {
2160
+ if (reclassified.length === 0) return false;
2161
+ this.#transaction(() => updateTasks(this.#database));
2162
+ } else {
2163
+ transitionWithUpdates(
2164
+ "approval-needed",
2165
+ retainedApprovals[0].pause_code ?? "approval-code-invalid",
2166
+ );
2167
+ }
2168
+ return true;
2169
+ }
2170
+ const first = reclassified[0];
2171
+ if (!first) return false;
2172
+ transitionWithUpdates("paused", first.code);
2173
+ return true;
2174
+ }
2175
+
2176
+ async #finishChange(
2177
+ runId: string,
2178
+ operationId: string,
2179
+ signal: AbortSignal,
2180
+ lease: OperationLease,
2181
+ onActivity?: (event: WorkflowActivityUpdate) => void,
2182
+ ): Promise<Record<string, unknown>> {
2183
+ const plan = this.#currentPlan(runId);
2184
+ let engineRun = this.#engineRun(runId);
2185
+ if (engineRun.current_revision === null)
2186
+ throw new Error("delivery-not-bound");
2187
+ const deliveryRevision = engineRun.current_revision;
2188
+ this.#transition(
2189
+ runId,
2190
+ "change-verifying",
2191
+ `${operationId}:change-verifying`,
2192
+ undefined,
2193
+ lease,
2194
+ );
2195
+ const projection = this.#runStore.status(runId);
2196
+ emitWorkflowActivity(onActivity, {
2197
+ state: "verifying",
2198
+ stage: projection.stage,
2199
+ runId,
2200
+ ...(projection.change ? { change: projection.change } : {}),
2201
+ objective: projection.change
2202
+ ? `verify change ${projection.change}`
2203
+ : "verify workflow change",
2204
+ legalCommands: ["status", "cancel", "discard"],
2205
+ });
2206
+ const verification = await this.#changeVerifier.verify({
2207
+ taskEvidence: this.#tasks(runId).map((row) => ({
2208
+ taskId: row.task_id,
2209
+ deliveryRevision: row.delivery_revision,
2210
+ })),
2211
+ runId,
2212
+ deliveryRevision,
2213
+ plan: structuredClone(plan),
2214
+ ...(engineRun.baseline_workspace_revision
2215
+ ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2216
+ : {}),
2217
+ ...(engineRun.current_workspace_revision
2218
+ ? {
2219
+ currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2220
+ }
2221
+ : {}),
2222
+ signal,
2223
+ });
2224
+ this.#runStore.assertLease(lease);
2225
+ if (verification.kind !== "verified") {
2226
+ if (verification.verification) {
2227
+ this.#setVerificationStatus(runId, verification.verification, lease);
2228
+ }
2229
+ if (
2230
+ verification.code === "verification-repair-required" &&
2231
+ verification.verification?.scope === "change-task-affected" &&
2232
+ verification.verification.attribution === "introduced" &&
2233
+ verification.verification.taskId
2234
+ ) {
2235
+ const task = this.#tasks(runId).find(
2236
+ (candidate) =>
2237
+ candidate.task_id === verification.verification?.taskId,
2238
+ );
2239
+ if (task) {
2240
+ const repairKey = recoveryKey(
2241
+ this.#taskPlan(task),
2242
+ task.phase,
2243
+ task,
2244
+ this.#engineRun(runId),
2245
+ );
2246
+ const used = (
2247
+ this.#database
2248
+ .prepare(
2249
+ "SELECT COUNT(*) AS count FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ? AND kind = 'repair'",
2250
+ )
2251
+ .get(runId, repairKey) as { count: number }
2252
+ ).count;
2253
+ if (used >= plan.verification.repair.maxAttempts) {
2254
+ this.#setTask(
2255
+ runId,
2256
+ task.task_id,
2257
+ {
2258
+ state: "paused",
2259
+ phase: task.phase,
2260
+ pauseCode: "repair-attempts-exhausted",
2261
+ attemptDiagnostic: {
2262
+ recovery: {
2263
+ key: recoveryKey(
2264
+ this.#taskPlan(task),
2265
+ task.phase,
2266
+ task,
2267
+ this.#engineRun(runId),
2268
+ ),
2269
+ failures: used,
2270
+ feedback: {
2271
+ code: "repair-attempts-exhausted",
2272
+ attempt: used + 1,
2273
+ maxAttempts: plan.verification.repair.maxAttempts,
2274
+ strategy: "repair-verification",
2275
+ failureIdentities:
2276
+ verification.verification.failureIdentities,
2277
+ },
2278
+ },
2279
+ },
2280
+ queuePosition: null,
2281
+ },
2282
+ lease,
2283
+ );
2284
+ this.#transition(
2285
+ runId,
2286
+ "paused",
2287
+ `${operationId}:repair-attempts-exhausted:${task.task_id}`,
2288
+ "repair-attempts-exhausted",
2289
+ lease,
2290
+ );
2291
+ return this.#statusByRun(runId);
2292
+ }
2293
+ this.#leasedTransaction(lease, () => {
2294
+ this.#database
2295
+ .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2296
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'repair' FROM workflow_recovery_events WHERE run_id = ?`)
2297
+ .run(runId, repairKey, runId);
2298
+ });
2299
+ const taskPlan = this.#taskPlan(task);
2300
+ this.#setTask(
2301
+ runId,
2302
+ task.task_id,
2303
+ {
2304
+ state: "repairable",
2305
+ phase: taskPlan.phases.refactor ? "refactor" : "green",
2306
+ pauseCode: verification.code,
2307
+ queuePosition: null,
2308
+ },
2309
+ lease,
2310
+ );
2311
+ this.#transition(
2312
+ runId,
2313
+ "running",
2314
+ `${operationId}:repairable:${task.task_id}`,
2315
+ undefined,
2316
+ lease,
2317
+ );
2318
+ return this.#advance(runId, operationId, signal, lease, onActivity);
2319
+ }
2320
+ }
2321
+ const approvalCodeInvalid =
2322
+ verification.kind === "approval-needed" &&
2323
+ !isWorkflowApprovalCode(verification.code);
2324
+ const state =
2325
+ verification.kind === "approval-needed" && !approvalCodeInvalid
2326
+ ? "approval-needed"
2327
+ : "paused";
2328
+ const code = approvalCodeInvalid
2329
+ ? "approval-code-invalid"
2330
+ : verification.code;
2331
+ this.#transition(
2332
+ runId,
2333
+ state,
2334
+ `${operationId}:change-verification-${state}`,
2335
+ code,
2336
+ lease,
2337
+ );
2338
+ return this.#statusByRun(runId);
2339
+ }
2340
+ if (verification.currentWorkspaceRevisionId !== undefined) {
2341
+ const currentWorkspaceRevisionId =
2342
+ verification.currentWorkspaceRevisionId;
2343
+ if (!SHA256.test(currentWorkspaceRevisionId)) {
2344
+ throw new Error("workflow-verification-workspace-fact-invalid");
2345
+ }
2346
+ this.#leasedTransaction(lease, () => {
2347
+ this.#database
2348
+ .prepare(
2349
+ `UPDATE workflow_engine_runs
2350
+ SET current_workspace_revision = ?, cleanup_state = 'retained'
2351
+ WHERE run_id = ?`,
2352
+ )
2353
+ .run(currentWorkspaceRevisionId, runId);
2354
+ });
2355
+ engineRun = this.#engineRun(runId);
2356
+ }
2357
+ this.#transition(
2358
+ runId,
2359
+ "ready-to-apply",
2360
+ `${operationId}:ready-to-apply`,
2361
+ undefined,
2362
+ lease,
2363
+ );
2364
+ if (!this.#application) {
2365
+ this.#transition(
2366
+ runId,
2367
+ "paused",
2368
+ `${operationId}:application-unavailable`,
2369
+ "application-unavailable",
2370
+ lease,
2371
+ );
2372
+ return this.#statusByRun(runId);
2373
+ }
2374
+ const transactionId = `apply-${hash(
2375
+ runId,
2376
+ String(deliveryRevision),
2377
+ operationId,
2378
+ ).slice(0, 40)}`;
2379
+ this.#leasedTransaction(lease, () => {
2380
+ this.#database
2381
+ .prepare(
2382
+ `UPDATE workflow_engine_runs SET transaction_id = ? WHERE run_id = ?`,
2383
+ )
2384
+ .run(transactionId, runId);
2385
+ });
2386
+ const applicationInput = {
2387
+ transactionId,
2388
+ runId,
2389
+ consumerRoot: this.#consumerRoot,
2390
+ deliveryRevision,
2391
+ plan: structuredClone(plan),
2392
+ contextReadPaths: this.#contextReads(runId),
2393
+ ...(engineRun.baseline_workspace_revision
2394
+ ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2395
+ : {}),
2396
+ ...(engineRun.current_workspace_revision
2397
+ ? {
2398
+ currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2399
+ }
2400
+ : {}),
2401
+ signal,
2402
+ };
2403
+ let application: Record<string, unknown>;
2404
+ if (this.#application.prepare && this.#application.applyPrepared) {
2405
+ const prepared = await this.#application.prepare(applicationInput);
2406
+ this.#runStore.assertLease(lease);
2407
+ if (prepared.state !== "prepared") {
2408
+ return this.#settleApplication(
2409
+ runId,
2410
+ operationId,
2411
+ prepared,
2412
+ lease,
2413
+ onActivity,
2414
+ );
2415
+ }
2416
+ this.#transition(
2417
+ runId,
2418
+ "applying",
2419
+ `${operationId}:applying`,
2420
+ undefined,
2421
+ lease,
2422
+ );
2423
+ emitWorkflowActivity(onActivity, {
2424
+ state: "applying",
2425
+ stage: projection.stage,
2426
+ runId,
2427
+ ...(projection.change ? { change: projection.change } : {}),
2428
+ objective: projection.change
2429
+ ? `apply change ${projection.change}`
2430
+ : "apply workflow change",
2431
+ legalCommands: ["status", "cancel", "discard"],
2432
+ });
2433
+ const active = this.#active.get(runId);
2434
+ if (active) active.kind = "applying";
2435
+ try {
2436
+ application = await this.#application.applyPrepared(applicationInput);
2437
+ } catch (error) {
2438
+ this.#runStore.assertLease(lease);
2439
+ if (this.#runStore.status(runId).state === "applying")
2440
+ this.#transition(
2441
+ runId,
2442
+ "recovering",
2443
+ `${operationId}:application-interrupted`,
2444
+ "operation-interrupted",
2445
+ lease,
2446
+ );
2447
+ throw error;
2448
+ }
2449
+ } else {
2450
+ this.#transition(
2451
+ runId,
2452
+ "applying",
2453
+ `${operationId}:applying`,
2454
+ undefined,
2455
+ lease,
2456
+ );
2457
+ emitWorkflowActivity(onActivity, {
2458
+ state: "applying",
2459
+ stage: projection.stage,
2460
+ runId,
2461
+ ...(projection.change ? { change: projection.change } : {}),
2462
+ objective: projection.change
2463
+ ? `apply change ${projection.change}`
2464
+ : "apply workflow change",
2465
+ legalCommands: ["status", "cancel", "discard"],
2466
+ });
2467
+ const active = this.#active.get(runId);
2468
+ if (active) active.kind = "applying";
2469
+ application = await this.#application.begin(applicationInput);
2470
+ }
2471
+ this.#runStore.assertLease(lease);
2472
+ return this.#settleApplication(
2473
+ runId,
2474
+ operationId,
2475
+ application,
2476
+ lease,
2477
+ onActivity,
2478
+ );
2479
+ }
2480
+
2481
+ #settleApplication(
2482
+ runId: string,
2483
+ operationId: string,
2484
+ outcome: Record<string, unknown>,
2485
+ lease: OperationLease,
2486
+ onActivity?: (event: WorkflowActivityUpdate) => void,
2487
+ ): Record<string, unknown> {
2488
+ const state = outcome.state;
2489
+ if (state === "completed") {
2490
+ const current = this.#runStore.status(runId);
2491
+ if (current.state === "recovering") {
2492
+ this.#transition(
2493
+ runId,
2494
+ "completed",
2495
+ `${operationId}:recovered-complete`,
2496
+ undefined,
2497
+ lease,
2498
+ );
2499
+ } else if (
2500
+ current.state === "applying" ||
2501
+ current.state === "ready-to-apply"
2502
+ ) {
2503
+ this.#transition(
2504
+ runId,
2505
+ "completed",
2506
+ `${operationId}:apply-complete`,
2507
+ undefined,
2508
+ lease,
2509
+ );
2510
+ }
2511
+ this.#cleanupTerminal(runId, operationId, lease);
2512
+ } else if (state === "paused") {
2513
+ if (this.#runStore.status(runId).state === "applying") {
2514
+ this.#transition(
2515
+ runId,
2516
+ "recovering",
2517
+ `${operationId}:apply-recovering`,
2518
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2519
+ lease,
2520
+ );
2521
+ const recovering = this.#runStore.status(runId);
2522
+ emitWorkflowActivity(onActivity, {
2523
+ state: "recovering",
2524
+ stage: recovering.stage,
2525
+ runId,
2526
+ ...(recovering.change ? { change: recovering.change } : {}),
2527
+ objective: recovering.change
2528
+ ? `recover change ${recovering.change}`
2529
+ : "recover workflow application",
2530
+ code:
2531
+ typeof outcome.code === "string"
2532
+ ? outcome.code
2533
+ : "recovery-required",
2534
+ legalCommands: ["status", "cancel", "discard"],
2535
+ });
2536
+ }
2537
+ this.#transition(
2538
+ runId,
2539
+ "paused",
2540
+ `${operationId}:apply-paused`,
2541
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2542
+ lease,
2543
+ );
2544
+ } else if (state === "discarded") {
2545
+ if (this.#runStore.status(runId).state === "applying") {
2546
+ this.#transition(
2547
+ runId,
2548
+ "recovering",
2549
+ `${operationId}:discard-recovering`,
2550
+ "run-discarded",
2551
+ lease,
2552
+ );
2553
+ const recovering = this.#runStore.status(runId);
2554
+ emitWorkflowActivity(onActivity, {
2555
+ state: "recovering",
2556
+ stage: recovering.stage,
2557
+ runId,
2558
+ ...(recovering.change ? { change: recovering.change } : {}),
2559
+ objective: recovering.change
2560
+ ? `recover change ${recovering.change}`
2561
+ : "recover workflow application",
2562
+ code: "run-discarded",
2563
+ legalCommands: ["status"],
2564
+ });
2565
+ }
2566
+ this.#transition(
2567
+ runId,
2568
+ "discarded",
2569
+ `${operationId}:apply-discarded`,
2570
+ undefined,
2571
+ lease,
2572
+ );
2573
+ this.#cleanupTerminal(runId, operationId, lease);
2574
+ } else if (state === "recovering") {
2575
+ this.#transition(
2576
+ runId,
2577
+ "recovering",
2578
+ `${operationId}:application-recovering`,
2579
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2580
+ lease,
2581
+ );
2582
+ const recovering = this.#runStore.status(runId);
2583
+ emitWorkflowActivity(onActivity, {
2584
+ state: "recovering",
2585
+ stage: recovering.stage,
2586
+ runId,
2587
+ ...(recovering.change ? { change: recovering.change } : {}),
2588
+ objective: recovering.change
2589
+ ? `recover change ${recovering.change}`
2590
+ : "recover workflow application",
2591
+ code:
2592
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2593
+ legalCommands: ["status", "cancel", "discard"],
2594
+ });
2595
+ } else {
2596
+ throw new Error("workflow-application-outcome-invalid");
2597
+ }
2598
+ return this.#statusByRun(runId);
2599
+ }
2600
+
2601
+ #applicationContext(runId: string): WorkflowApplicationContext {
2602
+ const current = this.#engineRun(runId);
2603
+ if (current.current_revision === null)
2604
+ throw new Error("delivery-not-bound");
2605
+ return {
2606
+ runId,
2607
+ deliveryRevision: current.current_revision,
2608
+ plan: structuredClone(this.#currentPlan(runId)),
2609
+ contextReadPaths: this.#contextReads(runId),
2610
+ ...(current.baseline_workspace_revision
2611
+ ? { baselineRevisionId: current.baseline_workspace_revision }
2612
+ : {}),
2613
+ ...(current.current_workspace_revision
2614
+ ? { currentWorkspaceRevisionId: current.current_workspace_revision }
2615
+ : {}),
2616
+ };
2617
+ }
2618
+
2619
+ #cleanupTerminal(
2620
+ runId: string,
2621
+ _currentOperationId: string,
2622
+ lease?: OperationLease,
2623
+ ): void {
2624
+ if (lease) this.#runStore.assertLease(lease);
2625
+ this.#lifecycle?.cleanup(runId);
2626
+ const cleanup = () => {
2627
+ this.#database
2628
+ .prepare("DELETE FROM workflow_engine_tasks WHERE run_id = ?")
2629
+ .run(runId);
2630
+ this.#database
2631
+ .prepare("DELETE FROM workflow_engine_deliveries WHERE run_id = ?")
2632
+ .run(runId);
2633
+ this.#database
2634
+ .prepare(`DELETE FROM workspace_revisions WHERE owner_run_id = ?`)
2635
+ .run(runId);
2636
+ this.#database
2637
+ .prepare(`DELETE FROM apply_transactions WHERE owner_run_id = ?`)
2638
+ .run(runId);
2639
+ this.#database
2640
+ .prepare(
2641
+ `UPDATE workflow_engine_runs
2642
+ SET baseline_workspace_revision = NULL,
2643
+ current_workspace_revision = NULL,
2644
+ transaction_id = NULL, verification_json = NULL,
2645
+ cleanup_state = 'complete'
2646
+ WHERE run_id = ?`,
2647
+ )
2648
+ .run(runId);
2649
+ };
2650
+ if (lease) this.#leasedTransaction(lease, cleanup);
2651
+ else this.#transaction(cleanup);
2652
+ }
2653
+
2654
+ #waitForCapacity(signal: AbortSignal): Promise<void> {
2655
+ if (signal.aborted || this.#activeTasks < LIMITS.maxActiveChildSessions)
2656
+ return Promise.resolve();
2657
+ return new Promise((resolve) => {
2658
+ const wake = () => {
2659
+ this.#capacityWaiters.delete(wake);
2660
+ signal.removeEventListener("abort", wake);
2661
+ resolve();
2662
+ };
2663
+ this.#capacityWaiters.add(wake);
2664
+ signal.addEventListener("abort", wake, { once: true });
2665
+ });
2666
+ }
2667
+
2668
+ async #advance(
2669
+ runId: string,
2670
+ operationId: string,
2671
+ signal: AbortSignal,
2672
+ lease: OperationLease,
2673
+ onActivity?: (event: WorkflowActivityUpdate) => void,
2674
+ ): Promise<Record<string, unknown>> {
2675
+ this.#transition(
2676
+ runId,
2677
+ "running",
2678
+ `${operationId}:running`,
2679
+ undefined,
2680
+ lease,
2681
+ );
2682
+ const run = this.#runStore.status(runId);
2683
+ emitWorkflowActivity(onActivity, {
2684
+ state: "running",
2685
+ stage: run.stage,
2686
+ runId,
2687
+ ...(run.change ? { change: run.change } : {}),
2688
+ objective: run.change ? `change ${run.change}` : "workflow run",
2689
+ legalCommands: ["status", "cancel", "discard"],
2690
+ });
2691
+ const existingVerification = this.#engineRun(runId).verification_json;
2692
+ const repairStatus = existingVerification
2693
+ ? normalizeWorkflowVerificationStatus(
2694
+ JSON.parse(existingVerification) as unknown,
2695
+ )
2696
+ : undefined;
2697
+ const retainsRepairContext =
2698
+ repairStatus?.scope === "change-task-affected" &&
2699
+ repairStatus.attribution === "introduced" &&
2700
+ repairStatus.taskId !== undefined &&
2701
+ this.#tasks(runId).some(
2702
+ (task) =>
2703
+ task.task_id === repairStatus.taskId && task.state !== "verified",
2704
+ );
2705
+ if (!retainsRepairContext) {
2706
+ this.#setVerificationStatus(runId, null, lease);
2707
+ }
2708
+ const attempted = new Set<string>();
2709
+ let madeProgress = true;
2710
+ while (madeProgress && !signal.aborted) {
2711
+ madeProgress = false;
2712
+ const rows = this.#tasks(runId);
2713
+ const selected = selectRunnableTasks({
2714
+ rows,
2715
+ attempted,
2716
+ activeTasks: this.#activeTasks,
2717
+ capacity: LIMITS.maxActiveChildSessions,
2718
+ nextQueuePosition: this.#engineRun(runId).next_queue_position,
2719
+ });
2720
+ const runnable = selected.runnable.map((taskId) => {
2721
+ const row = rows.find((candidate) => candidate.task_id === taskId);
2722
+ if (!row) throw new Error("scheduler-task-unavailable");
2723
+ attempted.add(taskId);
2724
+ return { row, before: `${row.state}:${row.phase}` };
2725
+ });
2726
+ for (const update of selected.updates) {
2727
+ const row = rows.find(
2728
+ (candidate) => candidate.task_id === update.taskId,
2729
+ );
2730
+ if (!row) throw new Error("scheduler-task-unavailable");
2731
+ if (update.kind === "pending")
2732
+ this.#setTask(
2733
+ runId,
2734
+ row.task_id,
2735
+ { state: "pending", pauseCode: null, queuePosition: null },
2736
+ lease,
2737
+ );
2738
+ else this.#queueTask(runId, row, lease, update.reason);
2739
+ }
2740
+ const capacityBlocked = selected.capacityBlocked;
2741
+ if (runnable.length === 0 && capacityBlocked) {
2742
+ await this.#waitForCapacity(signal);
2743
+ madeProgress = true;
2744
+ continue;
2745
+ }
2746
+ this.#activeTasks += runnable.length;
2747
+ const settled = await Promise.allSettled(
2748
+ runnable.map(({ row }) =>
2749
+ this.#runTask(
2750
+ runId,
2751
+ row,
2752
+ operationId,
2753
+ signal,
2754
+ lease,
2755
+ onActivity,
2756
+ ).finally(() => {
2757
+ this.#activeTasks--;
2758
+ for (const wake of this.#capacityWaiters) wake();
2759
+ }),
2760
+ ),
2761
+ );
2762
+ const rejected = settled.find(
2763
+ (outcome): outcome is PromiseRejectedResult =>
2764
+ outcome.status === "rejected",
2765
+ );
2766
+ if (rejected) throw rejected.reason;
2767
+ for (const { row, before } of runnable) {
2768
+ const after = this.#tasks(runId).find(
2769
+ (candidate) => candidate.task_id === row.task_id,
2770
+ );
2771
+ if (after && `${after.state}:${after.phase}` !== before) {
2772
+ madeProgress = true;
2773
+ }
2774
+ }
2775
+ }
2776
+
2777
+ if (signal.aborted) {
2778
+ const current = this.#runStore.status(runId);
2779
+ if (!["paused", "discarded", "completed"].includes(current.state)) {
2780
+ this.#transition(
2781
+ runId,
2782
+ "paused",
2783
+ `${operationId}:cancelled`,
2784
+ "operation-cancelled",
2785
+ lease,
2786
+ );
2787
+ }
2788
+ return this.#statusByRun(runId);
2789
+ }
2790
+ const rows = this.#tasks(runId);
2791
+ const paused = this.#firstPaused(rows);
2792
+ if (paused) {
2793
+ this.#transition(
2794
+ runId,
2795
+ paused.state === "approval-needed" ? "approval-needed" : "paused",
2796
+ `${operationId}:task-paused:${paused.task_id}`,
2797
+ paused.pause_code ?? "task-paused",
2798
+ lease,
2799
+ );
2800
+ return this.#statusByRun(runId);
2801
+ }
2802
+ if (rows.every((row) => row.state === "verified")) {
2803
+ return this.#finishChange(runId, operationId, signal, lease, onActivity);
2804
+ }
2805
+ this.#transition(
2806
+ runId,
2807
+ "paused",
2808
+ `${operationId}:scheduler-stalled`,
2809
+ "scheduler-integrity-paused",
2810
+ lease,
2811
+ );
2812
+ return this.#statusByRun(runId);
2813
+ }
2814
+
2815
+ #contextReads(runId: string, taskId?: string): string[] {
2816
+ const rows =
2817
+ taskId === undefined
2818
+ ? this.#database
2819
+ .prepare(
2820
+ "SELECT DISTINCT path FROM workflow_context_reads WHERE run_id = ? ORDER BY path",
2821
+ )
2822
+ .all(runId)
2823
+ : this.#database
2824
+ .prepare(
2825
+ "SELECT path FROM workflow_context_reads WHERE run_id = ? AND task_id = ? ORDER BY path",
2826
+ )
2827
+ .all(runId, taskId);
2828
+ return rows.map((row) => String(row.path));
2829
+ }
2830
+
2831
+ #phaseCount(plan: ImplementPlan): number {
2832
+ return plan.tasks.reduce((sum, task) => sum + phases(task).length, 0);
2833
+ }
2834
+
2835
+ #growWorkBudget(
2836
+ database: DatabaseSync,
2837
+ runId: string,
2838
+ plan: ImplementPlan,
2839
+ ): void {
2840
+ const count = this.#phaseCount(plan);
2841
+ database
2842
+ .prepare(`INSERT INTO workflow_work_budget(run_id, used, max_work, phase_high_water, hard_limit, recovery_policy)
2843
+ VALUES (?, 0, ?, ?, ?, 1) ON CONFLICT(run_id) DO UPDATE SET
2844
+ phase_high_water = MAX(phase_high_water, excluded.phase_high_water),
2845
+ max_work = MIN(hard_limit, MAX(max_work, 24 + 3 * MAX(phase_high_water, excluded.phase_high_water)))`)
2846
+ .run(
2847
+ runId,
2848
+ Math.min(this.#workHardLimit, 24 + 3 * count),
2849
+ count,
2850
+ this.#workHardLimit,
2851
+ );
2852
+ }
2853
+
2854
+ #hasRecoveryGrant(runId: string, operationId: string, key: string): boolean {
2855
+ return Boolean(
2856
+ this.#database
2857
+ .prepare(
2858
+ "SELECT 1 FROM workflow_recovery_grants WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
2859
+ )
2860
+ .get(runId, operationId, key),
2861
+ );
2862
+ }
2863
+
2864
+ #recoveryConditions(runId: string, key: string) {
2865
+ const run = this.#engineRun(runId);
2866
+ const task = this.#tasks(runId).find(
2867
+ (row) => recoveryKey(this.#taskPlan(row), row.phase, row, run) === key,
2868
+ );
2869
+ return {
2870
+ route: task?.route_fingerprint ?? run.route_fingerprint,
2871
+ context: task
2872
+ ? hash(
2873
+ canonicalJson(
2874
+ snapshotFiles(
2875
+ this.#consumerRoot,
2876
+ this.#contextReads(runId, task.task_id),
2877
+ ),
2878
+ ),
2879
+ )
2880
+ : null,
2881
+ };
2882
+ }
2883
+
2884
+ #failureSequence(runId: string, key: string): number {
2885
+ return (
2886
+ this.#database
2887
+ .prepare(
2888
+ "SELECT COALESCE(MAX(sequence), 0) AS sequence FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ?",
2889
+ )
2890
+ .get(runId, key) as { sequence: number }
2891
+ ).sequence;
2892
+ }
2893
+
2894
+ #grantRecovery(
2895
+ runId: string,
2896
+ operationId: string,
2897
+ request: NonNullable<
2898
+ Extract<ControlCommand, { command: "resume" }>["recovery"]
2899
+ >,
2900
+ lease: OperationLease,
2901
+ ): void {
2902
+ this.#leasedTransaction(lease, () => {
2903
+ const incident = this.#recovery(runId, request.incidentKey);
2904
+ const current = this.#tasks(runId).some(
2905
+ (row) =>
2906
+ ["paused", "retryable"].includes(row.state) &&
2907
+ recoveryKey(
2908
+ this.#taskPlan(row),
2909
+ row.phase,
2910
+ row,
2911
+ this.#engineRun(runId),
2912
+ ) === request.incidentKey,
2913
+ );
2914
+ if (
2915
+ !current ||
2916
+ !incident ||
2917
+ incident.failures < incident.feedback.maxAttempts ||
2918
+ this.#failureSequence(runId, request.incidentKey) !==
2919
+ request.failureSequence
2920
+ )
2921
+ throw new Error("recovery-request-stale");
2922
+ const conditions = this.#database
2923
+ .prepare(
2924
+ "SELECT conditions_json FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
2925
+ )
2926
+ .get(runId, request.incidentKey) as { conditions_json: string };
2927
+ const previous = JSON.parse(conditions.conditions_json);
2928
+ const observed = this.#recoveryConditions(runId, request.incidentKey);
2929
+ const budget = this.#database
2930
+ .prepare(
2931
+ "SELECT used, max_work FROM workflow_work_budget WHERE run_id = ?",
2932
+ )
2933
+ .get(runId) as { used: number; max_work: number };
2934
+ const rejected = assessRecoveryGrant({
2935
+ current,
2936
+ incident,
2937
+ failureSequence: this.#failureSequence(runId, request.incidentKey),
2938
+ requestedSequence: request.failureSequence,
2939
+ reason: request.reason,
2940
+ previous,
2941
+ observed,
2942
+ budget,
2943
+ });
2944
+ if (rejected) throw new Error(rejected);
2945
+ this.#database
2946
+ .prepare(
2947
+ "INSERT INTO workflow_recovery_grants(run_id, operation_id, incident_key, failure_sequence, reason) VALUES (?, ?, ?, ?, ?)",
2948
+ )
2949
+ .run(
2950
+ runId,
2951
+ operationId,
2952
+ request.incidentKey,
2953
+ request.failureSequence,
2954
+ request.reason,
2955
+ );
2956
+ });
2957
+ }
2958
+
2959
+ #reserveWork(
2960
+ runId: string,
2961
+ plan: ImplementPlan,
2962
+ key: string,
2963
+ lease: OperationLease,
2964
+ grantOperationId?: string,
2965
+ ): boolean {
2966
+ return this.#leasedTransaction(lease, () => {
2967
+ this.#growWorkBudget(this.#database, runId, plan);
2968
+ const reserved = this.#database
2969
+ .prepare(`UPDATE workflow_work_budget SET used = used + 1
2970
+ WHERE run_id = ? AND used < max_work`)
2971
+ .run(runId);
2972
+ if (reserved.changes !== 1) return false;
2973
+ if (grantOperationId) {
2974
+ const consumed = this.#database
2975
+ .prepare(
2976
+ "UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
2977
+ )
2978
+ .run(runId, grantOperationId, key);
2979
+ if (consumed.changes !== 1) throw new Error("recovery-request-stale");
2980
+ }
2981
+ this.#database
2982
+ .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2983
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
2984
+ .run(runId, key, runId);
2985
+ return true;
2986
+ });
2987
+ }
2988
+
2989
+ #recovery(runId: string, key: string): WorkflowRecoveryFact | undefined {
2990
+ const row = this.#database
2991
+ .prepare(
2992
+ `SELECT failures, max_attempts, feedback_json FROM workflow_recovery_incidents
2993
+ WHERE run_id = ? AND incident_key = ?`,
2994
+ )
2995
+ .get(runId, key) as
2996
+ | { failures: number; max_attempts: number; feedback_json: string }
2997
+ | undefined;
2998
+ return row
2999
+ ? {
3000
+ key,
3001
+ failures: row.failures,
3002
+ feedback: {
3003
+ ...JSON.parse(row.feedback_json),
3004
+ maxAttempts: row.max_attempts,
3005
+ },
3006
+ }
3007
+ : undefined;
3008
+ }
3009
+
3010
+ #recordRecovery(
3011
+ runId: string,
3012
+ fact: WorkflowRecoveryFact,
3013
+ lease: OperationLease,
3014
+ ): void {
3015
+ this.#leasedTransaction(lease, () => {
3016
+ this.#database
3017
+ .prepare(
3018
+ `INSERT INTO workflow_recovery_incidents(run_id, incident_key, failures, max_attempts, feedback_json, conditions_json)
3019
+ VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, incident_key) DO UPDATE SET
3020
+ failures = MAX(failures, excluded.failures), max_attempts = MIN(max_attempts, excluded.max_attempts), feedback_json = excluded.feedback_json, conditions_json = excluded.conditions_json`,
3021
+ )
3022
+ .run(
3023
+ runId,
3024
+ fact.key,
3025
+ fact.failures,
3026
+ fact.feedback.maxAttempts,
3027
+ JSON.stringify(fact.feedback),
3028
+ JSON.stringify(this.#recoveryConditions(runId, fact.key)),
3029
+ );
3030
+ this.#database
3031
+ .prepare(
3032
+ `INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
3033
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'failure' FROM workflow_recovery_events WHERE run_id = ?`,
3034
+ )
3035
+ .run(runId, fact.key, runId);
3036
+ });
3037
+ }
3038
+
3039
+ #safeAttemptDiagnostic(
3040
+ runId: string,
3041
+ row: EngineTaskRow | undefined,
3042
+ ): SafeAttemptDiagnostic | undefined {
3043
+ if (!row) return undefined;
3044
+ const diagnostic: SafeAttemptDiagnostic = row.attempt_diagnostic_json
3045
+ ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
3046
+ : {};
3047
+ delete diagnostic.recovery;
3048
+ const recovery = this.#recovery(
3049
+ runId,
3050
+ recoveryKey(this.#taskPlan(row), row.phase, row, this.#engineRun(runId)),
3051
+ );
3052
+ if (recovery) diagnostic.recovery = recovery;
3053
+ return Object.keys(diagnostic).length ? diagnostic : undefined;
3054
+ }
3055
+
3056
+ #statusByRun(runId: string): Record<string, unknown> {
3057
+ const projection = this.#runStore.status(runId);
3058
+ const rows = this.#tasks(runId);
3059
+ let engineRun: EngineRunRow | undefined;
3060
+ try {
3061
+ engineRun = this.#engineRun(runId);
3062
+ } catch {
3063
+ engineRun = undefined;
3064
+ }
3065
+ const attemptDiagnostics = new Map<string, SafeAttemptDiagnostic>();
3066
+ const failureSequences = new Map<string, number>();
3067
+ for (const row of rows) {
3068
+ const diagnostic = this.#safeAttemptDiagnostic(runId, row);
3069
+ if (!diagnostic) continue;
3070
+ attemptDiagnostics.set(row.task_id, diagnostic);
3071
+ if (diagnostic.recovery) {
3072
+ const key = diagnostic.recovery.key;
3073
+ failureSequences.set(key, this.#failureSequence(runId, key));
3074
+ }
3075
+ }
3076
+ const amendmentUsed =
3077
+ (
3078
+ this.#database
3079
+ .prepare(
3080
+ "SELECT used FROM workflow_amendment_budget WHERE run_id = ?",
3081
+ )
3082
+ .get(runId) as { used: number } | undefined
3083
+ )?.used ?? 0;
3084
+ const resourceBudget = this.#database
3085
+ .prepare(
3086
+ "SELECT used, max_work AS maximum, phase_high_water AS phaseHighWater, hard_limit AS hardLimit FROM workflow_work_budget WHERE run_id = ?",
3087
+ )
3088
+ .get(runId) as WorkflowResourceBudget | undefined;
3089
+ return projectWorkflowStatus({
3090
+ projection,
3091
+ rows,
3092
+ engineRun,
3093
+ attemptDiagnostics,
3094
+ failureSequences,
3095
+ amendmentUsed,
3096
+ resourceBudget,
3097
+ });
3098
+ }
3099
+
3100
+ async #withAvailableDelivery(
3101
+ outcome: Record<string, unknown>,
3102
+ signal?: AbortSignal,
3103
+ ): Promise<Record<string, unknown>> {
3104
+ if (
3105
+ outcome.stage !== "abel-implement" ||
3106
+ (isRecord(outcome.pause) &&
3107
+ outcome.pause.code === "operation-cancelled") ||
3108
+ (outcome.state !== "approval-needed" &&
3109
+ !(
3110
+ outcome.state === "paused" &&
3111
+ (isRecord(outcome.decisionBatch) ||
3112
+ (isRecord(outcome.recovery) && outcome.recovery.exhausted === true))
3113
+ )) ||
3114
+ typeof outcome.change !== "string" ||
3115
+ !this.#deliverySource.discoverLatest
3116
+ ) {
3117
+ return outcome;
3118
+ }
3119
+ let available: WorkflowAvailableDelivery | undefined;
3120
+ try {
3121
+ const discover = this.#deliverySource.discoverLatest;
3122
+ const change = outcome.change;
3123
+ available = await cancellableRead(
3124
+ () =>
3125
+ discover.call(this.#deliverySource, {
3126
+ stage: "abel-implement",
3127
+ change,
3128
+ ...(signal ? { signal } : {}),
3129
+ }),
3130
+ signal,
3131
+ );
3132
+ } catch {
3133
+ return outcome;
3134
+ }
3135
+ const currentRevision =
3136
+ typeof outcome.deliveryRevision === "number"
3137
+ ? outcome.deliveryRevision
3138
+ : 0;
3139
+ if (
3140
+ !available ||
3141
+ !Number.isSafeInteger(available.deliveryRevision) ||
3142
+ available.deliveryRevision <= currentRevision ||
3143
+ !SHA256.test(available.receiptHash)
3144
+ ) {
3145
+ return outcome;
3146
+ }
3147
+ if (
3148
+ this.#database
3149
+ .prepare(
3150
+ "SELECT 1 FROM workflow_rejected_deliveries WHERE run_id = ? AND delivery_revision = ? AND receipt_hash = ?",
3151
+ )
3152
+ .get(
3153
+ String(outcome.runId),
3154
+ available.deliveryRevision,
3155
+ available.receiptHash,
3156
+ )
3157
+ ) {
3158
+ return outcome;
3159
+ }
3160
+ const conditionalCommands = (
3161
+ Array.isArray(outcome.conditionalCommands)
3162
+ ? outcome.conditionalCommands
3163
+ : [{ command: "resume", stage: outcome.stage, change: outcome.change }]
3164
+ ).map((entry) =>
3165
+ isRecord(entry) && entry.command === "resume"
3166
+ ? {
3167
+ ...entry,
3168
+ satisfiedBy: {
3169
+ deliveryRevision: available.deliveryRevision,
3170
+ receiptHash: available.receiptHash,
3171
+ },
3172
+ }
3173
+ : entry,
3174
+ );
3175
+ return {
3176
+ ...outcome,
3177
+ legalCommands:
3178
+ outcome.state === "approval-needed"
3179
+ ? ["status", "resume", "discard"]
3180
+ : ["status", "resume", "rebind", "discard"],
3181
+ availableDelivery: structuredClone(available),
3182
+ continuation:
3183
+ (isRecord(outcome.resourceBudget) &&
3184
+ outcome.resourceBudget.remaining === 0) ||
3185
+ (isRecord(outcome.recovery) &&
3186
+ outcome.recovery.code === "change-recovery-budget-exhausted")
3187
+ ? undefined
3188
+ : {
3189
+ owner: "parent",
3190
+ automatic: true,
3191
+ command: "resume",
3192
+ stage: "abel-implement",
3193
+ change: outcome.change,
3194
+ },
3195
+ ...(conditionalCommands === undefined ? {} : { conditionalCommands }),
3196
+ };
3197
+ }
3198
+
3199
+ #launchAdvance(
3200
+ runId: string,
3201
+ operationId: string,
3202
+ externalSignal?: AbortSignal,
3203
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3204
+ ): Promise<Record<string, unknown>> {
3205
+ const lease = this.#operationLease(runId, operationId);
3206
+ const controller = new AbortController();
3207
+ const abortFromExternal = () => {
3208
+ controller.abort(
3209
+ externalSignal?.reason ?? new Error("operation-cancelled"),
3210
+ );
3211
+ };
3212
+ if (externalSignal?.aborted) abortFromExternal();
3213
+ else
3214
+ externalSignal?.addEventListener("abort", abortFromExternal, {
3215
+ once: true,
3216
+ });
3217
+ const active: ActiveOperation = {
3218
+ operationId,
3219
+ kind: "work",
3220
+ controller,
3221
+ settled: Promise.resolve({}),
3222
+ };
3223
+ this.#active.set(runId, active);
3224
+ const settled = (async () => {
3225
+ try {
3226
+ const outcome = await this.#advance(
3227
+ runId,
3228
+ operationId,
3229
+ controller.signal,
3230
+ lease,
3231
+ onActivity,
3232
+ );
3233
+ return this.#commitOperation(runId, operationId, outcome);
3234
+ } catch (error) {
3235
+ this.#interruptOperation(runId, operationId);
3236
+ const controlSettled =
3237
+ controller.signal.aborted &&
3238
+ controller.signal.reason instanceof Error &&
3239
+ ["operation-cancelled", "run-discarded"].includes(
3240
+ controller.signal.reason.message,
3241
+ ) &&
3242
+ (isCancellationException(error, controller.signal) ||
3243
+ (error instanceof Error &&
3244
+ ["lease-fenced", "operation-journal-fenced"].includes(
3245
+ error.message,
3246
+ )));
3247
+ if (controlSettled) return this.#statusByRun(runId);
3248
+ throw error;
3249
+ } finally {
3250
+ externalSignal?.removeEventListener("abort", abortFromExternal);
3251
+ if (this.#active.get(runId) === active) this.#active.delete(runId);
3252
+ }
3253
+ })();
3254
+ active.settled = settled;
3255
+ return settled;
3256
+ }
3257
+
3258
+ async #start(
3259
+ command: Extract<ControlCommand, { command: "start" }>,
3260
+ signal?: AbortSignal,
3261
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3262
+ ) {
3263
+ const existingRun = this.#lookupRun(command.stage, command.change);
3264
+ if (existingRun) {
3265
+ this.#ensureEngineRun(existingRun);
3266
+ const replay = this.#operationReplay(
3267
+ existingRun,
3268
+ command.operationId,
3269
+ command.command,
3270
+ );
3271
+ if (replay) return replay;
3272
+ const restarted = this.#runStore.startRun({
3273
+ stage: command.stage,
3274
+ change: command.change,
3275
+ operationId: command.operationId,
3276
+ });
3277
+ if (restarted.state !== "created") {
3278
+ const begun = this.#beginOperation(
3279
+ existingRun,
3280
+ command.operationId,
3281
+ command.command,
3282
+ );
3283
+ if (begun) return begun;
3284
+ return this.#commitOperation(
3285
+ existingRun,
3286
+ command.operationId,
3287
+ this.#statusByRun(existingRun),
3288
+ );
3289
+ }
3290
+ }
3291
+
3292
+ const runId =
3293
+ existingRun ??
3294
+ this.#runStore.startRun({
3295
+ stage: command.stage,
3296
+ change: command.change,
3297
+ operationId: command.operationId,
3298
+ }).runId;
3299
+ this.#ensureEngineRun(runId);
3300
+ const begun = this.#beginOperation(
3301
+ runId,
3302
+ command.operationId,
3303
+ command.command,
3304
+ );
3305
+ if (begun) return begun;
3306
+ const lease = this.#operationLease(runId, command.operationId);
3307
+ this.#transition(
3308
+ runId,
3309
+ "validating-delivery",
3310
+ `${command.operationId}:validating-delivery`,
3311
+ undefined,
3312
+ lease,
3313
+ );
3314
+ emitWorkflowActivity(onActivity, {
3315
+ state: "validating",
3316
+ stage: command.stage,
3317
+ runId,
3318
+ change: command.change,
3319
+ objective: `validate delivery ${command.change}`,
3320
+ legalCommands: ["status", "cancel", "discard"],
3321
+ });
3322
+ try {
3323
+ const delivery = await this.#loadDelivery(
3324
+ runId,
3325
+ lease,
3326
+ command.stage,
3327
+ command.change,
3328
+ undefined,
3329
+ undefined,
3330
+ signal,
3331
+ );
3332
+ this.#admitDelivery(runId, delivery, lease);
3333
+ this.#transition(
3334
+ runId,
3335
+ "ready",
3336
+ `${command.operationId}:delivery-ready`,
3337
+ undefined,
3338
+ lease,
3339
+ );
3340
+ return this.#launchAdvance(
3341
+ runId,
3342
+ command.operationId,
3343
+ signal,
3344
+ onActivity,
3345
+ );
3346
+ } catch (error) {
3347
+ const cancelled = this.#cancelledDelivery(
3348
+ runId,
3349
+ command.operationId,
3350
+ error,
3351
+ signal,
3352
+ lease,
3353
+ );
3354
+ if (cancelled) return cancelled;
3355
+ const failure = this.#deliveryFailure(
3356
+ runId,
3357
+ command.operationId,
3358
+ error,
3359
+ lease,
3360
+ );
3361
+ if (failure) {
3362
+ return this.#commitOperation(runId, command.operationId, failure);
3363
+ }
3364
+ this.#interruptOperation(runId, command.operationId);
3365
+ throw error;
3366
+ }
3367
+ }
3368
+
3369
+ async #resume(
3370
+ command: Extract<ControlCommand, { command: "resume" }>,
3371
+ signal?: AbortSignal,
3372
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3373
+ ) {
3374
+ const runId = this.#lookupRun(command.stage, command.change);
3375
+ if (!runId) throw new Error("run-not-found");
3376
+ this.#recoverLegacyContextApproval(runId);
3377
+ const replay = this.#beginOperation(
3378
+ runId,
3379
+ command.operationId,
3380
+ command.command,
3381
+ );
3382
+ if (replay) return replay;
3383
+ const lease = this.#operationLease(runId, command.operationId);
3384
+ const current = this.#runStore.status(runId);
3385
+ if (current.state === "recovering") {
3386
+ emitWorkflowActivity(onActivity, {
3387
+ state: "recovering",
3388
+ stage: current.stage,
3389
+ runId,
3390
+ ...(current.change ? { change: current.change } : {}),
3391
+ objective: current.change
3392
+ ? `recover change ${current.change}`
3393
+ : "recover workflow application",
3394
+ legalCommands: ["status", "cancel", "discard"],
3395
+ });
3396
+ const transactionId = this.#engineRun(runId).transaction_id;
3397
+ if (!transactionId || !this.#application) {
3398
+ this.#interruptOperation(runId, command.operationId);
3399
+ throw new Error("apply-recovery-unavailable");
3400
+ }
3401
+ const recovered = await this.#application.recover(
3402
+ transactionId,
3403
+ this.#applicationContext(runId),
3404
+ );
3405
+ this.#runStore.assertLease(lease);
3406
+ const outcome = this.#settleApplication(
3407
+ runId,
3408
+ command.operationId,
3409
+ recovered,
3410
+ lease,
3411
+ onActivity,
3412
+ );
3413
+ return this.#commitOperation(runId, command.operationId, outcome);
3414
+ }
3415
+ if (
3416
+ !["paused", "retryable", "approval-needed", "ready"].includes(
3417
+ current.state,
3418
+ )
3419
+ ) {
3420
+ this.#interruptOperation(runId, command.operationId);
3421
+ throw new Error("resume-not-allowed");
3422
+ }
3423
+ try {
3424
+ const engineRevision = this.#engineRun(runId).current_revision;
3425
+ const projectedRevision = current.deliveryRevision;
3426
+ if (
3427
+ command.deliveryRevision === undefined &&
3428
+ (current.state === "approval-needed" ||
3429
+ this.#statusByRun(runId).decisionBatch !== undefined ||
3430
+ this.#statusByRun(runId).recovery !== undefined) &&
3431
+ this.#deliverySource.discoverLatest
3432
+ ) {
3433
+ const available = await this.#deliverySource.discoverLatest({
3434
+ stage: command.stage,
3435
+ change: command.change,
3436
+ signal,
3437
+ });
3438
+ this.#runStore.assertLease(lease);
3439
+ if (available && available.deliveryRevision > (engineRevision ?? 0)) {
3440
+ command = {
3441
+ ...command,
3442
+ deliveryRevision: available.deliveryRevision,
3443
+ receiptHash: available.receiptHash,
3444
+ };
3445
+ }
3446
+ }
3447
+ const deliveryOutOfSync =
3448
+ projectedRevision !== undefined && projectedRevision !== engineRevision;
3449
+ if (
3450
+ current.state === "approval-needed" &&
3451
+ (command.deliveryRevision === undefined ||
3452
+ command.receiptHash === undefined ||
3453
+ command.deliveryRevision <=
3454
+ (engineRevision ?? projectedRevision ?? 0))
3455
+ ) {
3456
+ this.#interruptOperation(runId, command.operationId);
3457
+ throw new Error("approval-receipt-required");
3458
+ }
3459
+ if (
3460
+ command.deliveryRevision !== undefined ||
3461
+ engineRevision === null ||
3462
+ deliveryOutOfSync
3463
+ ) {
3464
+ this.#transition(
3465
+ runId,
3466
+ "validating-delivery",
3467
+ `${command.operationId}:validating-delivery`,
3468
+ undefined,
3469
+ lease,
3470
+ );
3471
+ emitWorkflowActivity(onActivity, {
3472
+ state: "validating",
3473
+ stage: command.stage,
3474
+ runId,
3475
+ change: command.change,
3476
+ objective: `validate delivery ${command.change}`,
3477
+ legalCommands: ["status", "cancel", "discard"],
3478
+ });
3479
+ const requestedRevision =
3480
+ command.deliveryRevision ??
3481
+ (deliveryOutOfSync ? projectedRevision : undefined);
3482
+ const projectedBinding = current.deliveryBindings.find(
3483
+ (binding) =>
3484
+ binding.gate === "gate-b" && binding.revision === requestedRevision,
3485
+ );
3486
+ const delivery = await this.#loadDelivery(
3487
+ runId,
3488
+ lease,
3489
+ command.stage,
3490
+ command.change,
3491
+ requestedRevision,
3492
+ command.receiptHash ?? projectedBinding?.receiptHash,
3493
+ signal,
3494
+ );
3495
+ this.#admitDelivery(runId, delivery, lease);
3496
+ }
3497
+ if (command.recovery)
3498
+ this.#grantRecovery(
3499
+ runId,
3500
+ command.operationId,
3501
+ command.recovery,
3502
+ lease,
3503
+ );
3504
+ this.#transition(
3505
+ runId,
3506
+ "ready",
3507
+ `${command.operationId}:resume-ready`,
3508
+ undefined,
3509
+ lease,
3510
+ );
3511
+ return this.#launchAdvance(
3512
+ runId,
3513
+ command.operationId,
3514
+ signal,
3515
+ onActivity,
3516
+ );
3517
+ } catch (error) {
3518
+ const cancelled = this.#cancelledDelivery(
3519
+ runId,
3520
+ command.operationId,
3521
+ error,
3522
+ signal,
3523
+ lease,
3524
+ );
3525
+ if (cancelled) return cancelled;
3526
+ const failure = this.#deliveryFailure(
3527
+ runId,
3528
+ command.operationId,
3529
+ error,
3530
+ lease,
3531
+ );
3532
+ if (failure) {
3533
+ return this.#commitOperation(runId, command.operationId, failure);
3534
+ }
3535
+ // A revised delivery may invalidate a grant. Retain its admitted facts,
3536
+ // but never leave a rejected resume looking ready or still validating.
3537
+ if (
3538
+ error instanceof Error &&
3539
+ [
3540
+ "recovery-request-stale",
3541
+ "recovery-evidence-unavailable",
3542
+ "change-work-budget-exhausted",
3543
+ ].includes(error.message) &&
3544
+ this.#runStore.status(runId).state === "validating-delivery"
3545
+ ) {
3546
+ this.#transition(
3547
+ runId,
3548
+ "paused",
3549
+ `${command.operationId}:recovery-rejected`,
3550
+ error.message,
3551
+ lease,
3552
+ );
3553
+ }
3554
+ this.#interruptOperation(runId, command.operationId);
3555
+ throw error;
3556
+ }
3557
+ }
3558
+
3559
+ #rebind(command: Extract<ControlCommand, { command: "rebind" }>) {
3560
+ const runId = this.#lookupRun(command.stage, command.change);
3561
+ if (!runId) throw new Error("run-not-found");
3562
+ const replay = this.#beginOperation(
3563
+ runId,
3564
+ command.operationId,
3565
+ command.command,
3566
+ );
3567
+ if (replay) return replay;
3568
+ const lease = this.#operationLease(runId, command.operationId);
3569
+ const state = this.#runStore.status(runId).state;
3570
+ if (!["paused", "retryable"].includes(state)) {
3571
+ this.#interruptOperation(runId, command.operationId);
3572
+ throw new Error("rebind-not-allowed");
3573
+ }
3574
+ const task = this.#tasks(runId).find((row) =>
3575
+ ["paused", "retryable"].includes(row.state),
3576
+ );
3577
+ if (!task) {
3578
+ this.#interruptOperation(runId, command.operationId);
3579
+ throw new Error("rebind-task-unavailable");
3580
+ }
3581
+ const rebound = this.#worker.rebind({
3582
+ runId,
3583
+ taskId: task.task_id,
3584
+ task: structuredClone(this.#taskPlan(task)),
3585
+ role: "implementation-worker",
3586
+ routeId: command.routeId,
3587
+ });
3588
+ if (!rebound.ok) {
3589
+ this.#interruptOperation(runId, command.operationId);
3590
+ throw new Error(rebound.code);
3591
+ }
3592
+ const routeId = rebound.routeId ?? rebound.route?.id ?? command.routeId;
3593
+ const routeFingerprint =
3594
+ rebound.routeFingerprint ?? rebound.route?.fingerprint;
3595
+ if (
3596
+ !IDENTIFIER.test(routeId) ||
3597
+ (routeFingerprint !== undefined && !SHA256.test(routeFingerprint))
3598
+ ) {
3599
+ this.#interruptOperation(runId, command.operationId);
3600
+ throw new Error("route-rebind-result-invalid");
3601
+ }
3602
+ const priorRouteId = task.route_id ?? this.#engineRun(runId).route_id;
3603
+ const priorRouteFingerprint =
3604
+ task.route_fingerprint ?? this.#engineRun(runId).route_fingerprint;
3605
+ this.#leasedTransaction(lease, () => {
3606
+ this.#setTask(runId, task.task_id, {
3607
+ routeId,
3608
+ routeFingerprint: routeFingerprint ?? null,
3609
+ });
3610
+ this.#database
3611
+ .prepare(
3612
+ `UPDATE workflow_engine_runs
3613
+ SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
3614
+ )
3615
+ .run(routeId, routeFingerprint ?? null, runId);
3616
+ });
3617
+ const outcome = {
3618
+ ...this.#statusByRun(runId),
3619
+ routeBinding: {
3620
+ routeId,
3621
+ ...(routeFingerprint ? { routeFingerprint } : {}),
3622
+ },
3623
+ operation: {
3624
+ kind: "route-rebound",
3625
+ ...(priorRouteId ? { priorRouteId } : {}),
3626
+ ...(priorRouteFingerprint ? { priorRouteFingerprint } : {}),
3627
+ routeId,
3628
+ ...(routeFingerprint ? { routeFingerprint } : {}),
3629
+ },
3630
+ };
3631
+ return this.#commitOperation(runId, command.operationId, outcome);
3632
+ }
3633
+
3634
+ async #controlApply(
3635
+ runId: string,
3636
+ operationId: string,
3637
+ intent: "cancel" | "discard",
3638
+ lease: OperationLease,
3639
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3640
+ ): Promise<Record<string, unknown>> {
3641
+ const transactionId = this.#engineRun(runId).transaction_id;
3642
+ if (!transactionId || !this.#application) {
3643
+ throw new Error("apply-recovery-unavailable");
3644
+ }
3645
+ const context = this.#applicationContext(runId);
3646
+ this.#application.requestControl(transactionId, intent, context);
3647
+ const active = this.#active.get(runId);
3648
+ if (active && active.operationId !== operationId) {
3649
+ active.controller.abort(
3650
+ new Error(
3651
+ intent === "cancel" ? "operation-cancelled" : "run-discarded",
3652
+ ),
3653
+ );
3654
+ }
3655
+ if (this.#runStore.status(runId).state === "applying") {
3656
+ this.#transition(
3657
+ runId,
3658
+ "recovering",
3659
+ `${operationId}:${intent}-recovering`,
3660
+ intent === "cancel" ? "operation-cancelled" : "run-discarded",
3661
+ lease,
3662
+ );
3663
+ const recovering = this.#runStore.status(runId);
3664
+ emitWorkflowActivity(onActivity, {
3665
+ state: "recovering",
3666
+ stage: recovering.stage,
3667
+ runId,
3668
+ ...(recovering.change ? { change: recovering.change } : {}),
3669
+ objective: recovering.change
3670
+ ? `recover change ${recovering.change}`
3671
+ : "recover workflow application",
3672
+ code: intent === "cancel" ? "operation-cancelled" : "run-discarded",
3673
+ legalCommands: ["status"],
3674
+ });
3675
+ }
3676
+ const recovered = await this.#application.recover(transactionId, context);
3677
+ this.#runStore.assertLease(lease);
3678
+ return this.#settleApplication(
3679
+ runId,
3680
+ operationId,
3681
+ recovered,
3682
+ lease,
3683
+ onActivity,
3684
+ );
3685
+ }
3686
+
3687
+ async #abortActiveOperation(runId: string, reason: Error): Promise<void> {
3688
+ const pending: Promise<unknown>[] = [];
3689
+ const amendment = this.#amendments.get(runId);
3690
+ if (amendment) pending.push(amendment);
3691
+ for (const [controller, entry] of this.#commands) {
3692
+ if (
3693
+ !["start", "resume"].includes(entry.command.command) ||
3694
+ this.#lookupRun(entry.command.stage, entry.command.change) !== runId
3695
+ )
3696
+ continue;
3697
+ controller.abort(reason);
3698
+ pending.push(entry.settled);
3699
+ }
3700
+ const active = this.#active.get(runId);
3701
+ if (active) {
3702
+ active.controller.abort(reason);
3703
+ pending.push(active.settled);
3704
+ }
3705
+ const results = await Promise.allSettled(pending);
3706
+ for (const result of results) {
3707
+ if (result.status !== "rejected") continue;
3708
+ const error = result.reason;
3709
+ if (
3710
+ error !== reason &&
3711
+ !(
3712
+ error instanceof Error &&
3713
+ ["lease-fenced", "operation-journal-fenced"].includes(error.message)
3714
+ )
3715
+ )
3716
+ throw error;
3717
+ }
3718
+ }
3719
+
3720
+ async #cancel(
3721
+ command: Extract<ControlCommand, { command: "cancel" }>,
3722
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3723
+ ) {
3724
+ const runId = this.#lookupRun(command.stage, command.change);
3725
+ if (!runId) throw new Error("run-not-found");
3726
+ const replay = this.#beginOperation(
3727
+ runId,
3728
+ command.operationId,
3729
+ command.command,
3730
+ );
3731
+ if (replay) return replay;
3732
+ const lease = this.#operationLease(runId, command.operationId);
3733
+ const state = this.#runStore.status(runId).state;
3734
+ let outcome: Record<string, unknown>;
3735
+ if (state === "applying" || state === "recovering") {
3736
+ outcome = await this.#controlApply(
3737
+ runId,
3738
+ command.operationId,
3739
+ "cancel",
3740
+ lease,
3741
+ onActivity,
3742
+ );
3743
+ } else {
3744
+ const activeSettlement = this.#abortActiveOperation(
3745
+ runId,
3746
+ new Error("operation-cancelled"),
3747
+ );
3748
+ const current = this.#runStore.status(runId);
3749
+ if (
3750
+ !["paused", "discarded", "completed", "rejected"].includes(
3751
+ current.state,
3752
+ )
3753
+ ) {
3754
+ this.#transition(
3755
+ runId,
3756
+ "paused",
3757
+ `${command.operationId}:cancelled`,
3758
+ "operation-cancelled",
3759
+ lease,
3760
+ );
3761
+ }
3762
+ outcome = this.#statusByRun(runId);
3763
+ await activeSettlement;
3764
+ }
3765
+ outcome = {
3766
+ ...outcome,
3767
+ operation: { kind: "operation-cancelled" },
3768
+ };
3769
+ return this.#commitOperation(runId, command.operationId, outcome);
3770
+ }
3771
+
3772
+ async #discard(
3773
+ command: Extract<ControlCommand, { command: "discard" }>,
3774
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3775
+ ) {
3776
+ const runId = this.#lookupRun(command.stage, command.change);
3777
+ if (!runId) throw new Error("run-not-found");
3778
+ const replay = this.#beginOperation(
3779
+ runId,
3780
+ command.operationId,
3781
+ command.command,
3782
+ );
3783
+ if (replay) return replay;
3784
+ const lease = this.#operationLease(runId, command.operationId);
3785
+ const state = this.#runStore.status(runId).state;
3786
+ let outcome: Record<string, unknown>;
3787
+ if (state === "applying" || state === "recovering") {
3788
+ outcome = await this.#controlApply(
3789
+ runId,
3790
+ command.operationId,
3791
+ "discard",
3792
+ lease,
3793
+ onActivity,
3794
+ );
3795
+ } else {
3796
+ const activeSettlement = this.#abortActiveOperation(
3797
+ runId,
3798
+ new Error("run-discarded"),
3799
+ );
3800
+ const current = this.#runStore.status(runId);
3801
+ if (current.state !== "discarded") {
3802
+ if (["completed", "rejected"].includes(current.state)) {
3803
+ this.#interruptOperation(runId, command.operationId);
3804
+ throw new Error("discard-not-allowed");
3805
+ }
3806
+ this.#transition(
3807
+ runId,
3808
+ "discarded",
3809
+ `${command.operationId}:discarded`,
3810
+ undefined,
3811
+ lease,
3812
+ );
3813
+ await activeSettlement;
3814
+ this.#cleanupTerminal(runId, command.operationId, lease);
3815
+ } else {
3816
+ await activeSettlement;
3817
+ }
3818
+ outcome = this.#statusByRun(runId);
3819
+ }
3820
+ return this.#commitOperation(runId, command.operationId, outcome);
3821
+ }
3822
+
3823
+ execute(
3824
+ value: unknown,
3825
+ signal?: AbortSignal,
3826
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3827
+ ): Promise<Record<string, unknown>> {
3828
+ let command: ControlCommand;
3829
+ try {
3830
+ this.#assertOpen();
3831
+ if (this.#closing) throw new Error("workflow-engine-closed");
3832
+ command = assertControlCommand(value);
3833
+ } catch (error) {
3834
+ return Promise.reject(error);
3835
+ }
3836
+ const controller = new AbortController();
3837
+ const onAbort = () => controller.abort(signal?.reason);
3838
+ if (signal?.aborted) onAbort();
3839
+ else signal?.addEventListener("abort", onAbort, { once: true });
3840
+ // Register before execution can yield or invoke external callbacks.
3841
+ const settled = Promise.resolve()
3842
+ .then(() => this.#executeCommand(command, controller.signal, onActivity))
3843
+ .finally(() => {
3844
+ signal?.removeEventListener("abort", onAbort);
3845
+ this.#commands.delete(controller);
3846
+ });
3847
+ this.#commands.set(controller, { command, settled });
3848
+ return settled;
3849
+ }
3850
+
3851
+ async #executeCommand(
3852
+ command: ControlCommand,
3853
+ signal?: AbortSignal,
3854
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3855
+ ): Promise<Record<string, unknown>> {
3856
+ switch (command.command) {
3857
+ case "start":
3858
+ return this.#withAvailableDelivery(
3859
+ await this.#start(command, signal, onActivity),
3860
+ signal,
3861
+ );
3862
+ case "status": {
3863
+ const runId = this.#lookupRun(command.stage, command.change);
3864
+ if (!runId) {
3865
+ return {
3866
+ stage: command.stage,
3867
+ change: command.change,
3868
+ state: "not-started",
3869
+ durable: true,
3870
+ completed: false,
3871
+ legalCommands: ["start"],
3872
+ tasks: [],
3873
+ queue: [],
3874
+ };
3875
+ }
3876
+ this.#recoverLegacyContextApproval(runId);
3877
+ return this.#withAvailableDelivery(this.#statusByRun(runId), signal);
3878
+ }
3879
+ case "resume":
3880
+ return this.#withAvailableDelivery(
3881
+ await this.#resume(command, signal, onActivity),
3882
+ signal,
3883
+ );
3884
+ case "rebind":
3885
+ return this.#rebind(command);
3886
+ case "cancel":
3887
+ return this.#cancel(command, onActivity);
3888
+ case "discard":
3889
+ return this.#discard(command, onActivity);
3890
+ }
3891
+ }
3892
+
3893
+ prepareBootstrapHandoff(
3894
+ input: Record<string, unknown>,
3895
+ ): Record<string, unknown> {
3896
+ this.#assertOpen();
3897
+ const acceptanceFacts = normalizeBootstrapAcceptanceFacts(
3898
+ input.acceptanceFacts,
3899
+ );
3900
+ const workspaceManifestHash =
3901
+ typeof input.workspaceManifestHash === "string" &&
3902
+ SHA256.test(input.workspaceManifestHash)
3903
+ ? input.workspaceManifestHash
3904
+ : undefined;
3905
+ if (
3906
+ typeof input.operationId !== "string" ||
3907
+ !IDENTIFIER.test(input.operationId) ||
3908
+ typeof input.runId !== "string" ||
3909
+ typeof input.change !== "string" ||
3910
+ !CHANGE_NAME.test(input.change) ||
3911
+ typeof input.receiptHash !== "string" ||
3912
+ !SHA256.test(input.receiptHash) ||
3913
+ typeof input.graphHash !== "string" ||
3914
+ !SHA256.test(input.graphHash) ||
3915
+ typeof input.acceptanceHash !== "string" ||
3916
+ !SHA256.test(input.acceptanceHash) ||
3917
+ (acceptanceFacts !== undefined) !==
3918
+ (workspaceManifestHash !== undefined) ||
3919
+ (acceptanceFacts !== undefined &&
3920
+ workspaceManifestHash !== undefined &&
3921
+ bootstrapAcceptanceHash(acceptanceFacts, workspaceManifestHash) !==
3922
+ input.acceptanceHash)
3923
+ ) {
3924
+ throw new Error("bootstrap-handoff-invalid");
3925
+ }
3926
+ this.#runStore.status(input.runId);
3927
+ const existing = this.#database
3928
+ .prepare(
3929
+ `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
3930
+ FROM bootstrap_handoffs WHERE receipt_hash = ?`,
3931
+ )
3932
+ .get(input.receiptHash) as BootstrapRow | undefined;
3933
+ if (existing) {
3934
+ const outcome = parseJsonRecord(existing.facts_json);
3935
+ const existingAcceptanceFacts = normalizeBootstrapAcceptanceFacts(
3936
+ outcome.acceptanceFacts,
3937
+ );
3938
+ const storedBindingValid =
3939
+ outcome.handoffId === existing.handoff_id &&
3940
+ outcome.runId === existing.owner_run_id &&
3941
+ outcome.receiptHash === existing.receipt_hash &&
3942
+ outcome.selectorBeforeCutover === "bootstrap" &&
3943
+ existing.state === "prepared" &&
3944
+ outcome.state === "prepared" &&
3945
+ outcome.selectorCasPending === true;
3946
+ if (!storedBindingValid) throw new Error("bootstrap-handoff-invalid");
3947
+ const stableBindingMatches =
3948
+ existing.owner_run_id === input.runId &&
3949
+ outcome.change === input.change &&
3950
+ outcome.graphHash === input.graphHash;
3951
+ if (!stableBindingMatches) {
3952
+ throw new Error("bootstrap-handoff-conflict");
3953
+ }
3954
+ const acceptanceMatches =
3955
+ outcome.acceptanceHash === input.acceptanceHash &&
3956
+ canonicalJson(existingAcceptanceFacts ?? null) ===
3957
+ canonicalJson(acceptanceFacts ?? null) &&
3958
+ outcome.workspaceManifestHash === workspaceManifestHash;
3959
+ if (acceptanceMatches) return structuredClone(outcome);
3960
+ if (
3961
+ existing.state !== "prepared" ||
3962
+ outcome.state !== "prepared" ||
3963
+ outcome.selectorBeforeCutover !== "bootstrap" ||
3964
+ outcome.selectorCasPending !== true ||
3965
+ acceptanceFacts === undefined ||
3966
+ workspaceManifestHash === undefined
3967
+ ) {
3968
+ throw new Error("bootstrap-handoff-conflict");
3969
+ }
3970
+ const refreshedHandoffId = `handoff-${hash(
3971
+ input.runId,
3972
+ input.receiptHash,
3973
+ input.graphHash,
3974
+ input.acceptanceHash,
3975
+ workspaceManifestHash,
3976
+ ).slice(0, 40)}`;
3977
+ const refreshed = {
3978
+ ...outcome,
3979
+ handoffId: refreshedHandoffId,
3980
+ acceptanceHash: input.acceptanceHash,
3981
+ acceptanceFacts,
3982
+ workspaceManifestHash,
3983
+ };
3984
+ this.#transaction(() => {
3985
+ const result = this.#database
3986
+ .prepare(
3987
+ `UPDATE bootstrap_handoffs
3988
+ SET handoff_id = ?, facts_json = ?
3989
+ WHERE handoff_id = ? AND state = 'prepared' AND facts_json = ?`,
3990
+ )
3991
+ .run(
3992
+ refreshedHandoffId,
3993
+ JSON.stringify(refreshed),
3994
+ existing.handoff_id,
3995
+ existing.facts_json,
3996
+ );
3997
+ if (Number(result.changes) !== 1) {
3998
+ throw new Error("bootstrap-handoff-conflict");
3999
+ }
4000
+ });
4001
+ return structuredClone(refreshed);
4002
+ }
4003
+ const handoffId = `handoff-${hash(
4004
+ input.runId,
4005
+ input.receiptHash,
4006
+ input.graphHash,
4007
+ input.acceptanceHash,
4008
+ workspaceManifestHash ?? "acceptance-pending",
4009
+ ).slice(0, 40)}`;
4010
+ const outcome = {
4011
+ handoffId,
4012
+ state: "prepared",
4013
+ runId: input.runId,
4014
+ change: input.change,
4015
+ receiptHash: input.receiptHash,
4016
+ graphHash: input.graphHash,
4017
+ acceptanceHash: input.acceptanceHash,
4018
+ ...(acceptanceFacts ? { acceptanceFacts } : {}),
4019
+ ...(workspaceManifestHash ? { workspaceManifestHash } : {}),
4020
+ selectorBeforeCutover: "bootstrap",
4021
+ selectorCasPending: true,
4022
+ };
4023
+ const preparedRows = this.#database
4024
+ .prepare(
4025
+ `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
4026
+ FROM bootstrap_handoffs WHERE state = 'prepared'
4027
+ ORDER BY handoff_id`,
4028
+ )
4029
+ .all() as unknown as BootstrapRow[];
4030
+ if (preparedRows.length > 0) throw new Error("bootstrap-handoff-conflict");
4031
+ this.#transaction(() => {
4032
+ this.#database
4033
+ .prepare(
4034
+ `INSERT INTO bootstrap_handoffs(
4035
+ handoff_id, owner_run_id, receipt_hash, state, facts_json
4036
+ ) VALUES (?, ?, ?, 'prepared', ?)`,
4037
+ )
4038
+ .run(
4039
+ handoffId,
4040
+ String(input.runId),
4041
+ String(input.receiptHash),
4042
+ JSON.stringify(outcome),
4043
+ );
4044
+ });
4045
+ return structuredClone(outcome);
4046
+ }
4047
+
4048
+ inspectBootstrapHandoff(receiptHash: string): Record<string, unknown> {
4049
+ this.#assertOpen();
4050
+ if (!SHA256.test(receiptHash)) throw new Error("bootstrap-handoff-invalid");
4051
+ const row = this.#database
4052
+ .prepare(
4053
+ `SELECT facts_json FROM bootstrap_handoffs WHERE receipt_hash = ?`,
4054
+ )
4055
+ .get(receiptHash) as { facts_json: string } | undefined;
4056
+ if (!row) throw new Error("bootstrap-handoff-not-found");
4057
+ return parseJsonRecord(row.facts_json);
4058
+ }
4059
+
4060
+ close(): Promise<void> {
4061
+ if (this.#closing) return this.#closing;
4062
+ if (this.#closed) return Promise.resolve();
4063
+ this.#closing = Promise.resolve().then(async () => {
4064
+ if (this.#orphanRecoveryTimer) {
4065
+ clearTimeout(this.#orphanRecoveryTimer);
4066
+ this.#orphanRecoveryTimer = undefined;
4067
+ }
4068
+ const reason = new Error("workflow-engine-closed");
4069
+ const commands = [...this.#commands];
4070
+ for (const [controller] of commands) controller.abort(reason);
4071
+ const active = [...this.#active.values()];
4072
+ for (const operation of active) operation.controller.abort(reason);
4073
+ await Promise.allSettled(commands.map(([, entry]) => entry.settled));
4074
+ await Promise.allSettled(active.map((operation) => operation.settled));
4075
+ await Promise.allSettled([...this.#amendments.values()]);
4076
+ for (const held of this.#operationLeases.values())
4077
+ clearInterval(held.timer);
4078
+ this.#operationLeases.clear();
4079
+ this.#lifecycle?.close?.();
4080
+ this.#database.close();
4081
+ this.#runStore.close();
4082
+ this.#closed = true;
4083
+ });
4084
+ return this.#closing;
4085
+ }
4086
+ }