@abelxiaoxing/cadence 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +115 -10
  2. package/package.json +5 -3
  3. package/prompts/abel-design.md +71 -17
  4. package/prompts/abel-diagnose.md +10 -1
  5. package/prompts/abel-implement.md +118 -20
  6. package/prompts/abel-init.md +9 -2
  7. package/src/apply-transaction.ts +16 -14
  8. package/src/artifact-store.ts +17 -0
  9. package/src/candidate-patch.ts +2 -1
  10. package/src/canonical.ts +16 -0
  11. package/src/change-contract.ts +268 -0
  12. package/src/child-session.ts +4 -3
  13. package/src/contracts.ts +141 -33
  14. package/src/control-contracts.ts +51 -1
  15. package/src/delivery-compiler.ts +148 -31
  16. package/src/design-control.ts +69 -11
  17. package/src/design-journal.ts +216 -72
  18. package/src/durable-workflow.ts +4385 -0
  19. package/src/implement-graph.ts +11 -16
  20. package/src/index.ts +380 -981
  21. package/src/isolation-backend.ts +84 -23
  22. package/src/package-delivery.ts +486 -0
  23. package/src/package-verification.ts +509 -0
  24. package/src/route-policy.ts +1 -11
  25. package/src/run-state.ts +4 -16
  26. package/src/run-store.ts +5 -144
  27. package/src/scoped-grep-worker.mjs +23 -0
  28. package/src/scoped-tools.ts +312 -116
  29. package/src/sqlite-schema.ts +171 -0
  30. package/src/storage-schema.ts +352 -0
  31. package/src/subagent-activity.ts +14 -1
  32. package/src/submit-tool.ts +20 -2
  33. package/src/task-ledger.ts +14 -55
  34. package/src/verification-capability.ts +298 -102
  35. package/src/verification-environment.ts +213 -0
  36. package/src/verification-identity.ts +77 -0
  37. package/src/worker-broker.ts +97 -34
  38. package/src/workflow-engine.ts +28 -8285
  39. package/src/workflow-policy.ts +1427 -0
  40. package/src/workflow-state-machine.ts +4478 -0
  41. package/src/workspace-io-entry.mjs +58 -0
  42. package/src/workspace-io-worker.mjs +984 -0
  43. package/src/workspace-io.ts +84 -0
  44. package/src/workspace-store.ts +125 -19
  45. package/skills/abel-workflow/SKILL.md +0 -205
@@ -0,0 +1,4478 @@
1
+ import { readFileSync } from "node:fs";
2
+ import path from "node:path";
3
+ import { DatabaseSync } from "node:sqlite";
4
+ import {
5
+ assertPlanWithinChangeContract,
6
+ normalizeChangeContract,
7
+ retainedChangeContract,
8
+ } from "./change-contract.ts";
9
+ import { LIMITS } from "./contracts.ts";
10
+ import {
11
+ assertControlCommand,
12
+ type ControlCommand,
13
+ type ControlStage,
14
+ } from "./control-contracts.ts";
15
+ import {
16
+ compileImplementPlan,
17
+ DeliveryValidationError,
18
+ type ImplementPlan,
19
+ type PlanTaskDraft,
20
+ } from "./delivery-compiler.ts";
21
+ import { snapshotFiles } from "./file-snapshot.ts";
22
+ import type { RoutePolicy } from "./route-policy.ts";
23
+ import {
24
+ canonicalJson,
25
+ legalControlCommands,
26
+ type RunProjection,
27
+ type RunState,
28
+ } from "./run-state.ts";
29
+ import { type OperationLease, RunStore } from "./run-store.ts";
30
+ import { isSafeRegularFile } from "./safe-path.ts";
31
+ import { configureSqlite, ensureSqliteSchema } from "./sqlite-schema.ts";
32
+ import type { ResolvedStateRoot } from "./state-root.ts";
33
+ import {
34
+ AMENDMENT_SCHEMA,
35
+ ENGINE_ADDITIONS,
36
+ ENGINE_SCHEMA,
37
+ RECOVERY_ADDITIONS,
38
+ RECOVERY_GRANT_SCHEMA,
39
+ RECOVERY_SCHEMA,
40
+ REJECTED_DELIVERY_SCHEMA,
41
+ } from "./storage-schema.ts";
42
+ import type { WorkflowActivityUpdate } from "./subagent-activity.ts";
43
+ import { permitsContextRead } from "./submit-tool.ts";
44
+ import {
45
+ type ActiveOperation,
46
+ approvalRequirement,
47
+ assertDelivery,
48
+ type BootstrapRow,
49
+ bootstrapAcceptanceHash,
50
+ CHANGE_NAME,
51
+ cancellableRead,
52
+ classifyPersistedContextRequest,
53
+ decideRecoveryAction,
54
+ deliveryBoundPaths,
55
+ type EngineOperationRow,
56
+ type EngineRunRow,
57
+ type EngineTaskRow,
58
+ emitWorkflowActivity,
59
+ expectedClassification,
60
+ type HeldOperationLease,
61
+ hash,
62
+ IDENTIFIER,
63
+ isCancellationException,
64
+ isRecord,
65
+ isWorkflowApprovalCode,
66
+ normalizeBootstrapAcceptanceFacts,
67
+ normalizeWorkflowContextRequest,
68
+ normalizeWorkflowVerificationStatus,
69
+ parseDeliveryDiagnostics,
70
+ parseJsonRecord,
71
+ parsePlan,
72
+ permitsPlanAmendment,
73
+ phases,
74
+ recoveryKey,
75
+ type SafeAttemptDiagnostic,
76
+ SHA256,
77
+ taskApprovalBoundary,
78
+ taskConflicts,
79
+ type WorkflowApplication,
80
+ type WorkflowApplicationContext,
81
+ type WorkflowAttemptOutcome,
82
+ type WorkflowAvailableDelivery,
83
+ type WorkflowChangeVerifier,
84
+ type WorkflowContextRequest,
85
+ type WorkflowDelivery,
86
+ type WorkflowDeliverySource,
87
+ type WorkflowEngineOptions,
88
+ type WorkflowRecoveryFact,
89
+ type WorkflowRunLifecycle,
90
+ type WorkflowVerificationStatus,
91
+ type WorkflowWorker,
92
+ } from "./workflow-policy.ts";
93
+ export class WorkflowEngine {
94
+ readonly #consumerRoot: string;
95
+ readonly #stateRoot: ResolvedStateRoot;
96
+ readonly #runStore: RunStore;
97
+ readonly #database: DatabaseSync;
98
+ readonly #deliverySource: WorkflowDeliverySource;
99
+ readonly #worker: WorkflowWorker;
100
+ readonly #changeVerifier: WorkflowChangeVerifier;
101
+ readonly #application?: WorkflowApplication;
102
+ readonly #lifecycle?: WorkflowRunLifecycle;
103
+ readonly #now: () => number;
104
+ readonly #leaseTtlMs: number;
105
+ readonly #workHardLimit: number;
106
+ #activeTasks = 0;
107
+ readonly #capacityWaiters = new Set<() => void>();
108
+ readonly #active = new Map<string, ActiveOperation>();
109
+ readonly #operationLeases = new Map<string, HeldOperationLease>();
110
+ #orphanRecoveryTimer?: ReturnType<typeof setTimeout>;
111
+ readonly #commands = new Map<
112
+ AbortController,
113
+ { command: ControlCommand; settled: Promise<Record<string, unknown>> }
114
+ >();
115
+ #closing?: Promise<void>;
116
+ readonly #amendments = new Map<string, Promise<Record<string, unknown>>>();
117
+ #closed = false;
118
+
119
+ private constructor(options: WorkflowEngineOptions) {
120
+ this.#consumerRoot = path.resolve(options.consumerRoot);
121
+ this.#stateRoot = options.stateRoot;
122
+ this.#deliverySource = options.deliverySource;
123
+ this.#worker = options.worker;
124
+ this.#changeVerifier = options.changeVerifier;
125
+ this.#application = options.application;
126
+ this.#lifecycle = options.lifecycle;
127
+ this.#now = options.now ?? Date.now;
128
+ this.#leaseTtlMs = options.leaseTtlMs ?? 60_000;
129
+ this.#workHardLimit = options.workHardLimit ?? 512;
130
+ if (!Number.isSafeInteger(this.#workHardLimit) || this.#workHardLimit < 1)
131
+ throw new Error("workflow-work-limit-invalid");
132
+ this.#runStore = RunStore.open(options.stateRoot, { now: this.#now });
133
+ this.#database = new DatabaseSync(options.stateRoot.databasePath);
134
+ try {
135
+ configureSqlite(this.#database);
136
+ ensureSqliteSchema(this.#database, ENGINE_SCHEMA, ENGINE_ADDITIONS);
137
+ ensureSqliteSchema(this.#database, RECOVERY_SCHEMA, RECOVERY_ADDITIONS);
138
+ ensureSqliteSchema(this.#database, RECOVERY_GRANT_SCHEMA);
139
+ ensureSqliteSchema(this.#database, AMENDMENT_SCHEMA);
140
+ ensureSqliteSchema(this.#database, REJECTED_DELIVERY_SCHEMA);
141
+ } catch (error) {
142
+ this.#database.close();
143
+ this.#runStore.close();
144
+ throw error;
145
+ }
146
+ this.#transaction(() => {
147
+ const budgets = this.#database
148
+ .prepare(
149
+ "SELECT run_id FROM workflow_work_budget WHERE recovery_policy = 0",
150
+ )
151
+ .all() as { run_id: string }[];
152
+ for (const { run_id } of budgets) {
153
+ const plans = this.#database
154
+ .prepare(
155
+ "SELECT plan_json FROM workflow_engine_deliveries WHERE run_id = ?",
156
+ )
157
+ .all(run_id) as { plan_json: string }[];
158
+ const high = Math.max(
159
+ 0,
160
+ ...plans.map((row) => this.#phaseCount(JSON.parse(row.plan_json))),
161
+ );
162
+ this.#database
163
+ .prepare(
164
+ "UPDATE workflow_work_budget SET phase_high_water = ?, hard_limit = MAX(hard_limit, max_work, used), recovery_policy = 1 WHERE run_id = ?",
165
+ )
166
+ .run(high, run_id);
167
+ }
168
+ });
169
+ this.#recoverInterruptedOperations();
170
+ this.#recoverTerminalCleanup();
171
+ }
172
+
173
+ static open(options: WorkflowEngineOptions): WorkflowEngine {
174
+ if (
175
+ path.resolve(options.consumerRoot) !== options.stateRoot.consumerRoot ||
176
+ options.stateRoot.consumerRootHash.length !== 64
177
+ ) {
178
+ throw new Error("workflow-engine-root-mismatch");
179
+ }
180
+ if (
181
+ !options.deliverySource ||
182
+ typeof options.deliverySource.load !== "function" ||
183
+ !options.worker ||
184
+ typeof options.worker.runAttempt !== "function" ||
185
+ typeof options.worker.rebind !== "function" ||
186
+ !options.changeVerifier ||
187
+ typeof options.changeVerifier.verify !== "function" ||
188
+ (options.application !== undefined &&
189
+ (typeof options.application.prepare === "function") !==
190
+ (typeof options.application.applyPrepared === "function"))
191
+ ) {
192
+ throw new Error("workflow-engine-service-invalid");
193
+ }
194
+ if (
195
+ options.leaseTtlMs !== undefined &&
196
+ (!Number.isSafeInteger(options.leaseTtlMs) || options.leaseTtlMs < 1)
197
+ ) {
198
+ throw new Error("workflow-engine-lease-invalid");
199
+ }
200
+ return new WorkflowEngine(options);
201
+ }
202
+
203
+ updateRoutePolicy(policy: RoutePolicy): void {
204
+ this.#assertOpen();
205
+ if (!this.#worker.updateRoutePolicy) {
206
+ throw new Error("route-policy-update-unavailable");
207
+ }
208
+ this.#worker.updateRoutePolicy(policy);
209
+ }
210
+
211
+ routePolicyStatus(): Record<string, unknown> | undefined {
212
+ this.#assertOpen();
213
+ return this.#worker.routePolicyStatus?.();
214
+ }
215
+
216
+ #assertOpen(): void {
217
+ if (this.#closed) throw new Error("workflow-engine-closed");
218
+ }
219
+
220
+ #transaction<T>(work: () => T): T {
221
+ this.#database.exec("BEGIN IMMEDIATE");
222
+ try {
223
+ const result = work();
224
+ this.#database.exec("COMMIT");
225
+ return result;
226
+ } catch (error) {
227
+ this.#database.exec("ROLLBACK");
228
+ throw error;
229
+ }
230
+ }
231
+
232
+ #operationKey(runId: string, operationId: string): string {
233
+ return `${runId}\0${operationId}`;
234
+ }
235
+
236
+ #leaseOperationId(runId: string, _operationId: string): string {
237
+ return `engine-run-${hash(runId).slice(0, 40)}`;
238
+ }
239
+
240
+ #holdOperationLease(lease: OperationLease, commandOperationId: string): void {
241
+ const key = this.#operationKey(lease.runId, commandOperationId);
242
+ const existing = this.#operationLeases.get(key);
243
+ if (existing) clearInterval(existing.timer);
244
+ const intervalMs = Math.max(1, Math.floor(this.#leaseTtlMs / 3));
245
+ const held: HeldOperationLease = {
246
+ lease,
247
+ timer: setInterval(() => {
248
+ if (this.#operationLeases.get(key) !== held) return;
249
+ try {
250
+ held.lease = this.#runStore.renewLease(held.lease, this.#leaseTtlMs);
251
+ this.#database
252
+ .prepare(
253
+ `UPDATE workflow_engine_operations
254
+ SET lease_expires_at = ?
255
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'
256
+ AND lease_token = ?`,
257
+ )
258
+ .run(
259
+ held.lease.expiresAt,
260
+ lease.runId,
261
+ commandOperationId,
262
+ lease.token,
263
+ );
264
+ } catch {
265
+ held.error = new Error("lease-fenced");
266
+ clearInterval(held.timer);
267
+ const active = this.#active.get(lease.runId);
268
+ if (active?.operationId === commandOperationId) {
269
+ active.controller.abort(held.error);
270
+ }
271
+ }
272
+ }, intervalMs),
273
+ };
274
+ held.timer.unref?.();
275
+ this.#operationLeases.set(key, held);
276
+ }
277
+
278
+ #operationLease(runId: string, operationId: string): OperationLease {
279
+ const held = this.#operationLeases.get(
280
+ this.#operationKey(runId, operationId),
281
+ );
282
+ if (!held || held.error) throw held?.error ?? new Error("lease-fenced");
283
+ this.#runStore.assertLease(held.lease);
284
+ return held.lease;
285
+ }
286
+
287
+ #releaseOperationLease(runId: string, operationId: string): void {
288
+ const key = this.#operationKey(runId, operationId);
289
+ const held = this.#operationLeases.get(key);
290
+ if (!held) return;
291
+ clearInterval(held.timer);
292
+ this.#operationLeases.delete(key);
293
+ }
294
+
295
+ #assertLeaseInTransaction(lease: OperationLease): void {
296
+ const row = this.#database
297
+ .prepare(
298
+ `SELECT state, lease_token, lease_expires_at
299
+ FROM operations WHERE run_id = ? AND operation_id = ?`,
300
+ )
301
+ .get(lease.runId, lease.operationId) as
302
+ | {
303
+ state: string;
304
+ lease_token: string | null;
305
+ lease_expires_at: number | null;
306
+ }
307
+ | undefined;
308
+ if (
309
+ row?.state !== "running" ||
310
+ row.lease_token !== lease.token ||
311
+ row.lease_expires_at === null ||
312
+ row.lease_expires_at < this.#now()
313
+ ) {
314
+ throw new Error("lease-fenced");
315
+ }
316
+ }
317
+
318
+ #leasedTransaction<T>(lease: OperationLease, work: () => T): T {
319
+ return this.#transaction(() => {
320
+ this.#assertLeaseInTransaction(lease);
321
+ return work();
322
+ });
323
+ }
324
+
325
+ amend(
326
+ change: string,
327
+ batchId: string,
328
+ request: unknown,
329
+ execute: (assertAuthority: () => void) => Promise<Record<string, unknown>>,
330
+ ): Promise<Record<string, unknown>> {
331
+ this.#assertOpen();
332
+ if (this.#closing) throw new Error("workflow-engine-closed");
333
+ const runId = this.#lookupRun("abel-implement", change);
334
+ if (!runId) throw new Error("run-not-found");
335
+ if (this.#amendments.has(runId)) throw new Error("amendment-busy");
336
+ const operationId = `amend-${hash(batchId, canonicalJson(request)).slice(0, 48)}`;
337
+ const pending = Promise.resolve()
338
+ .then(async () => {
339
+ const replay = this.#beginOperation(runId, operationId, "amend");
340
+ if (replay) return replay;
341
+ const lease = this.#operationLease(runId, operationId);
342
+ const assertAuthority = () => {
343
+ this.#assertOpen();
344
+ if (this.#closing) throw new Error("workflow-engine-closed");
345
+ this.#runStore.assertLease(lease);
346
+ const status = this.#statusByRun(runId);
347
+ if (
348
+ !["approval-needed", "paused"].includes(String(status.state)) ||
349
+ (status.decisionBatch as { id?: string } | undefined)?.id !==
350
+ batchId
351
+ )
352
+ throw new Error("amendment-batch-stale");
353
+ };
354
+ try {
355
+ assertAuthority();
356
+ if (
357
+ !isRecord(request) ||
358
+ !["status", "validate-plan-draft"].includes(
359
+ String(request.operation),
360
+ )
361
+ ) {
362
+ this.#leasedTransaction(lease, () => {
363
+ this.#database
364
+ .prepare(
365
+ "INSERT INTO workflow_amendment_budget(run_id, used) VALUES (?, 0) ON CONFLICT DO NOTHING",
366
+ )
367
+ .run(runId);
368
+ const reserved = this.#database
369
+ .prepare(
370
+ "UPDATE workflow_amendment_budget SET used = used + 1 WHERE run_id = ? AND used < 64",
371
+ )
372
+ .run(runId);
373
+ if (reserved.changes !== 1)
374
+ throw new Error("amendment-budget-exhausted");
375
+ });
376
+ }
377
+ this.#assertAmendmentContract(runId, change, request);
378
+ const outcome = await execute(assertAuthority);
379
+ assertAuthority();
380
+ return this.#commitOperation(runId, operationId, outcome);
381
+ } catch (error) {
382
+ this.#interruptOperation(runId, operationId);
383
+ throw error;
384
+ }
385
+ })
386
+ .finally(() => {
387
+ if (this.#amendments.get(runId) === pending)
388
+ this.#amendments.delete(runId);
389
+ });
390
+ this.#amendments.set(runId, pending);
391
+ return pending;
392
+ }
393
+
394
+ #assertAmendmentContract(
395
+ runId: string,
396
+ change: string,
397
+ request: unknown,
398
+ ): void {
399
+ if (!isRecord(request)) throw new Error("amendment-contract-invalid");
400
+ if (
401
+ (request.operation === "approve-gate" && request.gate === "gate-a") ||
402
+ (request.operation === "record-decision" &&
403
+ request.category === "behavior")
404
+ )
405
+ throw new Error("amendment-changes-accepted-behavior");
406
+ if (
407
+ ![
408
+ "write-artifact",
409
+ "delete-artifact",
410
+ "compile-plan",
411
+ "validate-plan-draft",
412
+ "finalize-delivery",
413
+ ].includes(String(request.operation))
414
+ )
415
+ return;
416
+ const relative = String(request.path);
417
+ if (relative === "proposal.md" || relative.startsWith("specs/")) {
418
+ const file = `openspec/changes/${change}/${relative}`;
419
+ if (
420
+ request.operation !== "write-artifact" ||
421
+ !isSafeRegularFile(this.#consumerRoot, file) ||
422
+ readFileSync(path.join(this.#consumerRoot, file), "utf8") !==
423
+ request.content
424
+ )
425
+ throw new Error("amendment-changes-accepted-behavior");
426
+ }
427
+ const compile = [
428
+ "compile-plan",
429
+ "validate-plan-draft",
430
+ "finalize-delivery",
431
+ ].includes(String(request.operation));
432
+ if (!compile) return;
433
+ const draftPath = `openspec/changes/${change}/plan-draft.json`;
434
+ if (!isSafeRegularFile(this.#consumerRoot, draftPath)) return;
435
+ const run = this.#engineRun(runId);
436
+ if (run.current_revision === null) return;
437
+ const prior = this.#planForRevision(runId, run.current_revision);
438
+ const contract = retainedChangeContract(prior);
439
+ const draft = JSON.parse(
440
+ readFileSync(path.join(this.#consumerRoot, draftPath), "utf8"),
441
+ );
442
+ if (
443
+ draft.changeContract !== undefined &&
444
+ canonicalJson(normalizeChangeContract(draft.changeContract)) !==
445
+ canonicalJson(contract)
446
+ )
447
+ throw new Error("change-contract-mismatch");
448
+ // Match DesignController's inheritance before validating authorized modes.
449
+ if (draft.changeContract === undefined && prior.changeContract)
450
+ draft.changeContract = contract;
451
+ const next = compileImplementPlan(draft, {
452
+ consumerRoot: this.#consumerRoot,
453
+ bindExecutionInputs: true,
454
+ }).plan;
455
+ assertPlanWithinChangeContract(contract, next.tasks, [
456
+ next.verification.change.fullSuite,
457
+ next.verification.change.postApply,
458
+ ]);
459
+ }
460
+
461
+ #lookupRun(stage: ControlStage, change: string): string | undefined {
462
+ return (
463
+ this.#database
464
+ .prepare(
465
+ `SELECT run_id FROM runs
466
+ WHERE root_hash = ? AND stage = ? AND lookup_key = ?`,
467
+ )
468
+ .get(this.#stateRoot.consumerRootHash, stage, `change:${change}`) as
469
+ | { run_id: string }
470
+ | undefined
471
+ )?.run_id;
472
+ }
473
+
474
+ #ensureEngineRun(runId: string): void {
475
+ this.#database
476
+ .prepare(
477
+ `INSERT INTO workflow_engine_runs(
478
+ run_id, current_revision, baseline_workspace_revision,
479
+ current_workspace_revision, cleanup_state, next_queue_position
480
+ ) VALUES (?, NULL, NULL, NULL, 'none', 1)
481
+ ON CONFLICT(run_id) DO NOTHING`,
482
+ )
483
+ .run(runId);
484
+ }
485
+
486
+ #engineRun(runId: string): EngineRunRow {
487
+ const row = this.#database
488
+ .prepare(
489
+ `SELECT current_revision, baseline_workspace_revision,
490
+ current_workspace_revision, cleanup_state, route_id,
491
+ route_fingerprint,
492
+ transaction_id, verification_json, delivery_diagnostics_json,
493
+ next_queue_position
494
+ FROM workflow_engine_runs WHERE run_id = ?`,
495
+ )
496
+ .get(runId) as EngineRunRow | undefined;
497
+ if (!row) throw new Error("workflow-engine-run-unavailable");
498
+ return row;
499
+ }
500
+
501
+ #operation(
502
+ runId: string,
503
+ operationId: string,
504
+ ): EngineOperationRow | undefined {
505
+ return this.#database
506
+ .prepare(
507
+ `SELECT command, state, outcome_json FROM workflow_engine_operations
508
+ WHERE run_id = ? AND operation_id = ?`,
509
+ )
510
+ .get(runId, operationId) as EngineOperationRow | undefined;
511
+ }
512
+
513
+ #operationReplay(
514
+ runId: string,
515
+ operationId: string,
516
+ command: string,
517
+ ): Record<string, unknown> | undefined {
518
+ const row = this.#operation(runId, operationId);
519
+ if (row && row.command !== command) {
520
+ throw new Error("operation-command-mismatch");
521
+ }
522
+ return row?.state === "committed" && row.outcome_json
523
+ ? parseJsonRecord(row.outcome_json)
524
+ : undefined;
525
+ }
526
+
527
+ #beginOperation(
528
+ runId: string,
529
+ operationId: string,
530
+ command: string,
531
+ ): Record<string, unknown> | undefined {
532
+ if (!IDENTIFIER.test(operationId)) throw new Error("invalid-operation-id");
533
+ const replay = this.#operationReplay(runId, operationId, command);
534
+ if (replay) return replay;
535
+ const existing = this.#operation(runId, operationId);
536
+ if (existing?.state === "running") {
537
+ const leaseStatus = this.#runStore.inspectOperation(
538
+ runId,
539
+ this.#leaseOperationId(runId, operationId),
540
+ );
541
+ if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
542
+ this.#database
543
+ .prepare(
544
+ `UPDATE workflow_engine_operations
545
+ SET state = 'committed', lease_token = NULL,
546
+ lease_expires_at = NULL, outcome_json = ?
547
+ WHERE run_id = ? AND operation_id = ?`,
548
+ )
549
+ .run(JSON.stringify(leaseStatus.outcome), runId, operationId);
550
+ return structuredClone(leaseStatus.outcome);
551
+ }
552
+ if (leaseStatus?.state === "running") {
553
+ throw new Error("operation-already-running");
554
+ }
555
+ this.#database
556
+ .prepare(
557
+ `UPDATE workflow_engine_operations
558
+ SET state = 'interrupted', lease_token = NULL,
559
+ lease_expires_at = NULL
560
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
561
+ )
562
+ .run(runId, operationId);
563
+ }
564
+ const leaseOperationId = this.#leaseOperationId(runId, operationId);
565
+ const priorLease = this.#runStore.inspectOperation(runId, leaseOperationId);
566
+ const preempt = command === "cancel" || command === "discard";
567
+ if (priorLease?.state === "running" && !preempt) {
568
+ throw new Error("operation-already-running");
569
+ }
570
+ const lease = this.#runStore.acquireLease({
571
+ runId,
572
+ operationId: leaseOperationId,
573
+ ttlMs: this.#leaseTtlMs,
574
+ exclusive: true,
575
+ ...(preempt ? { preempt: true } : {}),
576
+ });
577
+ if (preempt) {
578
+ this.#database
579
+ .prepare(
580
+ `UPDATE workflow_engine_operations
581
+ SET state = 'interrupted', lease_token = NULL,
582
+ lease_expires_at = NULL
583
+ WHERE run_id = ? AND operation_id <> ? AND state = 'running'`,
584
+ )
585
+ .run(runId, operationId);
586
+ }
587
+ this.#database
588
+ .prepare(
589
+ `INSERT INTO workflow_engine_operations(
590
+ run_id, operation_id, command, state, lease_token,
591
+ lease_expires_at, outcome_json
592
+ ) VALUES (?, ?, ?, 'running', ?, ?, NULL)
593
+ ON CONFLICT(run_id, operation_id) DO UPDATE SET
594
+ command = excluded.command, state = 'running',
595
+ lease_token = excluded.lease_token,
596
+ lease_expires_at = excluded.lease_expires_at,
597
+ outcome_json = NULL`,
598
+ )
599
+ .run(runId, operationId, command, lease.token, lease.expiresAt);
600
+ this.#holdOperationLease(lease, operationId);
601
+ return undefined;
602
+ }
603
+
604
+ #commitOperation(
605
+ runId: string,
606
+ operationId: string,
607
+ outcome: Record<string, unknown>,
608
+ ): Record<string, unknown> {
609
+ const lease = this.#operationLease(runId, operationId);
610
+ try {
611
+ const committed = this.#runStore.commitLease(lease, outcome);
612
+ const result = this.#database
613
+ .prepare(
614
+ `UPDATE workflow_engine_operations
615
+ SET state = 'committed', lease_token = NULL,
616
+ lease_expires_at = NULL, outcome_json = ?
617
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'
618
+ AND lease_token = ?`,
619
+ )
620
+ .run(JSON.stringify(committed), runId, operationId, lease.token);
621
+ if (Number(result.changes) !== 1) {
622
+ throw new Error("operation-journal-fenced");
623
+ }
624
+ return structuredClone(committed);
625
+ } finally {
626
+ this.#releaseOperationLease(runId, operationId);
627
+ }
628
+ }
629
+
630
+ #interruptOperation(runId: string, operationId: string): void {
631
+ let lease: OperationLease;
632
+ try {
633
+ lease = this.#operationLease(runId, operationId);
634
+ this.#runStore.interruptLease(lease);
635
+ this.#database
636
+ .prepare(
637
+ `UPDATE workflow_engine_operations
638
+ SET state = 'interrupted', lease_token = NULL,
639
+ lease_expires_at = NULL
640
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'
641
+ AND lease_token = ?`,
642
+ )
643
+ .run(runId, operationId, lease.token);
644
+ } catch (error) {
645
+ if (!(error instanceof Error) || error.message !== "lease-fenced") {
646
+ throw error;
647
+ }
648
+ } finally {
649
+ this.#releaseOperationLease(runId, operationId);
650
+ }
651
+ }
652
+
653
+ #transition(
654
+ runId: string,
655
+ to: RunState,
656
+ identity: string,
657
+ code?: string,
658
+ lease?: OperationLease,
659
+ ): RunProjection {
660
+ if (lease) this.#runStore.assertLease(lease);
661
+ const current = this.#runStore.status(runId);
662
+ if (
663
+ current.state === to &&
664
+ (to !== "paused" || code === undefined || current.pauseCode === code)
665
+ ) {
666
+ return current;
667
+ }
668
+ return this.#runStore.transition({
669
+ runId,
670
+ to,
671
+ operationId: `eng-${hash(runId, identity, String(current.sequence), to).slice(0, 40)}`,
672
+ ...(code ? { code } : {}),
673
+ ...(lease ? { lease } : {}),
674
+ });
675
+ }
676
+
677
+ #recoverInterruptedOperations(): void {
678
+ if (this.#orphanRecoveryTimer) {
679
+ clearTimeout(this.#orphanRecoveryTimer);
680
+ this.#orphanRecoveryTimer = undefined;
681
+ }
682
+ let nextExpiry: number | undefined;
683
+ const interrupted = this.#database
684
+ .prepare(
685
+ `SELECT run_id, operation_id FROM workflow_engine_operations
686
+ WHERE state = 'running'`,
687
+ )
688
+ .all() as unknown as Array<{ run_id: string; operation_id: string }>;
689
+ for (const row of interrupted) {
690
+ const leaseStatus = this.#runStore.inspectOperation(
691
+ row.run_id,
692
+ this.#leaseOperationId(row.run_id, row.operation_id),
693
+ );
694
+ if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
695
+ this.#database
696
+ .prepare(
697
+ `UPDATE workflow_engine_operations
698
+ SET state = 'committed', lease_token = NULL,
699
+ lease_expires_at = NULL, outcome_json = ?
700
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
701
+ )
702
+ .run(
703
+ JSON.stringify(leaseStatus.outcome),
704
+ row.run_id,
705
+ row.operation_id,
706
+ );
707
+ continue;
708
+ }
709
+ if (leaseStatus?.state === "running") {
710
+ if (leaseStatus.expiresAt !== undefined) {
711
+ nextExpiry = Math.min(nextExpiry ?? Infinity, leaseStatus.expiresAt);
712
+ }
713
+ continue;
714
+ }
715
+ this.#database
716
+ .prepare(
717
+ `UPDATE workflow_engine_operations
718
+ SET state = 'interrupted', lease_token = NULL,
719
+ lease_expires_at = NULL
720
+ WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
721
+ )
722
+ .run(row.run_id, row.operation_id);
723
+ const projection = this.#runStore.status(row.run_id);
724
+ if (["completed", "discarded", "rejected"].includes(projection.state)) {
725
+ continue;
726
+ }
727
+ this.#database
728
+ .prepare(
729
+ `UPDATE workflow_engine_tasks
730
+ SET state = 'paused',
731
+ pause_code = CASE
732
+ WHEN pause_code = 'red-artifact-constraint'
733
+ THEN pause_code
734
+ ELSE 'operation-interrupted'
735
+ END,
736
+ context_request_json = CASE
737
+ WHEN pause_code = 'red-artifact-constraint'
738
+ THEN context_request_json
739
+ ELSE NULL
740
+ END,
741
+ queue_position = NULL
742
+ WHERE run_id = ? AND state IN ('phase-running', 'validating')`,
743
+ )
744
+ .run(row.run_id);
745
+ if (projection.state === "applying") {
746
+ this.#transition(
747
+ row.run_id,
748
+ "recovering",
749
+ "restart-apply-recovery",
750
+ "operation-interrupted",
751
+ );
752
+ } else if (
753
+ [
754
+ "created",
755
+ "validating-delivery",
756
+ "ready",
757
+ "queued",
758
+ "connecting",
759
+ "running",
760
+ "validating",
761
+ "verifying",
762
+ "retryable",
763
+ "approval-needed",
764
+ "change-verifying",
765
+ "ready-to-apply",
766
+ ].includes(projection.state)
767
+ ) {
768
+ this.#transition(
769
+ row.run_id,
770
+ "paused",
771
+ "restart-operation-interrupted",
772
+ "operation-interrupted",
773
+ );
774
+ }
775
+ }
776
+ if (nextExpiry !== undefined && !this.#closed) {
777
+ const delay = Math.max(1, nextExpiry - this.#now() + 1);
778
+ this.#orphanRecoveryTimer = setTimeout(() => {
779
+ this.#orphanRecoveryTimer = undefined;
780
+ if (this.#closed) return;
781
+ try {
782
+ this.#recoverInterruptedOperations();
783
+ } catch {
784
+ // A later status/restart will surface durable journal corruption.
785
+ }
786
+ }, delay);
787
+ this.#orphanRecoveryTimer.unref?.();
788
+ }
789
+ }
790
+
791
+ #recoverTerminalCleanup(): void {
792
+ const rows = this.#database
793
+ .prepare(
794
+ `SELECT workflow_engine_runs.run_id
795
+ FROM workflow_engine_runs
796
+ JOIN runs ON runs.run_id = workflow_engine_runs.run_id
797
+ WHERE runs.state IN ('completed', 'discarded')
798
+ AND workflow_engine_runs.cleanup_state = 'retained'`,
799
+ )
800
+ .all() as unknown as Array<{ run_id: string }>;
801
+ for (const row of rows) {
802
+ try {
803
+ this.#cleanupTerminal(row.run_id, "restart-cleanup");
804
+ } catch {
805
+ // Terminal status remains truthful and a later reload retries cleanup.
806
+ }
807
+ }
808
+ }
809
+
810
+ #tasks(runId: string): EngineTaskRow[] {
811
+ return this.#database
812
+ .prepare(
813
+ `SELECT task_id, task_order, delivery_revision, plan_json, state,
814
+ phase, pause_code, context_request_json, attempt_diagnostic_json,
815
+ route_id, route_fingerprint, queue_position
816
+ FROM workflow_engine_tasks WHERE run_id = ? ORDER BY task_order`,
817
+ )
818
+ .all(runId) as unknown as EngineTaskRow[];
819
+ }
820
+
821
+ #taskPlan(row: EngineTaskRow): PlanTaskDraft {
822
+ return JSON.parse(row.plan_json) as PlanTaskDraft;
823
+ }
824
+
825
+ #currentPlan(runId: string): ImplementPlan {
826
+ const revision = this.#engineRun(runId).current_revision;
827
+ if (revision === null) throw new Error("delivery-not-bound");
828
+ return this.#planForRevision(runId, revision);
829
+ }
830
+
831
+ #planForRevision(runId: string, revision: number): ImplementPlan {
832
+ const row = this.#database
833
+ .prepare(
834
+ `SELECT plan_json FROM workflow_engine_deliveries
835
+ WHERE run_id = ? AND revision = ?`,
836
+ )
837
+ .get(runId, revision) as { plan_json: string } | undefined;
838
+ if (!row) throw new Error("delivery-not-bound");
839
+ return parsePlan(row.plan_json);
840
+ }
841
+
842
+ async #loadDelivery(
843
+ runId: string,
844
+ lease: OperationLease,
845
+ stage: ControlStage,
846
+ change: string,
847
+ deliveryRevision?: number,
848
+ receiptHash?: string,
849
+ signal?: AbortSignal,
850
+ ): Promise<WorkflowDelivery> {
851
+ if (deliveryRevision === undefined && this.#deliverySource.discoverLatest) {
852
+ try {
853
+ const discover = this.#deliverySource.discoverLatest;
854
+ const available = await cancellableRead(
855
+ () => discover.call(this.#deliverySource, { stage, change, signal }),
856
+ signal,
857
+ );
858
+ if (
859
+ available &&
860
+ Number.isSafeInteger(available.deliveryRevision) &&
861
+ available.deliveryRevision > 0 &&
862
+ SHA256.test(available.receiptHash)
863
+ ) {
864
+ deliveryRevision = available.deliveryRevision;
865
+ receiptHash = available.receiptHash;
866
+ }
867
+ } catch {
868
+ // Discovery is optional; authoritative loading still reports diagnostics.
869
+ signal?.throwIfAborted();
870
+ }
871
+ }
872
+ try {
873
+ const delivery = await cancellableRead(
874
+ () =>
875
+ this.#deliverySource.load({
876
+ stage,
877
+ change,
878
+ ...(deliveryRevision === undefined ? {} : { deliveryRevision }),
879
+ ...(receiptHash === undefined ? {} : { receiptHash }),
880
+ ...(signal ? { signal } : {}),
881
+ }),
882
+ signal,
883
+ );
884
+ signal?.throwIfAborted();
885
+ assertDelivery(delivery, stage, change, deliveryRevision, receiptHash);
886
+ return structuredClone(delivery);
887
+ } catch (error) {
888
+ if (
889
+ error instanceof DeliveryValidationError &&
890
+ deliveryRevision !== undefined &&
891
+ receiptHash !== undefined
892
+ ) {
893
+ this.#leasedTransaction(lease, () => {
894
+ this.#database
895
+ .prepare(
896
+ "INSERT INTO workflow_rejected_deliveries(run_id, delivery_revision, receipt_hash) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
897
+ )
898
+ .run(runId, deliveryRevision, receiptHash);
899
+ });
900
+ }
901
+ throw error;
902
+ }
903
+ }
904
+
905
+ #admitDelivery(
906
+ runId: string,
907
+ delivery: WorkflowDelivery,
908
+ lease: OperationLease,
909
+ ): void {
910
+ this.#runStore.assertLease(lease);
911
+ const existing = this.#database
912
+ .prepare(
913
+ `SELECT receipt_hash, plan_json FROM workflow_engine_deliveries
914
+ WHERE run_id = ? AND revision = ?`,
915
+ )
916
+ .get(runId, delivery.revision) as
917
+ | { receipt_hash: string; plan_json: string }
918
+ | undefined;
919
+ const capturedBudget = this.#database
920
+ .prepare("SELECT hard_limit FROM workflow_work_budget WHERE run_id = ?")
921
+ .get(runId) as { hard_limit: number } | undefined;
922
+ if (
923
+ this.#phaseCount(delivery.plan) >
924
+ (capturedBudget?.hard_limit ?? this.#workHardLimit) &&
925
+ !existing
926
+ )
927
+ throw new DeliveryValidationError([
928
+ "verification-work-budget-insufficient",
929
+ ]);
930
+ const serializedPlan = JSON.stringify(delivery.plan);
931
+ if (
932
+ existing &&
933
+ (existing.receipt_hash !== delivery.receiptHash ||
934
+ existing.plan_json !== serializedPlan)
935
+ ) {
936
+ throw new Error("delivery-revision-conflict");
937
+ }
938
+ const engineRun = this.#engineRun(runId);
939
+ if (
940
+ engineRun.current_revision !== null &&
941
+ delivery.revision < engineRun.current_revision
942
+ ) {
943
+ throw new Error("delivery-revision-stale");
944
+ }
945
+ const priorRows = this.#tasks(runId);
946
+ const priorById = new Map(priorRows.map((row) => [row.task_id, row]));
947
+ const priorPlan =
948
+ engineRun.current_revision === null
949
+ ? undefined
950
+ : this.#planForRevision(runId, engineRun.current_revision);
951
+ const priorTasks = new Map(
952
+ (priorPlan?.tasks ?? []).map((task) => [task.taskId, task]),
953
+ );
954
+ const nextTasks = new Map(
955
+ delivery.plan.tasks.map((task) => [task.taskId, task]),
956
+ );
957
+ const priorBoundPaths = new Set(
958
+ priorPlan ? deliveryBoundPaths(priorPlan) : [],
959
+ );
960
+ const boundaryExpanded = deliveryBoundPaths(delivery.plan).some(
961
+ (relative) => !priorBoundPaths.has(relative),
962
+ );
963
+ const compatible = new Set<string>();
964
+ if (priorPlan) {
965
+ for (const [taskId, priorTask] of priorTasks) {
966
+ const nextTask = nextTasks.get(taskId);
967
+ if (
968
+ nextTask &&
969
+ taskApprovalBoundary(priorPlan, priorTask) ===
970
+ taskApprovalBoundary(delivery.plan, nextTask)
971
+ ) {
972
+ compatible.add(taskId);
973
+ }
974
+ }
975
+ }
976
+ let invalidated = new Set(
977
+ priorRows
978
+ .filter((row) => !compatible.has(row.task_id))
979
+ .map((row) => row.task_id),
980
+ );
981
+ let revalidatedWorkspace:
982
+ | {
983
+ baselineRevisionId: string;
984
+ currentWorkspaceRevisionId: string;
985
+ }
986
+ | undefined;
987
+ if (
988
+ priorPlan &&
989
+ (invalidated.size > 0 || boundaryExpanded) &&
990
+ engineRun.baseline_workspace_revision &&
991
+ engineRun.current_workspace_revision
992
+ ) {
993
+ if (!this.#worker.revalidateDelivery) {
994
+ if (boundaryExpanded) {
995
+ throw new Error("workflow-delivery-revalidation-unavailable");
996
+ }
997
+ if (
998
+ engineRun.current_workspace_revision !==
999
+ engineRun.baseline_workspace_revision
1000
+ ) {
1001
+ throw new Error("workflow-delivery-revalidation-unavailable");
1002
+ }
1003
+ } else {
1004
+ const result = this.#worker.revalidateDelivery({
1005
+ runId,
1006
+ deliveryRevision: delivery.revision,
1007
+ plan: structuredClone(delivery.plan),
1008
+ tasks: priorRows
1009
+ .filter((row) => compatible.has(row.task_id))
1010
+ .map((row) => ({
1011
+ taskId: row.task_id,
1012
+ deliveryRevision: row.delivery_revision,
1013
+ state: row.state,
1014
+ phase: row.phase,
1015
+ contextReadPaths: this.#contextReads(runId, row.task_id),
1016
+ })),
1017
+ invalidatedTaskIds: [...invalidated].sort(),
1018
+ baselineRevisionId: engineRun.baseline_workspace_revision,
1019
+ currentWorkspaceRevisionId: engineRun.current_workspace_revision,
1020
+ });
1021
+ const resultIds = new Set(result.invalidatedTaskIds);
1022
+ if (
1023
+ !Array.isArray(result.invalidatedTaskIds) ||
1024
+ resultIds.size !== result.invalidatedTaskIds.length ||
1025
+ [...invalidated].some((taskId) => !resultIds.has(taskId)) ||
1026
+ [...resultIds].some((taskId) => !priorById.has(taskId)) ||
1027
+ !SHA256.test(result.baselineRevisionId) ||
1028
+ !SHA256.test(result.currentWorkspaceRevisionId)
1029
+ ) {
1030
+ throw new Error("workflow-delivery-revalidation-invalid");
1031
+ }
1032
+ invalidated = resultIds;
1033
+ revalidatedWorkspace = {
1034
+ baselineRevisionId: result.baselineRevisionId,
1035
+ currentWorkspaceRevisionId: result.currentWorkspaceRevisionId,
1036
+ };
1037
+ }
1038
+ }
1039
+ const deliveryBindings = delivery.approvalProofs
1040
+ ? ([
1041
+ {
1042
+ runId,
1043
+ gate: "gate-a" as const,
1044
+ revision: delivery.revision,
1045
+ receiptHash: delivery.receiptHash,
1046
+ approvalProof: structuredClone(delivery.approvalProofs.gateA),
1047
+ operationId: `bind-a-${hash(
1048
+ runId,
1049
+ String(delivery.revision),
1050
+ delivery.approvalProofs.gateA.recordHash,
1051
+ ).slice(0, 40)}`,
1052
+ lease,
1053
+ },
1054
+ {
1055
+ runId,
1056
+ gate: "gate-b" as const,
1057
+ revision: delivery.revision,
1058
+ receiptHash: delivery.receiptHash,
1059
+ approvalProof: structuredClone(delivery.approvalProofs.gateB),
1060
+ operationId: `bind-b-${hash(
1061
+ runId,
1062
+ String(delivery.revision),
1063
+ delivery.approvalProofs.gateB.recordHash,
1064
+ ).slice(0, 40)}`,
1065
+ lease,
1066
+ },
1067
+ ] as const)
1068
+ : ([
1069
+ {
1070
+ runId,
1071
+ gate: delivery.gate,
1072
+ revision: delivery.revision,
1073
+ receiptHash: delivery.receiptHash,
1074
+ operationId: `bind-${hash(
1075
+ runId,
1076
+ String(delivery.revision),
1077
+ delivery.receiptHash,
1078
+ ).slice(0, 40)}`,
1079
+ lease,
1080
+ },
1081
+ ] as const);
1082
+ this.#runStore.bindDeliveriesAtomically(deliveryBindings, (database) => {
1083
+ const engineLease = database
1084
+ .prepare(
1085
+ `SELECT state, lease_expires_at
1086
+ FROM workflow_engine_operations
1087
+ WHERE run_id = ? AND lease_token = ?`,
1088
+ )
1089
+ .get(runId, lease.token) as
1090
+ | { state: string; lease_expires_at: number | null }
1091
+ | undefined;
1092
+ if (
1093
+ engineLease?.state !== "running" ||
1094
+ engineLease.lease_expires_at === null ||
1095
+ engineLease.lease_expires_at < this.#now()
1096
+ ) {
1097
+ throw new Error("lease-fenced");
1098
+ }
1099
+ database
1100
+ .prepare(
1101
+ `INSERT INTO workflow_engine_runs(
1102
+ run_id, current_revision, delivery_diagnostics_json,
1103
+ next_queue_position
1104
+ ) VALUES (?, ?, NULL, 1)
1105
+ ON CONFLICT(run_id) DO UPDATE SET
1106
+ current_revision = excluded.current_revision,
1107
+ verification_json = NULL,
1108
+ delivery_diagnostics_json = NULL`,
1109
+ )
1110
+ .run(runId, delivery.revision);
1111
+ if (revalidatedWorkspace) {
1112
+ database
1113
+ .prepare(
1114
+ `UPDATE workflow_engine_runs
1115
+ SET baseline_workspace_revision = ?,
1116
+ current_workspace_revision = ?, cleanup_state = 'retained'
1117
+ WHERE run_id = ?`,
1118
+ )
1119
+ .run(
1120
+ revalidatedWorkspace.baselineRevisionId,
1121
+ revalidatedWorkspace.currentWorkspaceRevisionId,
1122
+ runId,
1123
+ );
1124
+ }
1125
+ database
1126
+ .prepare(
1127
+ `INSERT INTO workflow_engine_deliveries(
1128
+ run_id, revision, gate, receipt_hash, plan_json
1129
+ ) VALUES (?, ?, ?, ?, ?)
1130
+ ON CONFLICT(run_id, revision) DO NOTHING`,
1131
+ )
1132
+ .run(
1133
+ runId,
1134
+ delivery.revision,
1135
+ delivery.gate,
1136
+ delivery.receiptHash,
1137
+ serializedPlan,
1138
+ );
1139
+
1140
+ if (
1141
+ database
1142
+ .prepare("SELECT 1 FROM workflow_work_budget WHERE run_id = ?")
1143
+ .get(runId)
1144
+ )
1145
+ this.#growWorkBudget(database, runId, delivery.plan);
1146
+ const admitted = new Set<string>();
1147
+ delivery.plan.tasks.forEach((task, taskOrder) => {
1148
+ admitted.add(task.taskId);
1149
+ const taskJson = JSON.stringify(task);
1150
+ const prior = priorById.get(task.taskId);
1151
+ if (!prior) {
1152
+ database
1153
+ .prepare(
1154
+ `INSERT INTO workflow_engine_tasks(
1155
+ run_id, task_id, task_order, delivery_revision, plan_json,
1156
+ state, phase
1157
+ ) VALUES (?, ?, ?, ?, ?, 'pending', ?)`,
1158
+ )
1159
+ .run(
1160
+ runId,
1161
+ task.taskId,
1162
+ taskOrder,
1163
+ delivery.revision,
1164
+ taskJson,
1165
+ phases(task)[0],
1166
+ );
1167
+ return;
1168
+ }
1169
+ if (compatible.has(task.taskId) && !invalidated.has(task.taskId)) {
1170
+ database
1171
+ .prepare(
1172
+ `UPDATE workflow_engine_tasks
1173
+ SET task_order = ?, plan_json = ?
1174
+ WHERE run_id = ? AND task_id = ?`,
1175
+ )
1176
+ .run(taskOrder, taskJson, runId, task.taskId);
1177
+ return;
1178
+ }
1179
+ database
1180
+ .prepare(
1181
+ "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1182
+ )
1183
+ .run(runId, task.taskId);
1184
+ database
1185
+ .prepare(
1186
+ `UPDATE workflow_engine_tasks
1187
+ SET task_order = ?, delivery_revision = ?, plan_json = ?,
1188
+ state = 'pending', phase = ?, pause_code = NULL,
1189
+ context_request_json = NULL, attempt_diagnostic_json = NULL,
1190
+ queue_position = NULL
1191
+ WHERE run_id = ? AND task_id = ?`,
1192
+ )
1193
+ .run(
1194
+ taskOrder,
1195
+ delivery.revision,
1196
+ taskJson,
1197
+ phases(task)[0],
1198
+ runId,
1199
+ task.taskId,
1200
+ );
1201
+ });
1202
+ for (const row of priorRows) {
1203
+ if (!admitted.has(row.task_id)) {
1204
+ database
1205
+ .prepare(
1206
+ "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1207
+ )
1208
+ .run(runId, row.task_id);
1209
+ database
1210
+ .prepare(
1211
+ `DELETE FROM workflow_engine_tasks
1212
+ WHERE run_id = ? AND task_id = ?`,
1213
+ )
1214
+ .run(runId, row.task_id);
1215
+ }
1216
+ }
1217
+ });
1218
+ }
1219
+
1220
+ #cancelledDelivery(
1221
+ runId: string,
1222
+ operationId: string,
1223
+ error: unknown,
1224
+ signal: AbortSignal | undefined,
1225
+ lease: OperationLease,
1226
+ ): Record<string, unknown> | undefined {
1227
+ if (!signal || !isCancellationException(error, signal)) return undefined;
1228
+ try {
1229
+ this.#runStore.assertLease(lease);
1230
+ if (
1231
+ ["validating-delivery", "ready"].includes(
1232
+ this.#runStore.status(runId).state,
1233
+ )
1234
+ ) {
1235
+ this.#transition(
1236
+ runId,
1237
+ "paused",
1238
+ `${operationId}:delivery-cancelled`,
1239
+ "operation-cancelled",
1240
+ lease,
1241
+ );
1242
+ }
1243
+ } catch (failure) {
1244
+ if (!(failure instanceof Error) || failure.message !== "lease-fenced")
1245
+ throw failure;
1246
+ } finally {
1247
+ this.#interruptOperation(runId, operationId);
1248
+ }
1249
+ return this.#statusByRun(runId);
1250
+ }
1251
+
1252
+ #deliveryFailure(
1253
+ runId: string,
1254
+ operationId: string,
1255
+ error: unknown,
1256
+ lease: OperationLease,
1257
+ ): Record<string, unknown> | undefined {
1258
+ if (!(error instanceof DeliveryValidationError)) return undefined;
1259
+ this.#leasedTransaction(lease, () => {
1260
+ this.#database
1261
+ .prepare(
1262
+ `UPDATE workflow_engine_runs
1263
+ SET delivery_diagnostics_json = ? WHERE run_id = ?`,
1264
+ )
1265
+ .run(JSON.stringify(error.diagnostics), runId);
1266
+ });
1267
+ this.#transition(
1268
+ runId,
1269
+ "paused",
1270
+ `${operationId}:delivery-invalid`,
1271
+ "delivery-invalid",
1272
+ lease,
1273
+ );
1274
+ return {
1275
+ ...this.#statusByRun(runId),
1276
+ delivery: {
1277
+ code: "delivery-invalid",
1278
+ diagnostics: [...error.diagnostics],
1279
+ },
1280
+ };
1281
+ }
1282
+
1283
+ #setTask(
1284
+ runId: string,
1285
+ taskId: string,
1286
+ values: {
1287
+ state?: string;
1288
+ phase?: "red" | "green" | "refactor";
1289
+ pauseCode?: string | null;
1290
+ contextRequest?: WorkflowContextRequest | null;
1291
+ attemptDiagnostic?: SafeAttemptDiagnostic | null;
1292
+ routeId?: string | null;
1293
+ routeFingerprint?: string | null;
1294
+ queuePosition?: number | null;
1295
+ },
1296
+ lease?: OperationLease,
1297
+ ): void {
1298
+ if (values.attemptDiagnostic?.recovery && lease) {
1299
+ const fact = values.attemptDiagnostic.recovery;
1300
+ const prior = this.#recovery(runId, fact.key);
1301
+ if (
1302
+ !prior ||
1303
+ fact.failures > prior.failures ||
1304
+ fact.feedback.maxAttempts < prior.feedback.maxAttempts
1305
+ )
1306
+ this.#recordRecovery(runId, fact, lease);
1307
+ }
1308
+ const assignments: string[] = [];
1309
+ const parameters: Array<string | number | null> = [];
1310
+ if (values.state !== undefined) {
1311
+ assignments.push("state = ?");
1312
+ parameters.push(values.state);
1313
+ }
1314
+ if (values.phase !== undefined) {
1315
+ assignments.push("phase = ?");
1316
+ parameters.push(values.phase);
1317
+ }
1318
+ if (values.pauseCode !== undefined) {
1319
+ assignments.push("pause_code = ?");
1320
+ parameters.push(values.pauseCode);
1321
+ }
1322
+ if (values.contextRequest !== undefined) {
1323
+ assignments.push("context_request_json = ?");
1324
+ parameters.push(
1325
+ values.contextRequest === null
1326
+ ? null
1327
+ : JSON.stringify(
1328
+ normalizeWorkflowContextRequest(values.contextRequest),
1329
+ ),
1330
+ );
1331
+ }
1332
+ if (values.attemptDiagnostic !== undefined) {
1333
+ assignments.push("attempt_diagnostic_json = ?");
1334
+ parameters.push(
1335
+ values.attemptDiagnostic === null
1336
+ ? null
1337
+ : JSON.stringify(
1338
+ Object.fromEntries(
1339
+ Object.entries(values.attemptDiagnostic).filter(
1340
+ ([key]) => key !== "recovery",
1341
+ ),
1342
+ ),
1343
+ ),
1344
+ );
1345
+ }
1346
+ if (values.routeId !== undefined) {
1347
+ assignments.push("route_id = ?");
1348
+ parameters.push(values.routeId);
1349
+ }
1350
+ if (values.routeFingerprint !== undefined) {
1351
+ assignments.push("route_fingerprint = ?");
1352
+ parameters.push(values.routeFingerprint);
1353
+ }
1354
+ if (values.queuePosition !== undefined) {
1355
+ assignments.push("queue_position = ?");
1356
+ parameters.push(values.queuePosition);
1357
+ }
1358
+ if (assignments.length === 0) return;
1359
+ const write = () => {
1360
+ this.#database
1361
+ .prepare(
1362
+ `UPDATE workflow_engine_tasks SET ${assignments.join(", ")}
1363
+ WHERE run_id = ? AND task_id = ?`,
1364
+ )
1365
+ .run(...parameters, runId, taskId);
1366
+ };
1367
+ if (lease) this.#leasedTransaction(lease, write);
1368
+ else write();
1369
+ }
1370
+
1371
+ #setVerificationStatus(
1372
+ runId: string,
1373
+ status: WorkflowVerificationStatus | null,
1374
+ lease: OperationLease,
1375
+ ): void {
1376
+ const serialized =
1377
+ status === null
1378
+ ? null
1379
+ : JSON.stringify(normalizeWorkflowVerificationStatus(status));
1380
+ this.#leasedTransaction(lease, () => {
1381
+ this.#database
1382
+ .prepare(
1383
+ `UPDATE workflow_engine_runs SET verification_json = ?
1384
+ WHERE run_id = ?`,
1385
+ )
1386
+ .run(serialized, runId);
1387
+ });
1388
+ }
1389
+
1390
+ #queueTask(
1391
+ runId: string,
1392
+ row: EngineTaskRow,
1393
+ lease: OperationLease,
1394
+ reason: "conflict" | "capacity" = "conflict",
1395
+ ): void {
1396
+ const pauseCode = reason === "capacity" ? "worker-capacity" : null;
1397
+ if (row.state === "queued" && row.queue_position !== null) {
1398
+ if (row.pause_code !== pauseCode)
1399
+ this.#setTask(runId, row.task_id, { pauseCode }, lease);
1400
+ return;
1401
+ }
1402
+ const engineRun = this.#engineRun(runId);
1403
+ this.#leasedTransaction(lease, () => {
1404
+ this.#setTask(runId, row.task_id, {
1405
+ state: "queued",
1406
+ pauseCode,
1407
+ queuePosition: engineRun.next_queue_position,
1408
+ });
1409
+ this.#database
1410
+ .prepare(
1411
+ `UPDATE workflow_engine_runs
1412
+ SET next_queue_position = next_queue_position + 1
1413
+ WHERE run_id = ?`,
1414
+ )
1415
+ .run(runId);
1416
+ });
1417
+ }
1418
+
1419
+ #hasConflict(
1420
+ row: EngineTaskRow,
1421
+ rows: EngineTaskRow[],
1422
+ selectedTaskIds: ReadonlySet<string>,
1423
+ ): boolean {
1424
+ const candidate = this.#taskPlan(row);
1425
+ return rows.some((other) => {
1426
+ if (other.task_id === row.task_id || other.state === "verified") {
1427
+ return false;
1428
+ }
1429
+ if (other.state === "queued") {
1430
+ if (row.state !== "queued") return false;
1431
+ const candidatePosition = row.queue_position ?? Number.MAX_SAFE_INTEGER;
1432
+ const otherPosition = other.queue_position ?? Number.MAX_SAFE_INTEGER;
1433
+ if (
1434
+ otherPosition > candidatePosition ||
1435
+ (otherPosition === candidatePosition &&
1436
+ other.task_order > row.task_order)
1437
+ ) {
1438
+ return false;
1439
+ }
1440
+ }
1441
+ if (other.task_order > row.task_order && other.state === "pending") {
1442
+ return false;
1443
+ }
1444
+ if (
1445
+ other.state === "pending" &&
1446
+ !this.#dependenciesVerified(other, rows)
1447
+ ) {
1448
+ return false;
1449
+ }
1450
+ const conflict = taskConflicts(candidate, this.#taskPlan(other));
1451
+ return (
1452
+ conflict.taskLifetime ||
1453
+ (conflict.verification &&
1454
+ ([
1455
+ "pending",
1456
+ "phase-ready",
1457
+ "phase-running",
1458
+ "validating",
1459
+ "queued",
1460
+ ].includes(other.state) ||
1461
+ selectedTaskIds.has(other.task_id)))
1462
+ );
1463
+ });
1464
+ }
1465
+
1466
+ #dependenciesVerified(row: EngineTaskRow, rows: EngineTaskRow[]): boolean {
1467
+ const byId = new Map(
1468
+ rows.map((candidate) => [candidate.task_id, candidate]),
1469
+ );
1470
+ return this.#taskPlan(row).dependsOn.every(
1471
+ (dependency) => byId.get(dependency)?.state === "verified",
1472
+ );
1473
+ }
1474
+
1475
+ #recordWorkspaceFacts(
1476
+ runId: string,
1477
+ outcome: WorkflowAttemptOutcome,
1478
+ lease: OperationLease,
1479
+ ): void {
1480
+ const baselineRevisionId = outcome.baselineRevisionId;
1481
+ const currentWorkspaceRevisionId =
1482
+ outcome.kind === "phase-committed"
1483
+ ? outcome.isolatedRevisionId
1484
+ : outcome.currentWorkspaceRevisionId;
1485
+ if (
1486
+ (baselineRevisionId !== undefined && !SHA256.test(baselineRevisionId)) ||
1487
+ (currentWorkspaceRevisionId !== undefined &&
1488
+ !SHA256.test(currentWorkspaceRevisionId)) ||
1489
+ (outcome.kind !== "phase-committed" &&
1490
+ (baselineRevisionId === undefined) !==
1491
+ (currentWorkspaceRevisionId === undefined))
1492
+ ) {
1493
+ throw new Error("workflow-attempt-workspace-fact-invalid");
1494
+ }
1495
+ if (currentWorkspaceRevisionId === undefined) return;
1496
+ this.#leasedTransaction(lease, () => {
1497
+ this.#database
1498
+ .prepare(
1499
+ `UPDATE workflow_engine_runs
1500
+ SET baseline_workspace_revision = COALESCE(
1501
+ baseline_workspace_revision, ?
1502
+ ),
1503
+ current_workspace_revision = ?,
1504
+ cleanup_state = CASE
1505
+ WHEN ? IS NULL THEN cleanup_state ELSE 'retained'
1506
+ END
1507
+ WHERE run_id = ?`,
1508
+ )
1509
+ .run(
1510
+ baselineRevisionId ?? null,
1511
+ currentWorkspaceRevisionId,
1512
+ baselineRevisionId ?? null,
1513
+ runId,
1514
+ );
1515
+ });
1516
+ }
1517
+
1518
+ #recordRouteFacts(
1519
+ runId: string,
1520
+ taskId: string,
1521
+ outcome: WorkflowAttemptOutcome,
1522
+ lease: OperationLease,
1523
+ ): void {
1524
+ if (
1525
+ (outcome.routeId === undefined) !==
1526
+ (outcome.routeFingerprint === undefined) ||
1527
+ (outcome.routeId !== undefined && !IDENTIFIER.test(outcome.routeId)) ||
1528
+ (outcome.routeFingerprint !== undefined &&
1529
+ !SHA256.test(outcome.routeFingerprint))
1530
+ ) {
1531
+ throw new Error("workflow-attempt-route-fact-invalid");
1532
+ }
1533
+ if (!outcome.routeId || !outcome.routeFingerprint) return;
1534
+ const routeId = outcome.routeId;
1535
+ const routeFingerprint = outcome.routeFingerprint;
1536
+ this.#leasedTransaction(lease, () => {
1537
+ this.#setTask(runId, taskId, {
1538
+ routeId,
1539
+ routeFingerprint,
1540
+ });
1541
+ this.#database
1542
+ .prepare(
1543
+ `UPDATE workflow_engine_runs
1544
+ SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
1545
+ )
1546
+ .run(routeId, routeFingerprint, runId);
1547
+ });
1548
+ }
1549
+
1550
+ async #runTask(
1551
+ runId: string,
1552
+ row: EngineTaskRow,
1553
+ operationId: string,
1554
+ signal: AbortSignal,
1555
+ lease: OperationLease,
1556
+ onActivity?: (event: WorkflowActivityUpdate) => void,
1557
+ ): Promise<void> {
1558
+ const task = this.#taskPlan(row);
1559
+ const orderedPhases = phases(task);
1560
+ let phaseIndex = orderedPhases.indexOf(row.phase);
1561
+ let artifactAttempts = 0;
1562
+ const persistedContextRequest = row.context_request_json
1563
+ ? normalizeWorkflowContextRequest(
1564
+ JSON.parse(row.context_request_json) as unknown,
1565
+ )
1566
+ : undefined;
1567
+ const persistedRedArtifactCorrection =
1568
+ row.phase === "green" &&
1569
+ persistedContextRequest !== undefined &&
1570
+ classifyPersistedContextRequest(persistedContextRequest, task, "green")
1571
+ .code === "red-artifact-constraint";
1572
+ let redArtifactCorrection =
1573
+ row.phase === "green" &&
1574
+ (row.pause_code === "red-artifact-constraint" ||
1575
+ persistedRedArtifactCorrection);
1576
+ let requestedContext = persistedContextRequest;
1577
+ let correctionContextRequest = redArtifactCorrection
1578
+ ? persistedContextRequest
1579
+ : undefined;
1580
+ if (phaseIndex < 0) throw new Error("workflow-task-phase-invalid");
1581
+ while (phaseIndex < orderedPhases.length) {
1582
+ const phase = orderedPhases[phaseIndex];
1583
+ row.phase = phase;
1584
+ if (signal.aborted) {
1585
+ this.#setTask(
1586
+ runId,
1587
+ row.task_id,
1588
+ {
1589
+ state: "paused",
1590
+ phase,
1591
+ pauseCode: "operation-cancelled",
1592
+ contextRequest: null,
1593
+ queuePosition: null,
1594
+ },
1595
+ lease,
1596
+ );
1597
+ return;
1598
+ }
1599
+ const current = this.#engineRun(runId);
1600
+ if (current.current_revision === null)
1601
+ throw new Error("delivery-not-bound");
1602
+ const deliveryRevision = current.current_revision;
1603
+ const plan = this.#planForRevision(runId, deliveryRevision);
1604
+ this.#leasedTransaction(lease, () => {
1605
+ for (const ref of requestedContext?.refs ?? []) {
1606
+ if (
1607
+ ref.kind === "requested-path" &&
1608
+ ref.access === "read" &&
1609
+ permitsContextRead(ref.path, task.roots) &&
1610
+ (current.current_workspace_revision
1611
+ ? current.baseline_workspace_revision &&
1612
+ this.#worker.isContextReadAvailable?.({
1613
+ runId,
1614
+ deliveryRevision,
1615
+ plan,
1616
+ baselineRevisionId: current.baseline_workspace_revision,
1617
+ currentWorkspaceRevisionId:
1618
+ current.current_workspace_revision,
1619
+ path: ref.path,
1620
+ })
1621
+ : isSafeRegularFile(this.#consumerRoot, ref.path))
1622
+ ) {
1623
+ this.#database
1624
+ .prepare(
1625
+ "INSERT INTO workflow_context_reads(run_id, task_id, path) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
1626
+ )
1627
+ .run(runId, row.task_id, ref.path);
1628
+ }
1629
+ }
1630
+ });
1631
+ const verificationStatus = current.verification_json
1632
+ ? normalizeWorkflowVerificationStatus(
1633
+ JSON.parse(current.verification_json) as unknown,
1634
+ )
1635
+ : undefined;
1636
+ const priorDiagnostic = this.#safeAttemptDiagnostic(runId, row);
1637
+ const priorRecovery = priorDiagnostic?.recovery;
1638
+ const activeRecovery =
1639
+ priorRecovery?.key === recoveryKey(task, phase, row, current)
1640
+ ? priorRecovery
1641
+ : undefined;
1642
+ const maxRecoveryAttempts = Math.min(
1643
+ plan.verification.artifactCorrection.maxAttempts,
1644
+ activeRecovery?.feedback.maxAttempts ?? Infinity,
1645
+ );
1646
+ if (
1647
+ activeRecovery &&
1648
+ activeRecovery.feedback.maxAttempts !== maxRecoveryAttempts
1649
+ ) {
1650
+ this.#leasedTransaction(lease, () => {
1651
+ this.#database
1652
+ .prepare(
1653
+ "UPDATE workflow_recovery_incidents SET max_attempts = MIN(max_attempts, ?) WHERE run_id = ? AND incident_key = ?",
1654
+ )
1655
+ .run(maxRecoveryAttempts, runId, activeRecovery.key);
1656
+ });
1657
+ activeRecovery.feedback.maxAttempts = maxRecoveryAttempts;
1658
+ }
1659
+ const granted = this.#hasRecoveryGrant(
1660
+ runId,
1661
+ operationId,
1662
+ recoveryKey(task, phase, row, current),
1663
+ );
1664
+ const verificationOnly =
1665
+ !granted &&
1666
+ activeRecovery &&
1667
+ activeRecovery.failures >= maxRecoveryAttempts &&
1668
+ (await this.#worker.hasPendingVerification?.({
1669
+ runId,
1670
+ operationId,
1671
+ deliveryRevision: row.delivery_revision,
1672
+ taskId: row.task_id,
1673
+ phase,
1674
+ task,
1675
+ plan,
1676
+ signal,
1677
+ ...(current.baseline_workspace_revision
1678
+ ? { baselineRevisionId: current.baseline_workspace_revision }
1679
+ : {}),
1680
+ ...(current.current_workspace_revision
1681
+ ? { currentWorkspaceRevisionId: current.current_workspace_revision }
1682
+ : {}),
1683
+ })) === true;
1684
+ if (
1685
+ !granted &&
1686
+ !verificationOnly &&
1687
+ activeRecovery &&
1688
+ activeRecovery.failures >= maxRecoveryAttempts
1689
+ ) {
1690
+ this.#setTask(
1691
+ runId,
1692
+ row.task_id,
1693
+ {
1694
+ state: "paused",
1695
+ phase,
1696
+ pauseCode: activeRecovery.feedback.code,
1697
+ attemptDiagnostic: priorDiagnostic ?? null,
1698
+ queuePosition: null,
1699
+ },
1700
+ lease,
1701
+ );
1702
+ return;
1703
+ }
1704
+ if (activeRecovery)
1705
+ artifactAttempts = Math.max(artifactAttempts, activeRecovery.failures);
1706
+ const repair =
1707
+ verificationStatus?.scope === "change-task-affected" &&
1708
+ verificationStatus.attribution === "introduced" &&
1709
+ verificationStatus.taskId === row.task_id &&
1710
+ verificationStatus.failureIdentities &&
1711
+ verificationStatus.failureIdentities.length > 0
1712
+ ? {
1713
+ attribution: "introduced" as const,
1714
+ failureIdentities: [...verificationStatus.failureIdentities],
1715
+ }
1716
+ : undefined;
1717
+ const artifactCorrection =
1718
+ redArtifactCorrection && phase === "green"
1719
+ ? {
1720
+ code: "red-artifact-constraint" as const,
1721
+ attempt: artifactAttempts + 1,
1722
+ maxAttempts: maxRecoveryAttempts,
1723
+ ...(correctionContextRequest
1724
+ ? {
1725
+ contextRequest: structuredClone(correctionContextRequest),
1726
+ }
1727
+ : {}),
1728
+ }
1729
+ : undefined;
1730
+ this.#setTask(
1731
+ runId,
1732
+ row.task_id,
1733
+ {
1734
+ state: "phase-running",
1735
+ phase,
1736
+ pauseCode: artifactCorrection?.code ?? null,
1737
+ contextRequest: requestedContext ?? null,
1738
+ attemptDiagnostic: priorDiagnostic ?? null,
1739
+ queuePosition: null,
1740
+ },
1741
+ lease,
1742
+ );
1743
+ if (
1744
+ !this.#reserveWork(
1745
+ runId,
1746
+ plan,
1747
+ recoveryKey(task, phase, row, current),
1748
+ lease,
1749
+ granted ? operationId : undefined,
1750
+ )
1751
+ ) {
1752
+ this.#setTask(
1753
+ runId,
1754
+ row.task_id,
1755
+ {
1756
+ state: "paused",
1757
+ phase,
1758
+ pauseCode: "change-work-budget-exhausted",
1759
+ queuePosition: null,
1760
+ },
1761
+ lease,
1762
+ );
1763
+ return;
1764
+ }
1765
+ let initialReservation = true;
1766
+ let outcome: WorkflowAttemptOutcome;
1767
+ try {
1768
+ outcome = await this.#worker.runAttempt({
1769
+ runId,
1770
+ operationId,
1771
+ deliveryRevision: row.delivery_revision,
1772
+ taskId: row.task_id,
1773
+ phase,
1774
+ task: structuredClone(task),
1775
+ contextReadPaths: this.#contextReads(runId, row.task_id),
1776
+ plan: structuredClone(plan),
1777
+ ...(current.baseline_workspace_revision
1778
+ ? { baselineRevisionId: current.baseline_workspace_revision }
1779
+ : {}),
1780
+ ...(current.current_workspace_revision
1781
+ ? {
1782
+ currentWorkspaceRevisionId: current.current_workspace_revision,
1783
+ }
1784
+ : {}),
1785
+ ...(row.route_id || current.route_id
1786
+ ? { routeId: row.route_id ?? current.route_id ?? undefined }
1787
+ : {}),
1788
+ ...(row.route_fingerprint || current.route_fingerprint
1789
+ ? {
1790
+ routeFingerprint:
1791
+ row.route_fingerprint ??
1792
+ current.route_fingerprint ??
1793
+ undefined,
1794
+ }
1795
+ : {}),
1796
+ ...(repair ? { repair } : {}),
1797
+ ...(artifactCorrection ? { artifactCorrection } : {}),
1798
+ ...(requestedContext ? { contextRequest: requestedContext } : {}),
1799
+ ...(granted ? { additionalAttempt: true } : {}),
1800
+ ...(verificationOnly ? { verificationOnly: true } : {}),
1801
+ ...(activeRecovery
1802
+ ? { recoveryFeedback: activeRecovery.feedback }
1803
+ : {}),
1804
+ signal,
1805
+ recoveryDecision: (request) => {
1806
+ signal.throwIfAborted();
1807
+ this.#runStore.assertLease(lease);
1808
+ return decideRecoveryAction(plan, request, {
1809
+ additionalAttempt: granted,
1810
+ verificationOnly,
1811
+ });
1812
+ },
1813
+ reserveCandidate: () => {
1814
+ signal.throwIfAborted();
1815
+ if (verificationOnly) return false;
1816
+ if (initialReservation) {
1817
+ initialReservation = false;
1818
+ return true;
1819
+ }
1820
+ if (granted) return false;
1821
+ return this.#reserveWork(
1822
+ runId,
1823
+ plan,
1824
+ recoveryKey(task, phase, row, current),
1825
+ lease,
1826
+ );
1827
+ },
1828
+ onActivity: (event) => {
1829
+ const projection = this.#runStore.status(runId);
1830
+ emitWorkflowActivity(onActivity, {
1831
+ ...event,
1832
+ stage: projection.stage,
1833
+ runId,
1834
+ ...(projection.change ? { change: projection.change } : {}),
1835
+ taskId: row.task_id,
1836
+ phase,
1837
+ objective: task.objective,
1838
+ legalCommands: ["status", "cancel", "discard"],
1839
+ });
1840
+ },
1841
+ });
1842
+ } catch (error) {
1843
+ if (isCancellationException(error, signal)) {
1844
+ outcome = { kind: "operation-cancelled", code: "cancelled" };
1845
+ } else {
1846
+ throw error;
1847
+ }
1848
+ }
1849
+ if (!isRecord(outcome) || typeof outcome.kind !== "string") {
1850
+ throw new Error("workflow-attempt-outcome-invalid");
1851
+ }
1852
+ this.#recordWorkspaceFacts(runId, outcome, lease);
1853
+ this.#recordRouteFacts(runId, row.task_id, outcome, lease);
1854
+ if (outcome.routeId && outcome.routeFingerprint) {
1855
+ row.route_id = outcome.routeId;
1856
+ row.route_fingerprint = outcome.routeFingerprint;
1857
+ }
1858
+ if (
1859
+ outcome.kind !== "phase-committed" &&
1860
+ outcome.kind !== "operation-cancelled" &&
1861
+ outcome.verification
1862
+ ) {
1863
+ this.#setVerificationStatus(runId, outcome.verification, lease);
1864
+ }
1865
+ if (
1866
+ outcome.kind !== "phase-committed" &&
1867
+ outcome.kind !== "operation-cancelled" &&
1868
+ outcome.contextRequest
1869
+ ) {
1870
+ requestedContext = normalizeWorkflowContextRequest(
1871
+ outcome.contextRequest,
1872
+ );
1873
+ }
1874
+ if (
1875
+ outcome.kind !== "phase-committed" &&
1876
+ outcome.kind !== "operation-cancelled" &&
1877
+ outcome.attemptDiagnostic
1878
+ ) {
1879
+ const previous = row.attempt_diagnostic_json
1880
+ ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
1881
+ : undefined;
1882
+ const fingerprint = hash(
1883
+ outcome.code,
1884
+ outcome.attemptDiagnostic.finalCategory ?? "unknown",
1885
+ outcome.attemptDiagnostic.schema ?? "unknown",
1886
+ JSON.stringify(outcome.attemptDiagnostic.identityMismatch ?? []),
1887
+ ).slice(0, 16);
1888
+ const sameFailure = previous?.fingerprint === fingerprint;
1889
+ const sameFailureCount = sameFailure
1890
+ ? (previous?.sameFailureCount ?? 1) + 1
1891
+ : 1;
1892
+ const attemptDiagnostic = {
1893
+ ...outcome.attemptDiagnostic,
1894
+ ...(priorDiagnostic?.recovery
1895
+ ? { recovery: priorDiagnostic.recovery }
1896
+ : {}),
1897
+ fingerprint,
1898
+ sameFailureCount,
1899
+ ...(sameFailureCount > 1
1900
+ ? { action: "rebind-or-revise-delivery" as const }
1901
+ : {}),
1902
+ };
1903
+ this.#setTask(runId, row.task_id, { attemptDiagnostic }, lease);
1904
+ row.attempt_diagnostic_json = JSON.stringify(attemptDiagnostic);
1905
+ }
1906
+ if (
1907
+ outcome.kind === "paused" &&
1908
+ outcome.code === "repair-attempts-exhausted"
1909
+ ) {
1910
+ const maxAttempts = plan.verification.repair.maxAttempts;
1911
+ const failures = Math.max(
1912
+ maxAttempts,
1913
+ (activeRecovery?.failures ?? 0) +
1914
+ (granted || verificationOnly ? 1 : 0),
1915
+ );
1916
+ const diagnostic: SafeAttemptDiagnostic = {
1917
+ recovery: {
1918
+ key: recoveryKey(task, phase, row, this.#engineRun(runId)),
1919
+ failures,
1920
+ feedback: {
1921
+ code: outcome.code,
1922
+ attempt: failures + 1,
1923
+ maxAttempts,
1924
+ strategy: "repair-verification",
1925
+ },
1926
+ },
1927
+ };
1928
+ this.#setTask(
1929
+ runId,
1930
+ row.task_id,
1931
+ { attemptDiagnostic: diagnostic },
1932
+ lease,
1933
+ );
1934
+ row.attempt_diagnostic_json = JSON.stringify(diagnostic);
1935
+ }
1936
+ if (
1937
+ outcome.kind === "retryable" &&
1938
+ ["artifact", "stale", "verification"].includes(
1939
+ outcome.retryPolicy ?? "",
1940
+ )
1941
+ ) {
1942
+ const key = recoveryKey(task, phase, row, this.#engineRun(runId));
1943
+ const failures =
1944
+ (activeRecovery?.key === key ? activeRecovery.failures : 0) + 1;
1945
+ const strategy =
1946
+ outcome.code === "needs-task-split" ||
1947
+ outcome.code === "task-split-needed"
1948
+ ? ("compact-patch" as const)
1949
+ : outcome.retryPolicy === "stale"
1950
+ ? ("refresh-candidate" as const)
1951
+ : outcome.retryPolicy === "verification"
1952
+ ? ("repair-verification" as const)
1953
+ : ("revise-candidate" as const);
1954
+ const diagnostic: SafeAttemptDiagnostic = {
1955
+ ...this.#safeAttemptDiagnostic(runId, row),
1956
+ recovery: {
1957
+ key,
1958
+ failures,
1959
+ feedback: {
1960
+ code: outcome.code,
1961
+ attempt: failures + 1,
1962
+ maxAttempts: maxRecoveryAttempts,
1963
+ strategy,
1964
+ ...(outcome.verification?.failureIdentities
1965
+ ? { failureIdentities: outcome.verification.failureIdentities }
1966
+ : {}),
1967
+ },
1968
+ },
1969
+ };
1970
+ this.#setTask(
1971
+ runId,
1972
+ row.task_id,
1973
+ { attemptDiagnostic: diagnostic },
1974
+ lease,
1975
+ );
1976
+ row.attempt_diagnostic_json = JSON.stringify(diagnostic);
1977
+ if (outcome.code === "red-artifact-constraint") {
1978
+ redArtifactCorrection = true;
1979
+ correctionContextRequest = outcome.contextRequest
1980
+ ? normalizeWorkflowContextRequest(outcome.contextRequest)
1981
+ : correctionContextRequest;
1982
+ requestedContext = correctionContextRequest ?? requestedContext;
1983
+ }
1984
+ artifactAttempts = Math.max(artifactAttempts + 1, failures);
1985
+ const maxAttempts = maxRecoveryAttempts;
1986
+ if (artifactAttempts < maxAttempts) {
1987
+ this.#setTask(
1988
+ runId,
1989
+ row.task_id,
1990
+ {
1991
+ state: "retryable",
1992
+ phase,
1993
+ pauseCode: redArtifactCorrection
1994
+ ? "red-artifact-constraint"
1995
+ : outcome.code,
1996
+ contextRequest: requestedContext ?? null,
1997
+ queuePosition: null,
1998
+ },
1999
+ lease,
2000
+ );
2001
+ emitWorkflowActivity(onActivity, {
2002
+ state: "retrying",
2003
+ stage: this.#runStore.status(runId).stage,
2004
+ runId,
2005
+ taskId: row.task_id,
2006
+ phase,
2007
+ objective: task.objective,
2008
+ legalCommands: ["status", "cancel", "discard"],
2009
+ code: outcome.code,
2010
+ attempt: artifactAttempts,
2011
+ maxAttempts,
2012
+ wait: "artifact-correction",
2013
+ });
2014
+ continue;
2015
+ }
2016
+ }
2017
+ if (outcome.kind === "phase-committed") {
2018
+ if (
2019
+ !SHA256.test(outcome.artifactHash) ||
2020
+ !SHA256.test(outcome.isolatedRevisionId) ||
2021
+ (outcome.baselineRevisionId !== undefined &&
2022
+ !SHA256.test(outcome.baselineRevisionId)) ||
2023
+ outcome.classification !== expectedClassification(phase)
2024
+ ) {
2025
+ throw new Error("workflow-attempt-fact-invalid");
2026
+ }
2027
+ this.#leasedTransaction(lease, () => {
2028
+ const resolvedKey = recoveryKey(task, phase, row, current);
2029
+ if (this.#recovery(runId, resolvedKey)) {
2030
+ this.#database
2031
+ .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2032
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
2033
+ .run(runId, resolvedKey, runId);
2034
+ }
2035
+ this.#database
2036
+ .prepare(
2037
+ "DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
2038
+ )
2039
+ .run(runId, recoveryKey(task, phase, row, current));
2040
+ });
2041
+ if (repair) {
2042
+ this.#setVerificationStatus(runId, null, lease);
2043
+ this.#setTask(
2044
+ runId,
2045
+ row.task_id,
2046
+ {
2047
+ state: "verified",
2048
+ phase,
2049
+ pauseCode: null,
2050
+ contextRequest: null,
2051
+ attemptDiagnostic: null,
2052
+ queuePosition: null,
2053
+ },
2054
+ lease,
2055
+ );
2056
+ return;
2057
+ }
2058
+ artifactAttempts = 0;
2059
+ row.attempt_diagnostic_json = null;
2060
+ redArtifactCorrection = false;
2061
+ requestedContext = undefined;
2062
+ correctionContextRequest = undefined;
2063
+ phaseIndex += 1;
2064
+ if (phaseIndex >= orderedPhases.length) {
2065
+ this.#setTask(
2066
+ runId,
2067
+ row.task_id,
2068
+ {
2069
+ state: "verified",
2070
+ phase,
2071
+ pauseCode: null,
2072
+ contextRequest: null,
2073
+ attemptDiagnostic: null,
2074
+ queuePosition: null,
2075
+ },
2076
+ lease,
2077
+ );
2078
+ return;
2079
+ }
2080
+ this.#setTask(
2081
+ runId,
2082
+ row.task_id,
2083
+ {
2084
+ state: "phase-ready",
2085
+ phase: orderedPhases[phaseIndex],
2086
+ pauseCode: null,
2087
+ contextRequest: null,
2088
+ attemptDiagnostic: null,
2089
+ queuePosition: null,
2090
+ },
2091
+ lease,
2092
+ );
2093
+ continue;
2094
+ }
2095
+ if (outcome.kind === "operation-cancelled") {
2096
+ this.#setTask(
2097
+ runId,
2098
+ row.task_id,
2099
+ {
2100
+ state: "paused",
2101
+ phase,
2102
+ pauseCode: "operation-cancelled",
2103
+ contextRequest: null,
2104
+ queuePosition: null,
2105
+ },
2106
+ lease,
2107
+ );
2108
+ return;
2109
+ }
2110
+ if (
2111
+ outcome.kind === "paused" ||
2112
+ outcome.kind === "retryable" ||
2113
+ outcome.kind === "approval-needed"
2114
+ ) {
2115
+ if (typeof outcome.code !== "string" || outcome.code.length === 0) {
2116
+ throw new Error("workflow-attempt-outcome-invalid");
2117
+ }
2118
+ const approvalCodeInvalid =
2119
+ outcome.kind === "approval-needed" &&
2120
+ !isWorkflowApprovalCode(outcome.code);
2121
+ this.#setTask(
2122
+ runId,
2123
+ row.task_id,
2124
+ {
2125
+ state: approvalCodeInvalid ? "paused" : outcome.kind,
2126
+ phase,
2127
+ pauseCode: approvalCodeInvalid
2128
+ ? "approval-code-invalid"
2129
+ : outcome.code,
2130
+ contextRequest: approvalCodeInvalid
2131
+ ? null
2132
+ : (requestedContext ?? null),
2133
+ queuePosition: null,
2134
+ },
2135
+ lease,
2136
+ );
2137
+ return;
2138
+ }
2139
+ throw new Error("workflow-attempt-outcome-invalid");
2140
+ }
2141
+ }
2142
+
2143
+ #firstPaused(rows: EngineTaskRow[]): EngineTaskRow | undefined {
2144
+ return (
2145
+ rows.find((row) => row.state === "approval-needed") ??
2146
+ rows.find((row) => ["paused", "retryable"].includes(row.state))
2147
+ );
2148
+ }
2149
+
2150
+ #recoverLegacyContextApproval(runId: string): boolean {
2151
+ const projection = this.#runStore.status(runId);
2152
+ if (!["approval-needed", "paused"].includes(projection.state)) return false;
2153
+ const rows = this.#tasks(runId);
2154
+ const approvalRows = rows.filter((row) => row.state === "approval-needed");
2155
+ if (approvalRows.length === 0) return false;
2156
+ const reclassified = rows.flatMap((row) => {
2157
+ if (
2158
+ row.state !== "approval-needed" ||
2159
+ row.pause_code !== "boundary-review-needed" ||
2160
+ row.context_request_json === null
2161
+ ) {
2162
+ return [];
2163
+ }
2164
+ try {
2165
+ const request = normalizeWorkflowContextRequest(
2166
+ JSON.parse(row.context_request_json) as unknown,
2167
+ );
2168
+ const classified = classifyPersistedContextRequest(
2169
+ request,
2170
+ this.#taskPlan(row),
2171
+ row.phase,
2172
+ );
2173
+ if (classified.kind === "approval-needed") return [];
2174
+ return [
2175
+ {
2176
+ row,
2177
+ state: classified.kind === "retryable" ? "retryable" : "paused",
2178
+ code: classified.code,
2179
+ contextRequest: JSON.stringify(
2180
+ normalizeWorkflowContextRequest(classified.contextRequest),
2181
+ ),
2182
+ },
2183
+ ];
2184
+ } catch {
2185
+ return [];
2186
+ }
2187
+ });
2188
+ const reclassifiedTaskIds = new Set(
2189
+ reclassified.map((entry) => entry.row.task_id),
2190
+ );
2191
+ const retainedApprovals = approvalRows.filter(
2192
+ (row) => !reclassifiedTaskIds.has(row.task_id),
2193
+ );
2194
+ const updateTasks = (database: DatabaseSync) => {
2195
+ for (const entry of reclassified) {
2196
+ const updated = database
2197
+ .prepare(
2198
+ `UPDATE workflow_engine_tasks
2199
+ SET state = ?, phase = ?, pause_code = ?, context_request_json = ?,
2200
+ queue_position = NULL
2201
+ WHERE run_id = ? AND task_id = ?
2202
+ AND state = 'approval-needed'
2203
+ AND pause_code = 'boundary-review-needed'`,
2204
+ )
2205
+ .run(
2206
+ entry.state,
2207
+ entry.row.phase,
2208
+ entry.code,
2209
+ entry.contextRequest,
2210
+ runId,
2211
+ entry.row.task_id,
2212
+ );
2213
+ if (Number(updated.changes) !== 1) {
2214
+ throw new Error("workflow-legacy-context-race");
2215
+ }
2216
+ }
2217
+ };
2218
+ const transitionWithUpdates = (
2219
+ to: "paused" | "approval-needed",
2220
+ code: string,
2221
+ ) =>
2222
+ this.#runStore.transitionAtomically(
2223
+ {
2224
+ runId,
2225
+ to,
2226
+ operationId: `legacy-context-${hash(
2227
+ runId,
2228
+ String(projection.sequence),
2229
+ to,
2230
+ code,
2231
+ JSON.stringify(
2232
+ reclassified.map((entry) => [entry.row.task_id, entry.code]),
2233
+ ),
2234
+ ).slice(0, 40)}`,
2235
+ code,
2236
+ },
2237
+ updateTasks,
2238
+ );
2239
+ if (retainedApprovals.length > 0) {
2240
+ if (projection.state === "approval-needed") {
2241
+ if (reclassified.length === 0) return false;
2242
+ this.#transaction(() => updateTasks(this.#database));
2243
+ } else {
2244
+ transitionWithUpdates(
2245
+ "approval-needed",
2246
+ retainedApprovals[0].pause_code ?? "approval-code-invalid",
2247
+ );
2248
+ }
2249
+ return true;
2250
+ }
2251
+ const first = reclassified[0];
2252
+ if (!first) return false;
2253
+ transitionWithUpdates("paused", first.code);
2254
+ return true;
2255
+ }
2256
+
2257
+ async #finishChange(
2258
+ runId: string,
2259
+ operationId: string,
2260
+ signal: AbortSignal,
2261
+ lease: OperationLease,
2262
+ onActivity?: (event: WorkflowActivityUpdate) => void,
2263
+ ): Promise<Record<string, unknown>> {
2264
+ const plan = this.#currentPlan(runId);
2265
+ let engineRun = this.#engineRun(runId);
2266
+ if (engineRun.current_revision === null)
2267
+ throw new Error("delivery-not-bound");
2268
+ const deliveryRevision = engineRun.current_revision;
2269
+ this.#transition(
2270
+ runId,
2271
+ "change-verifying",
2272
+ `${operationId}:change-verifying`,
2273
+ undefined,
2274
+ lease,
2275
+ );
2276
+ const projection = this.#runStore.status(runId);
2277
+ emitWorkflowActivity(onActivity, {
2278
+ state: "verifying",
2279
+ stage: projection.stage,
2280
+ runId,
2281
+ ...(projection.change ? { change: projection.change } : {}),
2282
+ objective: projection.change
2283
+ ? `verify change ${projection.change}`
2284
+ : "verify workflow change",
2285
+ legalCommands: ["status", "cancel", "discard"],
2286
+ });
2287
+ const verification = await this.#changeVerifier.verify({
2288
+ taskEvidence: this.#tasks(runId).map((row) => ({
2289
+ taskId: row.task_id,
2290
+ deliveryRevision: row.delivery_revision,
2291
+ })),
2292
+ runId,
2293
+ deliveryRevision,
2294
+ plan: structuredClone(plan),
2295
+ ...(engineRun.baseline_workspace_revision
2296
+ ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2297
+ : {}),
2298
+ ...(engineRun.current_workspace_revision
2299
+ ? {
2300
+ currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2301
+ }
2302
+ : {}),
2303
+ signal,
2304
+ });
2305
+ this.#runStore.assertLease(lease);
2306
+ if (verification.kind !== "verified") {
2307
+ if (verification.verification) {
2308
+ this.#setVerificationStatus(runId, verification.verification, lease);
2309
+ }
2310
+ if (
2311
+ verification.code === "verification-repair-required" &&
2312
+ verification.verification?.scope === "change-task-affected" &&
2313
+ verification.verification.attribution === "introduced" &&
2314
+ verification.verification.taskId
2315
+ ) {
2316
+ const task = this.#tasks(runId).find(
2317
+ (candidate) =>
2318
+ candidate.task_id === verification.verification?.taskId,
2319
+ );
2320
+ if (task) {
2321
+ const repairKey = recoveryKey(
2322
+ this.#taskPlan(task),
2323
+ task.phase,
2324
+ task,
2325
+ this.#engineRun(runId),
2326
+ );
2327
+ const used = (
2328
+ this.#database
2329
+ .prepare(
2330
+ "SELECT COUNT(*) AS count FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ? AND kind = 'repair'",
2331
+ )
2332
+ .get(runId, repairKey) as { count: number }
2333
+ ).count;
2334
+ if (used >= plan.verification.repair.maxAttempts) {
2335
+ this.#setTask(
2336
+ runId,
2337
+ task.task_id,
2338
+ {
2339
+ state: "paused",
2340
+ phase: task.phase,
2341
+ pauseCode: "repair-attempts-exhausted",
2342
+ attemptDiagnostic: {
2343
+ recovery: {
2344
+ key: recoveryKey(
2345
+ this.#taskPlan(task),
2346
+ task.phase,
2347
+ task,
2348
+ this.#engineRun(runId),
2349
+ ),
2350
+ failures: used,
2351
+ feedback: {
2352
+ code: "repair-attempts-exhausted",
2353
+ attempt: used + 1,
2354
+ maxAttempts: plan.verification.repair.maxAttempts,
2355
+ strategy: "repair-verification",
2356
+ failureIdentities:
2357
+ verification.verification.failureIdentities,
2358
+ },
2359
+ },
2360
+ },
2361
+ queuePosition: null,
2362
+ },
2363
+ lease,
2364
+ );
2365
+ this.#transition(
2366
+ runId,
2367
+ "paused",
2368
+ `${operationId}:repair-attempts-exhausted:${task.task_id}`,
2369
+ "repair-attempts-exhausted",
2370
+ lease,
2371
+ );
2372
+ return this.#statusByRun(runId);
2373
+ }
2374
+ this.#leasedTransaction(lease, () => {
2375
+ this.#database
2376
+ .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2377
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'repair' FROM workflow_recovery_events WHERE run_id = ?`)
2378
+ .run(runId, repairKey, runId);
2379
+ });
2380
+ const taskPlan = this.#taskPlan(task);
2381
+ this.#setTask(
2382
+ runId,
2383
+ task.task_id,
2384
+ {
2385
+ state: "repairable",
2386
+ phase: taskPlan.phases.refactor ? "refactor" : "green",
2387
+ pauseCode: verification.code,
2388
+ queuePosition: null,
2389
+ },
2390
+ lease,
2391
+ );
2392
+ this.#transition(
2393
+ runId,
2394
+ "running",
2395
+ `${operationId}:repairable:${task.task_id}`,
2396
+ undefined,
2397
+ lease,
2398
+ );
2399
+ return this.#advance(runId, operationId, signal, lease, onActivity);
2400
+ }
2401
+ }
2402
+ const approvalCodeInvalid =
2403
+ verification.kind === "approval-needed" &&
2404
+ !isWorkflowApprovalCode(verification.code);
2405
+ const state =
2406
+ verification.kind === "approval-needed" && !approvalCodeInvalid
2407
+ ? "approval-needed"
2408
+ : "paused";
2409
+ const code = approvalCodeInvalid
2410
+ ? "approval-code-invalid"
2411
+ : verification.code;
2412
+ this.#transition(
2413
+ runId,
2414
+ state,
2415
+ `${operationId}:change-verification-${state}`,
2416
+ code,
2417
+ lease,
2418
+ );
2419
+ return this.#statusByRun(runId);
2420
+ }
2421
+ if (verification.currentWorkspaceRevisionId !== undefined) {
2422
+ const currentWorkspaceRevisionId =
2423
+ verification.currentWorkspaceRevisionId;
2424
+ if (!SHA256.test(currentWorkspaceRevisionId)) {
2425
+ throw new Error("workflow-verification-workspace-fact-invalid");
2426
+ }
2427
+ this.#leasedTransaction(lease, () => {
2428
+ this.#database
2429
+ .prepare(
2430
+ `UPDATE workflow_engine_runs
2431
+ SET current_workspace_revision = ?, cleanup_state = 'retained'
2432
+ WHERE run_id = ?`,
2433
+ )
2434
+ .run(currentWorkspaceRevisionId, runId);
2435
+ });
2436
+ engineRun = this.#engineRun(runId);
2437
+ }
2438
+ this.#transition(
2439
+ runId,
2440
+ "ready-to-apply",
2441
+ `${operationId}:ready-to-apply`,
2442
+ undefined,
2443
+ lease,
2444
+ );
2445
+ if (!this.#application) {
2446
+ this.#transition(
2447
+ runId,
2448
+ "paused",
2449
+ `${operationId}:application-unavailable`,
2450
+ "application-unavailable",
2451
+ lease,
2452
+ );
2453
+ return this.#statusByRun(runId);
2454
+ }
2455
+ const transactionId = `apply-${hash(
2456
+ runId,
2457
+ String(deliveryRevision),
2458
+ operationId,
2459
+ ).slice(0, 40)}`;
2460
+ this.#leasedTransaction(lease, () => {
2461
+ this.#database
2462
+ .prepare(
2463
+ `UPDATE workflow_engine_runs SET transaction_id = ? WHERE run_id = ?`,
2464
+ )
2465
+ .run(transactionId, runId);
2466
+ });
2467
+ const applicationInput = {
2468
+ transactionId,
2469
+ runId,
2470
+ consumerRoot: this.#consumerRoot,
2471
+ deliveryRevision,
2472
+ plan: structuredClone(plan),
2473
+ contextReadPaths: this.#contextReads(runId),
2474
+ ...(engineRun.baseline_workspace_revision
2475
+ ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2476
+ : {}),
2477
+ ...(engineRun.current_workspace_revision
2478
+ ? {
2479
+ currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2480
+ }
2481
+ : {}),
2482
+ signal,
2483
+ };
2484
+ let application: Record<string, unknown>;
2485
+ if (this.#application.prepare && this.#application.applyPrepared) {
2486
+ const prepared = await this.#application.prepare(applicationInput);
2487
+ this.#runStore.assertLease(lease);
2488
+ if (prepared.state !== "prepared") {
2489
+ return this.#settleApplication(
2490
+ runId,
2491
+ operationId,
2492
+ prepared,
2493
+ lease,
2494
+ onActivity,
2495
+ );
2496
+ }
2497
+ this.#transition(
2498
+ runId,
2499
+ "applying",
2500
+ `${operationId}:applying`,
2501
+ undefined,
2502
+ lease,
2503
+ );
2504
+ emitWorkflowActivity(onActivity, {
2505
+ state: "applying",
2506
+ stage: projection.stage,
2507
+ runId,
2508
+ ...(projection.change ? { change: projection.change } : {}),
2509
+ objective: projection.change
2510
+ ? `apply change ${projection.change}`
2511
+ : "apply workflow change",
2512
+ legalCommands: ["status", "cancel", "discard"],
2513
+ });
2514
+ const active = this.#active.get(runId);
2515
+ if (active) active.kind = "applying";
2516
+ try {
2517
+ application = await this.#application.applyPrepared(applicationInput);
2518
+ } catch (error) {
2519
+ this.#runStore.assertLease(lease);
2520
+ if (this.#runStore.status(runId).state === "applying")
2521
+ this.#transition(
2522
+ runId,
2523
+ "recovering",
2524
+ `${operationId}:application-interrupted`,
2525
+ "operation-interrupted",
2526
+ lease,
2527
+ );
2528
+ throw error;
2529
+ }
2530
+ } else {
2531
+ this.#transition(
2532
+ runId,
2533
+ "applying",
2534
+ `${operationId}:applying`,
2535
+ undefined,
2536
+ lease,
2537
+ );
2538
+ emitWorkflowActivity(onActivity, {
2539
+ state: "applying",
2540
+ stage: projection.stage,
2541
+ runId,
2542
+ ...(projection.change ? { change: projection.change } : {}),
2543
+ objective: projection.change
2544
+ ? `apply change ${projection.change}`
2545
+ : "apply workflow change",
2546
+ legalCommands: ["status", "cancel", "discard"],
2547
+ });
2548
+ const active = this.#active.get(runId);
2549
+ if (active) active.kind = "applying";
2550
+ application = await this.#application.begin(applicationInput);
2551
+ }
2552
+ this.#runStore.assertLease(lease);
2553
+ return this.#settleApplication(
2554
+ runId,
2555
+ operationId,
2556
+ application,
2557
+ lease,
2558
+ onActivity,
2559
+ );
2560
+ }
2561
+
2562
+ #settleApplication(
2563
+ runId: string,
2564
+ operationId: string,
2565
+ outcome: Record<string, unknown>,
2566
+ lease: OperationLease,
2567
+ onActivity?: (event: WorkflowActivityUpdate) => void,
2568
+ ): Record<string, unknown> {
2569
+ const state = outcome.state;
2570
+ if (state === "completed") {
2571
+ const current = this.#runStore.status(runId);
2572
+ if (current.state === "recovering") {
2573
+ this.#transition(
2574
+ runId,
2575
+ "completed",
2576
+ `${operationId}:recovered-complete`,
2577
+ undefined,
2578
+ lease,
2579
+ );
2580
+ } else if (
2581
+ current.state === "applying" ||
2582
+ current.state === "ready-to-apply"
2583
+ ) {
2584
+ this.#transition(
2585
+ runId,
2586
+ "completed",
2587
+ `${operationId}:apply-complete`,
2588
+ undefined,
2589
+ lease,
2590
+ );
2591
+ }
2592
+ this.#cleanupTerminal(runId, operationId, lease);
2593
+ } else if (state === "paused") {
2594
+ if (this.#runStore.status(runId).state === "applying") {
2595
+ this.#transition(
2596
+ runId,
2597
+ "recovering",
2598
+ `${operationId}:apply-recovering`,
2599
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2600
+ lease,
2601
+ );
2602
+ const recovering = this.#runStore.status(runId);
2603
+ emitWorkflowActivity(onActivity, {
2604
+ state: "recovering",
2605
+ stage: recovering.stage,
2606
+ runId,
2607
+ ...(recovering.change ? { change: recovering.change } : {}),
2608
+ objective: recovering.change
2609
+ ? `recover change ${recovering.change}`
2610
+ : "recover workflow application",
2611
+ code:
2612
+ typeof outcome.code === "string"
2613
+ ? outcome.code
2614
+ : "recovery-required",
2615
+ legalCommands: ["status", "cancel", "discard"],
2616
+ });
2617
+ }
2618
+ this.#transition(
2619
+ runId,
2620
+ "paused",
2621
+ `${operationId}:apply-paused`,
2622
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2623
+ lease,
2624
+ );
2625
+ } else if (state === "discarded") {
2626
+ if (this.#runStore.status(runId).state === "applying") {
2627
+ this.#transition(
2628
+ runId,
2629
+ "recovering",
2630
+ `${operationId}:discard-recovering`,
2631
+ "run-discarded",
2632
+ lease,
2633
+ );
2634
+ const recovering = this.#runStore.status(runId);
2635
+ emitWorkflowActivity(onActivity, {
2636
+ state: "recovering",
2637
+ stage: recovering.stage,
2638
+ runId,
2639
+ ...(recovering.change ? { change: recovering.change } : {}),
2640
+ objective: recovering.change
2641
+ ? `recover change ${recovering.change}`
2642
+ : "recover workflow application",
2643
+ code: "run-discarded",
2644
+ legalCommands: ["status"],
2645
+ });
2646
+ }
2647
+ this.#transition(
2648
+ runId,
2649
+ "discarded",
2650
+ `${operationId}:apply-discarded`,
2651
+ undefined,
2652
+ lease,
2653
+ );
2654
+ this.#cleanupTerminal(runId, operationId, lease);
2655
+ } else if (state === "recovering") {
2656
+ this.#transition(
2657
+ runId,
2658
+ "recovering",
2659
+ `${operationId}:application-recovering`,
2660
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2661
+ lease,
2662
+ );
2663
+ const recovering = this.#runStore.status(runId);
2664
+ emitWorkflowActivity(onActivity, {
2665
+ state: "recovering",
2666
+ stage: recovering.stage,
2667
+ runId,
2668
+ ...(recovering.change ? { change: recovering.change } : {}),
2669
+ objective: recovering.change
2670
+ ? `recover change ${recovering.change}`
2671
+ : "recover workflow application",
2672
+ code:
2673
+ typeof outcome.code === "string" ? outcome.code : "recovery-required",
2674
+ legalCommands: ["status", "cancel", "discard"],
2675
+ });
2676
+ } else {
2677
+ throw new Error("workflow-application-outcome-invalid");
2678
+ }
2679
+ return this.#statusByRun(runId);
2680
+ }
2681
+
2682
+ #applicationContext(runId: string): WorkflowApplicationContext {
2683
+ const current = this.#engineRun(runId);
2684
+ if (current.current_revision === null)
2685
+ throw new Error("delivery-not-bound");
2686
+ return {
2687
+ runId,
2688
+ deliveryRevision: current.current_revision,
2689
+ plan: structuredClone(this.#currentPlan(runId)),
2690
+ contextReadPaths: this.#contextReads(runId),
2691
+ ...(current.baseline_workspace_revision
2692
+ ? { baselineRevisionId: current.baseline_workspace_revision }
2693
+ : {}),
2694
+ ...(current.current_workspace_revision
2695
+ ? { currentWorkspaceRevisionId: current.current_workspace_revision }
2696
+ : {}),
2697
+ };
2698
+ }
2699
+
2700
+ #cleanupTerminal(
2701
+ runId: string,
2702
+ _currentOperationId: string,
2703
+ lease?: OperationLease,
2704
+ ): void {
2705
+ if (lease) this.#runStore.assertLease(lease);
2706
+ this.#lifecycle?.cleanup(runId);
2707
+ const cleanup = () => {
2708
+ this.#database
2709
+ .prepare("DELETE FROM workflow_engine_tasks WHERE run_id = ?")
2710
+ .run(runId);
2711
+ this.#database
2712
+ .prepare("DELETE FROM workflow_engine_deliveries WHERE run_id = ?")
2713
+ .run(runId);
2714
+ this.#database
2715
+ .prepare(`DELETE FROM workspace_revisions WHERE owner_run_id = ?`)
2716
+ .run(runId);
2717
+ this.#database
2718
+ .prepare(`DELETE FROM apply_transactions WHERE owner_run_id = ?`)
2719
+ .run(runId);
2720
+ this.#database
2721
+ .prepare(
2722
+ `UPDATE workflow_engine_runs
2723
+ SET baseline_workspace_revision = NULL,
2724
+ current_workspace_revision = NULL,
2725
+ transaction_id = NULL, verification_json = NULL,
2726
+ cleanup_state = 'complete'
2727
+ WHERE run_id = ?`,
2728
+ )
2729
+ .run(runId);
2730
+ };
2731
+ if (lease) this.#leasedTransaction(lease, cleanup);
2732
+ else this.#transaction(cleanup);
2733
+ }
2734
+
2735
+ #waitForCapacity(signal: AbortSignal): Promise<void> {
2736
+ if (signal.aborted || this.#activeTasks < LIMITS.maxActiveChildSessions)
2737
+ return Promise.resolve();
2738
+ return new Promise((resolve) => {
2739
+ const wake = () => {
2740
+ this.#capacityWaiters.delete(wake);
2741
+ signal.removeEventListener("abort", wake);
2742
+ resolve();
2743
+ };
2744
+ this.#capacityWaiters.add(wake);
2745
+ signal.addEventListener("abort", wake, { once: true });
2746
+ });
2747
+ }
2748
+
2749
+ async #advance(
2750
+ runId: string,
2751
+ operationId: string,
2752
+ signal: AbortSignal,
2753
+ lease: OperationLease,
2754
+ onActivity?: (event: WorkflowActivityUpdate) => void,
2755
+ ): Promise<Record<string, unknown>> {
2756
+ this.#transition(
2757
+ runId,
2758
+ "running",
2759
+ `${operationId}:running`,
2760
+ undefined,
2761
+ lease,
2762
+ );
2763
+ const run = this.#runStore.status(runId);
2764
+ emitWorkflowActivity(onActivity, {
2765
+ state: "running",
2766
+ stage: run.stage,
2767
+ runId,
2768
+ ...(run.change ? { change: run.change } : {}),
2769
+ objective: run.change ? `change ${run.change}` : "workflow run",
2770
+ legalCommands: ["status", "cancel", "discard"],
2771
+ });
2772
+ const existingVerification = this.#engineRun(runId).verification_json;
2773
+ const repairStatus = existingVerification
2774
+ ? normalizeWorkflowVerificationStatus(
2775
+ JSON.parse(existingVerification) as unknown,
2776
+ )
2777
+ : undefined;
2778
+ const retainsRepairContext =
2779
+ repairStatus?.scope === "change-task-affected" &&
2780
+ repairStatus.attribution === "introduced" &&
2781
+ repairStatus.taskId !== undefined &&
2782
+ this.#tasks(runId).some(
2783
+ (task) =>
2784
+ task.task_id === repairStatus.taskId && task.state !== "verified",
2785
+ );
2786
+ if (!retainsRepairContext) {
2787
+ this.#setVerificationStatus(runId, null, lease);
2788
+ }
2789
+ const attempted = new Set<string>();
2790
+ let madeProgress = true;
2791
+ while (madeProgress && !signal.aborted) {
2792
+ madeProgress = false;
2793
+ const rows = this.#tasks(runId);
2794
+ const runnable: Array<{
2795
+ row: EngineTaskRow;
2796
+ before: string;
2797
+ }> = [];
2798
+ const selectedTaskIds = new Set<string>();
2799
+ let capacityBlocked = false;
2800
+ for (const row of rows) {
2801
+ if (row.state === "verified" || attempted.has(row.task_id)) continue;
2802
+ if (!this.#dependenciesVerified(row, rows)) {
2803
+ if (row.state !== "pending") {
2804
+ this.#setTask(
2805
+ runId,
2806
+ row.task_id,
2807
+ {
2808
+ state: "pending",
2809
+ pauseCode: null,
2810
+ queuePosition: null,
2811
+ },
2812
+ lease,
2813
+ );
2814
+ }
2815
+ continue;
2816
+ }
2817
+ if (this.#hasConflict(row, this.#tasks(runId), selectedTaskIds)) {
2818
+ this.#queueTask(runId, row, lease);
2819
+ continue;
2820
+ }
2821
+ if (
2822
+ this.#activeTasks + runnable.length >=
2823
+ LIMITS.maxActiveChildSessions
2824
+ ) {
2825
+ this.#queueTask(runId, row, lease, "capacity");
2826
+ capacityBlocked = true;
2827
+ continue;
2828
+ }
2829
+ const before = `${row.state}:${row.phase}`;
2830
+ attempted.add(row.task_id);
2831
+ selectedTaskIds.add(row.task_id);
2832
+ runnable.push({ row, before });
2833
+ }
2834
+ if (runnable.length === 0 && capacityBlocked) {
2835
+ await this.#waitForCapacity(signal);
2836
+ madeProgress = true;
2837
+ continue;
2838
+ }
2839
+ this.#activeTasks += runnable.length;
2840
+ const settled = await Promise.allSettled(
2841
+ runnable.map(({ row }) =>
2842
+ this.#runTask(
2843
+ runId,
2844
+ row,
2845
+ operationId,
2846
+ signal,
2847
+ lease,
2848
+ onActivity,
2849
+ ).finally(() => {
2850
+ this.#activeTasks--;
2851
+ for (const wake of this.#capacityWaiters) wake();
2852
+ }),
2853
+ ),
2854
+ );
2855
+ const rejected = settled.find(
2856
+ (outcome): outcome is PromiseRejectedResult =>
2857
+ outcome.status === "rejected",
2858
+ );
2859
+ if (rejected) throw rejected.reason;
2860
+ for (const { row, before } of runnable) {
2861
+ const after = this.#tasks(runId).find(
2862
+ (candidate) => candidate.task_id === row.task_id,
2863
+ );
2864
+ if (after && `${after.state}:${after.phase}` !== before) {
2865
+ madeProgress = true;
2866
+ }
2867
+ }
2868
+ }
2869
+
2870
+ if (signal.aborted) {
2871
+ const current = this.#runStore.status(runId);
2872
+ if (!["paused", "discarded", "completed"].includes(current.state)) {
2873
+ this.#transition(
2874
+ runId,
2875
+ "paused",
2876
+ `${operationId}:cancelled`,
2877
+ "operation-cancelled",
2878
+ lease,
2879
+ );
2880
+ }
2881
+ return this.#statusByRun(runId);
2882
+ }
2883
+ const rows = this.#tasks(runId);
2884
+ const paused = this.#firstPaused(rows);
2885
+ if (paused) {
2886
+ this.#transition(
2887
+ runId,
2888
+ paused.state === "approval-needed" ? "approval-needed" : "paused",
2889
+ `${operationId}:task-paused:${paused.task_id}`,
2890
+ paused.pause_code ?? "task-paused",
2891
+ lease,
2892
+ );
2893
+ return this.#statusByRun(runId);
2894
+ }
2895
+ if (rows.every((row) => row.state === "verified")) {
2896
+ return this.#finishChange(runId, operationId, signal, lease, onActivity);
2897
+ }
2898
+ this.#transition(
2899
+ runId,
2900
+ "paused",
2901
+ `${operationId}:scheduler-stalled`,
2902
+ "scheduler-integrity-paused",
2903
+ lease,
2904
+ );
2905
+ return this.#statusByRun(runId);
2906
+ }
2907
+
2908
+ #contextReads(runId: string, taskId?: string): string[] {
2909
+ const rows =
2910
+ taskId === undefined
2911
+ ? this.#database
2912
+ .prepare(
2913
+ "SELECT DISTINCT path FROM workflow_context_reads WHERE run_id = ? ORDER BY path",
2914
+ )
2915
+ .all(runId)
2916
+ : this.#database
2917
+ .prepare(
2918
+ "SELECT path FROM workflow_context_reads WHERE run_id = ? AND task_id = ? ORDER BY path",
2919
+ )
2920
+ .all(runId, taskId);
2921
+ return rows.map((row) => String(row.path));
2922
+ }
2923
+
2924
+ #phaseCount(plan: ImplementPlan): number {
2925
+ return plan.tasks.reduce((sum, task) => sum + phases(task).length, 0);
2926
+ }
2927
+
2928
+ #growWorkBudget(
2929
+ database: DatabaseSync,
2930
+ runId: string,
2931
+ plan: ImplementPlan,
2932
+ ): void {
2933
+ const count = this.#phaseCount(plan);
2934
+ database
2935
+ .prepare(`INSERT INTO workflow_work_budget(run_id, used, max_work, phase_high_water, hard_limit, recovery_policy)
2936
+ VALUES (?, 0, ?, ?, ?, 1) ON CONFLICT(run_id) DO UPDATE SET
2937
+ phase_high_water = MAX(phase_high_water, excluded.phase_high_water),
2938
+ max_work = MIN(hard_limit, MAX(max_work, 24 + 3 * MAX(phase_high_water, excluded.phase_high_water)))`)
2939
+ .run(
2940
+ runId,
2941
+ Math.min(this.#workHardLimit, 24 + 3 * count),
2942
+ count,
2943
+ this.#workHardLimit,
2944
+ );
2945
+ }
2946
+
2947
+ #hasRecoveryGrant(runId: string, operationId: string, key: string): boolean {
2948
+ return Boolean(
2949
+ this.#database
2950
+ .prepare(
2951
+ "SELECT 1 FROM workflow_recovery_grants WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
2952
+ )
2953
+ .get(runId, operationId, key),
2954
+ );
2955
+ }
2956
+
2957
+ #recoveryConditions(runId: string, key: string) {
2958
+ const run = this.#engineRun(runId);
2959
+ const task = this.#tasks(runId).find(
2960
+ (row) => recoveryKey(this.#taskPlan(row), row.phase, row, run) === key,
2961
+ );
2962
+ return {
2963
+ route: task?.route_fingerprint ?? run.route_fingerprint,
2964
+ context: task
2965
+ ? hash(
2966
+ canonicalJson(
2967
+ snapshotFiles(
2968
+ this.#consumerRoot,
2969
+ this.#contextReads(runId, task.task_id),
2970
+ ),
2971
+ ),
2972
+ )
2973
+ : null,
2974
+ };
2975
+ }
2976
+
2977
+ #failureSequence(runId: string, key: string): number {
2978
+ return (
2979
+ this.#database
2980
+ .prepare(
2981
+ "SELECT COALESCE(MAX(sequence), 0) AS sequence FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ?",
2982
+ )
2983
+ .get(runId, key) as { sequence: number }
2984
+ ).sequence;
2985
+ }
2986
+
2987
+ #grantRecovery(
2988
+ runId: string,
2989
+ operationId: string,
2990
+ request: NonNullable<
2991
+ Extract<ControlCommand, { command: "resume" }>["recovery"]
2992
+ >,
2993
+ lease: OperationLease,
2994
+ ): void {
2995
+ this.#leasedTransaction(lease, () => {
2996
+ const incident = this.#recovery(runId, request.incidentKey);
2997
+ const current = this.#tasks(runId).some(
2998
+ (row) =>
2999
+ ["paused", "retryable"].includes(row.state) &&
3000
+ recoveryKey(
3001
+ this.#taskPlan(row),
3002
+ row.phase,
3003
+ row,
3004
+ this.#engineRun(runId),
3005
+ ) === request.incidentKey,
3006
+ );
3007
+ if (
3008
+ !current ||
3009
+ !incident ||
3010
+ incident.failures < incident.feedback.maxAttempts ||
3011
+ this.#failureSequence(runId, request.incidentKey) !==
3012
+ request.failureSequence
3013
+ )
3014
+ throw new Error("recovery-request-stale");
3015
+ const conditions = this.#database
3016
+ .prepare(
3017
+ "SELECT conditions_json FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
3018
+ )
3019
+ .get(runId, request.incidentKey) as { conditions_json: string };
3020
+ const previous = JSON.parse(conditions.conditions_json);
3021
+ const observed = this.#recoveryConditions(runId, request.incidentKey);
3022
+ if (
3023
+ (request.reason === "route-changed" &&
3024
+ (!previous.route || previous.route === observed.route)) ||
3025
+ (request.reason === "context-extended" &&
3026
+ (!previous.context || previous.context === observed.context))
3027
+ )
3028
+ throw new Error("recovery-evidence-unavailable");
3029
+ const budget = this.#database
3030
+ .prepare(
3031
+ "SELECT used, max_work FROM workflow_work_budget WHERE run_id = ?",
3032
+ )
3033
+ .get(runId) as { used: number; max_work: number };
3034
+ if (!budget || budget.used >= budget.max_work)
3035
+ throw new Error("change-work-budget-exhausted");
3036
+ this.#database
3037
+ .prepare(
3038
+ "INSERT INTO workflow_recovery_grants(run_id, operation_id, incident_key, failure_sequence, reason) VALUES (?, ?, ?, ?, ?)",
3039
+ )
3040
+ .run(
3041
+ runId,
3042
+ operationId,
3043
+ request.incidentKey,
3044
+ request.failureSequence,
3045
+ request.reason,
3046
+ );
3047
+ });
3048
+ }
3049
+
3050
+ #reserveWork(
3051
+ runId: string,
3052
+ plan: ImplementPlan,
3053
+ key: string,
3054
+ lease: OperationLease,
3055
+ grantOperationId?: string,
3056
+ ): boolean {
3057
+ return this.#leasedTransaction(lease, () => {
3058
+ this.#growWorkBudget(this.#database, runId, plan);
3059
+ const reserved = this.#database
3060
+ .prepare(`UPDATE workflow_work_budget SET used = used + 1
3061
+ WHERE run_id = ? AND used < max_work`)
3062
+ .run(runId);
3063
+ if (reserved.changes !== 1) return false;
3064
+ if (grantOperationId) {
3065
+ const consumed = this.#database
3066
+ .prepare(
3067
+ "UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
3068
+ )
3069
+ .run(runId, grantOperationId, key);
3070
+ if (consumed.changes !== 1) throw new Error("recovery-request-stale");
3071
+ }
3072
+ this.#database
3073
+ .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
3074
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
3075
+ .run(runId, key, runId);
3076
+ return true;
3077
+ });
3078
+ }
3079
+
3080
+ #recovery(runId: string, key: string): WorkflowRecoveryFact | undefined {
3081
+ const row = this.#database
3082
+ .prepare(
3083
+ `SELECT failures, max_attempts, feedback_json FROM workflow_recovery_incidents
3084
+ WHERE run_id = ? AND incident_key = ?`,
3085
+ )
3086
+ .get(runId, key) as
3087
+ | { failures: number; max_attempts: number; feedback_json: string }
3088
+ | undefined;
3089
+ return row
3090
+ ? {
3091
+ key,
3092
+ failures: row.failures,
3093
+ feedback: {
3094
+ ...JSON.parse(row.feedback_json),
3095
+ maxAttempts: row.max_attempts,
3096
+ },
3097
+ }
3098
+ : undefined;
3099
+ }
3100
+
3101
+ #recordRecovery(
3102
+ runId: string,
3103
+ fact: WorkflowRecoveryFact,
3104
+ lease: OperationLease,
3105
+ ): void {
3106
+ this.#leasedTransaction(lease, () => {
3107
+ this.#database
3108
+ .prepare(
3109
+ `INSERT INTO workflow_recovery_incidents(run_id, incident_key, failures, max_attempts, feedback_json, conditions_json)
3110
+ VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, incident_key) DO UPDATE SET
3111
+ failures = MAX(failures, excluded.failures), max_attempts = MIN(max_attempts, excluded.max_attempts), feedback_json = excluded.feedback_json, conditions_json = excluded.conditions_json`,
3112
+ )
3113
+ .run(
3114
+ runId,
3115
+ fact.key,
3116
+ fact.failures,
3117
+ fact.feedback.maxAttempts,
3118
+ JSON.stringify(fact.feedback),
3119
+ JSON.stringify(this.#recoveryConditions(runId, fact.key)),
3120
+ );
3121
+ this.#database
3122
+ .prepare(
3123
+ `INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
3124
+ SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'failure' FROM workflow_recovery_events WHERE run_id = ?`,
3125
+ )
3126
+ .run(runId, fact.key, runId);
3127
+ });
3128
+ }
3129
+
3130
+ #safeAttemptDiagnostic(
3131
+ runId: string,
3132
+ row: EngineTaskRow | undefined,
3133
+ ): SafeAttemptDiagnostic | undefined {
3134
+ if (!row) return undefined;
3135
+ const diagnostic: SafeAttemptDiagnostic = row.attempt_diagnostic_json
3136
+ ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
3137
+ : {};
3138
+ delete diagnostic.recovery;
3139
+ const recovery = this.#recovery(
3140
+ runId,
3141
+ recoveryKey(this.#taskPlan(row), row.phase, row, this.#engineRun(runId)),
3142
+ );
3143
+ if (recovery) diagnostic.recovery = recovery;
3144
+ return Object.keys(diagnostic).length ? diagnostic : undefined;
3145
+ }
3146
+
3147
+ #statusByRun(runId: string): Record<string, unknown> {
3148
+ const projection = this.#runStore.status(runId);
3149
+ const rows = this.#tasks(runId);
3150
+ const queued = rows
3151
+ .filter((row) => row.state === "queued")
3152
+ .sort(
3153
+ (left, right) =>
3154
+ (left.queue_position ?? Number.MAX_SAFE_INTEGER) -
3155
+ (right.queue_position ?? Number.MAX_SAFE_INTEGER) ||
3156
+ left.task_order - right.task_order,
3157
+ );
3158
+ const firstPaused = this.#firstPaused(rows);
3159
+ let engineRun: EngineRunRow | undefined;
3160
+ try {
3161
+ engineRun = this.#engineRun(runId);
3162
+ } catch {
3163
+ engineRun = undefined;
3164
+ }
3165
+ const contextRequest = firstPaused?.context_request_json
3166
+ ? normalizeWorkflowContextRequest(
3167
+ JSON.parse(firstPaused.context_request_json) as unknown,
3168
+ )
3169
+ : undefined;
3170
+ const pauseCode =
3171
+ projection.state === "approval-needed" &&
3172
+ firstPaused?.state === "approval-needed"
3173
+ ? (firstPaused.pause_code ?? projection.pauseCode ?? "task-paused")
3174
+ : (projection.pauseCode ?? firstPaused?.pause_code ?? "task-paused");
3175
+ const approval =
3176
+ projection.state === "approval-needed"
3177
+ ? approvalRequirement(pauseCode, contextRequest)
3178
+ : undefined;
3179
+ const attemptDiagnostic = this.#safeAttemptDiagnostic(runId, firstPaused);
3180
+ const recovery =
3181
+ rows
3182
+ .map((row) => this.#safeAttemptDiagnostic(runId, row)?.recovery)
3183
+ .find((fact) => fact && fact.failures >= fact.feedback.maxAttempts) ??
3184
+ attemptDiagnostic?.recovery;
3185
+ const recoveryExhausted =
3186
+ recovery && recovery.failures >= recovery.feedback.maxAttempts;
3187
+ const blockers = rows
3188
+ .filter((row) =>
3189
+ ["approval-needed", "paused", "retryable"].includes(row.state),
3190
+ )
3191
+ .map((row) => {
3192
+ const context = row.context_request_json
3193
+ ? normalizeWorkflowContextRequest(
3194
+ JSON.parse(row.context_request_json),
3195
+ )
3196
+ : undefined;
3197
+ const code = row.pause_code ?? "task-paused";
3198
+ const requirement =
3199
+ row.state === "approval-needed"
3200
+ ? approvalRequirement(code, context)
3201
+ : undefined;
3202
+ return {
3203
+ id: hash(runId, row.task_id, code, canonicalJson(context ?? {})),
3204
+ taskId: row.task_id,
3205
+ phase: row.phase,
3206
+ code,
3207
+ kind: requirement ? "decision" : "execution",
3208
+ ...(requirement ?? {}),
3209
+ ...(context ? { contextRequest: context } : {}),
3210
+ };
3211
+ });
3212
+ if (
3213
+ projection.state === "approval-needed" &&
3214
+ !blockers.some((item) => item.kind === "decision") &&
3215
+ approval
3216
+ ) {
3217
+ blockers.push({
3218
+ id: hash(runId, pauseCode),
3219
+ taskId: "change",
3220
+ phase: "green",
3221
+ code: pauseCode,
3222
+ kind: "decision",
3223
+ ...approval,
3224
+ });
3225
+ }
3226
+ const decisions = blockers.filter((item) => item.kind === "decision");
3227
+ const amendments = blockers.filter(
3228
+ (item) => item.kind === "decision" || permitsPlanAmendment(item.code),
3229
+ );
3230
+ if (
3231
+ projection.state === "paused" &&
3232
+ permitsPlanAmendment(pauseCode) &&
3233
+ !amendments.some((item) => item.code === pauseCode)
3234
+ ) {
3235
+ const item = {
3236
+ id: hash(runId, pauseCode, engineRun?.delivery_diagnostics_json ?? ""),
3237
+ taskId: "change",
3238
+ phase: "red" as const,
3239
+ code: pauseCode,
3240
+ kind: "execution",
3241
+ };
3242
+ blockers.push(item);
3243
+ amendments.push(item);
3244
+ }
3245
+ const amendmentUsed =
3246
+ (
3247
+ this.#database
3248
+ .prepare(
3249
+ "SELECT used FROM workflow_amendment_budget WHERE run_id = ?",
3250
+ )
3251
+ .get(runId) as { used: number } | undefined
3252
+ )?.used ?? 0;
3253
+ const amendmentBudget = {
3254
+ used: amendmentUsed,
3255
+ maximum: 64,
3256
+ remaining: 64 - amendmentUsed,
3257
+ exhausted: amendmentUsed >= 64,
3258
+ };
3259
+ const decisionBatch =
3260
+ amendments.length &&
3261
+ pauseCode !== "operation-cancelled" &&
3262
+ ["approval-needed", "paused"].includes(projection.state)
3263
+ ? {
3264
+ id: hash(
3265
+ runId,
3266
+ String(projection.deliveryRevision ?? 0),
3267
+ canonicalJson(amendments),
3268
+ ),
3269
+ continuation: {
3270
+ action: "amend",
3271
+ change: projection.change,
3272
+ batchId: hash(
3273
+ runId,
3274
+ String(projection.deliveryRevision ?? 0),
3275
+ canonicalJson(amendments),
3276
+ ),
3277
+ },
3278
+ resolution: {
3279
+ owner: "parent",
3280
+ strategy: "recommended",
3281
+ requiresUserInput: false,
3282
+ },
3283
+ items: amendments,
3284
+ requiredGates: [
3285
+ ...new Set(
3286
+ amendments.flatMap((item) => item.requiredGates ?? ["gate-b"]),
3287
+ ),
3288
+ ].sort(),
3289
+ }
3290
+ : undefined;
3291
+ const resourceBudget = this.#database
3292
+ .prepare(
3293
+ "SELECT used, max_work AS maximum, phase_high_water AS phaseHighWater, hard_limit AS hardLimit FROM workflow_work_budget WHERE run_id = ?",
3294
+ )
3295
+ .get(runId) as { used: number; maximum: number } | undefined;
3296
+ const budgetCode = rows.find(
3297
+ (row) =>
3298
+ row.pause_code === "change-work-budget-exhausted" ||
3299
+ row.pause_code === "change-recovery-budget-exhausted",
3300
+ )?.pause_code;
3301
+ const budgetExhausted = budgetCode !== undefined;
3302
+ const currentRevision = projection.deliveryRevision ?? 0;
3303
+ return {
3304
+ runId: projection.runId,
3305
+ stage: projection.stage,
3306
+ ...(projection.change ? { change: projection.change } : {}),
3307
+ state: projection.state,
3308
+ ...(projection.deliveryRevision === undefined
3309
+ ? {}
3310
+ : { deliveryRevision: projection.deliveryRevision }),
3311
+ ...(projection.deliveryBindings.length > 0
3312
+ ? { deliveryBindings: structuredClone(projection.deliveryBindings) }
3313
+ : {}),
3314
+ completed: projection.state === "completed",
3315
+ blockers,
3316
+ ...(resourceBudget
3317
+ ? {
3318
+ resourceBudget: {
3319
+ ...resourceBudget,
3320
+ remaining: resourceBudget.maximum - resourceBudget.used,
3321
+ },
3322
+ }
3323
+ : {}),
3324
+ ...(budgetExhausted
3325
+ ? {
3326
+ recovery: {
3327
+ automatic: false,
3328
+ exhausted: true,
3329
+ code: budgetCode,
3330
+ },
3331
+ }
3332
+ : {}),
3333
+ ...(decisionBatch
3334
+ ? {
3335
+ decisionBatch,
3336
+ amendmentBudget,
3337
+ ...(!amendmentBudget.exhausted &&
3338
+ !budgetExhausted &&
3339
+ (!resourceBudget || resourceBudget.used < resourceBudget.maximum)
3340
+ ? {
3341
+ continuation: {
3342
+ owner: "parent",
3343
+ automatic: true,
3344
+ ...decisionBatch.continuation,
3345
+ },
3346
+ }
3347
+ : {}),
3348
+ }
3349
+ : {}),
3350
+ ...(recoveryExhausted
3351
+ ? {
3352
+ recovery: {
3353
+ automatic: false,
3354
+ exhausted: true,
3355
+ code: recovery.feedback.code,
3356
+ attempts: recovery.failures,
3357
+ maxAttempts: recovery.feedback.maxAttempts,
3358
+ automaticRetryExhausted: true,
3359
+ ...(!budgetExhausted &&
3360
+ resourceBudget &&
3361
+ resourceBudget.used < resourceBudget.maximum
3362
+ ? {
3363
+ additionalAttempt: {
3364
+ incidentKey: recovery.key,
3365
+ failureSequence: this.#failureSequence(
3366
+ runId,
3367
+ recovery.key,
3368
+ ),
3369
+ reason: "parent-directed-retry",
3370
+ },
3371
+ }
3372
+ : {}),
3373
+ },
3374
+ }
3375
+ : {}),
3376
+ ...(projection.terminal ? { terminal: projection.terminal } : {}),
3377
+ ...(projection.pauseCode || firstPaused?.pause_code
3378
+ ? {
3379
+ pause: {
3380
+ code: pauseCode,
3381
+ ...(attemptDiagnostic ? { diagnostic: attemptDiagnostic } : {}),
3382
+ },
3383
+ }
3384
+ : {}),
3385
+ legalCommands:
3386
+ projection.state === "approval-needed"
3387
+ ? ["status", "discard"]
3388
+ : legalControlCommands(projection.state).filter(
3389
+ (command) =>
3390
+ (command !== "rebind" || firstPaused !== undefined) &&
3391
+ (command !== "resume" ||
3392
+ (!recoveryExhausted && !budgetExhausted)),
3393
+ ),
3394
+ ...(approval && projection.change
3395
+ ? {
3396
+ approval: {
3397
+ category: approval.category,
3398
+ requiredGates:
3399
+ decisionBatch?.requiredGates ?? approval.requiredGates,
3400
+ categories: [
3401
+ ...new Set(
3402
+ decisions.flatMap((item) =>
3403
+ item.category ? [item.category] : [],
3404
+ ),
3405
+ ),
3406
+ ],
3407
+ refs: [...new Set(decisions.flatMap((item) => item.refs ?? []))],
3408
+ continuation: decisionBatch?.continuation,
3409
+ retainedRun: {
3410
+ runId: projection.runId,
3411
+ deliveryRevision: currentRevision,
3412
+ },
3413
+ receiptPrecondition: {
3414
+ deliveryRevision: { greaterThan: currentRevision },
3415
+ receiptHash: "matching-ready-receipt",
3416
+ },
3417
+ },
3418
+ conditionalCommands: [
3419
+ {
3420
+ command: "resume",
3421
+ stage: "abel-implement",
3422
+ change: projection.change,
3423
+ requires: {
3424
+ deliveryRevision: { greaterThan: currentRevision },
3425
+ receiptHash: "matching-ready-receipt",
3426
+ },
3427
+ },
3428
+ ],
3429
+ }
3430
+ : {}),
3431
+ tasks: rows.map((row) => ({
3432
+ taskId: row.task_id,
3433
+ state: row.state,
3434
+ ...(row.state === "verified" ? {} : { phase: row.phase }),
3435
+ ...(row.context_request_json
3436
+ ? {
3437
+ contextRequest: normalizeWorkflowContextRequest(
3438
+ JSON.parse(row.context_request_json) as unknown,
3439
+ ),
3440
+ }
3441
+ : {}),
3442
+ })),
3443
+ queue: queued.map((row, index) => ({
3444
+ taskId: row.task_id,
3445
+ position: index + 1,
3446
+ reason: row.pause_code === "worker-capacity" ? "capacity" : "conflict",
3447
+ })),
3448
+ ...(engineRun?.verification_json
3449
+ ? {
3450
+ verification: normalizeWorkflowVerificationStatus(
3451
+ JSON.parse(engineRun.verification_json) as unknown,
3452
+ ),
3453
+ }
3454
+ : {}),
3455
+ ...(engineRun?.delivery_diagnostics_json
3456
+ ? {
3457
+ delivery: {
3458
+ code: "delivery-invalid",
3459
+ diagnostics: parseDeliveryDiagnostics(
3460
+ engineRun.delivery_diagnostics_json,
3461
+ ),
3462
+ },
3463
+ }
3464
+ : {}),
3465
+ ...(engineRun?.route_id
3466
+ ? {
3467
+ routeBinding: {
3468
+ routeId: engineRun.route_id,
3469
+ ...(engineRun.route_fingerprint
3470
+ ? { routeFingerprint: engineRun.route_fingerprint }
3471
+ : {}),
3472
+ },
3473
+ }
3474
+ : {}),
3475
+ ...(engineRun?.cleanup_state === "retained"
3476
+ ? {
3477
+ privateData: {
3478
+ retained: true,
3479
+ cleanup: "retained",
3480
+ ...(engineRun.baseline_workspace_revision
3481
+ ? {
3482
+ baselineRevisionId: engineRun.baseline_workspace_revision,
3483
+ }
3484
+ : {}),
3485
+ ...(engineRun.current_workspace_revision
3486
+ ? {
3487
+ currentRevisionId: engineRun.current_workspace_revision,
3488
+ }
3489
+ : {}),
3490
+ },
3491
+ }
3492
+ : engineRun?.cleanup_state === "complete"
3493
+ ? {
3494
+ privateData: {
3495
+ retained: false,
3496
+ cleanup: "complete",
3497
+ },
3498
+ }
3499
+ : {}),
3500
+ };
3501
+ }
3502
+
3503
+ async #withAvailableDelivery(
3504
+ outcome: Record<string, unknown>,
3505
+ signal?: AbortSignal,
3506
+ ): Promise<Record<string, unknown>> {
3507
+ if (
3508
+ outcome.stage !== "abel-implement" ||
3509
+ (isRecord(outcome.pause) &&
3510
+ outcome.pause.code === "operation-cancelled") ||
3511
+ (outcome.state !== "approval-needed" &&
3512
+ !(
3513
+ outcome.state === "paused" &&
3514
+ (isRecord(outcome.decisionBatch) ||
3515
+ (isRecord(outcome.recovery) && outcome.recovery.exhausted === true))
3516
+ )) ||
3517
+ typeof outcome.change !== "string" ||
3518
+ !this.#deliverySource.discoverLatest
3519
+ ) {
3520
+ return outcome;
3521
+ }
3522
+ let available: WorkflowAvailableDelivery | undefined;
3523
+ try {
3524
+ const discover = this.#deliverySource.discoverLatest;
3525
+ const change = outcome.change;
3526
+ available = await cancellableRead(
3527
+ () =>
3528
+ discover.call(this.#deliverySource, {
3529
+ stage: "abel-implement",
3530
+ change,
3531
+ ...(signal ? { signal } : {}),
3532
+ }),
3533
+ signal,
3534
+ );
3535
+ } catch {
3536
+ return outcome;
3537
+ }
3538
+ const currentRevision =
3539
+ typeof outcome.deliveryRevision === "number"
3540
+ ? outcome.deliveryRevision
3541
+ : 0;
3542
+ if (
3543
+ !available ||
3544
+ !Number.isSafeInteger(available.deliveryRevision) ||
3545
+ available.deliveryRevision <= currentRevision ||
3546
+ !SHA256.test(available.receiptHash)
3547
+ ) {
3548
+ return outcome;
3549
+ }
3550
+ if (
3551
+ this.#database
3552
+ .prepare(
3553
+ "SELECT 1 FROM workflow_rejected_deliveries WHERE run_id = ? AND delivery_revision = ? AND receipt_hash = ?",
3554
+ )
3555
+ .get(
3556
+ String(outcome.runId),
3557
+ available.deliveryRevision,
3558
+ available.receiptHash,
3559
+ )
3560
+ ) {
3561
+ return outcome;
3562
+ }
3563
+ const conditionalCommands = (
3564
+ Array.isArray(outcome.conditionalCommands)
3565
+ ? outcome.conditionalCommands
3566
+ : [{ command: "resume", stage: outcome.stage, change: outcome.change }]
3567
+ ).map((entry) =>
3568
+ isRecord(entry) && entry.command === "resume"
3569
+ ? {
3570
+ ...entry,
3571
+ satisfiedBy: {
3572
+ deliveryRevision: available.deliveryRevision,
3573
+ receiptHash: available.receiptHash,
3574
+ },
3575
+ }
3576
+ : entry,
3577
+ );
3578
+ return {
3579
+ ...outcome,
3580
+ legalCommands:
3581
+ outcome.state === "approval-needed"
3582
+ ? ["status", "resume", "discard"]
3583
+ : ["status", "resume", "rebind", "discard"],
3584
+ availableDelivery: structuredClone(available),
3585
+ continuation:
3586
+ (isRecord(outcome.resourceBudget) &&
3587
+ outcome.resourceBudget.remaining === 0) ||
3588
+ (isRecord(outcome.recovery) &&
3589
+ outcome.recovery.code === "change-recovery-budget-exhausted")
3590
+ ? undefined
3591
+ : {
3592
+ owner: "parent",
3593
+ automatic: true,
3594
+ command: "resume",
3595
+ stage: "abel-implement",
3596
+ change: outcome.change,
3597
+ },
3598
+ ...(conditionalCommands === undefined ? {} : { conditionalCommands }),
3599
+ };
3600
+ }
3601
+
3602
+ #launchAdvance(
3603
+ runId: string,
3604
+ operationId: string,
3605
+ externalSignal?: AbortSignal,
3606
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3607
+ ): Promise<Record<string, unknown>> {
3608
+ const lease = this.#operationLease(runId, operationId);
3609
+ const controller = new AbortController();
3610
+ const abortFromExternal = () => {
3611
+ controller.abort(
3612
+ externalSignal?.reason ?? new Error("operation-cancelled"),
3613
+ );
3614
+ };
3615
+ if (externalSignal?.aborted) abortFromExternal();
3616
+ else
3617
+ externalSignal?.addEventListener("abort", abortFromExternal, {
3618
+ once: true,
3619
+ });
3620
+ const active: ActiveOperation = {
3621
+ operationId,
3622
+ kind: "work",
3623
+ controller,
3624
+ settled: Promise.resolve({}),
3625
+ };
3626
+ this.#active.set(runId, active);
3627
+ const settled = (async () => {
3628
+ try {
3629
+ const outcome = await this.#advance(
3630
+ runId,
3631
+ operationId,
3632
+ controller.signal,
3633
+ lease,
3634
+ onActivity,
3635
+ );
3636
+ return this.#commitOperation(runId, operationId, outcome);
3637
+ } catch (error) {
3638
+ this.#interruptOperation(runId, operationId);
3639
+ const controlSettled =
3640
+ controller.signal.aborted &&
3641
+ controller.signal.reason instanceof Error &&
3642
+ ["operation-cancelled", "run-discarded"].includes(
3643
+ controller.signal.reason.message,
3644
+ ) &&
3645
+ (isCancellationException(error, controller.signal) ||
3646
+ (error instanceof Error &&
3647
+ ["lease-fenced", "operation-journal-fenced"].includes(
3648
+ error.message,
3649
+ )));
3650
+ if (controlSettled) return this.#statusByRun(runId);
3651
+ throw error;
3652
+ } finally {
3653
+ externalSignal?.removeEventListener("abort", abortFromExternal);
3654
+ if (this.#active.get(runId) === active) this.#active.delete(runId);
3655
+ }
3656
+ })();
3657
+ active.settled = settled;
3658
+ return settled;
3659
+ }
3660
+
3661
+ async #start(
3662
+ command: Extract<ControlCommand, { command: "start" }>,
3663
+ signal?: AbortSignal,
3664
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3665
+ ) {
3666
+ const existingRun = this.#lookupRun(command.stage, command.change);
3667
+ if (existingRun) {
3668
+ this.#ensureEngineRun(existingRun);
3669
+ const replay = this.#operationReplay(
3670
+ existingRun,
3671
+ command.operationId,
3672
+ command.command,
3673
+ );
3674
+ if (replay) return replay;
3675
+ const restarted = this.#runStore.startRun({
3676
+ stage: command.stage,
3677
+ change: command.change,
3678
+ operationId: command.operationId,
3679
+ });
3680
+ if (restarted.state !== "created") {
3681
+ const begun = this.#beginOperation(
3682
+ existingRun,
3683
+ command.operationId,
3684
+ command.command,
3685
+ );
3686
+ if (begun) return begun;
3687
+ return this.#commitOperation(
3688
+ existingRun,
3689
+ command.operationId,
3690
+ this.#statusByRun(existingRun),
3691
+ );
3692
+ }
3693
+ }
3694
+
3695
+ const runId =
3696
+ existingRun ??
3697
+ this.#runStore.startRun({
3698
+ stage: command.stage,
3699
+ change: command.change,
3700
+ operationId: command.operationId,
3701
+ }).runId;
3702
+ this.#ensureEngineRun(runId);
3703
+ const begun = this.#beginOperation(
3704
+ runId,
3705
+ command.operationId,
3706
+ command.command,
3707
+ );
3708
+ if (begun) return begun;
3709
+ const lease = this.#operationLease(runId, command.operationId);
3710
+ this.#transition(
3711
+ runId,
3712
+ "validating-delivery",
3713
+ `${command.operationId}:validating-delivery`,
3714
+ undefined,
3715
+ lease,
3716
+ );
3717
+ emitWorkflowActivity(onActivity, {
3718
+ state: "validating",
3719
+ stage: command.stage,
3720
+ runId,
3721
+ change: command.change,
3722
+ objective: `validate delivery ${command.change}`,
3723
+ legalCommands: ["status", "cancel", "discard"],
3724
+ });
3725
+ try {
3726
+ const delivery = await this.#loadDelivery(
3727
+ runId,
3728
+ lease,
3729
+ command.stage,
3730
+ command.change,
3731
+ undefined,
3732
+ undefined,
3733
+ signal,
3734
+ );
3735
+ this.#admitDelivery(runId, delivery, lease);
3736
+ this.#transition(
3737
+ runId,
3738
+ "ready",
3739
+ `${command.operationId}:delivery-ready`,
3740
+ undefined,
3741
+ lease,
3742
+ );
3743
+ return this.#launchAdvance(
3744
+ runId,
3745
+ command.operationId,
3746
+ signal,
3747
+ onActivity,
3748
+ );
3749
+ } catch (error) {
3750
+ const cancelled = this.#cancelledDelivery(
3751
+ runId,
3752
+ command.operationId,
3753
+ error,
3754
+ signal,
3755
+ lease,
3756
+ );
3757
+ if (cancelled) return cancelled;
3758
+ const failure = this.#deliveryFailure(
3759
+ runId,
3760
+ command.operationId,
3761
+ error,
3762
+ lease,
3763
+ );
3764
+ if (failure) {
3765
+ return this.#commitOperation(runId, command.operationId, failure);
3766
+ }
3767
+ this.#interruptOperation(runId, command.operationId);
3768
+ throw error;
3769
+ }
3770
+ }
3771
+
3772
+ async #resume(
3773
+ command: Extract<ControlCommand, { command: "resume" }>,
3774
+ signal?: AbortSignal,
3775
+ onActivity?: (event: WorkflowActivityUpdate) => void,
3776
+ ) {
3777
+ const runId = this.#lookupRun(command.stage, command.change);
3778
+ if (!runId) throw new Error("run-not-found");
3779
+ this.#recoverLegacyContextApproval(runId);
3780
+ const replay = this.#beginOperation(
3781
+ runId,
3782
+ command.operationId,
3783
+ command.command,
3784
+ );
3785
+ if (replay) return replay;
3786
+ const lease = this.#operationLease(runId, command.operationId);
3787
+ const current = this.#runStore.status(runId);
3788
+ if (current.state === "recovering") {
3789
+ emitWorkflowActivity(onActivity, {
3790
+ state: "recovering",
3791
+ stage: current.stage,
3792
+ runId,
3793
+ ...(current.change ? { change: current.change } : {}),
3794
+ objective: current.change
3795
+ ? `recover change ${current.change}`
3796
+ : "recover workflow application",
3797
+ legalCommands: ["status", "cancel", "discard"],
3798
+ });
3799
+ const transactionId = this.#engineRun(runId).transaction_id;
3800
+ if (!transactionId || !this.#application) {
3801
+ this.#interruptOperation(runId, command.operationId);
3802
+ throw new Error("apply-recovery-unavailable");
3803
+ }
3804
+ const recovered = await this.#application.recover(
3805
+ transactionId,
3806
+ this.#applicationContext(runId),
3807
+ );
3808
+ this.#runStore.assertLease(lease);
3809
+ const outcome = this.#settleApplication(
3810
+ runId,
3811
+ command.operationId,
3812
+ recovered,
3813
+ lease,
3814
+ onActivity,
3815
+ );
3816
+ return this.#commitOperation(runId, command.operationId, outcome);
3817
+ }
3818
+ if (
3819
+ !["paused", "retryable", "approval-needed", "ready"].includes(
3820
+ current.state,
3821
+ )
3822
+ ) {
3823
+ this.#interruptOperation(runId, command.operationId);
3824
+ throw new Error("resume-not-allowed");
3825
+ }
3826
+ try {
3827
+ const engineRevision = this.#engineRun(runId).current_revision;
3828
+ const projectedRevision = current.deliveryRevision;
3829
+ if (
3830
+ command.deliveryRevision === undefined &&
3831
+ (current.state === "approval-needed" ||
3832
+ this.#statusByRun(runId).decisionBatch !== undefined ||
3833
+ this.#statusByRun(runId).recovery !== undefined) &&
3834
+ this.#deliverySource.discoverLatest
3835
+ ) {
3836
+ const available = await this.#deliverySource.discoverLatest({
3837
+ stage: command.stage,
3838
+ change: command.change,
3839
+ signal,
3840
+ });
3841
+ this.#runStore.assertLease(lease);
3842
+ if (available && available.deliveryRevision > (engineRevision ?? 0)) {
3843
+ command = {
3844
+ ...command,
3845
+ deliveryRevision: available.deliveryRevision,
3846
+ receiptHash: available.receiptHash,
3847
+ };
3848
+ }
3849
+ }
3850
+ const deliveryOutOfSync =
3851
+ projectedRevision !== undefined && projectedRevision !== engineRevision;
3852
+ if (
3853
+ current.state === "approval-needed" &&
3854
+ (command.deliveryRevision === undefined ||
3855
+ command.receiptHash === undefined ||
3856
+ command.deliveryRevision <=
3857
+ (engineRevision ?? projectedRevision ?? 0))
3858
+ ) {
3859
+ this.#interruptOperation(runId, command.operationId);
3860
+ throw new Error("approval-receipt-required");
3861
+ }
3862
+ if (
3863
+ command.deliveryRevision !== undefined ||
3864
+ engineRevision === null ||
3865
+ deliveryOutOfSync
3866
+ ) {
3867
+ this.#transition(
3868
+ runId,
3869
+ "validating-delivery",
3870
+ `${command.operationId}:validating-delivery`,
3871
+ undefined,
3872
+ lease,
3873
+ );
3874
+ emitWorkflowActivity(onActivity, {
3875
+ state: "validating",
3876
+ stage: command.stage,
3877
+ runId,
3878
+ change: command.change,
3879
+ objective: `validate delivery ${command.change}`,
3880
+ legalCommands: ["status", "cancel", "discard"],
3881
+ });
3882
+ const requestedRevision =
3883
+ command.deliveryRevision ??
3884
+ (deliveryOutOfSync ? projectedRevision : undefined);
3885
+ const projectedBinding = current.deliveryBindings.find(
3886
+ (binding) =>
3887
+ binding.gate === "gate-b" && binding.revision === requestedRevision,
3888
+ );
3889
+ const delivery = await this.#loadDelivery(
3890
+ runId,
3891
+ lease,
3892
+ command.stage,
3893
+ command.change,
3894
+ requestedRevision,
3895
+ command.receiptHash ?? projectedBinding?.receiptHash,
3896
+ signal,
3897
+ );
3898
+ this.#admitDelivery(runId, delivery, lease);
3899
+ this.#transition(
3900
+ runId,
3901
+ "ready",
3902
+ `${command.operationId}:delivery-ready`,
3903
+ undefined,
3904
+ lease,
3905
+ );
3906
+ } else if (current.state !== "ready") {
3907
+ this.#transition(
3908
+ runId,
3909
+ "ready",
3910
+ `${command.operationId}:resume-ready`,
3911
+ undefined,
3912
+ lease,
3913
+ );
3914
+ }
3915
+ if (command.recovery)
3916
+ this.#grantRecovery(
3917
+ runId,
3918
+ command.operationId,
3919
+ command.recovery,
3920
+ lease,
3921
+ );
3922
+ return this.#launchAdvance(
3923
+ runId,
3924
+ command.operationId,
3925
+ signal,
3926
+ onActivity,
3927
+ );
3928
+ } catch (error) {
3929
+ const cancelled = this.#cancelledDelivery(
3930
+ runId,
3931
+ command.operationId,
3932
+ error,
3933
+ signal,
3934
+ lease,
3935
+ );
3936
+ if (cancelled) return cancelled;
3937
+ const failure = this.#deliveryFailure(
3938
+ runId,
3939
+ command.operationId,
3940
+ error,
3941
+ lease,
3942
+ );
3943
+ if (failure) {
3944
+ return this.#commitOperation(runId, command.operationId, failure);
3945
+ }
3946
+ this.#interruptOperation(runId, command.operationId);
3947
+ throw error;
3948
+ }
3949
+ }
3950
+
3951
+ #rebind(command: Extract<ControlCommand, { command: "rebind" }>) {
3952
+ const runId = this.#lookupRun(command.stage, command.change);
3953
+ if (!runId) throw new Error("run-not-found");
3954
+ const replay = this.#beginOperation(
3955
+ runId,
3956
+ command.operationId,
3957
+ command.command,
3958
+ );
3959
+ if (replay) return replay;
3960
+ const lease = this.#operationLease(runId, command.operationId);
3961
+ const state = this.#runStore.status(runId).state;
3962
+ if (!["paused", "retryable"].includes(state)) {
3963
+ this.#interruptOperation(runId, command.operationId);
3964
+ throw new Error("rebind-not-allowed");
3965
+ }
3966
+ const task = this.#tasks(runId).find((row) =>
3967
+ ["paused", "retryable"].includes(row.state),
3968
+ );
3969
+ if (!task) {
3970
+ this.#interruptOperation(runId, command.operationId);
3971
+ throw new Error("rebind-task-unavailable");
3972
+ }
3973
+ const rebound = this.#worker.rebind({
3974
+ runId,
3975
+ taskId: task.task_id,
3976
+ task: structuredClone(this.#taskPlan(task)),
3977
+ role: "implementation-worker",
3978
+ routeId: command.routeId,
3979
+ });
3980
+ if (!rebound.ok) {
3981
+ this.#interruptOperation(runId, command.operationId);
3982
+ throw new Error(rebound.code);
3983
+ }
3984
+ const routeId = rebound.routeId ?? rebound.route?.id ?? command.routeId;
3985
+ const routeFingerprint =
3986
+ rebound.routeFingerprint ?? rebound.route?.fingerprint;
3987
+ if (
3988
+ !IDENTIFIER.test(routeId) ||
3989
+ (routeFingerprint !== undefined && !SHA256.test(routeFingerprint))
3990
+ ) {
3991
+ this.#interruptOperation(runId, command.operationId);
3992
+ throw new Error("route-rebind-result-invalid");
3993
+ }
3994
+ const priorRouteId = task.route_id ?? this.#engineRun(runId).route_id;
3995
+ const priorRouteFingerprint =
3996
+ task.route_fingerprint ?? this.#engineRun(runId).route_fingerprint;
3997
+ this.#leasedTransaction(lease, () => {
3998
+ this.#setTask(runId, task.task_id, {
3999
+ routeId,
4000
+ routeFingerprint: routeFingerprint ?? null,
4001
+ });
4002
+ this.#database
4003
+ .prepare(
4004
+ `UPDATE workflow_engine_runs
4005
+ SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
4006
+ )
4007
+ .run(routeId, routeFingerprint ?? null, runId);
4008
+ });
4009
+ const outcome = {
4010
+ ...this.#statusByRun(runId),
4011
+ routeBinding: {
4012
+ routeId,
4013
+ ...(routeFingerprint ? { routeFingerprint } : {}),
4014
+ },
4015
+ operation: {
4016
+ kind: "route-rebound",
4017
+ ...(priorRouteId ? { priorRouteId } : {}),
4018
+ ...(priorRouteFingerprint ? { priorRouteFingerprint } : {}),
4019
+ routeId,
4020
+ ...(routeFingerprint ? { routeFingerprint } : {}),
4021
+ },
4022
+ };
4023
+ return this.#commitOperation(runId, command.operationId, outcome);
4024
+ }
4025
+
4026
+ async #controlApply(
4027
+ runId: string,
4028
+ operationId: string,
4029
+ intent: "cancel" | "discard",
4030
+ lease: OperationLease,
4031
+ onActivity?: (event: WorkflowActivityUpdate) => void,
4032
+ ): Promise<Record<string, unknown>> {
4033
+ const transactionId = this.#engineRun(runId).transaction_id;
4034
+ if (!transactionId || !this.#application) {
4035
+ throw new Error("apply-recovery-unavailable");
4036
+ }
4037
+ const context = this.#applicationContext(runId);
4038
+ this.#application.requestControl(transactionId, intent, context);
4039
+ const active = this.#active.get(runId);
4040
+ if (active && active.operationId !== operationId) {
4041
+ active.controller.abort(
4042
+ new Error(
4043
+ intent === "cancel" ? "operation-cancelled" : "run-discarded",
4044
+ ),
4045
+ );
4046
+ }
4047
+ if (this.#runStore.status(runId).state === "applying") {
4048
+ this.#transition(
4049
+ runId,
4050
+ "recovering",
4051
+ `${operationId}:${intent}-recovering`,
4052
+ intent === "cancel" ? "operation-cancelled" : "run-discarded",
4053
+ lease,
4054
+ );
4055
+ const recovering = this.#runStore.status(runId);
4056
+ emitWorkflowActivity(onActivity, {
4057
+ state: "recovering",
4058
+ stage: recovering.stage,
4059
+ runId,
4060
+ ...(recovering.change ? { change: recovering.change } : {}),
4061
+ objective: recovering.change
4062
+ ? `recover change ${recovering.change}`
4063
+ : "recover workflow application",
4064
+ code: intent === "cancel" ? "operation-cancelled" : "run-discarded",
4065
+ legalCommands: ["status"],
4066
+ });
4067
+ }
4068
+ const recovered = await this.#application.recover(transactionId, context);
4069
+ this.#runStore.assertLease(lease);
4070
+ return this.#settleApplication(
4071
+ runId,
4072
+ operationId,
4073
+ recovered,
4074
+ lease,
4075
+ onActivity,
4076
+ );
4077
+ }
4078
+
4079
+ async #abortActiveOperation(runId: string, reason: Error): Promise<void> {
4080
+ const pending: Promise<unknown>[] = [];
4081
+ const amendment = this.#amendments.get(runId);
4082
+ if (amendment) pending.push(amendment);
4083
+ for (const [controller, entry] of this.#commands) {
4084
+ if (
4085
+ !["start", "resume"].includes(entry.command.command) ||
4086
+ this.#lookupRun(entry.command.stage, entry.command.change) !== runId
4087
+ )
4088
+ continue;
4089
+ controller.abort(reason);
4090
+ pending.push(entry.settled);
4091
+ }
4092
+ const active = this.#active.get(runId);
4093
+ if (active) {
4094
+ active.controller.abort(reason);
4095
+ pending.push(active.settled);
4096
+ }
4097
+ const results = await Promise.allSettled(pending);
4098
+ for (const result of results) {
4099
+ if (result.status !== "rejected") continue;
4100
+ const error = result.reason;
4101
+ if (
4102
+ error !== reason &&
4103
+ !(
4104
+ error instanceof Error &&
4105
+ ["lease-fenced", "operation-journal-fenced"].includes(error.message)
4106
+ )
4107
+ )
4108
+ throw error;
4109
+ }
4110
+ }
4111
+
4112
+ async #cancel(
4113
+ command: Extract<ControlCommand, { command: "cancel" }>,
4114
+ onActivity?: (event: WorkflowActivityUpdate) => void,
4115
+ ) {
4116
+ const runId = this.#lookupRun(command.stage, command.change);
4117
+ if (!runId) throw new Error("run-not-found");
4118
+ const replay = this.#beginOperation(
4119
+ runId,
4120
+ command.operationId,
4121
+ command.command,
4122
+ );
4123
+ if (replay) return replay;
4124
+ const lease = this.#operationLease(runId, command.operationId);
4125
+ const state = this.#runStore.status(runId).state;
4126
+ let outcome: Record<string, unknown>;
4127
+ if (state === "applying" || state === "recovering") {
4128
+ outcome = await this.#controlApply(
4129
+ runId,
4130
+ command.operationId,
4131
+ "cancel",
4132
+ lease,
4133
+ onActivity,
4134
+ );
4135
+ } else {
4136
+ const activeSettlement = this.#abortActiveOperation(
4137
+ runId,
4138
+ new Error("operation-cancelled"),
4139
+ );
4140
+ const current = this.#runStore.status(runId);
4141
+ if (
4142
+ !["paused", "discarded", "completed", "rejected"].includes(
4143
+ current.state,
4144
+ )
4145
+ ) {
4146
+ this.#transition(
4147
+ runId,
4148
+ "paused",
4149
+ `${command.operationId}:cancelled`,
4150
+ "operation-cancelled",
4151
+ lease,
4152
+ );
4153
+ }
4154
+ outcome = this.#statusByRun(runId);
4155
+ await activeSettlement;
4156
+ }
4157
+ outcome = {
4158
+ ...outcome,
4159
+ operation: { kind: "operation-cancelled" },
4160
+ };
4161
+ return this.#commitOperation(runId, command.operationId, outcome);
4162
+ }
4163
+
4164
+ async #discard(
4165
+ command: Extract<ControlCommand, { command: "discard" }>,
4166
+ onActivity?: (event: WorkflowActivityUpdate) => void,
4167
+ ) {
4168
+ const runId = this.#lookupRun(command.stage, command.change);
4169
+ if (!runId) throw new Error("run-not-found");
4170
+ const replay = this.#beginOperation(
4171
+ runId,
4172
+ command.operationId,
4173
+ command.command,
4174
+ );
4175
+ if (replay) return replay;
4176
+ const lease = this.#operationLease(runId, command.operationId);
4177
+ const state = this.#runStore.status(runId).state;
4178
+ let outcome: Record<string, unknown>;
4179
+ if (state === "applying" || state === "recovering") {
4180
+ outcome = await this.#controlApply(
4181
+ runId,
4182
+ command.operationId,
4183
+ "discard",
4184
+ lease,
4185
+ onActivity,
4186
+ );
4187
+ } else {
4188
+ const activeSettlement = this.#abortActiveOperation(
4189
+ runId,
4190
+ new Error("run-discarded"),
4191
+ );
4192
+ const current = this.#runStore.status(runId);
4193
+ if (current.state !== "discarded") {
4194
+ if (["completed", "rejected"].includes(current.state)) {
4195
+ this.#interruptOperation(runId, command.operationId);
4196
+ throw new Error("discard-not-allowed");
4197
+ }
4198
+ this.#transition(
4199
+ runId,
4200
+ "discarded",
4201
+ `${command.operationId}:discarded`,
4202
+ undefined,
4203
+ lease,
4204
+ );
4205
+ await activeSettlement;
4206
+ this.#cleanupTerminal(runId, command.operationId, lease);
4207
+ } else {
4208
+ await activeSettlement;
4209
+ }
4210
+ outcome = this.#statusByRun(runId);
4211
+ }
4212
+ return this.#commitOperation(runId, command.operationId, outcome);
4213
+ }
4214
+
4215
+ execute(
4216
+ value: unknown,
4217
+ signal?: AbortSignal,
4218
+ onActivity?: (event: WorkflowActivityUpdate) => void,
4219
+ ): Promise<Record<string, unknown>> {
4220
+ let command: ControlCommand;
4221
+ try {
4222
+ this.#assertOpen();
4223
+ if (this.#closing) throw new Error("workflow-engine-closed");
4224
+ command = assertControlCommand(value);
4225
+ } catch (error) {
4226
+ return Promise.reject(error);
4227
+ }
4228
+ const controller = new AbortController();
4229
+ const onAbort = () => controller.abort(signal?.reason);
4230
+ if (signal?.aborted) onAbort();
4231
+ else signal?.addEventListener("abort", onAbort, { once: true });
4232
+ // Register before execution can yield or invoke external callbacks.
4233
+ const settled = Promise.resolve()
4234
+ .then(() => this.#executeCommand(command, controller.signal, onActivity))
4235
+ .finally(() => {
4236
+ signal?.removeEventListener("abort", onAbort);
4237
+ this.#commands.delete(controller);
4238
+ });
4239
+ this.#commands.set(controller, { command, settled });
4240
+ return settled;
4241
+ }
4242
+
4243
+ async #executeCommand(
4244
+ command: ControlCommand,
4245
+ signal?: AbortSignal,
4246
+ onActivity?: (event: WorkflowActivityUpdate) => void,
4247
+ ): Promise<Record<string, unknown>> {
4248
+ switch (command.command) {
4249
+ case "start":
4250
+ return this.#withAvailableDelivery(
4251
+ await this.#start(command, signal, onActivity),
4252
+ signal,
4253
+ );
4254
+ case "status": {
4255
+ const runId = this.#lookupRun(command.stage, command.change);
4256
+ if (!runId) {
4257
+ return {
4258
+ stage: command.stage,
4259
+ change: command.change,
4260
+ state: "not-started",
4261
+ durable: true,
4262
+ completed: false,
4263
+ legalCommands: ["start"],
4264
+ tasks: [],
4265
+ queue: [],
4266
+ };
4267
+ }
4268
+ this.#recoverLegacyContextApproval(runId);
4269
+ return this.#withAvailableDelivery(this.#statusByRun(runId), signal);
4270
+ }
4271
+ case "resume":
4272
+ return this.#withAvailableDelivery(
4273
+ await this.#resume(command, signal, onActivity),
4274
+ signal,
4275
+ );
4276
+ case "rebind":
4277
+ return this.#rebind(command);
4278
+ case "cancel":
4279
+ return this.#cancel(command, onActivity);
4280
+ case "discard":
4281
+ return this.#discard(command, onActivity);
4282
+ }
4283
+ }
4284
+
4285
+ prepareBootstrapHandoff(
4286
+ input: Record<string, unknown>,
4287
+ ): Record<string, unknown> {
4288
+ this.#assertOpen();
4289
+ const acceptanceFacts = normalizeBootstrapAcceptanceFacts(
4290
+ input.acceptanceFacts,
4291
+ );
4292
+ const workspaceManifestHash =
4293
+ typeof input.workspaceManifestHash === "string" &&
4294
+ SHA256.test(input.workspaceManifestHash)
4295
+ ? input.workspaceManifestHash
4296
+ : undefined;
4297
+ if (
4298
+ typeof input.operationId !== "string" ||
4299
+ !IDENTIFIER.test(input.operationId) ||
4300
+ typeof input.runId !== "string" ||
4301
+ typeof input.change !== "string" ||
4302
+ !CHANGE_NAME.test(input.change) ||
4303
+ typeof input.receiptHash !== "string" ||
4304
+ !SHA256.test(input.receiptHash) ||
4305
+ typeof input.graphHash !== "string" ||
4306
+ !SHA256.test(input.graphHash) ||
4307
+ typeof input.acceptanceHash !== "string" ||
4308
+ !SHA256.test(input.acceptanceHash) ||
4309
+ (acceptanceFacts !== undefined) !==
4310
+ (workspaceManifestHash !== undefined) ||
4311
+ (acceptanceFacts !== undefined &&
4312
+ workspaceManifestHash !== undefined &&
4313
+ bootstrapAcceptanceHash(acceptanceFacts, workspaceManifestHash) !==
4314
+ input.acceptanceHash)
4315
+ ) {
4316
+ throw new Error("bootstrap-handoff-invalid");
4317
+ }
4318
+ this.#runStore.status(input.runId);
4319
+ const existing = this.#database
4320
+ .prepare(
4321
+ `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
4322
+ FROM bootstrap_handoffs WHERE receipt_hash = ?`,
4323
+ )
4324
+ .get(input.receiptHash) as BootstrapRow | undefined;
4325
+ if (existing) {
4326
+ const outcome = parseJsonRecord(existing.facts_json);
4327
+ const existingAcceptanceFacts = normalizeBootstrapAcceptanceFacts(
4328
+ outcome.acceptanceFacts,
4329
+ );
4330
+ const storedBindingValid =
4331
+ outcome.handoffId === existing.handoff_id &&
4332
+ outcome.runId === existing.owner_run_id &&
4333
+ outcome.receiptHash === existing.receipt_hash &&
4334
+ outcome.selectorBeforeCutover === "bootstrap" &&
4335
+ existing.state === "prepared" &&
4336
+ outcome.state === "prepared" &&
4337
+ outcome.selectorCasPending === true;
4338
+ if (!storedBindingValid) throw new Error("bootstrap-handoff-invalid");
4339
+ const stableBindingMatches =
4340
+ existing.owner_run_id === input.runId &&
4341
+ outcome.change === input.change &&
4342
+ outcome.graphHash === input.graphHash;
4343
+ if (!stableBindingMatches) {
4344
+ throw new Error("bootstrap-handoff-conflict");
4345
+ }
4346
+ const acceptanceMatches =
4347
+ outcome.acceptanceHash === input.acceptanceHash &&
4348
+ canonicalJson(existingAcceptanceFacts ?? null) ===
4349
+ canonicalJson(acceptanceFacts ?? null) &&
4350
+ outcome.workspaceManifestHash === workspaceManifestHash;
4351
+ if (acceptanceMatches) return structuredClone(outcome);
4352
+ if (
4353
+ existing.state !== "prepared" ||
4354
+ outcome.state !== "prepared" ||
4355
+ outcome.selectorBeforeCutover !== "bootstrap" ||
4356
+ outcome.selectorCasPending !== true ||
4357
+ acceptanceFacts === undefined ||
4358
+ workspaceManifestHash === undefined
4359
+ ) {
4360
+ throw new Error("bootstrap-handoff-conflict");
4361
+ }
4362
+ const refreshedHandoffId = `handoff-${hash(
4363
+ input.runId,
4364
+ input.receiptHash,
4365
+ input.graphHash,
4366
+ input.acceptanceHash,
4367
+ workspaceManifestHash,
4368
+ ).slice(0, 40)}`;
4369
+ const refreshed = {
4370
+ ...outcome,
4371
+ handoffId: refreshedHandoffId,
4372
+ acceptanceHash: input.acceptanceHash,
4373
+ acceptanceFacts,
4374
+ workspaceManifestHash,
4375
+ };
4376
+ this.#transaction(() => {
4377
+ const result = this.#database
4378
+ .prepare(
4379
+ `UPDATE bootstrap_handoffs
4380
+ SET handoff_id = ?, facts_json = ?
4381
+ WHERE handoff_id = ? AND state = 'prepared' AND facts_json = ?`,
4382
+ )
4383
+ .run(
4384
+ refreshedHandoffId,
4385
+ JSON.stringify(refreshed),
4386
+ existing.handoff_id,
4387
+ existing.facts_json,
4388
+ );
4389
+ if (Number(result.changes) !== 1) {
4390
+ throw new Error("bootstrap-handoff-conflict");
4391
+ }
4392
+ });
4393
+ return structuredClone(refreshed);
4394
+ }
4395
+ const handoffId = `handoff-${hash(
4396
+ input.runId,
4397
+ input.receiptHash,
4398
+ input.graphHash,
4399
+ input.acceptanceHash,
4400
+ workspaceManifestHash ?? "acceptance-pending",
4401
+ ).slice(0, 40)}`;
4402
+ const outcome = {
4403
+ handoffId,
4404
+ state: "prepared",
4405
+ runId: input.runId,
4406
+ change: input.change,
4407
+ receiptHash: input.receiptHash,
4408
+ graphHash: input.graphHash,
4409
+ acceptanceHash: input.acceptanceHash,
4410
+ ...(acceptanceFacts ? { acceptanceFacts } : {}),
4411
+ ...(workspaceManifestHash ? { workspaceManifestHash } : {}),
4412
+ selectorBeforeCutover: "bootstrap",
4413
+ selectorCasPending: true,
4414
+ };
4415
+ const preparedRows = this.#database
4416
+ .prepare(
4417
+ `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
4418
+ FROM bootstrap_handoffs WHERE state = 'prepared'
4419
+ ORDER BY handoff_id`,
4420
+ )
4421
+ .all() as unknown as BootstrapRow[];
4422
+ if (preparedRows.length > 0) throw new Error("bootstrap-handoff-conflict");
4423
+ this.#transaction(() => {
4424
+ this.#database
4425
+ .prepare(
4426
+ `INSERT INTO bootstrap_handoffs(
4427
+ handoff_id, owner_run_id, receipt_hash, state, facts_json
4428
+ ) VALUES (?, ?, ?, 'prepared', ?)`,
4429
+ )
4430
+ .run(
4431
+ handoffId,
4432
+ String(input.runId),
4433
+ String(input.receiptHash),
4434
+ JSON.stringify(outcome),
4435
+ );
4436
+ });
4437
+ return structuredClone(outcome);
4438
+ }
4439
+
4440
+ inspectBootstrapHandoff(receiptHash: string): Record<string, unknown> {
4441
+ this.#assertOpen();
4442
+ if (!SHA256.test(receiptHash)) throw new Error("bootstrap-handoff-invalid");
4443
+ const row = this.#database
4444
+ .prepare(
4445
+ `SELECT facts_json FROM bootstrap_handoffs WHERE receipt_hash = ?`,
4446
+ )
4447
+ .get(receiptHash) as { facts_json: string } | undefined;
4448
+ if (!row) throw new Error("bootstrap-handoff-not-found");
4449
+ return parseJsonRecord(row.facts_json);
4450
+ }
4451
+
4452
+ close(): Promise<void> {
4453
+ if (this.#closing) return this.#closing;
4454
+ if (this.#closed) return Promise.resolve();
4455
+ this.#closing = Promise.resolve().then(async () => {
4456
+ if (this.#orphanRecoveryTimer) {
4457
+ clearTimeout(this.#orphanRecoveryTimer);
4458
+ this.#orphanRecoveryTimer = undefined;
4459
+ }
4460
+ const reason = new Error("workflow-engine-closed");
4461
+ const commands = [...this.#commands];
4462
+ for (const [controller] of commands) controller.abort(reason);
4463
+ const active = [...this.#active.values()];
4464
+ for (const operation of active) operation.controller.abort(reason);
4465
+ await Promise.allSettled(commands.map(([, entry]) => entry.settled));
4466
+ await Promise.allSettled(active.map((operation) => operation.settled));
4467
+ await Promise.allSettled([...this.#amendments.values()]);
4468
+ for (const held of this.#operationLeases.values())
4469
+ clearInterval(held.timer);
4470
+ this.#operationLeases.clear();
4471
+ this.#lifecycle?.close?.();
4472
+ this.#database.close();
4473
+ this.#runStore.close();
4474
+ this.#closed = true;
4475
+ });
4476
+ return this.#closing;
4477
+ }
4478
+ }