@abelxiaoxing/cadence 1.8.4 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/README.md +324 -117
  2. package/THIRD_PARTY_NOTICES.md +15 -0
  3. package/agents/design-explorer.md +14 -18
  4. package/agents/diagnosis-worker.md +14 -14
  5. package/agents/implementation-worker.md +16 -17
  6. package/config/workflow-task.example.json +7 -0
  7. package/docs/remote-access.md +60 -0
  8. package/docs/reuse-and-distribution.md +44 -0
  9. package/docs/service-management.md +108 -0
  10. package/licenses/bundled-web.txt +6245 -0
  11. package/licenses/pi-web-ui-MIT.txt +21 -0
  12. package/licenses/pi-web-ui-reuse.md +22 -0
  13. package/package.json +92 -20
  14. package/src/index.ts +21 -1802
  15. package/src/native-build.ts +6 -0
  16. package/src/native-client.ts +103 -0
  17. package/src/native-export.ts +58 -0
  18. package/src/native-history.ts +359 -0
  19. package/src/operator-cli.mjs +453 -1937
  20. package/src/plan-proposal.ts +87 -0
  21. package/src/sdk-worker.mjs +549 -0
  22. package/src/service/contracts/active.ts +82 -0
  23. package/src/service/contracts/budget.ts +148 -0
  24. package/src/service/contracts/build.ts +67 -0
  25. package/src/service/contracts/capture.ts +48 -0
  26. package/src/service/contracts/core.ts +33 -0
  27. package/src/service/contracts/current.ts +261 -0
  28. package/src/service/contracts/domain.ts +68 -0
  29. package/src/service/contracts/history.ts +2245 -0
  30. package/src/service/contracts/models.ts +139 -0
  31. package/src/service/coordination.ts +862 -0
  32. package/src/service/private-files.ts +204 -0
  33. package/src/service-cli.mjs +5592 -0
  34. package/src/service-client.ts +747 -0
  35. package/src/service-contract.ts +245 -0
  36. package/src/service-host.mjs +22382 -0
  37. package/src/service-http-client.ts +425 -0
  38. package/src/service-wire.ts +372 -0
  39. package/src/task-shell.ts +1045 -0
  40. package/src/task-store.ts +479 -0
  41. package/src/task-types.ts +184 -0
  42. package/src/verification-diagnostics.ts +29 -72
  43. package/src/worker-capture.mjs +1197 -0
  44. package/src/worker-public.mjs +1753 -0
  45. package/src/worker-redaction.mjs +608 -0
  46. package/src/worker-tools.mjs +202 -0
  47. package/web/dist/assets/KaTeX_AMS-Regular.ttf +0 -0
  48. package/web/dist/assets/KaTeX_AMS-Regular.woff +0 -0
  49. package/web/dist/assets/KaTeX_AMS-Regular.woff2 +0 -0
  50. package/web/dist/assets/KaTeX_Caligraphic-Bold.ttf +0 -0
  51. package/web/dist/assets/KaTeX_Caligraphic-Bold.woff +0 -0
  52. package/web/dist/assets/KaTeX_Caligraphic-Bold.woff2 +0 -0
  53. package/web/dist/assets/KaTeX_Caligraphic-Regular.ttf +0 -0
  54. package/web/dist/assets/KaTeX_Caligraphic-Regular.woff +0 -0
  55. package/web/dist/assets/KaTeX_Caligraphic-Regular.woff2 +0 -0
  56. package/web/dist/assets/KaTeX_Fraktur-Bold.ttf +0 -0
  57. package/web/dist/assets/KaTeX_Fraktur-Bold.woff +0 -0
  58. package/web/dist/assets/KaTeX_Fraktur-Bold.woff2 +0 -0
  59. package/web/dist/assets/KaTeX_Fraktur-Regular.ttf +0 -0
  60. package/web/dist/assets/KaTeX_Fraktur-Regular.woff +0 -0
  61. package/web/dist/assets/KaTeX_Fraktur-Regular.woff2 +0 -0
  62. package/web/dist/assets/KaTeX_Main-Bold.ttf +0 -0
  63. package/web/dist/assets/KaTeX_Main-Bold.woff +0 -0
  64. package/web/dist/assets/KaTeX_Main-Bold.woff2 +0 -0
  65. package/web/dist/assets/KaTeX_Main-BoldItalic.ttf +0 -0
  66. package/web/dist/assets/KaTeX_Main-BoldItalic.woff +0 -0
  67. package/web/dist/assets/KaTeX_Main-BoldItalic.woff2 +0 -0
  68. package/web/dist/assets/KaTeX_Main-Italic.ttf +0 -0
  69. package/web/dist/assets/KaTeX_Main-Italic.woff +0 -0
  70. package/web/dist/assets/KaTeX_Main-Italic.woff2 +0 -0
  71. package/web/dist/assets/KaTeX_Main-Regular.ttf +0 -0
  72. package/web/dist/assets/KaTeX_Main-Regular.woff +0 -0
  73. package/web/dist/assets/KaTeX_Main-Regular.woff2 +0 -0
  74. package/web/dist/assets/KaTeX_Math-BoldItalic.ttf +0 -0
  75. package/web/dist/assets/KaTeX_Math-BoldItalic.woff +0 -0
  76. package/web/dist/assets/KaTeX_Math-BoldItalic.woff2 +0 -0
  77. package/web/dist/assets/KaTeX_Math-Italic.ttf +0 -0
  78. package/web/dist/assets/KaTeX_Math-Italic.woff +0 -0
  79. package/web/dist/assets/KaTeX_Math-Italic.woff2 +0 -0
  80. package/web/dist/assets/KaTeX_SansSerif-Bold.ttf +0 -0
  81. package/web/dist/assets/KaTeX_SansSerif-Bold.woff +0 -0
  82. package/web/dist/assets/KaTeX_SansSerif-Bold.woff2 +0 -0
  83. package/web/dist/assets/KaTeX_SansSerif-Italic.ttf +0 -0
  84. package/web/dist/assets/KaTeX_SansSerif-Italic.woff +0 -0
  85. package/web/dist/assets/KaTeX_SansSerif-Italic.woff2 +0 -0
  86. package/web/dist/assets/KaTeX_SansSerif-Regular.ttf +0 -0
  87. package/web/dist/assets/KaTeX_SansSerif-Regular.woff +0 -0
  88. package/web/dist/assets/KaTeX_SansSerif-Regular.woff2 +0 -0
  89. package/web/dist/assets/KaTeX_Script-Regular.ttf +0 -0
  90. package/web/dist/assets/KaTeX_Script-Regular.woff +0 -0
  91. package/web/dist/assets/KaTeX_Script-Regular.woff2 +0 -0
  92. package/web/dist/assets/KaTeX_Size1-Regular.ttf +0 -0
  93. package/web/dist/assets/KaTeX_Size1-Regular.woff +0 -0
  94. package/web/dist/assets/KaTeX_Size1-Regular.woff2 +0 -0
  95. package/web/dist/assets/KaTeX_Size2-Regular.ttf +0 -0
  96. package/web/dist/assets/KaTeX_Size2-Regular.woff +0 -0
  97. package/web/dist/assets/KaTeX_Size2-Regular.woff2 +0 -0
  98. package/web/dist/assets/KaTeX_Size3-Regular.ttf +0 -0
  99. package/web/dist/assets/KaTeX_Size3-Regular.woff +0 -0
  100. package/web/dist/assets/KaTeX_Size3-Regular.woff2 +0 -0
  101. package/web/dist/assets/KaTeX_Size4-Regular.ttf +0 -0
  102. package/web/dist/assets/KaTeX_Size4-Regular.woff +0 -0
  103. package/web/dist/assets/KaTeX_Size4-Regular.woff2 +0 -0
  104. package/web/dist/assets/KaTeX_Typewriter-Regular.ttf +0 -0
  105. package/web/dist/assets/KaTeX_Typewriter-Regular.woff +0 -0
  106. package/web/dist/assets/KaTeX_Typewriter-Regular.woff2 +0 -0
  107. package/web/dist/assets/cadence.css +2 -0
  108. package/web/dist/assets/cadence.js +7 -0
  109. package/web/dist/assets/markdown.js +296 -0
  110. package/web/dist/assets/react.js +9 -0
  111. package/web/dist/assets/rolldown-runtime.js +1 -0
  112. package/web/dist/assets/timeline-client.js +41 -0
  113. package/web/dist/assets/vis-timeline.css +1274 -0
  114. package/web/dist/assets/vis-timeline.js +32 -0
  115. package/web/dist/index.html +18 -0
  116. package/web/dist/manifest.json +505 -0
  117. package/config/plan-draft.example.json +0 -125
  118. package/config/plan-draft.multiple-tasks.example.json +0 -375
  119. package/config/plan-draft.quick.example.json +0 -78
  120. package/prompts/abel-design.md +0 -382
  121. package/prompts/abel-diagnose.md +0 -71
  122. package/prompts/abel-implement.md +0 -284
  123. package/prompts/abel-init.md +0 -52
  124. package/src/activation.ts +0 -44
  125. package/src/activity-contracts.ts +0 -82
  126. package/src/agent-registry.ts +0 -53
  127. package/src/apply-transaction.ts +0 -1437
  128. package/src/artifact-store.ts +0 -213
  129. package/src/build-windows-job.mjs +0 -95
  130. package/src/candidate-artifact.ts +0 -143
  131. package/src/candidate-context.ts +0 -289
  132. package/src/candidate-patch.ts +0 -271
  133. package/src/candidate-workspace.ts +0 -80
  134. package/src/canonical.ts +0 -16
  135. package/src/change-contract-schema.ts +0 -191
  136. package/src/change-contract.ts +0 -324
  137. package/src/change-verification.ts +0 -1617
  138. package/src/contracts.ts +0 -2466
  139. package/src/control-contracts.ts +0 -382
  140. package/src/delivery-compiler.ts +0 -1805
  141. package/src/delivery-revision.ts +0 -42
  142. package/src/design-control.ts +0 -1299
  143. package/src/design-diagnostics.ts +0 -181
  144. package/src/design-journal.ts +0 -1773
  145. package/src/directory-sync.ts +0 -36
  146. package/src/durable-contracts.ts +0 -227
  147. package/src/durable-workflow.ts +0 -822
  148. package/src/evidence-draft.ts +0 -122
  149. package/src/execution-profile.ts +0 -64
  150. package/src/execution-retention.ts +0 -77
  151. package/src/file-snapshot.ts +0 -163
  152. package/src/implement-continuation.ts +0 -559
  153. package/src/implement-graph.ts +0 -690
  154. package/src/implement-plan.ts +0 -72
  155. package/src/install-config.d.mts +0 -6
  156. package/src/isolation-backend.ts +0 -353
  157. package/src/model-source.ts +0 -22
  158. package/src/openspec-cli.ts +0 -494
  159. package/src/operator-cli-entry.mjs +0 -24
  160. package/src/operator-tools.ts +0 -309
  161. package/src/package-candidate-process.ts +0 -206
  162. package/src/package-candidate.ts +0 -24
  163. package/src/package-delivery.ts +0 -486
  164. package/src/package-state.ts +0 -320
  165. package/src/package-verification.ts +0 -635
  166. package/src/package-workflow.ts +0 -305
  167. package/src/packet-runtime.ts +0 -459
  168. package/src/phase-execution.ts +0 -2163
  169. package/src/pi-adapter.ts +0 -19
  170. package/src/plan-draft-summary.ts +0 -527
  171. package/src/plan-draft.ts +0 -502
  172. package/src/remove-path.ts +0 -32
  173. package/src/run-state.ts +0 -420
  174. package/src/run-store.ts +0 -1179
  175. package/src/safe-path.ts +0 -77
  176. package/src/scoped-grep-worker.mjs +0 -23
  177. package/src/scoped-tools.ts +0 -775
  178. package/src/single-task-draft.ts +0 -142
  179. package/src/sqlite-schema.ts +0 -171
  180. package/src/state-root.ts +0 -215
  181. package/src/storage-schema.ts +0 -360
  182. package/src/subagent-activity.ts +0 -1417
  183. package/src/subagent-process.ts +0 -476
  184. package/src/task-ledger.ts +0 -1338
  185. package/src/transport-budget.ts +0 -50
  186. package/src/verification-capability.ts +0 -1191
  187. package/src/verification-environment-io.ts +0 -389
  188. package/src/verification-environment.ts +0 -162
  189. package/src/verification-identity.ts +0 -77
  190. package/src/verification-scan.ts +0 -64
  191. package/src/windows-job-backend.ts +0 -419
  192. package/src/windows-job.c +0 -149
  193. package/src/workflow-engine.ts +0 -28
  194. package/src/workflow-policy.ts +0 -1670
  195. package/src/workflow-recovery-policy.ts +0 -97
  196. package/src/workflow-scheduling.ts +0 -105
  197. package/src/workflow-state-machine.ts +0 -4341
  198. package/src/workflow-status.ts +0 -576
  199. package/src/workspace-io-entry.mjs +0 -76
  200. package/src/workspace-io-worker.mjs +0 -1312
  201. package/src/workspace-io.ts +0 -89
  202. package/src/workspace-store.ts +0 -634
@@ -1,4341 +0,0 @@
1
- import { readFileSync } from "node:fs";
2
- import path from "node:path";
3
- import { DatabaseSync } from "node:sqlite";
4
- import type { WorkflowActivityUpdate } from "./activity-contracts.ts";
5
- import { permitsContextRead } from "./candidate-context.ts";
6
- import {
7
- assertPlanWithinChangeContract,
8
- normalizeChangeContract,
9
- retainedChangeContract,
10
- } from "./change-contract.ts";
11
- import { LIMITS, verificationInputPaths } from "./contracts.ts";
12
- import {
13
- assertControlCommand,
14
- type ControlCommand,
15
- type ControlStage,
16
- } from "./control-contracts.ts";
17
- import {
18
- compileImplementPlan,
19
- DeliveryValidationError,
20
- type ImplementPlan,
21
- type PlanTaskDraft,
22
- } from "./delivery-compiler.ts";
23
- import { compareDeliveryRevision } from "./delivery-revision.ts";
24
- import { snapshotFiles } from "./file-snapshot.ts";
25
- import {
26
- canonicalJson,
27
- type RunProjection,
28
- type RunState,
29
- } from "./run-state.ts";
30
- import { type OperationLease, RunStore } from "./run-store.ts";
31
- import { isSafeRegularFile } from "./safe-path.ts";
32
- import { configureSqlite, ensureSqliteSchema } from "./sqlite-schema.ts";
33
- import type { ResolvedStateRoot } from "./state-root.ts";
34
- import {
35
- AMENDMENT_SCHEMA,
36
- ENGINE_ADDITIONS,
37
- ENGINE_SCHEMA,
38
- RECOVERY_ADDITIONS,
39
- RECOVERY_GRANT_SCHEMA,
40
- RECOVERY_SCHEMA,
41
- REJECTED_DELIVERY_SCHEMA,
42
- } from "./storage-schema.ts";
43
- import {
44
- type ActiveOperation,
45
- assertDelivery,
46
- type BootstrapRow,
47
- baselineRecoveryKey,
48
- bootstrapAcceptanceHash,
49
- CHANGE_NAME,
50
- cancellableRead,
51
- classifyPersistedContextRequest,
52
- decideRecoveryAction,
53
- type EngineOperationRow,
54
- type EngineRunRow,
55
- type EngineTaskRow,
56
- emitWorkflowActivity,
57
- expectedClassification,
58
- type HeldOperationLease,
59
- hash,
60
- IDENTIFIER,
61
- isCancellationException,
62
- isRecord,
63
- isWorkflowApprovalCode,
64
- normalizeBootstrapAcceptanceFacts,
65
- normalizeVerificationPrerequisite,
66
- normalizeWorkflowContextRequest,
67
- normalizeWorkflowVerificationStatus,
68
- parseJsonRecord,
69
- parsePlan,
70
- phases,
71
- recoveryKey,
72
- type SafeAttemptDiagnostic,
73
- SHA256,
74
- type WorkflowApplication,
75
- type WorkflowApplicationContext,
76
- type WorkflowAttemptOutcome,
77
- type WorkflowAvailableDelivery,
78
- type WorkflowChangeVerifier,
79
- type WorkflowContextRequest,
80
- type WorkflowDelivery,
81
- type WorkflowDeliverySource,
82
- type WorkflowEngineOptions,
83
- type WorkflowRecoveryFact,
84
- type WorkflowRunLifecycle,
85
- type WorkflowVerificationStatus,
86
- type WorkflowWorker,
87
- } from "./workflow-policy.ts";
88
- import {
89
- assessRecoveryGrant,
90
- recoveryExhausted,
91
- } from "./workflow-recovery-policy.ts";
92
- import { selectRunnableTasks } from "./workflow-scheduling.ts";
93
- import {
94
- firstPausedTask,
95
- projectWorkflowStatus,
96
- type WorkflowResourceBudget,
97
- } from "./workflow-status.ts";
98
- export class WorkflowEngine {
99
- readonly #consumerRoot: string;
100
- readonly #stateRoot: ResolvedStateRoot;
101
- readonly #runStore: RunStore;
102
- readonly #database: DatabaseSync;
103
- readonly #deliverySource: WorkflowDeliverySource;
104
- readonly #worker: WorkflowWorker;
105
- readonly #changeVerifier: WorkflowChangeVerifier;
106
- readonly #application?: WorkflowApplication;
107
- readonly #lifecycle?: WorkflowRunLifecycle;
108
- readonly #now: () => number;
109
- readonly #leaseTtlMs: number;
110
- readonly #workHardLimit: number;
111
- #activeTasks = 0;
112
- readonly #capacityWaiters = new Set<() => void>();
113
- readonly #active = new Map<string, ActiveOperation>();
114
- readonly #operationLeases = new Map<string, HeldOperationLease>();
115
- #orphanRecoveryTimer?: ReturnType<typeof setTimeout>;
116
- readonly #commands = new Map<
117
- AbortController,
118
- { command: ControlCommand; settled: Promise<Record<string, unknown>> }
119
- >();
120
- #closing?: Promise<void>;
121
- readonly #amendments = new Map<string, Promise<Record<string, unknown>>>();
122
- #closed = false;
123
-
124
- private constructor(options: WorkflowEngineOptions) {
125
- this.#consumerRoot = path.resolve(options.consumerRoot);
126
- this.#stateRoot = options.stateRoot;
127
- this.#deliverySource = options.deliverySource;
128
- this.#worker = options.worker;
129
- this.#changeVerifier = options.changeVerifier;
130
- this.#application = options.application;
131
- this.#lifecycle = options.lifecycle;
132
- this.#now = options.now ?? Date.now;
133
- this.#leaseTtlMs = options.leaseTtlMs ?? 60_000;
134
- this.#workHardLimit = options.workHardLimit ?? 512;
135
- if (!Number.isSafeInteger(this.#workHardLimit) || this.#workHardLimit < 1)
136
- throw new Error("workflow-work-limit-invalid");
137
- this.#runStore = RunStore.open(options.stateRoot, { now: this.#now });
138
- this.#database = new DatabaseSync(options.stateRoot.databasePath);
139
- try {
140
- configureSqlite(this.#database);
141
- ensureSqliteSchema(this.#database, ENGINE_SCHEMA, ENGINE_ADDITIONS);
142
- ensureSqliteSchema(this.#database, RECOVERY_SCHEMA, RECOVERY_ADDITIONS);
143
- ensureSqliteSchema(this.#database, RECOVERY_GRANT_SCHEMA);
144
- ensureSqliteSchema(this.#database, AMENDMENT_SCHEMA);
145
- ensureSqliteSchema(this.#database, REJECTED_DELIVERY_SCHEMA);
146
- } catch (error) {
147
- this.#database.close();
148
- this.#runStore.close();
149
- throw error;
150
- }
151
- this.#transaction(() => {
152
- const budgets = this.#database
153
- .prepare(
154
- "SELECT run_id FROM workflow_work_budget WHERE recovery_policy = 0",
155
- )
156
- .all() as { run_id: string }[];
157
- for (const { run_id } of budgets) {
158
- const plans = this.#database
159
- .prepare(
160
- "SELECT plan_json FROM workflow_engine_deliveries WHERE run_id = ?",
161
- )
162
- .all(run_id) as { plan_json: string }[];
163
- const high = Math.max(
164
- 0,
165
- ...plans.map((row) => this.#phaseCount(JSON.parse(row.plan_json))),
166
- );
167
- this.#database
168
- .prepare(
169
- "UPDATE workflow_work_budget SET phase_high_water = ?, hard_limit = MAX(hard_limit, max_work, used), recovery_policy = 1 WHERE run_id = ?",
170
- )
171
- .run(high, run_id);
172
- }
173
- });
174
- this.#recoverInterruptedOperations();
175
- this.#recoverTerminalCleanup();
176
- }
177
-
178
- static open(options: WorkflowEngineOptions): WorkflowEngine {
179
- if (
180
- path.resolve(options.consumerRoot) !== options.stateRoot.consumerRoot ||
181
- options.stateRoot.consumerRootHash.length !== 64
182
- ) {
183
- throw new Error("workflow-engine-root-mismatch");
184
- }
185
- if (
186
- !options.deliverySource ||
187
- typeof options.deliverySource.load !== "function" ||
188
- !options.worker ||
189
- typeof options.worker.runAttempt !== "function" ||
190
- typeof options.worker.rebind !== "function" ||
191
- !options.changeVerifier ||
192
- typeof options.changeVerifier.verify !== "function" ||
193
- (options.application !== undefined &&
194
- (typeof options.application.prepare === "function") !==
195
- (typeof options.application.applyPrepared === "function"))
196
- ) {
197
- throw new Error("workflow-engine-service-invalid");
198
- }
199
- if (
200
- options.leaseTtlMs !== undefined &&
201
- (!Number.isSafeInteger(options.leaseTtlMs) || options.leaseTtlMs < 1)
202
- ) {
203
- throw new Error("workflow-engine-lease-invalid");
204
- }
205
- return new WorkflowEngine(options);
206
- }
207
-
208
- #assertOpen(): void {
209
- if (this.#closed) throw new Error("workflow-engine-closed");
210
- }
211
-
212
- #transaction<T>(work: () => T): T {
213
- this.#database.exec("BEGIN IMMEDIATE");
214
- try {
215
- const result = work();
216
- this.#database.exec("COMMIT");
217
- return result;
218
- } catch (error) {
219
- this.#database.exec("ROLLBACK");
220
- throw error;
221
- }
222
- }
223
-
224
- #operationKey(runId: string, operationId: string): string {
225
- return `${runId}\0${operationId}`;
226
- }
227
-
228
- #leaseOperationId(runId: string, _operationId: string): string {
229
- return `engine-run-${hash(runId).slice(0, 40)}`;
230
- }
231
-
232
- #holdOperationLease(lease: OperationLease, commandOperationId: string): void {
233
- const key = this.#operationKey(lease.runId, commandOperationId);
234
- const existing = this.#operationLeases.get(key);
235
- if (existing) clearInterval(existing.timer);
236
- const intervalMs = Math.max(1, Math.floor(this.#leaseTtlMs / 3));
237
- const held: HeldOperationLease = {
238
- lease,
239
- timer: setInterval(() => {
240
- if (this.#operationLeases.get(key) !== held) return;
241
- try {
242
- held.lease = this.#runStore.renewLease(held.lease, this.#leaseTtlMs);
243
- this.#database
244
- .prepare(
245
- `UPDATE workflow_engine_operations
246
- SET lease_expires_at = ?
247
- WHERE run_id = ? AND operation_id = ? AND state = 'running'
248
- AND lease_token = ?`,
249
- )
250
- .run(
251
- held.lease.expiresAt,
252
- lease.runId,
253
- commandOperationId,
254
- lease.token,
255
- );
256
- } catch {
257
- held.error = new Error("lease-fenced");
258
- clearInterval(held.timer);
259
- const active = this.#active.get(lease.runId);
260
- if (active?.operationId === commandOperationId) {
261
- active.controller.abort(held.error);
262
- }
263
- }
264
- }, intervalMs),
265
- };
266
- held.timer.unref?.();
267
- this.#operationLeases.set(key, held);
268
- }
269
-
270
- #operationLease(runId: string, operationId: string): OperationLease {
271
- const held = this.#operationLeases.get(
272
- this.#operationKey(runId, operationId),
273
- );
274
- if (!held || held.error) throw held?.error ?? new Error("lease-fenced");
275
- this.#runStore.assertLease(held.lease);
276
- return held.lease;
277
- }
278
-
279
- #releaseOperationLease(runId: string, operationId: string): void {
280
- const key = this.#operationKey(runId, operationId);
281
- const held = this.#operationLeases.get(key);
282
- if (!held) return;
283
- clearInterval(held.timer);
284
- this.#operationLeases.delete(key);
285
- }
286
-
287
- #assertLeaseInTransaction(lease: OperationLease): void {
288
- const row = this.#database
289
- .prepare(
290
- `SELECT state, lease_token, lease_expires_at
291
- FROM operations WHERE run_id = ? AND operation_id = ?`,
292
- )
293
- .get(lease.runId, lease.operationId) as
294
- | {
295
- state: string;
296
- lease_token: string | null;
297
- lease_expires_at: number | null;
298
- }
299
- | undefined;
300
- if (
301
- row?.state !== "running" ||
302
- row.lease_token !== lease.token ||
303
- row.lease_expires_at === null ||
304
- row.lease_expires_at < this.#now()
305
- ) {
306
- throw new Error("lease-fenced");
307
- }
308
- }
309
-
310
- #leasedTransaction<T>(lease: OperationLease, work: () => T): T {
311
- return this.#transaction(() => {
312
- this.#assertLeaseInTransaction(lease);
313
- return work();
314
- });
315
- }
316
-
317
- amend(
318
- change: string,
319
- batchId: string,
320
- request: unknown,
321
- execute: (assertAuthority: () => void) => Promise<Record<string, unknown>>,
322
- ): Promise<Record<string, unknown>> {
323
- this.#assertOpen();
324
- if (this.#closing) throw new Error("workflow-engine-closed");
325
- const runId = this.#lookupRun("abel-implement", change);
326
- if (!runId) throw new Error("run-not-found");
327
- if (this.#amendments.has(runId)) throw new Error("amendment-busy");
328
- const operationId = `amend-${hash(batchId, canonicalJson(request)).slice(0, 48)}`;
329
- const pending = Promise.resolve()
330
- .then(async () => {
331
- const replay = this.#beginOperation(runId, operationId, "amend");
332
- if (replay) return replay;
333
- const lease = this.#operationLease(runId, operationId);
334
- const assertAuthority = () => {
335
- this.#assertOpen();
336
- if (this.#closing) throw new Error("workflow-engine-closed");
337
- this.#runStore.assertLease(lease);
338
- const status = this.#statusByRun(runId);
339
- if (
340
- !["approval-needed", "paused"].includes(String(status.state)) ||
341
- (status.decisionBatch as { id?: string } | undefined)?.id !==
342
- batchId
343
- )
344
- throw new Error("amendment-batch-stale");
345
- };
346
- try {
347
- assertAuthority();
348
- if (
349
- !isRecord(request) ||
350
- !["status", "validate-plan-draft"].includes(
351
- String(request.operation),
352
- )
353
- ) {
354
- this.#leasedTransaction(lease, () => {
355
- this.#database
356
- .prepare(
357
- "INSERT INTO workflow_amendment_budget(run_id, used) VALUES (?, 0) ON CONFLICT DO NOTHING",
358
- )
359
- .run(runId);
360
- const reserved = this.#database
361
- .prepare(
362
- "UPDATE workflow_amendment_budget SET used = used + 1 WHERE run_id = ? AND used < 64",
363
- )
364
- .run(runId);
365
- if (reserved.changes !== 1)
366
- throw new Error("amendment-budget-exhausted");
367
- });
368
- }
369
- this.#assertAmendmentContract(runId, change, request);
370
- const outcome = await execute(assertAuthority);
371
- assertAuthority();
372
- return this.#commitOperation(runId, operationId, outcome);
373
- } catch (error) {
374
- this.#interruptOperation(runId, operationId);
375
- throw error;
376
- }
377
- })
378
- .finally(() => {
379
- if (this.#amendments.get(runId) === pending)
380
- this.#amendments.delete(runId);
381
- });
382
- this.#amendments.set(runId, pending);
383
- return pending;
384
- }
385
-
386
- #assertAmendmentContract(
387
- runId: string,
388
- change: string,
389
- request: unknown,
390
- ): void {
391
- if (!isRecord(request)) throw new Error("amendment-contract-invalid");
392
- if (
393
- (request.operation === "approve-gate" && request.gate === "gate-a") ||
394
- (request.operation === "record-decision" &&
395
- request.category === "behavior")
396
- )
397
- throw new Error("amendment-changes-accepted-behavior");
398
- if (
399
- ![
400
- "write-artifact",
401
- "delete-artifact",
402
- "compile-plan",
403
- "validate-plan-draft",
404
- "finalize-delivery",
405
- ].includes(String(request.operation))
406
- )
407
- return;
408
- const relative = String(request.path);
409
- if (relative === "proposal.md" || relative.startsWith("specs/")) {
410
- const file = `openspec/changes/${change}/${relative}`;
411
- if (
412
- request.operation !== "write-artifact" ||
413
- !isSafeRegularFile(this.#consumerRoot, file) ||
414
- readFileSync(path.join(this.#consumerRoot, file), "utf8") !==
415
- request.content
416
- )
417
- throw new Error("amendment-changes-accepted-behavior");
418
- }
419
- const compile = [
420
- "compile-plan",
421
- "validate-plan-draft",
422
- "finalize-delivery",
423
- ].includes(String(request.operation));
424
- if (!compile) return;
425
- const draftPath = `openspec/changes/${change}/plan-draft.json`;
426
- if (!isSafeRegularFile(this.#consumerRoot, draftPath)) return;
427
- const run = this.#engineRun(runId);
428
- if (run.current_revision === null) return;
429
- const prior = this.#planForRevision(runId, run.current_revision);
430
- const contract = retainedChangeContract(prior);
431
- const draft = JSON.parse(
432
- readFileSync(path.join(this.#consumerRoot, draftPath), "utf8"),
433
- );
434
- if (
435
- draft.changeContract !== undefined &&
436
- canonicalJson(normalizeChangeContract(draft.changeContract)) !==
437
- canonicalJson(contract)
438
- )
439
- throw new Error("change-contract-mismatch");
440
- // Match DesignController's inheritance before validating authorized modes.
441
- if (draft.changeContract === undefined && prior.changeContract)
442
- draft.changeContract = contract;
443
- const next = compileImplementPlan(draft, {
444
- consumerRoot: this.#consumerRoot,
445
- bindExecutionInputs: true,
446
- }).plan;
447
- assertPlanWithinChangeContract(contract, next.tasks, [
448
- next.verification.change.fullSuite,
449
- next.verification.change.postApply,
450
- ]);
451
- }
452
-
453
- #lookupRun(stage: ControlStage, change: string): string | undefined {
454
- return (
455
- this.#database
456
- .prepare(
457
- `SELECT run_id FROM runs
458
- WHERE root_hash = ? AND stage = ? AND lookup_key = ?`,
459
- )
460
- .get(this.#stateRoot.consumerRootHash, stage, `change:${change}`) as
461
- | { run_id: string }
462
- | undefined
463
- )?.run_id;
464
- }
465
-
466
- #ensureEngineRun(runId: string): void {
467
- this.#database
468
- .prepare(
469
- `INSERT INTO workflow_engine_runs(
470
- run_id, current_revision, baseline_workspace_revision,
471
- current_workspace_revision, cleanup_state, next_queue_position
472
- ) VALUES (?, NULL, NULL, NULL, 'none', 1)
473
- ON CONFLICT(run_id) DO NOTHING`,
474
- )
475
- .run(runId);
476
- }
477
-
478
- #engineRun(runId: string): EngineRunRow {
479
- const row = this.#database
480
- .prepare(
481
- `SELECT current_revision, baseline_workspace_revision,
482
- current_workspace_revision, cleanup_state, route_id,
483
- route_fingerprint,
484
- transaction_id, verification_json, delivery_diagnostics_json,
485
- next_queue_position
486
- FROM workflow_engine_runs WHERE run_id = ?`,
487
- )
488
- .get(runId) as EngineRunRow | undefined;
489
- if (!row) throw new Error("workflow-engine-run-unavailable");
490
- return row;
491
- }
492
-
493
- #operation(
494
- runId: string,
495
- operationId: string,
496
- ): EngineOperationRow | undefined {
497
- return this.#database
498
- .prepare(
499
- `SELECT command, state, outcome_json FROM workflow_engine_operations
500
- WHERE run_id = ? AND operation_id = ?`,
501
- )
502
- .get(runId, operationId) as EngineOperationRow | undefined;
503
- }
504
-
505
- #operationReplay(
506
- runId: string,
507
- operationId: string,
508
- command: string,
509
- ): Record<string, unknown> | undefined {
510
- const row = this.#operation(runId, operationId);
511
- if (row && row.command !== command) {
512
- throw new Error("operation-command-mismatch");
513
- }
514
- return row?.state === "committed" && row.outcome_json
515
- ? parseJsonRecord(row.outcome_json)
516
- : undefined;
517
- }
518
-
519
- #beginOperation(
520
- runId: string,
521
- operationId: string,
522
- command: string,
523
- ): Record<string, unknown> | undefined {
524
- if (!IDENTIFIER.test(operationId)) throw new Error("invalid-operation-id");
525
- const replay = this.#operationReplay(runId, operationId, command);
526
- if (replay) return replay;
527
- const existing = this.#operation(runId, operationId);
528
- if (existing?.state === "running") {
529
- const leaseStatus = this.#runStore.inspectOperation(
530
- runId,
531
- this.#leaseOperationId(runId, operationId),
532
- );
533
- if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
534
- this.#database
535
- .prepare(
536
- `UPDATE workflow_engine_operations
537
- SET state = 'committed', lease_token = NULL,
538
- lease_expires_at = NULL, outcome_json = ?
539
- WHERE run_id = ? AND operation_id = ?`,
540
- )
541
- .run(JSON.stringify(leaseStatus.outcome), runId, operationId);
542
- return structuredClone(leaseStatus.outcome);
543
- }
544
- if (leaseStatus?.state === "running") {
545
- throw new Error("operation-already-running");
546
- }
547
- this.#database
548
- .prepare(
549
- `UPDATE workflow_engine_operations
550
- SET state = 'interrupted', lease_token = NULL,
551
- lease_expires_at = NULL
552
- WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
553
- )
554
- .run(runId, operationId);
555
- }
556
- const leaseOperationId = this.#leaseOperationId(runId, operationId);
557
- const priorLease = this.#runStore.inspectOperation(runId, leaseOperationId);
558
- const preempt = command === "cancel" || command === "discard";
559
- if (priorLease?.state === "running" && !preempt) {
560
- throw new Error("operation-already-running");
561
- }
562
- const lease = this.#runStore.acquireLease({
563
- runId,
564
- operationId: leaseOperationId,
565
- ttlMs: this.#leaseTtlMs,
566
- exclusive: true,
567
- ...(preempt ? { preempt: true } : {}),
568
- });
569
- if (preempt) {
570
- this.#database
571
- .prepare(
572
- `UPDATE workflow_engine_operations
573
- SET state = 'interrupted', lease_token = NULL,
574
- lease_expires_at = NULL
575
- WHERE run_id = ? AND operation_id <> ? AND state = 'running'`,
576
- )
577
- .run(runId, operationId);
578
- }
579
- this.#database
580
- .prepare(
581
- `INSERT INTO workflow_engine_operations(
582
- run_id, operation_id, command, state, lease_token,
583
- lease_expires_at, outcome_json
584
- ) VALUES (?, ?, ?, 'running', ?, ?, NULL)
585
- ON CONFLICT(run_id, operation_id) DO UPDATE SET
586
- command = excluded.command, state = 'running',
587
- lease_token = excluded.lease_token,
588
- lease_expires_at = excluded.lease_expires_at,
589
- outcome_json = NULL`,
590
- )
591
- .run(runId, operationId, command, lease.token, lease.expiresAt);
592
- this.#holdOperationLease(lease, operationId);
593
- return undefined;
594
- }
595
-
596
- #commitOperation(
597
- runId: string,
598
- operationId: string,
599
- outcome: Record<string, unknown>,
600
- ): Record<string, unknown> {
601
- const lease = this.#operationLease(runId, operationId);
602
- try {
603
- const committed = this.#runStore.commitLease(lease, outcome);
604
- const result = this.#database
605
- .prepare(
606
- `UPDATE workflow_engine_operations
607
- SET state = 'committed', lease_token = NULL,
608
- lease_expires_at = NULL, outcome_json = ?
609
- WHERE run_id = ? AND operation_id = ? AND state = 'running'
610
- AND lease_token = ?`,
611
- )
612
- .run(JSON.stringify(committed), runId, operationId, lease.token);
613
- if (Number(result.changes) !== 1) {
614
- throw new Error("operation-journal-fenced");
615
- }
616
- return structuredClone(committed);
617
- } finally {
618
- this.#releaseOperationLease(runId, operationId);
619
- }
620
- }
621
-
622
- #interruptOperation(runId: string, operationId: string): void {
623
- let lease: OperationLease;
624
- try {
625
- lease = this.#operationLease(runId, operationId);
626
- if (this.#operation(runId, operationId)?.command !== "amend")
627
- this.#pauseInterruptedRun(runId, operationId, lease);
628
- this.#runStore.interruptLease(lease);
629
- this.#database
630
- .prepare(
631
- `UPDATE workflow_engine_operations
632
- SET state = 'interrupted', lease_token = NULL,
633
- lease_expires_at = NULL
634
- WHERE run_id = ? AND operation_id = ? AND state = 'running'
635
- AND lease_token = ?`,
636
- )
637
- .run(runId, operationId, lease.token);
638
- } catch (error) {
639
- if (!(error instanceof Error) || error.message !== "lease-fenced") {
640
- throw error;
641
- }
642
- } finally {
643
- this.#releaseOperationLease(runId, operationId);
644
- }
645
- }
646
-
647
- #transition(
648
- runId: string,
649
- to: RunState,
650
- identity: string,
651
- code?: string,
652
- lease?: OperationLease,
653
- ): RunProjection {
654
- if (lease) this.#runStore.assertLease(lease);
655
- const current = this.#runStore.status(runId);
656
- if (
657
- current.state === to &&
658
- (to !== "paused" || code === undefined || current.pauseCode === code)
659
- ) {
660
- return current;
661
- }
662
- return this.#runStore.transition({
663
- runId,
664
- to,
665
- operationId: `eng-${hash(runId, identity, String(current.sequence), to).slice(0, 40)}`,
666
- ...(code ? { code } : {}),
667
- ...(lease ? { lease } : {}),
668
- });
669
- }
670
-
671
- #pauseInterruptedRun(
672
- runId: string,
673
- operationId: string,
674
- lease: OperationLease,
675
- ): void {
676
- const projection = this.#runStore.status(runId);
677
- if (["completed", "discarded", "rejected"].includes(projection.state))
678
- return;
679
- const write = (database: DatabaseSync) => {
680
- database
681
- .prepare(`UPDATE workflow_engine_tasks
682
- SET state = 'paused',
683
- pause_code = CASE WHEN pause_code = 'red-artifact-constraint'
684
- THEN pause_code ELSE 'operation-interrupted' END,
685
- context_request_json = CASE WHEN pause_code = 'red-artifact-constraint'
686
- THEN context_request_json ELSE NULL END,
687
- queue_position = NULL
688
- WHERE run_id = ? AND state IN ('phase-running', 'validating')`)
689
- .run(runId);
690
- };
691
- const active = [
692
- "created",
693
- "validating-delivery",
694
- "ready",
695
- "queued",
696
- "connecting",
697
- "running",
698
- "validating",
699
- "verifying",
700
- "retryable",
701
- "approval-needed",
702
- "change-verifying",
703
- "ready-to-apply",
704
- ];
705
- const to =
706
- projection.state === "applying"
707
- ? "recovering"
708
- : active.includes(projection.state)
709
- ? "paused"
710
- : undefined;
711
- if (to) {
712
- // Keep task and run projections atomic; retained evidence and budgets are
713
- // deliberately untouched. The original exception still reaches the caller.
714
- this.#runStore.transitionAtomically(
715
- {
716
- runId,
717
- to,
718
- lease,
719
- code: "operation-interrupted",
720
- operationId: `eng-${hash(runId, operationId, String(projection.sequence), "interrupted").slice(0, 40)}`,
721
- },
722
- write,
723
- );
724
- } else {
725
- this.#leasedTransaction(lease, () => write(this.#database));
726
- }
727
- }
728
-
729
- #recoverInterruptedOperations(): void {
730
- if (this.#orphanRecoveryTimer) {
731
- clearTimeout(this.#orphanRecoveryTimer);
732
- this.#orphanRecoveryTimer = undefined;
733
- }
734
- let nextExpiry: number | undefined;
735
- const interrupted = this.#database
736
- .prepare(
737
- `SELECT run_id, operation_id FROM workflow_engine_operations
738
- WHERE state = 'running' OR (state = 'interrupted' AND command <> 'amend'
739
- AND (run_id IN (SELECT run_id FROM workflow_engine_tasks
740
- WHERE state IN ('phase-running', 'validating'))
741
- OR run_id IN (SELECT run_id FROM runs WHERE state IN
742
- ('running', 'validating', 'verifying', 'change-verifying', 'applying'))))`,
743
- )
744
- .all() as unknown as Array<{ run_id: string; operation_id: string }>;
745
- for (const row of interrupted) {
746
- const leaseStatus = this.#runStore.inspectOperation(
747
- row.run_id,
748
- this.#leaseOperationId(row.run_id, row.operation_id),
749
- );
750
- if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
751
- this.#database
752
- .prepare(
753
- `UPDATE workflow_engine_operations
754
- SET state = 'committed', lease_token = NULL,
755
- lease_expires_at = NULL, outcome_json = ?
756
- WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
757
- )
758
- .run(
759
- JSON.stringify(leaseStatus.outcome),
760
- row.run_id,
761
- row.operation_id,
762
- );
763
- continue;
764
- }
765
- if (leaseStatus?.state === "running") {
766
- if (leaseStatus.expiresAt !== undefined) {
767
- nextExpiry = Math.min(nextExpiry ?? Infinity, leaseStatus.expiresAt);
768
- }
769
- continue;
770
- }
771
- this.#database
772
- .prepare(
773
- `UPDATE workflow_engine_operations
774
- SET state = 'interrupted', lease_token = NULL,
775
- lease_expires_at = NULL
776
- WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
777
- )
778
- .run(row.run_id, row.operation_id);
779
- // Claim the same run-exclusive lease before repairing projections. A live
780
- // owner (including a different host) must never be mistaken for an orphan.
781
- let recoveryLease: OperationLease;
782
- try {
783
- recoveryLease = this.#runStore.acquireLease({
784
- runId: row.run_id,
785
- operationId: this.#leaseOperationId(row.run_id, row.operation_id),
786
- ttlMs: this.#leaseTtlMs,
787
- exclusive: true,
788
- });
789
- } catch (error) {
790
- if (
791
- error instanceof Error &&
792
- error.message === "operation-already-running"
793
- )
794
- continue;
795
- throw error;
796
- }
797
- try {
798
- this.#pauseInterruptedRun(row.run_id, row.operation_id, recoveryLease);
799
- } finally {
800
- this.#runStore.interruptLease(recoveryLease);
801
- }
802
- }
803
- if (nextExpiry !== undefined && !this.#closed) {
804
- const delay = Math.max(1, nextExpiry - this.#now() + 1);
805
- this.#orphanRecoveryTimer = setTimeout(() => {
806
- this.#orphanRecoveryTimer = undefined;
807
- if (this.#closed) return;
808
- try {
809
- this.#recoverInterruptedOperations();
810
- } catch {
811
- // A later status/restart will surface durable journal corruption.
812
- }
813
- }, delay);
814
- this.#orphanRecoveryTimer.unref?.();
815
- }
816
- }
817
-
818
- #recoverTerminalCleanup(): void {
819
- const rows = this.#database
820
- .prepare(
821
- `SELECT workflow_engine_runs.run_id
822
- FROM workflow_engine_runs
823
- JOIN runs ON runs.run_id = workflow_engine_runs.run_id
824
- WHERE runs.state IN ('completed', 'discarded')
825
- AND workflow_engine_runs.cleanup_state = 'retained'`,
826
- )
827
- .all() as unknown as Array<{ run_id: string }>;
828
- for (const row of rows) {
829
- try {
830
- this.#cleanupTerminal(row.run_id, "restart-cleanup");
831
- } catch {
832
- // Terminal status remains truthful and a later reload retries cleanup.
833
- }
834
- }
835
- }
836
-
837
- #tasks(runId: string): EngineTaskRow[] {
838
- return this.#database
839
- .prepare(
840
- `SELECT task_id, task_order, delivery_revision, plan_json, state,
841
- phase, pause_code, context_request_json, attempt_diagnostic_json,
842
- route_id, route_fingerprint, queue_position
843
- FROM workflow_engine_tasks WHERE run_id = ? ORDER BY task_order`,
844
- )
845
- .all(runId) as unknown as EngineTaskRow[];
846
- }
847
-
848
- #taskPlan(row: EngineTaskRow): PlanTaskDraft {
849
- return JSON.parse(row.plan_json) as PlanTaskDraft;
850
- }
851
-
852
- #currentPlan(runId: string): ImplementPlan {
853
- const revision = this.#engineRun(runId).current_revision;
854
- if (revision === null) throw new Error("delivery-not-bound");
855
- return this.#planForRevision(runId, revision);
856
- }
857
-
858
- #planForRevision(runId: string, revision: number): ImplementPlan {
859
- const row = this.#database
860
- .prepare(
861
- `SELECT plan_json FROM workflow_engine_deliveries
862
- WHERE run_id = ? AND revision = ?`,
863
- )
864
- .get(runId, revision) as { plan_json: string } | undefined;
865
- if (!row) throw new Error("delivery-not-bound");
866
- return parsePlan(row.plan_json);
867
- }
868
-
869
- async #loadDelivery(
870
- runId: string,
871
- lease: OperationLease,
872
- stage: ControlStage,
873
- change: string,
874
- deliveryRevision?: number,
875
- receiptHash?: string,
876
- signal?: AbortSignal,
877
- ): Promise<WorkflowDelivery> {
878
- if (deliveryRevision === undefined && this.#deliverySource.discoverLatest) {
879
- try {
880
- const discover = this.#deliverySource.discoverLatest;
881
- const available = await cancellableRead(
882
- () => discover.call(this.#deliverySource, { stage, change, signal }),
883
- signal,
884
- );
885
- if (
886
- available &&
887
- Number.isSafeInteger(available.deliveryRevision) &&
888
- available.deliveryRevision > 0 &&
889
- SHA256.test(available.receiptHash)
890
- ) {
891
- deliveryRevision = available.deliveryRevision;
892
- receiptHash = available.receiptHash;
893
- }
894
- } catch {
895
- // Discovery is optional; authoritative loading still reports diagnostics.
896
- signal?.throwIfAborted();
897
- }
898
- }
899
- try {
900
- const delivery = await cancellableRead(
901
- () =>
902
- this.#deliverySource.load({
903
- stage,
904
- change,
905
- ...(deliveryRevision === undefined ? {} : { deliveryRevision }),
906
- ...(receiptHash === undefined ? {} : { receiptHash }),
907
- ...(signal ? { signal } : {}),
908
- }),
909
- signal,
910
- );
911
- signal?.throwIfAborted();
912
- assertDelivery(delivery, stage, change, deliveryRevision, receiptHash);
913
- return structuredClone(delivery);
914
- } catch (error) {
915
- if (
916
- error instanceof DeliveryValidationError &&
917
- deliveryRevision !== undefined &&
918
- receiptHash !== undefined
919
- ) {
920
- this.#leasedTransaction(lease, () => {
921
- this.#database
922
- .prepare(
923
- "INSERT INTO workflow_rejected_deliveries(run_id, delivery_revision, receipt_hash) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
924
- )
925
- .run(runId, deliveryRevision, receiptHash);
926
- });
927
- }
928
- throw error;
929
- }
930
- }
931
-
932
- #admitDelivery(
933
- runId: string,
934
- delivery: WorkflowDelivery,
935
- lease: OperationLease,
936
- ): void {
937
- this.#runStore.assertLease(lease);
938
- const existing = this.#database
939
- .prepare(
940
- `SELECT receipt_hash, plan_json FROM workflow_engine_deliveries
941
- WHERE run_id = ? AND revision = ?`,
942
- )
943
- .get(runId, delivery.revision) as
944
- | { receipt_hash: string; plan_json: string }
945
- | undefined;
946
- const capturedBudget = this.#database
947
- .prepare("SELECT hard_limit FROM workflow_work_budget WHERE run_id = ?")
948
- .get(runId) as { hard_limit: number } | undefined;
949
- if (
950
- this.#phaseCount(delivery.plan) >
951
- (capturedBudget?.hard_limit ?? this.#workHardLimit) &&
952
- !existing
953
- )
954
- throw new DeliveryValidationError([
955
- "verification-work-budget-insufficient",
956
- ]);
957
- const serializedPlan = JSON.stringify(delivery.plan);
958
- if (
959
- existing &&
960
- (existing.receipt_hash !== delivery.receiptHash ||
961
- existing.plan_json !== serializedPlan)
962
- ) {
963
- throw new Error("delivery-revision-conflict");
964
- }
965
- const engineRun = this.#engineRun(runId);
966
- if (
967
- engineRun.current_revision !== null &&
968
- delivery.revision < engineRun.current_revision
969
- ) {
970
- throw new Error("delivery-revision-stale");
971
- }
972
- const priorRows = this.#tasks(runId);
973
- const priorById = new Map(priorRows.map((row) => [row.task_id, row]));
974
- const priorPlan =
975
- engineRun.current_revision === null
976
- ? undefined
977
- : this.#planForRevision(runId, engineRun.current_revision);
978
- const comparison = compareDeliveryRevision(
979
- priorPlan,
980
- delivery.plan,
981
- priorRows,
982
- );
983
- const { compatible, boundaryExpanded } = comparison;
984
- let invalidated = comparison.invalidated;
985
- let revalidatedWorkspace:
986
- | {
987
- baselineRevisionId: string;
988
- currentWorkspaceRevisionId: string;
989
- }
990
- | undefined;
991
- if (
992
- priorPlan &&
993
- (invalidated.size > 0 || boundaryExpanded) &&
994
- engineRun.baseline_workspace_revision &&
995
- engineRun.current_workspace_revision
996
- ) {
997
- if (!this.#worker.revalidateDelivery) {
998
- if (boundaryExpanded) {
999
- throw new Error("workflow-delivery-revalidation-unavailable");
1000
- }
1001
- if (
1002
- engineRun.current_workspace_revision !==
1003
- engineRun.baseline_workspace_revision
1004
- ) {
1005
- throw new Error("workflow-delivery-revalidation-unavailable");
1006
- }
1007
- } else {
1008
- const result = this.#worker.revalidateDelivery({
1009
- runId,
1010
- deliveryRevision: delivery.revision,
1011
- plan: structuredClone(delivery.plan),
1012
- tasks: priorRows
1013
- .filter((row) => compatible.has(row.task_id))
1014
- .map((row) => ({
1015
- taskId: row.task_id,
1016
- deliveryRevision: row.delivery_revision,
1017
- state: row.state,
1018
- phase: row.phase,
1019
- contextReadPaths: this.#contextReads(runId, row.task_id),
1020
- })),
1021
- invalidatedTaskIds: [...invalidated].sort(),
1022
- baselineRevisionId: engineRun.baseline_workspace_revision,
1023
- currentWorkspaceRevisionId: engineRun.current_workspace_revision,
1024
- });
1025
- const resultIds = new Set(result.invalidatedTaskIds);
1026
- if (
1027
- !Array.isArray(result.invalidatedTaskIds) ||
1028
- resultIds.size !== result.invalidatedTaskIds.length ||
1029
- [...invalidated].some((taskId) => !resultIds.has(taskId)) ||
1030
- [...resultIds].some((taskId) => !priorById.has(taskId)) ||
1031
- !SHA256.test(result.baselineRevisionId) ||
1032
- !SHA256.test(result.currentWorkspaceRevisionId)
1033
- ) {
1034
- throw new Error("workflow-delivery-revalidation-invalid");
1035
- }
1036
- invalidated = resultIds;
1037
- revalidatedWorkspace = {
1038
- baselineRevisionId: result.baselineRevisionId,
1039
- currentWorkspaceRevisionId: result.currentWorkspaceRevisionId,
1040
- };
1041
- }
1042
- }
1043
- const deliveryBindings = delivery.approvalProofs
1044
- ? ([
1045
- {
1046
- runId,
1047
- gate: "gate-a" as const,
1048
- revision: delivery.revision,
1049
- receiptHash: delivery.receiptHash,
1050
- approvalProof: structuredClone(delivery.approvalProofs.gateA),
1051
- operationId: `bind-a-${hash(
1052
- runId,
1053
- String(delivery.revision),
1054
- delivery.approvalProofs.gateA.recordHash,
1055
- ).slice(0, 40)}`,
1056
- lease,
1057
- },
1058
- {
1059
- runId,
1060
- gate: "gate-b" as const,
1061
- revision: delivery.revision,
1062
- receiptHash: delivery.receiptHash,
1063
- approvalProof: structuredClone(delivery.approvalProofs.gateB),
1064
- operationId: `bind-b-${hash(
1065
- runId,
1066
- String(delivery.revision),
1067
- delivery.approvalProofs.gateB.recordHash,
1068
- ).slice(0, 40)}`,
1069
- lease,
1070
- },
1071
- ] as const)
1072
- : ([
1073
- {
1074
- runId,
1075
- gate: delivery.gate,
1076
- revision: delivery.revision,
1077
- receiptHash: delivery.receiptHash,
1078
- operationId: `bind-${hash(
1079
- runId,
1080
- String(delivery.revision),
1081
- delivery.receiptHash,
1082
- ).slice(0, 40)}`,
1083
- lease,
1084
- },
1085
- ] as const);
1086
- this.#runStore.bindDeliveriesAtomically(deliveryBindings, (database) => {
1087
- const engineLease = database
1088
- .prepare(
1089
- `SELECT state, lease_expires_at
1090
- FROM workflow_engine_operations
1091
- WHERE run_id = ? AND lease_token = ?`,
1092
- )
1093
- .get(runId, lease.token) as
1094
- | { state: string; lease_expires_at: number | null }
1095
- | undefined;
1096
- if (
1097
- engineLease?.state !== "running" ||
1098
- engineLease.lease_expires_at === null ||
1099
- engineLease.lease_expires_at < this.#now()
1100
- ) {
1101
- throw new Error("lease-fenced");
1102
- }
1103
- database
1104
- .prepare(
1105
- `INSERT INTO workflow_engine_runs(
1106
- run_id, current_revision, delivery_diagnostics_json,
1107
- next_queue_position
1108
- ) VALUES (?, ?, NULL, 1)
1109
- ON CONFLICT(run_id) DO UPDATE SET
1110
- current_revision = excluded.current_revision,
1111
- verification_json = NULL,
1112
- delivery_diagnostics_json = NULL`,
1113
- )
1114
- .run(runId, delivery.revision);
1115
- if (revalidatedWorkspace) {
1116
- database
1117
- .prepare(
1118
- `UPDATE workflow_engine_runs
1119
- SET baseline_workspace_revision = ?,
1120
- current_workspace_revision = ?, cleanup_state = 'retained'
1121
- WHERE run_id = ?`,
1122
- )
1123
- .run(
1124
- revalidatedWorkspace.baselineRevisionId,
1125
- revalidatedWorkspace.currentWorkspaceRevisionId,
1126
- runId,
1127
- );
1128
- }
1129
- database
1130
- .prepare(
1131
- `INSERT INTO workflow_engine_deliveries(
1132
- run_id, revision, gate, receipt_hash, plan_json
1133
- ) VALUES (?, ?, ?, ?, ?)
1134
- ON CONFLICT(run_id, revision) DO NOTHING`,
1135
- )
1136
- .run(
1137
- runId,
1138
- delivery.revision,
1139
- delivery.gate,
1140
- delivery.receiptHash,
1141
- serializedPlan,
1142
- );
1143
-
1144
- this.#growWorkBudget(database, runId, delivery.plan);
1145
- const admitted = new Set<string>();
1146
- delivery.plan.tasks.forEach((task, taskOrder) => {
1147
- admitted.add(task.taskId);
1148
- const taskJson = JSON.stringify(task);
1149
- const prior = priorById.get(task.taskId);
1150
- if (!prior) {
1151
- database
1152
- .prepare(
1153
- `INSERT INTO workflow_engine_tasks(
1154
- run_id, task_id, task_order, delivery_revision, plan_json,
1155
- state, phase
1156
- ) VALUES (?, ?, ?, ?, ?, 'pending', ?)`,
1157
- )
1158
- .run(
1159
- runId,
1160
- task.taskId,
1161
- taskOrder,
1162
- delivery.revision,
1163
- taskJson,
1164
- phases(task)[0],
1165
- );
1166
- return;
1167
- }
1168
- if (compatible.has(task.taskId) && !invalidated.has(task.taskId)) {
1169
- database
1170
- .prepare(
1171
- `UPDATE workflow_engine_tasks
1172
- SET task_order = ?, plan_json = ?
1173
- WHERE run_id = ? AND task_id = ?`,
1174
- )
1175
- .run(taskOrder, taskJson, runId, task.taskId);
1176
- return;
1177
- }
1178
- database
1179
- .prepare(
1180
- "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1181
- )
1182
- .run(runId, task.taskId);
1183
- database
1184
- .prepare(
1185
- `UPDATE workflow_engine_tasks
1186
- SET task_order = ?, delivery_revision = ?, plan_json = ?,
1187
- state = 'pending', phase = ?, pause_code = NULL,
1188
- context_request_json = NULL, attempt_diagnostic_json = NULL,
1189
- queue_position = NULL
1190
- WHERE run_id = ? AND task_id = ?`,
1191
- )
1192
- .run(
1193
- taskOrder,
1194
- delivery.revision,
1195
- taskJson,
1196
- phases(task)[0],
1197
- runId,
1198
- task.taskId,
1199
- );
1200
- });
1201
- for (const row of priorRows) {
1202
- if (!admitted.has(row.task_id)) {
1203
- database
1204
- .prepare(
1205
- "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1206
- )
1207
- .run(runId, row.task_id);
1208
- database
1209
- .prepare(
1210
- `DELETE FROM workflow_engine_tasks
1211
- WHERE run_id = ? AND task_id = ?`,
1212
- )
1213
- .run(runId, row.task_id);
1214
- }
1215
- }
1216
- });
1217
- }
1218
-
1219
- #cancelledDelivery(
1220
- runId: string,
1221
- operationId: string,
1222
- error: unknown,
1223
- signal: AbortSignal | undefined,
1224
- lease: OperationLease,
1225
- ): Record<string, unknown> | undefined {
1226
- if (!signal || !isCancellationException(error, signal)) return undefined;
1227
- try {
1228
- this.#runStore.assertLease(lease);
1229
- if (
1230
- ["validating-delivery", "ready"].includes(
1231
- this.#runStore.status(runId).state,
1232
- )
1233
- ) {
1234
- this.#transition(
1235
- runId,
1236
- "paused",
1237
- `${operationId}:delivery-cancelled`,
1238
- "operation-cancelled",
1239
- lease,
1240
- );
1241
- }
1242
- } catch (failure) {
1243
- if (!(failure instanceof Error) || failure.message !== "lease-fenced")
1244
- throw failure;
1245
- } finally {
1246
- this.#interruptOperation(runId, operationId);
1247
- }
1248
- return this.#statusByRun(runId);
1249
- }
1250
-
1251
- #deliveryFailure(
1252
- runId: string,
1253
- operationId: string,
1254
- error: unknown,
1255
- lease: OperationLease,
1256
- ): Record<string, unknown> | undefined {
1257
- if (!(error instanceof DeliveryValidationError)) return undefined;
1258
- this.#leasedTransaction(lease, () => {
1259
- this.#database
1260
- .prepare(
1261
- `UPDATE workflow_engine_runs
1262
- SET delivery_diagnostics_json = ? WHERE run_id = ?`,
1263
- )
1264
- .run(JSON.stringify(error.diagnostics), runId);
1265
- });
1266
- this.#transition(
1267
- runId,
1268
- "paused",
1269
- `${operationId}:delivery-invalid`,
1270
- "delivery-invalid",
1271
- lease,
1272
- );
1273
- return {
1274
- ...this.#statusByRun(runId),
1275
- delivery: {
1276
- code: "delivery-invalid",
1277
- diagnostics: [...error.diagnostics],
1278
- },
1279
- };
1280
- }
1281
-
1282
- #setTask(
1283
- runId: string,
1284
- taskId: string,
1285
- values: {
1286
- state?: string;
1287
- phase?: "red" | "green" | "refactor";
1288
- pauseCode?: string | null;
1289
- contextRequest?: WorkflowContextRequest | null;
1290
- attemptDiagnostic?: SafeAttemptDiagnostic | null;
1291
- routeId?: string | null;
1292
- routeFingerprint?: string | null;
1293
- queuePosition?: number | null;
1294
- },
1295
- lease?: OperationLease,
1296
- ): void {
1297
- if (values.attemptDiagnostic?.recovery && lease) {
1298
- const fact = values.attemptDiagnostic.recovery;
1299
- const prior = this.#recovery(runId, fact.key);
1300
- if (
1301
- !prior ||
1302
- fact.failures > prior.failures ||
1303
- fact.feedback.maxAttempts < prior.feedback.maxAttempts
1304
- )
1305
- this.#recordRecovery(runId, fact, lease);
1306
- }
1307
- const assignments: string[] = [];
1308
- const parameters: Array<string | number | null> = [];
1309
- if (values.state !== undefined) {
1310
- assignments.push("state = ?");
1311
- parameters.push(values.state);
1312
- }
1313
- if (values.phase !== undefined) {
1314
- assignments.push("phase = ?");
1315
- parameters.push(values.phase);
1316
- }
1317
- if (values.pauseCode !== undefined) {
1318
- assignments.push("pause_code = ?");
1319
- parameters.push(values.pauseCode);
1320
- }
1321
- if (values.contextRequest !== undefined) {
1322
- assignments.push("context_request_json = ?");
1323
- parameters.push(
1324
- values.contextRequest === null
1325
- ? null
1326
- : JSON.stringify(
1327
- normalizeWorkflowContextRequest(values.contextRequest),
1328
- ),
1329
- );
1330
- }
1331
- if (values.attemptDiagnostic !== undefined) {
1332
- assignments.push("attempt_diagnostic_json = ?");
1333
- parameters.push(
1334
- values.attemptDiagnostic === null
1335
- ? null
1336
- : JSON.stringify(
1337
- Object.fromEntries(
1338
- Object.entries(values.attemptDiagnostic).filter(
1339
- ([key]) => key !== "recovery",
1340
- ),
1341
- ),
1342
- ),
1343
- );
1344
- }
1345
- if (values.routeId !== undefined) {
1346
- assignments.push("route_id = ?");
1347
- parameters.push(values.routeId);
1348
- }
1349
- if (values.routeFingerprint !== undefined) {
1350
- assignments.push("route_fingerprint = ?");
1351
- parameters.push(values.routeFingerprint);
1352
- }
1353
- if (values.queuePosition !== undefined) {
1354
- assignments.push("queue_position = ?");
1355
- parameters.push(values.queuePosition);
1356
- }
1357
- if (assignments.length === 0) return;
1358
- const write = () => {
1359
- this.#database
1360
- .prepare(
1361
- `UPDATE workflow_engine_tasks SET ${assignments.join(", ")}
1362
- WHERE run_id = ? AND task_id = ?`,
1363
- )
1364
- .run(...parameters, runId, taskId);
1365
- };
1366
- if (lease) this.#leasedTransaction(lease, write);
1367
- else write();
1368
- }
1369
-
1370
- #setVerificationStatus(
1371
- runId: string,
1372
- status: WorkflowVerificationStatus | null,
1373
- lease: OperationLease,
1374
- ): void {
1375
- const serialized =
1376
- status === null
1377
- ? null
1378
- : JSON.stringify(normalizeWorkflowVerificationStatus(status));
1379
- this.#leasedTransaction(lease, () => {
1380
- this.#database
1381
- .prepare(
1382
- `UPDATE workflow_engine_runs SET verification_json = ?
1383
- WHERE run_id = ?`,
1384
- )
1385
- .run(serialized, runId);
1386
- });
1387
- }
1388
-
1389
- #queueTask(
1390
- runId: string,
1391
- row: EngineTaskRow,
1392
- lease: OperationLease,
1393
- reason: "conflict" | "capacity" = "conflict",
1394
- ): void {
1395
- const pauseCode = reason === "capacity" ? "worker-capacity" : null;
1396
- if (row.state === "queued" && row.queue_position !== null) {
1397
- if (row.pause_code !== pauseCode)
1398
- this.#setTask(runId, row.task_id, { pauseCode }, lease);
1399
- return;
1400
- }
1401
- const engineRun = this.#engineRun(runId);
1402
- this.#leasedTransaction(lease, () => {
1403
- this.#setTask(runId, row.task_id, {
1404
- state: "queued",
1405
- pauseCode,
1406
- queuePosition: engineRun.next_queue_position,
1407
- });
1408
- this.#database
1409
- .prepare(
1410
- `UPDATE workflow_engine_runs
1411
- SET next_queue_position = next_queue_position + 1
1412
- WHERE run_id = ?`,
1413
- )
1414
- .run(runId);
1415
- });
1416
- }
1417
-
1418
- #recordWorkspaceFacts(
1419
- runId: string,
1420
- outcome: WorkflowAttemptOutcome,
1421
- lease: OperationLease,
1422
- ): void {
1423
- const baselineRevisionId = outcome.baselineRevisionId;
1424
- const currentWorkspaceRevisionId =
1425
- outcome.kind === "phase-committed"
1426
- ? outcome.isolatedRevisionId
1427
- : outcome.currentWorkspaceRevisionId;
1428
- if (
1429
- (baselineRevisionId !== undefined && !SHA256.test(baselineRevisionId)) ||
1430
- (currentWorkspaceRevisionId !== undefined &&
1431
- !SHA256.test(currentWorkspaceRevisionId)) ||
1432
- (outcome.kind !== "phase-committed" &&
1433
- (baselineRevisionId === undefined) !==
1434
- (currentWorkspaceRevisionId === undefined))
1435
- ) {
1436
- throw new Error("workflow-attempt-workspace-fact-invalid");
1437
- }
1438
- if (currentWorkspaceRevisionId === undefined) return;
1439
- this.#leasedTransaction(lease, () => {
1440
- this.#database
1441
- .prepare(
1442
- `UPDATE workflow_engine_runs
1443
- SET baseline_workspace_revision = COALESCE(
1444
- baseline_workspace_revision, ?
1445
- ),
1446
- current_workspace_revision = ?,
1447
- cleanup_state = CASE
1448
- WHEN ? IS NULL THEN cleanup_state ELSE 'retained'
1449
- END
1450
- WHERE run_id = ?`,
1451
- )
1452
- .run(
1453
- baselineRevisionId ?? null,
1454
- currentWorkspaceRevisionId,
1455
- baselineRevisionId ?? null,
1456
- runId,
1457
- );
1458
- });
1459
- }
1460
-
1461
- #recordRouteFacts(
1462
- runId: string,
1463
- taskId: string,
1464
- outcome: WorkflowAttemptOutcome,
1465
- lease: OperationLease,
1466
- ): void {
1467
- if (
1468
- (outcome.routeId === undefined) !==
1469
- (outcome.routeFingerprint === undefined) ||
1470
- (outcome.routeId !== undefined && !IDENTIFIER.test(outcome.routeId)) ||
1471
- (outcome.routeFingerprint !== undefined &&
1472
- !SHA256.test(outcome.routeFingerprint))
1473
- ) {
1474
- throw new Error("workflow-attempt-route-fact-invalid");
1475
- }
1476
- if (!outcome.routeId || !outcome.routeFingerprint) return;
1477
- const routeId = outcome.routeId;
1478
- const routeFingerprint = outcome.routeFingerprint;
1479
- this.#leasedTransaction(lease, () => {
1480
- this.#setTask(runId, taskId, {
1481
- routeId,
1482
- routeFingerprint,
1483
- });
1484
- this.#database
1485
- .prepare(
1486
- `UPDATE workflow_engine_runs
1487
- SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
1488
- )
1489
- .run(routeId, routeFingerprint, runId);
1490
- });
1491
- }
1492
-
1493
- async #runTask(
1494
- runId: string,
1495
- row: EngineTaskRow,
1496
- operationId: string,
1497
- signal: AbortSignal,
1498
- lease: OperationLease,
1499
- onActivity?: (event: WorkflowActivityUpdate) => void,
1500
- ): Promise<void> {
1501
- const task = this.#taskPlan(row);
1502
- const orderedPhases = phases(task);
1503
- let phaseIndex = orderedPhases.indexOf(row.phase);
1504
- let artifactAttempts = 0;
1505
- const persistedContextRequest = row.context_request_json
1506
- ? normalizeWorkflowContextRequest(
1507
- JSON.parse(row.context_request_json) as unknown,
1508
- )
1509
- : undefined;
1510
- const persistedRedArtifactCorrection =
1511
- row.phase === "green" &&
1512
- persistedContextRequest !== undefined &&
1513
- classifyPersistedContextRequest(persistedContextRequest, task, "green")
1514
- .code === "red-artifact-constraint";
1515
- let redArtifactCorrection =
1516
- row.phase === "green" &&
1517
- (row.pause_code === "red-artifact-constraint" ||
1518
- persistedRedArtifactCorrection);
1519
- let requestedContext = persistedContextRequest;
1520
- let correctionContextRequest = redArtifactCorrection
1521
- ? persistedContextRequest
1522
- : undefined;
1523
- if (phaseIndex < 0) throw new Error("workflow-task-phase-invalid");
1524
- while (phaseIndex < orderedPhases.length) {
1525
- const phase = orderedPhases[phaseIndex];
1526
- row.phase = phase;
1527
- if (signal.aborted) {
1528
- this.#setTask(
1529
- runId,
1530
- row.task_id,
1531
- {
1532
- state: "paused",
1533
- phase,
1534
- pauseCode: "operation-cancelled",
1535
- contextRequest: null,
1536
- queuePosition: null,
1537
- },
1538
- lease,
1539
- );
1540
- return;
1541
- }
1542
- const current = this.#engineRun(runId);
1543
- if (current.current_revision === null)
1544
- throw new Error("delivery-not-bound");
1545
- const deliveryRevision = current.current_revision;
1546
- const plan = this.#planForRevision(runId, deliveryRevision);
1547
- this.#leasedTransaction(lease, () => {
1548
- for (const ref of requestedContext?.refs ?? []) {
1549
- if (
1550
- ref.kind === "requested-path" &&
1551
- ref.access === "read" &&
1552
- permitsContextRead(ref.path, task.roots) &&
1553
- (current.current_workspace_revision
1554
- ? current.baseline_workspace_revision &&
1555
- this.#worker.isContextReadAvailable?.({
1556
- runId,
1557
- deliveryRevision,
1558
- plan,
1559
- baselineRevisionId: current.baseline_workspace_revision,
1560
- currentWorkspaceRevisionId:
1561
- current.current_workspace_revision,
1562
- path: ref.path,
1563
- })
1564
- : isSafeRegularFile(this.#consumerRoot, ref.path))
1565
- ) {
1566
- this.#database
1567
- .prepare(
1568
- "INSERT INTO workflow_context_reads(run_id, task_id, path) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
1569
- )
1570
- .run(runId, row.task_id, ref.path);
1571
- }
1572
- }
1573
- });
1574
- const verificationStatus = current.verification_json
1575
- ? normalizeWorkflowVerificationStatus(
1576
- JSON.parse(current.verification_json) as unknown,
1577
- )
1578
- : undefined;
1579
- const priorDiagnostic = this.#safeAttemptDiagnostic(runId, row);
1580
- const priorRecovery = priorDiagnostic?.recovery;
1581
- const preparationKey = baselineRecoveryKey(task, phase);
1582
- const activeRecovery =
1583
- priorRecovery &&
1584
- [recoveryKey(task, phase, row, current), preparationKey].includes(
1585
- priorRecovery.key,
1586
- )
1587
- ? priorRecovery
1588
- : undefined;
1589
- const maxRecoveryAttempts = Math.min(
1590
- plan.verification.artifactCorrection.maxAttempts,
1591
- activeRecovery?.feedback.maxAttempts ?? Infinity,
1592
- );
1593
- if (
1594
- activeRecovery &&
1595
- activeRecovery.feedback.maxAttempts !== maxRecoveryAttempts
1596
- ) {
1597
- this.#leasedTransaction(lease, () => {
1598
- this.#database
1599
- .prepare(
1600
- "UPDATE workflow_recovery_incidents SET max_attempts = MIN(max_attempts, ?) WHERE run_id = ? AND incident_key = ?",
1601
- )
1602
- .run(maxRecoveryAttempts, runId, activeRecovery.key);
1603
- });
1604
- activeRecovery.feedback.maxAttempts = maxRecoveryAttempts;
1605
- }
1606
- const granted = this.#hasRecoveryGrant(
1607
- runId,
1608
- operationId,
1609
- activeRecovery?.key ?? recoveryKey(task, phase, row, current),
1610
- );
1611
- const verificationOnly =
1612
- activeRecovery?.key !== preparationKey &&
1613
- recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted) &&
1614
- (await this.#worker.hasPendingVerification?.({
1615
- runId,
1616
- operationId,
1617
- deliveryRevision: row.delivery_revision,
1618
- taskId: row.task_id,
1619
- phase,
1620
- task,
1621
- plan,
1622
- signal,
1623
- ...(current.baseline_workspace_revision
1624
- ? { baselineRevisionId: current.baseline_workspace_revision }
1625
- : {}),
1626
- ...(current.current_workspace_revision
1627
- ? { currentWorkspaceRevisionId: current.current_workspace_revision }
1628
- : {}),
1629
- })) === true;
1630
- if (
1631
- !verificationOnly &&
1632
- activeRecovery &&
1633
- recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted)
1634
- ) {
1635
- this.#setTask(
1636
- runId,
1637
- row.task_id,
1638
- {
1639
- state: "paused",
1640
- phase,
1641
- pauseCode: activeRecovery.feedback.code,
1642
- attemptDiagnostic: priorDiagnostic ?? null,
1643
- queuePosition: null,
1644
- },
1645
- lease,
1646
- );
1647
- return;
1648
- }
1649
- if (activeRecovery && activeRecovery.key !== preparationKey)
1650
- artifactAttempts = Math.max(artifactAttempts, activeRecovery.failures);
1651
- const repair =
1652
- verificationStatus?.scope === "change-task-affected" &&
1653
- verificationStatus.attribution === "introduced" &&
1654
- verificationStatus.taskId === row.task_id &&
1655
- verificationStatus.failureIdentities &&
1656
- verificationStatus.failureIdentities.length > 0
1657
- ? {
1658
- attribution: "introduced" as const,
1659
- failureIdentities: [...verificationStatus.failureIdentities],
1660
- }
1661
- : undefined;
1662
- const artifactCorrection =
1663
- redArtifactCorrection && phase === "green"
1664
- ? {
1665
- code: "red-artifact-constraint" as const,
1666
- attempt: artifactAttempts + 1,
1667
- maxAttempts: maxRecoveryAttempts,
1668
- ...(correctionContextRequest
1669
- ? {
1670
- contextRequest: structuredClone(correctionContextRequest),
1671
- }
1672
- : {}),
1673
- }
1674
- : undefined;
1675
- if (this.#worker.prepareTask) {
1676
- // Reserve this selected task's conflicts while asynchronous baseline
1677
- // preparation runs, without reserving a Worker attempt.
1678
- this.#setTask(
1679
- runId,
1680
- row.task_id,
1681
- { state: "phase-ready", phase, queuePosition: null },
1682
- lease,
1683
- );
1684
- if (granted && activeRecovery?.key === preparationKey) {
1685
- this.#leasedTransaction(lease, () => {
1686
- const consumed = this.#database
1687
- .prepare(
1688
- "UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
1689
- )
1690
- .run(runId, operationId, preparationKey);
1691
- if (consumed.changes !== 1)
1692
- throw new Error("recovery-request-stale");
1693
- this.#database
1694
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
1695
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
1696
- .run(runId, preparationKey, runId);
1697
- });
1698
- }
1699
- const prepared = await this.#worker.prepareTask({
1700
- runId,
1701
- operationId,
1702
- deliveryRevision: row.delivery_revision,
1703
- taskId: row.task_id,
1704
- phase,
1705
- task: structuredClone(task),
1706
- plan: structuredClone(plan),
1707
- ...(current.baseline_workspace_revision
1708
- ? { baselineRevisionId: current.baseline_workspace_revision }
1709
- : {}),
1710
- ...(current.current_workspace_revision
1711
- ? { currentWorkspaceRevisionId: current.current_workspace_revision }
1712
- : {}),
1713
- signal,
1714
- });
1715
- this.#runStore.assertLease(lease);
1716
- if (prepared.kind === "phase-committed")
1717
- throw new Error("workflow-preparation-outcome-invalid");
1718
- this.#recordWorkspaceFacts(
1719
- runId,
1720
- {
1721
- ...prepared,
1722
- kind: "paused",
1723
- code: "task-prepared",
1724
- // Preparation has no merge authority over a sibling's newer commit.
1725
- ...(prepared.currentWorkspaceRevisionId
1726
- ? {
1727
- currentWorkspaceRevisionId:
1728
- this.#engineRun(runId).current_workspace_revision ??
1729
- prepared.currentWorkspaceRevisionId,
1730
- }
1731
- : {}),
1732
- },
1733
- lease,
1734
- );
1735
- // Preparation may finish successfully while close is settling it.
1736
- // Re-enter the cancellation guard before granting any Worker work.
1737
- if (signal.aborted) continue;
1738
- if (prepared.kind !== "prepared") {
1739
- const prerequisite =
1740
- prepared.kind !== "operation-cancelled"
1741
- ? normalizeVerificationPrerequisite(prepared.prerequisite)
1742
- : undefined;
1743
- if (prerequisite) {
1744
- const verification =
1745
- prerequisite.scope === "baseline-full-suite"
1746
- ? plan.verification.baseline?.fullSuite
1747
- : (task.baselineVerification ?? task.affectedVerification);
1748
- if (
1749
- !verification ||
1750
- verification.id !== prerequisite.verificationId ||
1751
- (prerequisite.scope === "baseline-task-affected" &&
1752
- prerequisite.taskId !== row.task_id) ||
1753
- (prerequisite.input &&
1754
- !verificationInputPaths(verification).includes(
1755
- prerequisite.input.path,
1756
- )) ||
1757
- (prerequisite.producer &&
1758
- !plan.outputs.some(
1759
- (output) =>
1760
- output.path === prerequisite.input?.path &&
1761
- output.producer.taskId === prerequisite.producer?.taskId &&
1762
- output.producer.phase === prerequisite.producer?.phase,
1763
- ))
1764
- )
1765
- throw new Error("workflow-prerequisite-evidence-invalid");
1766
- }
1767
- if (prerequisite && prepared.kind === "paused") {
1768
- const identity = hash(
1769
- canonicalJson({
1770
- ...prerequisite,
1771
- verificationId: undefined,
1772
- taskId: undefined,
1773
- producer: prerequisite.producer
1774
- ? { phase: prerequisite.producer.phase }
1775
- : undefined,
1776
- }),
1777
- );
1778
- const priorPreparation = this.#recovery(runId, preparationKey);
1779
- if (priorPreparation?.feedback.prerequisiteIdentity !== identity) {
1780
- const failures = (priorPreparation?.failures ?? 0) + 1;
1781
- this.#recordRecovery(
1782
- runId,
1783
- {
1784
- key: preparationKey,
1785
- failures,
1786
- feedback: {
1787
- code: prepared.code,
1788
- attempt: failures + 1,
1789
- maxAttempts: maxRecoveryAttempts,
1790
- strategy: "repair-verification",
1791
- prerequisiteIdentity: identity,
1792
- },
1793
- },
1794
- lease,
1795
- );
1796
- }
1797
- }
1798
- if (prepared.kind !== "operation-cancelled" && prepared.verification)
1799
- this.#setVerificationStatus(runId, prepared.verification, lease);
1800
- this.#setTask(
1801
- runId,
1802
- row.task_id,
1803
- {
1804
- state:
1805
- prepared.kind === "approval-needed"
1806
- ? "approval-needed"
1807
- : "paused",
1808
- phase,
1809
- pauseCode:
1810
- prepared.kind === "operation-cancelled"
1811
- ? "operation-cancelled"
1812
- : prepared.code,
1813
- attemptDiagnostic: {
1814
- ...priorDiagnostic,
1815
- prerequisite,
1816
- },
1817
- queuePosition: null,
1818
- },
1819
- lease,
1820
- );
1821
- return;
1822
- }
1823
- if (priorDiagnostic) delete priorDiagnostic.prerequisite;
1824
- if (activeRecovery?.key === preparationKey) {
1825
- // A successful original-baseline observation resolves this incident.
1826
- // Keep its events and immutable failure evidence, then re-enter the
1827
- // ordinary phase recovery checks before reserving Worker work.
1828
- this.#leasedTransaction(lease, () => {
1829
- this.#database
1830
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
1831
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
1832
- .run(runId, preparationKey, runId);
1833
- this.#database
1834
- .prepare(
1835
- "DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
1836
- )
1837
- .run(runId, preparationKey);
1838
- });
1839
- if (priorDiagnostic) delete priorDiagnostic.recovery;
1840
- const retainedDiagnostic =
1841
- priorDiagnostic && Object.keys(priorDiagnostic).length
1842
- ? priorDiagnostic
1843
- : null;
1844
- this.#setTask(
1845
- runId,
1846
- row.task_id,
1847
- { attemptDiagnostic: retainedDiagnostic },
1848
- lease,
1849
- );
1850
- row.attempt_diagnostic_json = retainedDiagnostic
1851
- ? JSON.stringify(retainedDiagnostic)
1852
- : null;
1853
- continue;
1854
- }
1855
- }
1856
- this.#setTask(
1857
- runId,
1858
- row.task_id,
1859
- {
1860
- state: "phase-running",
1861
- phase,
1862
- pauseCode: artifactCorrection?.code ?? null,
1863
- contextRequest: requestedContext ?? null,
1864
- attemptDiagnostic: priorDiagnostic ?? null,
1865
- queuePosition: null,
1866
- },
1867
- lease,
1868
- );
1869
- if (
1870
- !this.#reserveWork(
1871
- runId,
1872
- plan,
1873
- recoveryKey(task, phase, row, current),
1874
- lease,
1875
- granted ? operationId : undefined,
1876
- )
1877
- ) {
1878
- this.#setTask(
1879
- runId,
1880
- row.task_id,
1881
- {
1882
- state: "paused",
1883
- phase,
1884
- pauseCode: "change-work-budget-exhausted",
1885
- queuePosition: null,
1886
- },
1887
- lease,
1888
- );
1889
- return;
1890
- }
1891
- let initialReservation = true;
1892
- let outcome: WorkflowAttemptOutcome;
1893
- try {
1894
- outcome = await this.#worker.runAttempt({
1895
- runId,
1896
- operationId,
1897
- deliveryRevision: row.delivery_revision,
1898
- taskId: row.task_id,
1899
- phase,
1900
- task: structuredClone(task),
1901
- contextReadPaths: this.#contextReads(runId, row.task_id),
1902
- plan: structuredClone(plan),
1903
- ...(current.baseline_workspace_revision
1904
- ? { baselineRevisionId: current.baseline_workspace_revision }
1905
- : {}),
1906
- ...(current.current_workspace_revision
1907
- ? {
1908
- currentWorkspaceRevisionId: current.current_workspace_revision,
1909
- }
1910
- : {}),
1911
- ...(row.route_id || current.route_id
1912
- ? { routeId: row.route_id ?? current.route_id ?? undefined }
1913
- : {}),
1914
- ...(row.route_fingerprint || current.route_fingerprint
1915
- ? {
1916
- routeFingerprint:
1917
- row.route_fingerprint ??
1918
- current.route_fingerprint ??
1919
- undefined,
1920
- }
1921
- : {}),
1922
- ...(repair ? { repair } : {}),
1923
- ...(artifactCorrection ? { artifactCorrection } : {}),
1924
- ...(requestedContext ? { contextRequest: requestedContext } : {}),
1925
- ...(granted ? { additionalAttempt: true } : {}),
1926
- ...(verificationOnly ? { verificationOnly: true } : {}),
1927
- ...(activeRecovery
1928
- ? { recoveryFeedback: activeRecovery.feedback }
1929
- : {}),
1930
- signal,
1931
- recoveryDecision: (request) => {
1932
- signal.throwIfAborted();
1933
- this.#runStore.assertLease(lease);
1934
- return decideRecoveryAction(plan, request, {
1935
- additionalAttempt: granted,
1936
- verificationOnly,
1937
- });
1938
- },
1939
- reserveCandidate: () => {
1940
- signal.throwIfAborted();
1941
- if (verificationOnly) return false;
1942
- if (initialReservation) {
1943
- initialReservation = false;
1944
- return true;
1945
- }
1946
- if (granted) return false;
1947
- return this.#reserveWork(
1948
- runId,
1949
- plan,
1950
- recoveryKey(task, phase, row, current),
1951
- lease,
1952
- );
1953
- },
1954
- onActivity: (event) => {
1955
- const projection = this.#runStore.status(runId);
1956
- emitWorkflowActivity(onActivity, {
1957
- ...event,
1958
- stage: projection.stage,
1959
- runId,
1960
- ...(projection.change ? { change: projection.change } : {}),
1961
- taskId: row.task_id,
1962
- phase,
1963
- objective: task.objective,
1964
- legalCommands: ["status", "cancel", "discard"],
1965
- });
1966
- },
1967
- });
1968
- } catch (error) {
1969
- if (isCancellationException(error, signal)) {
1970
- outcome = { kind: "operation-cancelled", code: "cancelled" };
1971
- } else {
1972
- throw error;
1973
- }
1974
- }
1975
- if (!isRecord(outcome) || typeof outcome.kind !== "string") {
1976
- throw new Error("workflow-attempt-outcome-invalid");
1977
- }
1978
- this.#recordWorkspaceFacts(runId, outcome, lease);
1979
- this.#recordRouteFacts(runId, row.task_id, outcome, lease);
1980
- if (outcome.routeId && outcome.routeFingerprint) {
1981
- row.route_id = outcome.routeId;
1982
- row.route_fingerprint = outcome.routeFingerprint;
1983
- }
1984
- if (
1985
- outcome.kind !== "phase-committed" &&
1986
- outcome.kind !== "operation-cancelled" &&
1987
- outcome.verification
1988
- ) {
1989
- this.#setVerificationStatus(runId, outcome.verification, lease);
1990
- }
1991
- if (
1992
- outcome.kind !== "phase-committed" &&
1993
- outcome.kind !== "operation-cancelled" &&
1994
- outcome.contextRequest
1995
- ) {
1996
- requestedContext = normalizeWorkflowContextRequest(
1997
- outcome.contextRequest,
1998
- );
1999
- }
2000
- if (
2001
- outcome.kind !== "phase-committed" &&
2002
- outcome.kind !== "operation-cancelled" &&
2003
- outcome.attemptDiagnostic
2004
- ) {
2005
- const previous = row.attempt_diagnostic_json
2006
- ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
2007
- : undefined;
2008
- const fingerprint = hash(
2009
- outcome.code,
2010
- outcome.attemptDiagnostic.finalCategory ?? "unknown",
2011
- outcome.attemptDiagnostic.schema ?? "unknown",
2012
- JSON.stringify(outcome.attemptDiagnostic.identityMismatch ?? []),
2013
- ).slice(0, 16);
2014
- const sameFailure = previous?.fingerprint === fingerprint;
2015
- const sameFailureCount = sameFailure
2016
- ? (previous?.sameFailureCount ?? 1) + 1
2017
- : 1;
2018
- const attemptDiagnostic = {
2019
- ...outcome.attemptDiagnostic,
2020
- ...(priorDiagnostic?.recovery
2021
- ? { recovery: priorDiagnostic.recovery }
2022
- : {}),
2023
- fingerprint,
2024
- sameFailureCount,
2025
- ...(sameFailureCount > 1
2026
- ? { action: "rebind-or-revise-delivery" as const }
2027
- : {}),
2028
- };
2029
- // Only parent preparation can contribute original-baseline authority.
2030
- delete attemptDiagnostic.prerequisite;
2031
- this.#setTask(runId, row.task_id, { attemptDiagnostic }, lease);
2032
- row.attempt_diagnostic_json = JSON.stringify(attemptDiagnostic);
2033
- }
2034
- if (
2035
- outcome.kind === "paused" &&
2036
- outcome.code === "repair-attempts-exhausted"
2037
- ) {
2038
- const maxAttempts = plan.verification.repair.maxAttempts;
2039
- const failures = Math.max(
2040
- maxAttempts,
2041
- (activeRecovery?.failures ?? 0) +
2042
- (granted || verificationOnly ? 1 : 0),
2043
- );
2044
- const diagnostic: SafeAttemptDiagnostic = {
2045
- recovery: {
2046
- key: recoveryKey(task, phase, row, this.#engineRun(runId)),
2047
- failures,
2048
- feedback: {
2049
- code: outcome.code,
2050
- attempt: failures + 1,
2051
- maxAttempts,
2052
- strategy: "repair-verification",
2053
- },
2054
- },
2055
- };
2056
- this.#setTask(
2057
- runId,
2058
- row.task_id,
2059
- { attemptDiagnostic: diagnostic },
2060
- lease,
2061
- );
2062
- row.attempt_diagnostic_json = JSON.stringify(diagnostic);
2063
- }
2064
- if (
2065
- outcome.kind === "retryable" &&
2066
- ["artifact", "stale", "verification"].includes(
2067
- outcome.retryPolicy ?? "",
2068
- )
2069
- ) {
2070
- const key = recoveryKey(task, phase, row, this.#engineRun(runId));
2071
- const failures =
2072
- (activeRecovery?.key === key ? activeRecovery.failures : 0) + 1;
2073
- const strategy =
2074
- outcome.code === "needs-task-split" ||
2075
- outcome.code === "task-split-needed"
2076
- ? ("compact-patch" as const)
2077
- : outcome.retryPolicy === "stale"
2078
- ? ("refresh-candidate" as const)
2079
- : outcome.retryPolicy === "verification"
2080
- ? ("repair-verification" as const)
2081
- : ("revise-candidate" as const);
2082
- const diagnostic: SafeAttemptDiagnostic = {
2083
- ...this.#safeAttemptDiagnostic(runId, row),
2084
- recovery: {
2085
- key,
2086
- failures,
2087
- feedback: {
2088
- code: outcome.code,
2089
- attempt: failures + 1,
2090
- maxAttempts: maxRecoveryAttempts,
2091
- strategy,
2092
- ...(outcome.verification?.failureIdentities
2093
- ? { failureIdentities: outcome.verification.failureIdentities }
2094
- : {}),
2095
- },
2096
- },
2097
- };
2098
- this.#setTask(
2099
- runId,
2100
- row.task_id,
2101
- { attemptDiagnostic: diagnostic },
2102
- lease,
2103
- );
2104
- row.attempt_diagnostic_json = JSON.stringify(diagnostic);
2105
- if (outcome.code === "red-artifact-constraint") {
2106
- redArtifactCorrection = true;
2107
- correctionContextRequest = outcome.contextRequest
2108
- ? normalizeWorkflowContextRequest(outcome.contextRequest)
2109
- : correctionContextRequest;
2110
- requestedContext = correctionContextRequest ?? requestedContext;
2111
- }
2112
- artifactAttempts = Math.max(artifactAttempts + 1, failures);
2113
- const maxAttempts = maxRecoveryAttempts;
2114
- if (artifactAttempts < maxAttempts) {
2115
- this.#setTask(
2116
- runId,
2117
- row.task_id,
2118
- {
2119
- state: "retryable",
2120
- phase,
2121
- pauseCode: redArtifactCorrection
2122
- ? "red-artifact-constraint"
2123
- : outcome.code,
2124
- contextRequest: requestedContext ?? null,
2125
- queuePosition: null,
2126
- },
2127
- lease,
2128
- );
2129
- emitWorkflowActivity(onActivity, {
2130
- state: "retrying",
2131
- stage: this.#runStore.status(runId).stage,
2132
- runId,
2133
- taskId: row.task_id,
2134
- phase,
2135
- objective: task.objective,
2136
- legalCommands: ["status", "cancel", "discard"],
2137
- code: outcome.code,
2138
- attempt: artifactAttempts,
2139
- maxAttempts,
2140
- wait: "artifact-correction",
2141
- });
2142
- continue;
2143
- }
2144
- }
2145
- if (outcome.kind === "phase-committed") {
2146
- if (
2147
- !SHA256.test(outcome.artifactHash) ||
2148
- !SHA256.test(outcome.isolatedRevisionId) ||
2149
- (outcome.baselineRevisionId !== undefined &&
2150
- !SHA256.test(outcome.baselineRevisionId)) ||
2151
- outcome.classification !== expectedClassification(phase)
2152
- ) {
2153
- throw new Error("workflow-attempt-fact-invalid");
2154
- }
2155
- this.#leasedTransaction(lease, () => {
2156
- const resolvedKey = recoveryKey(task, phase, row, current);
2157
- if (this.#recovery(runId, resolvedKey)) {
2158
- this.#database
2159
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2160
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
2161
- .run(runId, resolvedKey, runId);
2162
- }
2163
- this.#database
2164
- .prepare(
2165
- "DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
2166
- )
2167
- .run(runId, recoveryKey(task, phase, row, current));
2168
- });
2169
- if (repair) {
2170
- this.#setVerificationStatus(runId, null, lease);
2171
- this.#setTask(
2172
- runId,
2173
- row.task_id,
2174
- {
2175
- state: "verified",
2176
- phase,
2177
- pauseCode: null,
2178
- contextRequest: null,
2179
- attemptDiagnostic: null,
2180
- queuePosition: null,
2181
- },
2182
- lease,
2183
- );
2184
- return;
2185
- }
2186
- artifactAttempts = 0;
2187
- row.attempt_diagnostic_json = null;
2188
- redArtifactCorrection = false;
2189
- requestedContext = undefined;
2190
- correctionContextRequest = undefined;
2191
- phaseIndex += 1;
2192
- if (phaseIndex >= orderedPhases.length) {
2193
- this.#setTask(
2194
- runId,
2195
- row.task_id,
2196
- {
2197
- state: "verified",
2198
- phase,
2199
- pauseCode: null,
2200
- contextRequest: null,
2201
- attemptDiagnostic: null,
2202
- queuePosition: null,
2203
- },
2204
- lease,
2205
- );
2206
- return;
2207
- }
2208
- this.#setTask(
2209
- runId,
2210
- row.task_id,
2211
- {
2212
- state: "phase-ready",
2213
- phase: orderedPhases[phaseIndex],
2214
- pauseCode: null,
2215
- contextRequest: null,
2216
- attemptDiagnostic: null,
2217
- queuePosition: null,
2218
- },
2219
- lease,
2220
- );
2221
- continue;
2222
- }
2223
- if (outcome.kind === "operation-cancelled") {
2224
- this.#setTask(
2225
- runId,
2226
- row.task_id,
2227
- {
2228
- state: "paused",
2229
- phase,
2230
- pauseCode: "operation-cancelled",
2231
- contextRequest: null,
2232
- queuePosition: null,
2233
- },
2234
- lease,
2235
- );
2236
- return;
2237
- }
2238
- if (
2239
- outcome.kind === "paused" ||
2240
- outcome.kind === "retryable" ||
2241
- outcome.kind === "approval-needed"
2242
- ) {
2243
- if (typeof outcome.code !== "string" || outcome.code.length === 0) {
2244
- throw new Error("workflow-attempt-outcome-invalid");
2245
- }
2246
- const approvalCodeInvalid =
2247
- outcome.kind === "approval-needed" &&
2248
- !isWorkflowApprovalCode(outcome.code);
2249
- this.#setTask(
2250
- runId,
2251
- row.task_id,
2252
- {
2253
- state: approvalCodeInvalid ? "paused" : outcome.kind,
2254
- phase,
2255
- pauseCode: approvalCodeInvalid
2256
- ? "approval-code-invalid"
2257
- : outcome.code,
2258
- contextRequest: approvalCodeInvalid
2259
- ? null
2260
- : (requestedContext ?? null),
2261
- queuePosition: null,
2262
- },
2263
- lease,
2264
- );
2265
- return;
2266
- }
2267
- throw new Error("workflow-attempt-outcome-invalid");
2268
- }
2269
- }
2270
-
2271
- #firstPaused(rows: EngineTaskRow[]): EngineTaskRow | undefined {
2272
- return firstPausedTask(rows);
2273
- }
2274
-
2275
- #recoverLegacyContextApproval(runId: string): boolean {
2276
- const projection = this.#runStore.status(runId);
2277
- if (!["approval-needed", "paused"].includes(projection.state)) return false;
2278
- const rows = this.#tasks(runId);
2279
- const approvalRows = rows.filter((row) => row.state === "approval-needed");
2280
- if (approvalRows.length === 0) return false;
2281
- const reclassified = rows.flatMap((row) => {
2282
- if (
2283
- row.state !== "approval-needed" ||
2284
- row.pause_code !== "boundary-review-needed" ||
2285
- row.context_request_json === null
2286
- ) {
2287
- return [];
2288
- }
2289
- try {
2290
- const request = normalizeWorkflowContextRequest(
2291
- JSON.parse(row.context_request_json) as unknown,
2292
- );
2293
- const classified = classifyPersistedContextRequest(
2294
- request,
2295
- this.#taskPlan(row),
2296
- row.phase,
2297
- );
2298
- if (classified.kind === "approval-needed") return [];
2299
- return [
2300
- {
2301
- row,
2302
- state: classified.kind === "retryable" ? "retryable" : "paused",
2303
- code: classified.code,
2304
- contextRequest: JSON.stringify(
2305
- normalizeWorkflowContextRequest(classified.contextRequest),
2306
- ),
2307
- },
2308
- ];
2309
- } catch {
2310
- return [];
2311
- }
2312
- });
2313
- const reclassifiedTaskIds = new Set(
2314
- reclassified.map((entry) => entry.row.task_id),
2315
- );
2316
- const retainedApprovals = approvalRows.filter(
2317
- (row) => !reclassifiedTaskIds.has(row.task_id),
2318
- );
2319
- const updateTasks = (database: DatabaseSync) => {
2320
- for (const entry of reclassified) {
2321
- const updated = database
2322
- .prepare(
2323
- `UPDATE workflow_engine_tasks
2324
- SET state = ?, phase = ?, pause_code = ?, context_request_json = ?,
2325
- queue_position = NULL
2326
- WHERE run_id = ? AND task_id = ?
2327
- AND state = 'approval-needed'
2328
- AND pause_code = 'boundary-review-needed'`,
2329
- )
2330
- .run(
2331
- entry.state,
2332
- entry.row.phase,
2333
- entry.code,
2334
- entry.contextRequest,
2335
- runId,
2336
- entry.row.task_id,
2337
- );
2338
- if (Number(updated.changes) !== 1) {
2339
- throw new Error("workflow-legacy-context-race");
2340
- }
2341
- }
2342
- };
2343
- const transitionWithUpdates = (
2344
- to: "paused" | "approval-needed",
2345
- code: string,
2346
- ) =>
2347
- this.#runStore.transitionAtomically(
2348
- {
2349
- runId,
2350
- to,
2351
- operationId: `legacy-context-${hash(
2352
- runId,
2353
- String(projection.sequence),
2354
- to,
2355
- code,
2356
- JSON.stringify(
2357
- reclassified.map((entry) => [entry.row.task_id, entry.code]),
2358
- ),
2359
- ).slice(0, 40)}`,
2360
- code,
2361
- },
2362
- updateTasks,
2363
- );
2364
- if (retainedApprovals.length > 0) {
2365
- if (projection.state === "approval-needed") {
2366
- if (reclassified.length === 0) return false;
2367
- this.#transaction(() => updateTasks(this.#database));
2368
- } else {
2369
- transitionWithUpdates(
2370
- "approval-needed",
2371
- retainedApprovals[0].pause_code ?? "approval-code-invalid",
2372
- );
2373
- }
2374
- return true;
2375
- }
2376
- const first = reclassified[0];
2377
- if (!first) return false;
2378
- transitionWithUpdates("paused", first.code);
2379
- return true;
2380
- }
2381
-
2382
- async #finishChange(
2383
- runId: string,
2384
- operationId: string,
2385
- signal: AbortSignal,
2386
- lease: OperationLease,
2387
- onActivity?: (event: WorkflowActivityUpdate) => void,
2388
- ): Promise<Record<string, unknown>> {
2389
- const plan = this.#currentPlan(runId);
2390
- let engineRun = this.#engineRun(runId);
2391
- if (engineRun.current_revision === null)
2392
- throw new Error("delivery-not-bound");
2393
- const deliveryRevision = engineRun.current_revision;
2394
- this.#transition(
2395
- runId,
2396
- "change-verifying",
2397
- `${operationId}:change-verifying`,
2398
- undefined,
2399
- lease,
2400
- );
2401
- const projection = this.#runStore.status(runId);
2402
- emitWorkflowActivity(onActivity, {
2403
- state: "verifying",
2404
- stage: projection.stage,
2405
- runId,
2406
- ...(projection.change ? { change: projection.change } : {}),
2407
- objective: projection.change
2408
- ? `verify change ${projection.change}`
2409
- : "verify workflow change",
2410
- legalCommands: ["status", "cancel", "discard"],
2411
- });
2412
- const verification = await this.#changeVerifier.verify({
2413
- taskEvidence: this.#tasks(runId).map((row) => ({
2414
- taskId: row.task_id,
2415
- deliveryRevision: row.delivery_revision,
2416
- })),
2417
- runId,
2418
- deliveryRevision,
2419
- plan: structuredClone(plan),
2420
- ...(engineRun.baseline_workspace_revision
2421
- ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2422
- : {}),
2423
- ...(engineRun.current_workspace_revision
2424
- ? {
2425
- currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2426
- }
2427
- : {}),
2428
- signal,
2429
- });
2430
- this.#runStore.assertLease(lease);
2431
- if (verification.kind !== "verified") {
2432
- if (verification.verification) {
2433
- this.#setVerificationStatus(runId, verification.verification, lease);
2434
- }
2435
- if (
2436
- verification.code === "verification-repair-required" &&
2437
- verification.verification?.scope === "change-task-affected" &&
2438
- verification.verification.attribution === "introduced" &&
2439
- verification.verification.taskId
2440
- ) {
2441
- const task = this.#tasks(runId).find(
2442
- (candidate) =>
2443
- candidate.task_id === verification.verification?.taskId,
2444
- );
2445
- if (task) {
2446
- const repairKey = recoveryKey(
2447
- this.#taskPlan(task),
2448
- task.phase,
2449
- task,
2450
- this.#engineRun(runId),
2451
- );
2452
- const used = (
2453
- this.#database
2454
- .prepare(
2455
- "SELECT COUNT(*) AS count FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ? AND kind = 'repair'",
2456
- )
2457
- .get(runId, repairKey) as { count: number }
2458
- ).count;
2459
- if (used >= plan.verification.repair.maxAttempts) {
2460
- this.#setTask(
2461
- runId,
2462
- task.task_id,
2463
- {
2464
- state: "paused",
2465
- phase: task.phase,
2466
- pauseCode: "repair-attempts-exhausted",
2467
- attemptDiagnostic: {
2468
- recovery: {
2469
- key: recoveryKey(
2470
- this.#taskPlan(task),
2471
- task.phase,
2472
- task,
2473
- this.#engineRun(runId),
2474
- ),
2475
- failures: used,
2476
- feedback: {
2477
- code: "repair-attempts-exhausted",
2478
- attempt: used + 1,
2479
- maxAttempts: plan.verification.repair.maxAttempts,
2480
- strategy: "repair-verification",
2481
- failureIdentities:
2482
- verification.verification.failureIdentities,
2483
- },
2484
- },
2485
- },
2486
- queuePosition: null,
2487
- },
2488
- lease,
2489
- );
2490
- this.#transition(
2491
- runId,
2492
- "paused",
2493
- `${operationId}:repair-attempts-exhausted:${task.task_id}`,
2494
- "repair-attempts-exhausted",
2495
- lease,
2496
- );
2497
- return this.#statusByRun(runId);
2498
- }
2499
- this.#leasedTransaction(lease, () => {
2500
- this.#database
2501
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2502
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'repair' FROM workflow_recovery_events WHERE run_id = ?`)
2503
- .run(runId, repairKey, runId);
2504
- });
2505
- const taskPlan = this.#taskPlan(task);
2506
- this.#setTask(
2507
- runId,
2508
- task.task_id,
2509
- {
2510
- state: "repairable",
2511
- phase: taskPlan.phases.refactor ? "refactor" : "green",
2512
- pauseCode: verification.code,
2513
- queuePosition: null,
2514
- },
2515
- lease,
2516
- );
2517
- this.#transition(
2518
- runId,
2519
- "running",
2520
- `${operationId}:repairable:${task.task_id}`,
2521
- undefined,
2522
- lease,
2523
- );
2524
- return this.#advance(runId, operationId, signal, lease, onActivity);
2525
- }
2526
- }
2527
- const approvalCodeInvalid =
2528
- verification.kind === "approval-needed" &&
2529
- !isWorkflowApprovalCode(verification.code);
2530
- const state =
2531
- verification.kind === "approval-needed" && !approvalCodeInvalid
2532
- ? "approval-needed"
2533
- : "paused";
2534
- const code = approvalCodeInvalid
2535
- ? "approval-code-invalid"
2536
- : verification.code;
2537
- this.#transition(
2538
- runId,
2539
- state,
2540
- `${operationId}:change-verification-${state}`,
2541
- code,
2542
- lease,
2543
- );
2544
- return this.#statusByRun(runId);
2545
- }
2546
- if (verification.currentWorkspaceRevisionId !== undefined) {
2547
- const currentWorkspaceRevisionId =
2548
- verification.currentWorkspaceRevisionId;
2549
- if (!SHA256.test(currentWorkspaceRevisionId)) {
2550
- throw new Error("workflow-verification-workspace-fact-invalid");
2551
- }
2552
- this.#leasedTransaction(lease, () => {
2553
- this.#database
2554
- .prepare(
2555
- `UPDATE workflow_engine_runs
2556
- SET current_workspace_revision = ?, cleanup_state = 'retained'
2557
- WHERE run_id = ?`,
2558
- )
2559
- .run(currentWorkspaceRevisionId, runId);
2560
- });
2561
- engineRun = this.#engineRun(runId);
2562
- }
2563
- this.#transition(
2564
- runId,
2565
- "ready-to-apply",
2566
- `${operationId}:ready-to-apply`,
2567
- undefined,
2568
- lease,
2569
- );
2570
- if (!this.#application) {
2571
- this.#transition(
2572
- runId,
2573
- "paused",
2574
- `${operationId}:application-unavailable`,
2575
- "application-unavailable",
2576
- lease,
2577
- );
2578
- return this.#statusByRun(runId);
2579
- }
2580
- const transactionId = `apply-${hash(
2581
- runId,
2582
- String(deliveryRevision),
2583
- operationId,
2584
- ).slice(0, 40)}`;
2585
- this.#leasedTransaction(lease, () => {
2586
- this.#database
2587
- .prepare(
2588
- `UPDATE workflow_engine_runs SET transaction_id = ? WHERE run_id = ?`,
2589
- )
2590
- .run(transactionId, runId);
2591
- });
2592
- const applicationInput = {
2593
- transactionId,
2594
- runId,
2595
- consumerRoot: this.#consumerRoot,
2596
- deliveryRevision,
2597
- plan: structuredClone(plan),
2598
- contextReadPaths: this.#contextReads(runId),
2599
- ...(engineRun.baseline_workspace_revision
2600
- ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2601
- : {}),
2602
- ...(engineRun.current_workspace_revision
2603
- ? {
2604
- currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2605
- }
2606
- : {}),
2607
- signal,
2608
- };
2609
- let application: Record<string, unknown>;
2610
- if (this.#application.prepare && this.#application.applyPrepared) {
2611
- const prepared = await this.#application.prepare(applicationInput);
2612
- this.#runStore.assertLease(lease);
2613
- if (prepared.state !== "prepared") {
2614
- return this.#settleApplication(
2615
- runId,
2616
- operationId,
2617
- prepared,
2618
- lease,
2619
- onActivity,
2620
- );
2621
- }
2622
- this.#transition(
2623
- runId,
2624
- "applying",
2625
- `${operationId}:applying`,
2626
- undefined,
2627
- lease,
2628
- );
2629
- emitWorkflowActivity(onActivity, {
2630
- state: "applying",
2631
- stage: projection.stage,
2632
- runId,
2633
- ...(projection.change ? { change: projection.change } : {}),
2634
- objective: projection.change
2635
- ? `apply change ${projection.change}`
2636
- : "apply workflow change",
2637
- legalCommands: ["status", "cancel", "discard"],
2638
- });
2639
- const active = this.#active.get(runId);
2640
- if (active) active.kind = "applying";
2641
- try {
2642
- application = await this.#application.applyPrepared(applicationInput);
2643
- } catch (error) {
2644
- this.#runStore.assertLease(lease);
2645
- if (this.#runStore.status(runId).state === "applying")
2646
- this.#transition(
2647
- runId,
2648
- "recovering",
2649
- `${operationId}:application-interrupted`,
2650
- "operation-interrupted",
2651
- lease,
2652
- );
2653
- throw error;
2654
- }
2655
- } else {
2656
- this.#transition(
2657
- runId,
2658
- "applying",
2659
- `${operationId}:applying`,
2660
- undefined,
2661
- lease,
2662
- );
2663
- emitWorkflowActivity(onActivity, {
2664
- state: "applying",
2665
- stage: projection.stage,
2666
- runId,
2667
- ...(projection.change ? { change: projection.change } : {}),
2668
- objective: projection.change
2669
- ? `apply change ${projection.change}`
2670
- : "apply workflow change",
2671
- legalCommands: ["status", "cancel", "discard"],
2672
- });
2673
- const active = this.#active.get(runId);
2674
- if (active) active.kind = "applying";
2675
- application = await this.#application.begin(applicationInput);
2676
- }
2677
- this.#runStore.assertLease(lease);
2678
- return this.#settleApplication(
2679
- runId,
2680
- operationId,
2681
- application,
2682
- lease,
2683
- onActivity,
2684
- );
2685
- }
2686
-
2687
- #settleApplication(
2688
- runId: string,
2689
- operationId: string,
2690
- outcome: Record<string, unknown>,
2691
- lease: OperationLease,
2692
- onActivity?: (event: WorkflowActivityUpdate) => void,
2693
- ): Record<string, unknown> {
2694
- const state = outcome.state;
2695
- if (state === "completed") {
2696
- const current = this.#runStore.status(runId);
2697
- if (current.state === "recovering") {
2698
- this.#transition(
2699
- runId,
2700
- "completed",
2701
- `${operationId}:recovered-complete`,
2702
- undefined,
2703
- lease,
2704
- );
2705
- } else if (
2706
- current.state === "applying" ||
2707
- current.state === "ready-to-apply"
2708
- ) {
2709
- this.#transition(
2710
- runId,
2711
- "completed",
2712
- `${operationId}:apply-complete`,
2713
- undefined,
2714
- lease,
2715
- );
2716
- }
2717
- this.#cleanupTerminal(runId, operationId, lease);
2718
- } else if (state === "paused") {
2719
- if (this.#runStore.status(runId).state === "applying") {
2720
- this.#transition(
2721
- runId,
2722
- "recovering",
2723
- `${operationId}:apply-recovering`,
2724
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2725
- lease,
2726
- );
2727
- const recovering = this.#runStore.status(runId);
2728
- emitWorkflowActivity(onActivity, {
2729
- state: "recovering",
2730
- stage: recovering.stage,
2731
- runId,
2732
- ...(recovering.change ? { change: recovering.change } : {}),
2733
- objective: recovering.change
2734
- ? `recover change ${recovering.change}`
2735
- : "recover workflow application",
2736
- code:
2737
- typeof outcome.code === "string"
2738
- ? outcome.code
2739
- : "recovery-required",
2740
- legalCommands: ["status", "cancel", "discard"],
2741
- });
2742
- }
2743
- this.#transition(
2744
- runId,
2745
- "paused",
2746
- `${operationId}:apply-paused`,
2747
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2748
- lease,
2749
- );
2750
- } else if (state === "discarded") {
2751
- if (this.#runStore.status(runId).state === "applying") {
2752
- this.#transition(
2753
- runId,
2754
- "recovering",
2755
- `${operationId}:discard-recovering`,
2756
- "run-discarded",
2757
- lease,
2758
- );
2759
- const recovering = this.#runStore.status(runId);
2760
- emitWorkflowActivity(onActivity, {
2761
- state: "recovering",
2762
- stage: recovering.stage,
2763
- runId,
2764
- ...(recovering.change ? { change: recovering.change } : {}),
2765
- objective: recovering.change
2766
- ? `recover change ${recovering.change}`
2767
- : "recover workflow application",
2768
- code: "run-discarded",
2769
- legalCommands: ["status"],
2770
- });
2771
- }
2772
- this.#transition(
2773
- runId,
2774
- "discarded",
2775
- `${operationId}:apply-discarded`,
2776
- undefined,
2777
- lease,
2778
- );
2779
- this.#cleanupTerminal(runId, operationId, lease);
2780
- } else if (state === "recovering") {
2781
- this.#transition(
2782
- runId,
2783
- "recovering",
2784
- `${operationId}:application-recovering`,
2785
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2786
- lease,
2787
- );
2788
- const recovering = this.#runStore.status(runId);
2789
- emitWorkflowActivity(onActivity, {
2790
- state: "recovering",
2791
- stage: recovering.stage,
2792
- runId,
2793
- ...(recovering.change ? { change: recovering.change } : {}),
2794
- objective: recovering.change
2795
- ? `recover change ${recovering.change}`
2796
- : "recover workflow application",
2797
- code:
2798
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2799
- legalCommands: ["status", "cancel", "discard"],
2800
- });
2801
- } else {
2802
- throw new Error("workflow-application-outcome-invalid");
2803
- }
2804
- return this.#statusByRun(runId);
2805
- }
2806
-
2807
- #applicationContext(runId: string): WorkflowApplicationContext {
2808
- const current = this.#engineRun(runId);
2809
- if (current.current_revision === null)
2810
- throw new Error("delivery-not-bound");
2811
- return {
2812
- runId,
2813
- deliveryRevision: current.current_revision,
2814
- plan: structuredClone(this.#currentPlan(runId)),
2815
- contextReadPaths: this.#contextReads(runId),
2816
- ...(current.baseline_workspace_revision
2817
- ? { baselineRevisionId: current.baseline_workspace_revision }
2818
- : {}),
2819
- ...(current.current_workspace_revision
2820
- ? { currentWorkspaceRevisionId: current.current_workspace_revision }
2821
- : {}),
2822
- };
2823
- }
2824
-
2825
- #cleanupTerminal(
2826
- runId: string,
2827
- _currentOperationId: string,
2828
- lease?: OperationLease,
2829
- ): void {
2830
- if (lease) this.#runStore.assertLease(lease);
2831
- this.#lifecycle?.cleanup(runId);
2832
- const cleanup = () => {
2833
- this.#database
2834
- .prepare("DELETE FROM workflow_engine_tasks WHERE run_id = ?")
2835
- .run(runId);
2836
- this.#database
2837
- .prepare("DELETE FROM workflow_engine_deliveries WHERE run_id = ?")
2838
- .run(runId);
2839
- this.#database
2840
- .prepare(`DELETE FROM workspace_revisions WHERE owner_run_id = ?`)
2841
- .run(runId);
2842
- this.#database
2843
- .prepare(`DELETE FROM apply_transactions WHERE owner_run_id = ?`)
2844
- .run(runId);
2845
- this.#database
2846
- .prepare(
2847
- `UPDATE workflow_engine_runs
2848
- SET baseline_workspace_revision = NULL,
2849
- current_workspace_revision = NULL,
2850
- transaction_id = NULL, verification_json = NULL,
2851
- cleanup_state = 'complete'
2852
- WHERE run_id = ?`,
2853
- )
2854
- .run(runId);
2855
- };
2856
- if (lease) this.#leasedTransaction(lease, cleanup);
2857
- else this.#transaction(cleanup);
2858
- }
2859
-
2860
- #waitForCapacity(signal: AbortSignal): Promise<void> {
2861
- if (signal.aborted || this.#activeTasks < LIMITS.maxActiveChildSessions)
2862
- return Promise.resolve();
2863
- return new Promise((resolve) => {
2864
- const wake = () => {
2865
- this.#capacityWaiters.delete(wake);
2866
- signal.removeEventListener("abort", wake);
2867
- resolve();
2868
- };
2869
- this.#capacityWaiters.add(wake);
2870
- signal.addEventListener("abort", wake, { once: true });
2871
- });
2872
- }
2873
-
2874
- async #advance(
2875
- runId: string,
2876
- operationId: string,
2877
- signal: AbortSignal,
2878
- lease: OperationLease,
2879
- onActivity?: (event: WorkflowActivityUpdate) => void,
2880
- ): Promise<Record<string, unknown>> {
2881
- this.#transition(
2882
- runId,
2883
- "running",
2884
- `${operationId}:running`,
2885
- undefined,
2886
- lease,
2887
- );
2888
- const run = this.#runStore.status(runId);
2889
- emitWorkflowActivity(onActivity, {
2890
- state: "running",
2891
- stage: run.stage,
2892
- runId,
2893
- ...(run.change ? { change: run.change } : {}),
2894
- objective: run.change ? `change ${run.change}` : "workflow run",
2895
- legalCommands: ["status", "cancel", "discard"],
2896
- });
2897
- const existingVerification = this.#engineRun(runId).verification_json;
2898
- const repairStatus = existingVerification
2899
- ? normalizeWorkflowVerificationStatus(
2900
- JSON.parse(existingVerification) as unknown,
2901
- )
2902
- : undefined;
2903
- const retainsRepairContext =
2904
- repairStatus?.scope === "change-task-affected" &&
2905
- repairStatus.attribution === "introduced" &&
2906
- repairStatus.taskId !== undefined &&
2907
- this.#tasks(runId).some(
2908
- (task) =>
2909
- task.task_id === repairStatus.taskId && task.state !== "verified",
2910
- );
2911
- if (!retainsRepairContext) {
2912
- this.#setVerificationStatus(runId, null, lease);
2913
- }
2914
- const attempted = new Set<string>();
2915
- // Keep every launched operation owned until it settles, including failures.
2916
- // A fulfilled envelope avoids unhandled rejections while another task wins
2917
- // the race; the finally barrier also protects cancellation and storage close.
2918
- type Settlement =
2919
- | { taskId: string; status: "fulfilled" }
2920
- | { taskId: string; status: "rejected"; reason: unknown };
2921
- const pending = new Map<string, Promise<Settlement>>();
2922
- let firstFailure: { reason: unknown } | undefined;
2923
- try {
2924
- while (!signal.aborted) {
2925
- if (firstFailure) throw firstFailure.reason;
2926
- const rows = this.#tasks(runId);
2927
- const selected = selectRunnableTasks({
2928
- rows,
2929
- attempted,
2930
- activeTasks: this.#activeTasks,
2931
- capacity: LIMITS.maxActiveChildSessions,
2932
- nextQueuePosition: this.#engineRun(runId).next_queue_position,
2933
- });
2934
- const runnable = selected.runnable.map((taskId) => {
2935
- const row = rows.find((candidate) => candidate.task_id === taskId);
2936
- if (!row) throw new Error("scheduler-task-unavailable");
2937
- attempted.add(taskId);
2938
- return row;
2939
- });
2940
- for (const update of selected.updates) {
2941
- const row = rows.find(
2942
- (candidate) => candidate.task_id === update.taskId,
2943
- );
2944
- if (!row) throw new Error("scheduler-task-unavailable");
2945
- if (update.kind === "pending")
2946
- this.#setTask(
2947
- runId,
2948
- row.task_id,
2949
- { state: "pending", pauseCode: null, queuePosition: null },
2950
- lease,
2951
- );
2952
- else this.#queueTask(runId, row, lease, update.reason);
2953
- }
2954
- this.#activeTasks += runnable.length;
2955
- for (const row of runnable) {
2956
- const operation = this.#runTask(
2957
- runId,
2958
- row,
2959
- operationId,
2960
- signal,
2961
- lease,
2962
- onActivity,
2963
- )
2964
- .then<Settlement, Settlement>(
2965
- () => ({ taskId: row.task_id, status: "fulfilled" }),
2966
- (reason: unknown) => {
2967
- firstFailure ??= { reason };
2968
- return { taskId: row.task_id, status: "rejected", reason };
2969
- },
2970
- )
2971
- .finally(() => {
2972
- this.#activeTasks--;
2973
- for (const wake of this.#capacityWaiters) wake();
2974
- });
2975
- pending.set(row.task_id, operation);
2976
- }
2977
- if (pending.size > 0) {
2978
- const settled = await Promise.race([
2979
- ...pending.values(),
2980
- ...(selected.capacityBlocked
2981
- ? [this.#waitForCapacity(signal).then(() => null)]
2982
- : []),
2983
- ]);
2984
- if (settled === null) continue;
2985
- pending.delete(settled.taskId);
2986
- if (settled.status === "rejected") throw settled.reason;
2987
- continue;
2988
- }
2989
- if (selected.capacityBlocked) {
2990
- await this.#waitForCapacity(signal);
2991
- continue;
2992
- }
2993
- break;
2994
- }
2995
- } finally {
2996
- await Promise.all(pending.values());
2997
- }
2998
- if (firstFailure) throw firstFailure.reason;
2999
-
3000
- if (signal.aborted) {
3001
- const current = this.#runStore.status(runId);
3002
- if (!["paused", "discarded", "completed"].includes(current.state)) {
3003
- this.#transition(
3004
- runId,
3005
- "paused",
3006
- `${operationId}:cancelled`,
3007
- "operation-cancelled",
3008
- lease,
3009
- );
3010
- }
3011
- return this.#statusByRun(runId);
3012
- }
3013
- const rows = this.#tasks(runId);
3014
- const paused = this.#firstPaused(rows);
3015
- if (paused) {
3016
- this.#transition(
3017
- runId,
3018
- paused.state === "approval-needed" ? "approval-needed" : "paused",
3019
- `${operationId}:task-paused:${paused.task_id}`,
3020
- paused.pause_code ?? "task-paused",
3021
- lease,
3022
- );
3023
- return this.#statusByRun(runId);
3024
- }
3025
- if (rows.every((row) => row.state === "verified")) {
3026
- return this.#finishChange(runId, operationId, signal, lease, onActivity);
3027
- }
3028
- this.#transition(
3029
- runId,
3030
- "paused",
3031
- `${operationId}:scheduler-stalled`,
3032
- "scheduler-integrity-paused",
3033
- lease,
3034
- );
3035
- return this.#statusByRun(runId);
3036
- }
3037
-
3038
- #contextReads(runId: string, taskId?: string): string[] {
3039
- const rows =
3040
- taskId === undefined
3041
- ? this.#database
3042
- .prepare(
3043
- "SELECT DISTINCT path FROM workflow_context_reads WHERE run_id = ? ORDER BY path",
3044
- )
3045
- .all(runId)
3046
- : this.#database
3047
- .prepare(
3048
- "SELECT path FROM workflow_context_reads WHERE run_id = ? AND task_id = ? ORDER BY path",
3049
- )
3050
- .all(runId, taskId);
3051
- return rows.map((row) => String(row.path));
3052
- }
3053
-
3054
- #phaseCount(plan: ImplementPlan): number {
3055
- return plan.tasks.reduce((sum, task) => sum + phases(task).length, 0);
3056
- }
3057
-
3058
- #growWorkBudget(
3059
- database: DatabaseSync,
3060
- runId: string,
3061
- plan: ImplementPlan,
3062
- ): void {
3063
- const count = this.#phaseCount(plan);
3064
- database
3065
- .prepare(`INSERT INTO workflow_work_budget(run_id, used, max_work, phase_high_water, hard_limit, recovery_policy)
3066
- VALUES (?, 0, ?, ?, ?, 1) ON CONFLICT(run_id) DO UPDATE SET
3067
- phase_high_water = MAX(phase_high_water, excluded.phase_high_water),
3068
- max_work = MIN(hard_limit, MAX(max_work, 24 + 3 * MAX(phase_high_water, excluded.phase_high_water)))`)
3069
- .run(
3070
- runId,
3071
- Math.min(this.#workHardLimit, 24 + 3 * count),
3072
- count,
3073
- this.#workHardLimit,
3074
- );
3075
- }
3076
-
3077
- #hasRecoveryGrant(runId: string, operationId: string, key: string): boolean {
3078
- return Boolean(
3079
- this.#database
3080
- .prepare(
3081
- "SELECT 1 FROM workflow_recovery_grants WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
3082
- )
3083
- .get(runId, operationId, key),
3084
- );
3085
- }
3086
-
3087
- #recoveryConditions(runId: string, key: string) {
3088
- const run = this.#engineRun(runId);
3089
- const task = this.#tasks(runId).find((row) =>
3090
- [
3091
- recoveryKey(this.#taskPlan(row), row.phase, row, run),
3092
- baselineRecoveryKey(this.#taskPlan(row), row.phase),
3093
- ].includes(key),
3094
- );
3095
- return {
3096
- route: task?.route_fingerprint ?? run.route_fingerprint,
3097
- context: task
3098
- ? hash(
3099
- canonicalJson(
3100
- snapshotFiles(
3101
- this.#consumerRoot,
3102
- this.#contextReads(runId, task.task_id),
3103
- ),
3104
- ),
3105
- )
3106
- : null,
3107
- };
3108
- }
3109
-
3110
- #failureSequence(runId: string, key: string): number {
3111
- return (
3112
- this.#database
3113
- .prepare(
3114
- "SELECT COALESCE(MAX(sequence), 0) AS sequence FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ?",
3115
- )
3116
- .get(runId, key) as { sequence: number }
3117
- ).sequence;
3118
- }
3119
-
3120
- #grantRecovery(
3121
- runId: string,
3122
- operationId: string,
3123
- request: NonNullable<
3124
- Extract<ControlCommand, { command: "resume" }>["recovery"]
3125
- >,
3126
- lease: OperationLease,
3127
- ): void {
3128
- this.#leasedTransaction(lease, () => {
3129
- const incident = this.#recovery(runId, request.incidentKey);
3130
- const current = this.#tasks(runId).some(
3131
- (row) =>
3132
- ["paused", "retryable"].includes(row.state) &&
3133
- [
3134
- recoveryKey(
3135
- this.#taskPlan(row),
3136
- row.phase,
3137
- row,
3138
- this.#engineRun(runId),
3139
- ),
3140
- baselineRecoveryKey(this.#taskPlan(row), row.phase),
3141
- ].includes(request.incidentKey),
3142
- );
3143
- if (
3144
- !current ||
3145
- !incident ||
3146
- incident.failures < incident.feedback.maxAttempts ||
3147
- this.#failureSequence(runId, request.incidentKey) !==
3148
- request.failureSequence
3149
- )
3150
- throw new Error("recovery-request-stale");
3151
- const conditions = this.#database
3152
- .prepare(
3153
- "SELECT conditions_json FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
3154
- )
3155
- .get(runId, request.incidentKey) as { conditions_json: string };
3156
- const previous = JSON.parse(conditions.conditions_json);
3157
- const observed = this.#recoveryConditions(runId, request.incidentKey);
3158
- const budget = this.#database
3159
- .prepare(
3160
- "SELECT used, max_work FROM workflow_work_budget WHERE run_id = ?",
3161
- )
3162
- .get(runId) as { used: number; max_work: number };
3163
- const rejected = assessRecoveryGrant({
3164
- current,
3165
- incident,
3166
- failureSequence: this.#failureSequence(runId, request.incidentKey),
3167
- requestedSequence: request.failureSequence,
3168
- reason: request.reason,
3169
- previous,
3170
- observed,
3171
- budget,
3172
- });
3173
- if (rejected) throw new Error(rejected);
3174
- this.#database
3175
- .prepare(
3176
- "INSERT INTO workflow_recovery_grants(run_id, operation_id, incident_key, failure_sequence, reason) VALUES (?, ?, ?, ?, ?)",
3177
- )
3178
- .run(
3179
- runId,
3180
- operationId,
3181
- request.incidentKey,
3182
- request.failureSequence,
3183
- request.reason,
3184
- );
3185
- });
3186
- }
3187
-
3188
- #reserveWork(
3189
- runId: string,
3190
- plan: ImplementPlan,
3191
- key: string,
3192
- lease: OperationLease,
3193
- grantOperationId?: string,
3194
- ): boolean {
3195
- return this.#leasedTransaction(lease, () => {
3196
- this.#growWorkBudget(this.#database, runId, plan);
3197
- const reserved = this.#database
3198
- .prepare(`UPDATE workflow_work_budget SET used = used + 1
3199
- WHERE run_id = ? AND used < max_work`)
3200
- .run(runId);
3201
- if (reserved.changes !== 1) return false;
3202
- if (grantOperationId) {
3203
- const consumed = this.#database
3204
- .prepare(
3205
- "UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
3206
- )
3207
- .run(runId, grantOperationId, key);
3208
- if (consumed.changes !== 1) throw new Error("recovery-request-stale");
3209
- }
3210
- this.#database
3211
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
3212
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
3213
- .run(runId, key, runId);
3214
- return true;
3215
- });
3216
- }
3217
-
3218
- #recovery(runId: string, key: string): WorkflowRecoveryFact | undefined {
3219
- const row = this.#database
3220
- .prepare(
3221
- `SELECT failures, max_attempts, feedback_json FROM workflow_recovery_incidents
3222
- WHERE run_id = ? AND incident_key = ?`,
3223
- )
3224
- .get(runId, key) as
3225
- | { failures: number; max_attempts: number; feedback_json: string }
3226
- | undefined;
3227
- return row
3228
- ? {
3229
- key,
3230
- failures: row.failures,
3231
- feedback: {
3232
- ...JSON.parse(row.feedback_json),
3233
- maxAttempts: row.max_attempts,
3234
- },
3235
- }
3236
- : undefined;
3237
- }
3238
-
3239
- #recordRecovery(
3240
- runId: string,
3241
- fact: WorkflowRecoveryFact,
3242
- lease: OperationLease,
3243
- ): void {
3244
- this.#leasedTransaction(lease, () => {
3245
- this.#database
3246
- .prepare(
3247
- `INSERT INTO workflow_recovery_incidents(run_id, incident_key, failures, max_attempts, feedback_json, conditions_json)
3248
- VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, incident_key) DO UPDATE SET
3249
- failures = MAX(failures, excluded.failures), max_attempts = MIN(max_attempts, excluded.max_attempts), feedback_json = excluded.feedback_json, conditions_json = excluded.conditions_json`,
3250
- )
3251
- .run(
3252
- runId,
3253
- fact.key,
3254
- fact.failures,
3255
- fact.feedback.maxAttempts,
3256
- JSON.stringify(fact.feedback),
3257
- JSON.stringify(this.#recoveryConditions(runId, fact.key)),
3258
- );
3259
- this.#database
3260
- .prepare(
3261
- `INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
3262
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'failure' FROM workflow_recovery_events WHERE run_id = ?`,
3263
- )
3264
- .run(runId, fact.key, runId);
3265
- });
3266
- }
3267
-
3268
- #safeAttemptDiagnostic(
3269
- runId: string,
3270
- row: EngineTaskRow | undefined,
3271
- ): SafeAttemptDiagnostic | undefined {
3272
- if (!row) return undefined;
3273
- const diagnostic: SafeAttemptDiagnostic = row.attempt_diagnostic_json
3274
- ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
3275
- : {};
3276
- delete diagnostic.recovery;
3277
- const recovery =
3278
- this.#recovery(
3279
- runId,
3280
- baselineRecoveryKey(this.#taskPlan(row), row.phase),
3281
- ) ??
3282
- this.#recovery(
3283
- runId,
3284
- recoveryKey(
3285
- this.#taskPlan(row),
3286
- row.phase,
3287
- row,
3288
- this.#engineRun(runId),
3289
- ),
3290
- );
3291
- if (recovery) diagnostic.recovery = recovery;
3292
- return Object.keys(diagnostic).length ? diagnostic : undefined;
3293
- }
3294
-
3295
- #statusByRun(runId: string): Record<string, unknown> {
3296
- const projection = this.#runStore.status(runId);
3297
- const rows = this.#tasks(runId);
3298
- let engineRun: EngineRunRow | undefined;
3299
- try {
3300
- engineRun = this.#engineRun(runId);
3301
- } catch {
3302
- engineRun = undefined;
3303
- }
3304
- const attemptDiagnostics = new Map<string, SafeAttemptDiagnostic>();
3305
- const failureSequences = new Map<string, number>();
3306
- for (const row of rows) {
3307
- const diagnostic = this.#safeAttemptDiagnostic(runId, row);
3308
- if (!diagnostic) continue;
3309
- attemptDiagnostics.set(row.task_id, diagnostic);
3310
- if (diagnostic.recovery) {
3311
- const key = diagnostic.recovery.key;
3312
- failureSequences.set(key, this.#failureSequence(runId, key));
3313
- }
3314
- }
3315
- const amendmentUsed =
3316
- (
3317
- this.#database
3318
- .prepare(
3319
- "SELECT used FROM workflow_amendment_budget WHERE run_id = ?",
3320
- )
3321
- .get(runId) as { used: number } | undefined
3322
- )?.used ?? 0;
3323
- const resourceBudget = this.#database
3324
- .prepare(
3325
- "SELECT used, max_work AS maximum, phase_high_water AS phaseHighWater, hard_limit AS hardLimit FROM workflow_work_budget WHERE run_id = ?",
3326
- )
3327
- .get(runId) as WorkflowResourceBudget | undefined;
3328
- return projectWorkflowStatus({
3329
- projection,
3330
- rows,
3331
- engineRun,
3332
- attemptDiagnostics,
3333
- failureSequences,
3334
- amendmentUsed,
3335
- resourceBudget,
3336
- });
3337
- }
3338
-
3339
- async #withAvailableDelivery(
3340
- outcome: Record<string, unknown>,
3341
- signal?: AbortSignal,
3342
- ): Promise<Record<string, unknown>> {
3343
- if (
3344
- outcome.stage !== "abel-implement" ||
3345
- (isRecord(outcome.pause) &&
3346
- outcome.pause.code === "operation-cancelled") ||
3347
- (outcome.state !== "approval-needed" &&
3348
- !(
3349
- outcome.state === "paused" &&
3350
- (isRecord(outcome.decisionBatch) ||
3351
- (isRecord(outcome.recovery) && outcome.recovery.exhausted === true))
3352
- )) ||
3353
- typeof outcome.change !== "string" ||
3354
- !this.#deliverySource.discoverLatest
3355
- ) {
3356
- return outcome;
3357
- }
3358
- let available: WorkflowAvailableDelivery | undefined;
3359
- try {
3360
- const discover = this.#deliverySource.discoverLatest;
3361
- const change = outcome.change;
3362
- available = await cancellableRead(
3363
- () =>
3364
- discover.call(this.#deliverySource, {
3365
- stage: "abel-implement",
3366
- change,
3367
- ...(signal ? { signal } : {}),
3368
- }),
3369
- signal,
3370
- );
3371
- } catch {
3372
- return outcome;
3373
- }
3374
- const currentRevision =
3375
- typeof outcome.deliveryRevision === "number"
3376
- ? outcome.deliveryRevision
3377
- : 0;
3378
- if (
3379
- !available ||
3380
- !Number.isSafeInteger(available.deliveryRevision) ||
3381
- available.deliveryRevision <= currentRevision ||
3382
- !SHA256.test(available.receiptHash)
3383
- ) {
3384
- return outcome;
3385
- }
3386
- if (
3387
- this.#database
3388
- .prepare(
3389
- "SELECT 1 FROM workflow_rejected_deliveries WHERE run_id = ? AND delivery_revision = ? AND receipt_hash = ?",
3390
- )
3391
- .get(
3392
- String(outcome.runId),
3393
- available.deliveryRevision,
3394
- available.receiptHash,
3395
- )
3396
- ) {
3397
- return outcome;
3398
- }
3399
- const conditionalCommands = (
3400
- Array.isArray(outcome.conditionalCommands)
3401
- ? outcome.conditionalCommands
3402
- : [{ command: "resume", stage: outcome.stage, change: outcome.change }]
3403
- ).map((entry) =>
3404
- isRecord(entry) && entry.command === "resume"
3405
- ? {
3406
- ...entry,
3407
- satisfiedBy: {
3408
- deliveryRevision: available.deliveryRevision,
3409
- receiptHash: available.receiptHash,
3410
- },
3411
- }
3412
- : entry,
3413
- );
3414
- return {
3415
- ...outcome,
3416
- legalCommands:
3417
- outcome.state === "approval-needed"
3418
- ? ["status", "resume", "discard"]
3419
- : ["status", "resume", "rebind", "discard"],
3420
- availableDelivery: structuredClone(available),
3421
- continuation:
3422
- (isRecord(outcome.resourceBudget) &&
3423
- outcome.resourceBudget.remaining === 0) ||
3424
- (isRecord(outcome.recovery) &&
3425
- outcome.recovery.code === "change-recovery-budget-exhausted")
3426
- ? undefined
3427
- : {
3428
- owner: "parent",
3429
- automatic: true,
3430
- command: "resume",
3431
- stage: "abel-implement",
3432
- change: outcome.change,
3433
- },
3434
- ...(conditionalCommands === undefined ? {} : { conditionalCommands }),
3435
- };
3436
- }
3437
-
3438
- #launchAdvance(
3439
- runId: string,
3440
- operationId: string,
3441
- externalSignal?: AbortSignal,
3442
- onActivity?: (event: WorkflowActivityUpdate) => void,
3443
- ): Promise<Record<string, unknown>> {
3444
- const lease = this.#operationLease(runId, operationId);
3445
- const controller = new AbortController();
3446
- const abortFromExternal = () => {
3447
- controller.abort(
3448
- externalSignal?.reason ?? new Error("operation-cancelled"),
3449
- );
3450
- };
3451
- if (externalSignal?.aborted) abortFromExternal();
3452
- else
3453
- externalSignal?.addEventListener("abort", abortFromExternal, {
3454
- once: true,
3455
- });
3456
- const active: ActiveOperation = {
3457
- operationId,
3458
- kind: "work",
3459
- controller,
3460
- settled: Promise.resolve({}),
3461
- };
3462
- this.#active.set(runId, active);
3463
- const settled = (async () => {
3464
- try {
3465
- const outcome = await this.#advance(
3466
- runId,
3467
- operationId,
3468
- controller.signal,
3469
- lease,
3470
- onActivity,
3471
- );
3472
- return this.#commitOperation(runId, operationId, outcome);
3473
- } catch (error) {
3474
- this.#interruptOperation(runId, operationId);
3475
- const controlSettled =
3476
- controller.signal.aborted &&
3477
- controller.signal.reason instanceof Error &&
3478
- ["operation-cancelled", "run-discarded"].includes(
3479
- controller.signal.reason.message,
3480
- ) &&
3481
- (isCancellationException(error, controller.signal) ||
3482
- (error instanceof Error &&
3483
- ["lease-fenced", "operation-journal-fenced"].includes(
3484
- error.message,
3485
- )));
3486
- if (controlSettled) return this.#statusByRun(runId);
3487
- throw error;
3488
- } finally {
3489
- externalSignal?.removeEventListener("abort", abortFromExternal);
3490
- if (this.#active.get(runId) === active) this.#active.delete(runId);
3491
- }
3492
- })();
3493
- active.settled = settled;
3494
- return settled;
3495
- }
3496
-
3497
- async #start(
3498
- command: Extract<ControlCommand, { command: "start" }>,
3499
- signal?: AbortSignal,
3500
- onActivity?: (event: WorkflowActivityUpdate) => void,
3501
- ) {
3502
- const existingRun = this.#lookupRun(command.stage, command.change);
3503
- if (existingRun) {
3504
- this.#ensureEngineRun(existingRun);
3505
- const replay = this.#operationReplay(
3506
- existingRun,
3507
- command.operationId,
3508
- command.command,
3509
- );
3510
- if (replay) return replay;
3511
- const restarted = this.#runStore.startRun({
3512
- stage: command.stage,
3513
- change: command.change,
3514
- operationId: command.operationId,
3515
- });
3516
- if (restarted.state !== "created") {
3517
- const begun = this.#beginOperation(
3518
- existingRun,
3519
- command.operationId,
3520
- command.command,
3521
- );
3522
- if (begun) return begun;
3523
- return this.#commitOperation(
3524
- existingRun,
3525
- command.operationId,
3526
- this.#statusByRun(existingRun),
3527
- );
3528
- }
3529
- }
3530
-
3531
- const runId =
3532
- existingRun ??
3533
- this.#runStore.startRun({
3534
- stage: command.stage,
3535
- change: command.change,
3536
- operationId: command.operationId,
3537
- }).runId;
3538
- this.#ensureEngineRun(runId);
3539
- const begun = this.#beginOperation(
3540
- runId,
3541
- command.operationId,
3542
- command.command,
3543
- );
3544
- if (begun) return begun;
3545
- const lease = this.#operationLease(runId, command.operationId);
3546
- this.#transition(
3547
- runId,
3548
- "validating-delivery",
3549
- `${command.operationId}:validating-delivery`,
3550
- undefined,
3551
- lease,
3552
- );
3553
- emitWorkflowActivity(onActivity, {
3554
- state: "validating",
3555
- stage: command.stage,
3556
- runId,
3557
- change: command.change,
3558
- objective: `validate delivery ${command.change}`,
3559
- legalCommands: ["status", "cancel", "discard"],
3560
- });
3561
- try {
3562
- const delivery = await this.#loadDelivery(
3563
- runId,
3564
- lease,
3565
- command.stage,
3566
- command.change,
3567
- undefined,
3568
- undefined,
3569
- signal,
3570
- );
3571
- this.#admitDelivery(runId, delivery, lease);
3572
- this.#transition(
3573
- runId,
3574
- "ready",
3575
- `${command.operationId}:delivery-ready`,
3576
- undefined,
3577
- lease,
3578
- );
3579
- return this.#launchAdvance(
3580
- runId,
3581
- command.operationId,
3582
- signal,
3583
- onActivity,
3584
- );
3585
- } catch (error) {
3586
- const cancelled = this.#cancelledDelivery(
3587
- runId,
3588
- command.operationId,
3589
- error,
3590
- signal,
3591
- lease,
3592
- );
3593
- if (cancelled) return cancelled;
3594
- const failure = this.#deliveryFailure(
3595
- runId,
3596
- command.operationId,
3597
- error,
3598
- lease,
3599
- );
3600
- if (failure) {
3601
- return this.#commitOperation(runId, command.operationId, failure);
3602
- }
3603
- this.#interruptOperation(runId, command.operationId);
3604
- throw error;
3605
- }
3606
- }
3607
-
3608
- async #resume(
3609
- command: Extract<ControlCommand, { command: "resume" }>,
3610
- signal?: AbortSignal,
3611
- onActivity?: (event: WorkflowActivityUpdate) => void,
3612
- ) {
3613
- const runId = this.#lookupRun(command.stage, command.change);
3614
- if (!runId) throw new Error("run-not-found");
3615
- this.#recoverLegacyContextApproval(runId);
3616
- const replay = this.#beginOperation(
3617
- runId,
3618
- command.operationId,
3619
- command.command,
3620
- );
3621
- if (replay) return replay;
3622
- const lease = this.#operationLease(runId, command.operationId);
3623
- const current = this.#runStore.status(runId);
3624
- if (current.state === "recovering") {
3625
- emitWorkflowActivity(onActivity, {
3626
- state: "recovering",
3627
- stage: current.stage,
3628
- runId,
3629
- ...(current.change ? { change: current.change } : {}),
3630
- objective: current.change
3631
- ? `recover change ${current.change}`
3632
- : "recover workflow application",
3633
- legalCommands: ["status", "cancel", "discard"],
3634
- });
3635
- const transactionId = this.#engineRun(runId).transaction_id;
3636
- if (!transactionId || !this.#application) {
3637
- this.#interruptOperation(runId, command.operationId);
3638
- throw new Error("apply-recovery-unavailable");
3639
- }
3640
- const recovered = await this.#application.recover(
3641
- transactionId,
3642
- this.#applicationContext(runId),
3643
- );
3644
- this.#runStore.assertLease(lease);
3645
- const outcome = this.#settleApplication(
3646
- runId,
3647
- command.operationId,
3648
- recovered,
3649
- lease,
3650
- onActivity,
3651
- );
3652
- return this.#commitOperation(runId, command.operationId, outcome);
3653
- }
3654
- if (
3655
- !["paused", "retryable", "approval-needed", "ready"].includes(
3656
- current.state,
3657
- )
3658
- ) {
3659
- this.#interruptOperation(runId, command.operationId);
3660
- throw new Error("resume-not-allowed");
3661
- }
3662
- try {
3663
- const engineRevision = this.#engineRun(runId).current_revision;
3664
- const projectedRevision = current.deliveryRevision;
3665
- if (
3666
- command.deliveryRevision === undefined &&
3667
- (current.state === "approval-needed" ||
3668
- this.#statusByRun(runId).decisionBatch !== undefined ||
3669
- this.#statusByRun(runId).recovery !== undefined) &&
3670
- this.#deliverySource.discoverLatest
3671
- ) {
3672
- const available = await this.#deliverySource.discoverLatest({
3673
- stage: command.stage,
3674
- change: command.change,
3675
- signal,
3676
- });
3677
- this.#runStore.assertLease(lease);
3678
- if (available && available.deliveryRevision > (engineRevision ?? 0)) {
3679
- command = {
3680
- ...command,
3681
- deliveryRevision: available.deliveryRevision,
3682
- receiptHash: available.receiptHash,
3683
- };
3684
- }
3685
- }
3686
- const deliveryOutOfSync =
3687
- projectedRevision !== undefined && projectedRevision !== engineRevision;
3688
- if (
3689
- current.state === "approval-needed" &&
3690
- (command.deliveryRevision === undefined ||
3691
- command.receiptHash === undefined ||
3692
- command.deliveryRevision <=
3693
- (engineRevision ?? projectedRevision ?? 0))
3694
- ) {
3695
- this.#interruptOperation(runId, command.operationId);
3696
- throw new Error("approval-receipt-required");
3697
- }
3698
- if (
3699
- command.deliveryRevision !== undefined ||
3700
- engineRevision === null ||
3701
- deliveryOutOfSync
3702
- ) {
3703
- this.#transition(
3704
- runId,
3705
- "validating-delivery",
3706
- `${command.operationId}:validating-delivery`,
3707
- undefined,
3708
- lease,
3709
- );
3710
- emitWorkflowActivity(onActivity, {
3711
- state: "validating",
3712
- stage: command.stage,
3713
- runId,
3714
- change: command.change,
3715
- objective: `validate delivery ${command.change}`,
3716
- legalCommands: ["status", "cancel", "discard"],
3717
- });
3718
- const requestedRevision =
3719
- command.deliveryRevision ??
3720
- (deliveryOutOfSync ? projectedRevision : undefined);
3721
- const projectedBinding = current.deliveryBindings.find(
3722
- (binding) =>
3723
- binding.gate === "gate-b" && binding.revision === requestedRevision,
3724
- );
3725
- const delivery = await this.#loadDelivery(
3726
- runId,
3727
- lease,
3728
- command.stage,
3729
- command.change,
3730
- requestedRevision,
3731
- command.receiptHash ?? projectedBinding?.receiptHash,
3732
- signal,
3733
- );
3734
- this.#admitDelivery(runId, delivery, lease);
3735
- }
3736
- if (command.recovery)
3737
- this.#grantRecovery(
3738
- runId,
3739
- command.operationId,
3740
- command.recovery,
3741
- lease,
3742
- );
3743
- this.#transition(
3744
- runId,
3745
- "ready",
3746
- `${command.operationId}:resume-ready`,
3747
- undefined,
3748
- lease,
3749
- );
3750
- return this.#launchAdvance(
3751
- runId,
3752
- command.operationId,
3753
- signal,
3754
- onActivity,
3755
- );
3756
- } catch (error) {
3757
- const cancelled = this.#cancelledDelivery(
3758
- runId,
3759
- command.operationId,
3760
- error,
3761
- signal,
3762
- lease,
3763
- );
3764
- if (cancelled) return cancelled;
3765
- const failure = this.#deliveryFailure(
3766
- runId,
3767
- command.operationId,
3768
- error,
3769
- lease,
3770
- );
3771
- if (failure) {
3772
- return this.#commitOperation(runId, command.operationId, failure);
3773
- }
3774
- // A revised delivery may invalidate a grant. Retain its admitted facts,
3775
- // but never leave a rejected resume looking ready or still validating.
3776
- if (
3777
- error instanceof Error &&
3778
- [
3779
- "recovery-request-stale",
3780
- "recovery-evidence-unavailable",
3781
- "change-work-budget-exhausted",
3782
- ].includes(error.message) &&
3783
- this.#runStore.status(runId).state === "validating-delivery"
3784
- ) {
3785
- this.#transition(
3786
- runId,
3787
- "paused",
3788
- `${command.operationId}:recovery-rejected`,
3789
- error.message,
3790
- lease,
3791
- );
3792
- }
3793
- this.#interruptOperation(runId, command.operationId);
3794
- throw error;
3795
- }
3796
- }
3797
-
3798
- #rebind(command: Extract<ControlCommand, { command: "rebind" }>) {
3799
- const runId = this.#lookupRun(command.stage, command.change);
3800
- if (!runId) throw new Error("run-not-found");
3801
- const replay = this.#beginOperation(
3802
- runId,
3803
- command.operationId,
3804
- command.command,
3805
- );
3806
- if (replay) return replay;
3807
- const lease = this.#operationLease(runId, command.operationId);
3808
- const state = this.#runStore.status(runId).state;
3809
- if (!["paused", "retryable"].includes(state)) {
3810
- this.#interruptOperation(runId, command.operationId);
3811
- throw new Error("rebind-not-allowed");
3812
- }
3813
- const task = this.#tasks(runId).find((row) =>
3814
- ["paused", "retryable"].includes(row.state),
3815
- );
3816
- if (!task) {
3817
- this.#interruptOperation(runId, command.operationId);
3818
- throw new Error("rebind-task-unavailable");
3819
- }
3820
- const rebound = this.#worker.rebind({
3821
- runId,
3822
- taskId: task.task_id,
3823
- task: structuredClone(this.#taskPlan(task)),
3824
- role: "implementation-worker",
3825
- routeId: command.routeId,
3826
- });
3827
- if (!rebound.ok) {
3828
- this.#interruptOperation(runId, command.operationId);
3829
- throw new Error(rebound.code);
3830
- }
3831
- const routeId = rebound.routeId ?? rebound.route?.id ?? command.routeId;
3832
- const routeFingerprint =
3833
- rebound.routeFingerprint ?? rebound.route?.fingerprint;
3834
- if (
3835
- !IDENTIFIER.test(routeId) ||
3836
- (routeFingerprint !== undefined && !SHA256.test(routeFingerprint))
3837
- ) {
3838
- this.#interruptOperation(runId, command.operationId);
3839
- throw new Error("route-rebind-result-invalid");
3840
- }
3841
- const priorRouteId = task.route_id ?? this.#engineRun(runId).route_id;
3842
- const priorRouteFingerprint =
3843
- task.route_fingerprint ?? this.#engineRun(runId).route_fingerprint;
3844
- this.#leasedTransaction(lease, () => {
3845
- this.#setTask(runId, task.task_id, {
3846
- routeId,
3847
- routeFingerprint: routeFingerprint ?? null,
3848
- });
3849
- this.#database
3850
- .prepare(
3851
- `UPDATE workflow_engine_runs
3852
- SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
3853
- )
3854
- .run(routeId, routeFingerprint ?? null, runId);
3855
- });
3856
- const outcome = {
3857
- ...this.#statusByRun(runId),
3858
- routeBinding: {
3859
- routeId,
3860
- ...(routeFingerprint ? { routeFingerprint } : {}),
3861
- },
3862
- operation: {
3863
- kind: "route-rebound",
3864
- ...(priorRouteId ? { priorRouteId } : {}),
3865
- ...(priorRouteFingerprint ? { priorRouteFingerprint } : {}),
3866
- routeId,
3867
- ...(routeFingerprint ? { routeFingerprint } : {}),
3868
- },
3869
- };
3870
- return this.#commitOperation(runId, command.operationId, outcome);
3871
- }
3872
-
3873
- async #controlApply(
3874
- runId: string,
3875
- operationId: string,
3876
- intent: "cancel" | "discard",
3877
- lease: OperationLease,
3878
- onActivity?: (event: WorkflowActivityUpdate) => void,
3879
- ): Promise<Record<string, unknown>> {
3880
- const transactionId = this.#engineRun(runId).transaction_id;
3881
- if (!transactionId || !this.#application) {
3882
- throw new Error("apply-recovery-unavailable");
3883
- }
3884
- const context = this.#applicationContext(runId);
3885
- this.#application.requestControl(transactionId, intent, context);
3886
- const active = this.#active.get(runId);
3887
- if (active && active.operationId !== operationId) {
3888
- active.controller.abort(
3889
- new Error(
3890
- intent === "cancel" ? "operation-cancelled" : "run-discarded",
3891
- ),
3892
- );
3893
- }
3894
- if (this.#runStore.status(runId).state === "applying") {
3895
- this.#transition(
3896
- runId,
3897
- "recovering",
3898
- `${operationId}:${intent}-recovering`,
3899
- intent === "cancel" ? "operation-cancelled" : "run-discarded",
3900
- lease,
3901
- );
3902
- const recovering = this.#runStore.status(runId);
3903
- emitWorkflowActivity(onActivity, {
3904
- state: "recovering",
3905
- stage: recovering.stage,
3906
- runId,
3907
- ...(recovering.change ? { change: recovering.change } : {}),
3908
- objective: recovering.change
3909
- ? `recover change ${recovering.change}`
3910
- : "recover workflow application",
3911
- code: intent === "cancel" ? "operation-cancelled" : "run-discarded",
3912
- legalCommands: ["status"],
3913
- });
3914
- }
3915
- const recovered = await this.#application.recover(transactionId, context);
3916
- this.#runStore.assertLease(lease);
3917
- return this.#settleApplication(
3918
- runId,
3919
- operationId,
3920
- recovered,
3921
- lease,
3922
- onActivity,
3923
- );
3924
- }
3925
-
3926
- async #abortActiveOperation(runId: string, reason: Error): Promise<void> {
3927
- const pending: Promise<unknown>[] = [];
3928
- const amendment = this.#amendments.get(runId);
3929
- if (amendment) pending.push(amendment);
3930
- for (const [controller, entry] of this.#commands) {
3931
- if (
3932
- !["start", "resume"].includes(entry.command.command) ||
3933
- this.#lookupRun(entry.command.stage, entry.command.change) !== runId
3934
- )
3935
- continue;
3936
- controller.abort(reason);
3937
- pending.push(entry.settled);
3938
- }
3939
- const active = this.#active.get(runId);
3940
- if (active) {
3941
- active.controller.abort(reason);
3942
- pending.push(active.settled);
3943
- }
3944
- const results = await Promise.allSettled(pending);
3945
- for (const result of results) {
3946
- if (result.status !== "rejected") continue;
3947
- const error = result.reason;
3948
- if (
3949
- error !== reason &&
3950
- !(
3951
- error instanceof Error &&
3952
- ["lease-fenced", "operation-journal-fenced"].includes(error.message)
3953
- )
3954
- )
3955
- throw error;
3956
- }
3957
- }
3958
-
3959
- async #cancel(
3960
- command: Extract<ControlCommand, { command: "cancel" }>,
3961
- onActivity?: (event: WorkflowActivityUpdate) => void,
3962
- ) {
3963
- const runId = this.#lookupRun(command.stage, command.change);
3964
- if (!runId) throw new Error("run-not-found");
3965
- const replay = this.#beginOperation(
3966
- runId,
3967
- command.operationId,
3968
- command.command,
3969
- );
3970
- if (replay) return replay;
3971
- const lease = this.#operationLease(runId, command.operationId);
3972
- const state = this.#runStore.status(runId).state;
3973
- let outcome: Record<string, unknown>;
3974
- if (state === "applying" || state === "recovering") {
3975
- outcome = await this.#controlApply(
3976
- runId,
3977
- command.operationId,
3978
- "cancel",
3979
- lease,
3980
- onActivity,
3981
- );
3982
- } else {
3983
- const activeSettlement = this.#abortActiveOperation(
3984
- runId,
3985
- new Error("operation-cancelled"),
3986
- );
3987
- const current = this.#runStore.status(runId);
3988
- if (
3989
- !["paused", "discarded", "completed", "rejected"].includes(
3990
- current.state,
3991
- )
3992
- ) {
3993
- this.#transition(
3994
- runId,
3995
- "paused",
3996
- `${command.operationId}:cancelled`,
3997
- "operation-cancelled",
3998
- lease,
3999
- );
4000
- }
4001
- await activeSettlement;
4002
- for (const task of this.#tasks(runId)) {
4003
- if (
4004
- ["phase-ready", "phase-running", "validating"].includes(task.state)
4005
- ) {
4006
- this.#setTask(
4007
- runId,
4008
- task.task_id,
4009
- {
4010
- state: "paused",
4011
- pauseCode: "operation-cancelled",
4012
- queuePosition: null,
4013
- },
4014
- lease,
4015
- );
4016
- }
4017
- }
4018
- outcome = this.#statusByRun(runId);
4019
- }
4020
- outcome = {
4021
- ...outcome,
4022
- operation: { kind: "operation-cancelled" },
4023
- };
4024
- return this.#commitOperation(runId, command.operationId, outcome);
4025
- }
4026
-
4027
- async #discard(
4028
- command: Extract<ControlCommand, { command: "discard" }>,
4029
- onActivity?: (event: WorkflowActivityUpdate) => void,
4030
- ) {
4031
- const runId = this.#lookupRun(command.stage, command.change);
4032
- if (!runId) throw new Error("run-not-found");
4033
- const replay = this.#beginOperation(
4034
- runId,
4035
- command.operationId,
4036
- command.command,
4037
- );
4038
- if (replay) return replay;
4039
- const lease = this.#operationLease(runId, command.operationId);
4040
- const state = this.#runStore.status(runId).state;
4041
- let outcome: Record<string, unknown>;
4042
- if (state === "applying" || state === "recovering") {
4043
- outcome = await this.#controlApply(
4044
- runId,
4045
- command.operationId,
4046
- "discard",
4047
- lease,
4048
- onActivity,
4049
- );
4050
- } else {
4051
- const activeSettlement = this.#abortActiveOperation(
4052
- runId,
4053
- new Error("run-discarded"),
4054
- );
4055
- const current = this.#runStore.status(runId);
4056
- if (current.state !== "discarded") {
4057
- if (["completed", "rejected"].includes(current.state)) {
4058
- this.#interruptOperation(runId, command.operationId);
4059
- throw new Error("discard-not-allowed");
4060
- }
4061
- this.#transition(
4062
- runId,
4063
- "discarded",
4064
- `${command.operationId}:discarded`,
4065
- undefined,
4066
- lease,
4067
- );
4068
- await activeSettlement;
4069
- this.#cleanupTerminal(runId, command.operationId, lease);
4070
- } else {
4071
- await activeSettlement;
4072
- }
4073
- outcome = this.#statusByRun(runId);
4074
- }
4075
- return this.#commitOperation(runId, command.operationId, outcome);
4076
- }
4077
-
4078
- execute(
4079
- value: unknown,
4080
- signal?: AbortSignal,
4081
- onActivity?: (event: WorkflowActivityUpdate) => void,
4082
- ): Promise<Record<string, unknown>> {
4083
- let command: ControlCommand;
4084
- try {
4085
- this.#assertOpen();
4086
- if (this.#closing) throw new Error("workflow-engine-closed");
4087
- command = assertControlCommand(value);
4088
- } catch (error) {
4089
- return Promise.reject(error);
4090
- }
4091
- const controller = new AbortController();
4092
- const onAbort = () => controller.abort(signal?.reason);
4093
- if (signal?.aborted) onAbort();
4094
- else signal?.addEventListener("abort", onAbort, { once: true });
4095
- // Register before execution can yield or invoke external callbacks.
4096
- const settled = Promise.resolve()
4097
- .then(() => this.#executeCommand(command, controller.signal, onActivity))
4098
- .finally(() => {
4099
- signal?.removeEventListener("abort", onAbort);
4100
- this.#commands.delete(controller);
4101
- });
4102
- this.#commands.set(controller, { command, settled });
4103
- return settled;
4104
- }
4105
-
4106
- async #executeCommand(
4107
- command: ControlCommand,
4108
- signal?: AbortSignal,
4109
- onActivity?: (event: WorkflowActivityUpdate) => void,
4110
- ): Promise<Record<string, unknown>> {
4111
- switch (command.command) {
4112
- case "start":
4113
- return this.#withAvailableDelivery(
4114
- await this.#start(command, signal, onActivity),
4115
- signal,
4116
- );
4117
- case "status": {
4118
- const runId = this.#lookupRun(command.stage, command.change);
4119
- if (!runId) {
4120
- return {
4121
- stage: command.stage,
4122
- change: command.change,
4123
- state: "not-started",
4124
- durable: true,
4125
- completed: false,
4126
- legalCommands: ["start"],
4127
- tasks: [],
4128
- queue: [],
4129
- };
4130
- }
4131
- this.#recoverLegacyContextApproval(runId);
4132
- return this.#withAvailableDelivery(this.#statusByRun(runId), signal);
4133
- }
4134
- case "resume":
4135
- return this.#withAvailableDelivery(
4136
- await this.#resume(command, signal, onActivity),
4137
- signal,
4138
- );
4139
- case "rebind":
4140
- return this.#rebind(command);
4141
- case "cancel":
4142
- return this.#cancel(command, onActivity);
4143
- case "discard":
4144
- return this.#discard(command, onActivity);
4145
- }
4146
- }
4147
-
4148
- prepareBootstrapHandoff(
4149
- input: Record<string, unknown>,
4150
- ): Record<string, unknown> {
4151
- this.#assertOpen();
4152
- const acceptanceFacts = normalizeBootstrapAcceptanceFacts(
4153
- input.acceptanceFacts,
4154
- );
4155
- const workspaceManifestHash =
4156
- typeof input.workspaceManifestHash === "string" &&
4157
- SHA256.test(input.workspaceManifestHash)
4158
- ? input.workspaceManifestHash
4159
- : undefined;
4160
- if (
4161
- typeof input.operationId !== "string" ||
4162
- !IDENTIFIER.test(input.operationId) ||
4163
- typeof input.runId !== "string" ||
4164
- typeof input.change !== "string" ||
4165
- !CHANGE_NAME.test(input.change) ||
4166
- typeof input.receiptHash !== "string" ||
4167
- !SHA256.test(input.receiptHash) ||
4168
- typeof input.graphHash !== "string" ||
4169
- !SHA256.test(input.graphHash) ||
4170
- typeof input.acceptanceHash !== "string" ||
4171
- !SHA256.test(input.acceptanceHash) ||
4172
- (acceptanceFacts !== undefined) !==
4173
- (workspaceManifestHash !== undefined) ||
4174
- (acceptanceFacts !== undefined &&
4175
- workspaceManifestHash !== undefined &&
4176
- bootstrapAcceptanceHash(acceptanceFacts, workspaceManifestHash) !==
4177
- input.acceptanceHash)
4178
- ) {
4179
- throw new Error("bootstrap-handoff-invalid");
4180
- }
4181
- this.#runStore.status(input.runId);
4182
- const existing = this.#database
4183
- .prepare(
4184
- `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
4185
- FROM bootstrap_handoffs WHERE receipt_hash = ?`,
4186
- )
4187
- .get(input.receiptHash) as BootstrapRow | undefined;
4188
- if (existing) {
4189
- const outcome = parseJsonRecord(existing.facts_json);
4190
- const existingAcceptanceFacts = normalizeBootstrapAcceptanceFacts(
4191
- outcome.acceptanceFacts,
4192
- );
4193
- const storedBindingValid =
4194
- outcome.handoffId === existing.handoff_id &&
4195
- outcome.runId === existing.owner_run_id &&
4196
- outcome.receiptHash === existing.receipt_hash &&
4197
- outcome.selectorBeforeCutover === "bootstrap" &&
4198
- existing.state === "prepared" &&
4199
- outcome.state === "prepared" &&
4200
- outcome.selectorCasPending === true;
4201
- if (!storedBindingValid) throw new Error("bootstrap-handoff-invalid");
4202
- const stableBindingMatches =
4203
- existing.owner_run_id === input.runId &&
4204
- outcome.change === input.change &&
4205
- outcome.graphHash === input.graphHash;
4206
- if (!stableBindingMatches) {
4207
- throw new Error("bootstrap-handoff-conflict");
4208
- }
4209
- const acceptanceMatches =
4210
- outcome.acceptanceHash === input.acceptanceHash &&
4211
- canonicalJson(existingAcceptanceFacts ?? null) ===
4212
- canonicalJson(acceptanceFacts ?? null) &&
4213
- outcome.workspaceManifestHash === workspaceManifestHash;
4214
- if (acceptanceMatches) return structuredClone(outcome);
4215
- if (
4216
- existing.state !== "prepared" ||
4217
- outcome.state !== "prepared" ||
4218
- outcome.selectorBeforeCutover !== "bootstrap" ||
4219
- outcome.selectorCasPending !== true ||
4220
- acceptanceFacts === undefined ||
4221
- workspaceManifestHash === undefined
4222
- ) {
4223
- throw new Error("bootstrap-handoff-conflict");
4224
- }
4225
- const refreshedHandoffId = `handoff-${hash(
4226
- input.runId,
4227
- input.receiptHash,
4228
- input.graphHash,
4229
- input.acceptanceHash,
4230
- workspaceManifestHash,
4231
- ).slice(0, 40)}`;
4232
- const refreshed = {
4233
- ...outcome,
4234
- handoffId: refreshedHandoffId,
4235
- acceptanceHash: input.acceptanceHash,
4236
- acceptanceFacts,
4237
- workspaceManifestHash,
4238
- };
4239
- this.#transaction(() => {
4240
- const result = this.#database
4241
- .prepare(
4242
- `UPDATE bootstrap_handoffs
4243
- SET handoff_id = ?, facts_json = ?
4244
- WHERE handoff_id = ? AND state = 'prepared' AND facts_json = ?`,
4245
- )
4246
- .run(
4247
- refreshedHandoffId,
4248
- JSON.stringify(refreshed),
4249
- existing.handoff_id,
4250
- existing.facts_json,
4251
- );
4252
- if (Number(result.changes) !== 1) {
4253
- throw new Error("bootstrap-handoff-conflict");
4254
- }
4255
- });
4256
- return structuredClone(refreshed);
4257
- }
4258
- const handoffId = `handoff-${hash(
4259
- input.runId,
4260
- input.receiptHash,
4261
- input.graphHash,
4262
- input.acceptanceHash,
4263
- workspaceManifestHash ?? "acceptance-pending",
4264
- ).slice(0, 40)}`;
4265
- const outcome = {
4266
- handoffId,
4267
- state: "prepared",
4268
- runId: input.runId,
4269
- change: input.change,
4270
- receiptHash: input.receiptHash,
4271
- graphHash: input.graphHash,
4272
- acceptanceHash: input.acceptanceHash,
4273
- ...(acceptanceFacts ? { acceptanceFacts } : {}),
4274
- ...(workspaceManifestHash ? { workspaceManifestHash } : {}),
4275
- selectorBeforeCutover: "bootstrap",
4276
- selectorCasPending: true,
4277
- };
4278
- const preparedRows = this.#database
4279
- .prepare(
4280
- `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
4281
- FROM bootstrap_handoffs WHERE state = 'prepared'
4282
- ORDER BY handoff_id`,
4283
- )
4284
- .all() as unknown as BootstrapRow[];
4285
- if (preparedRows.length > 0) throw new Error("bootstrap-handoff-conflict");
4286
- this.#transaction(() => {
4287
- this.#database
4288
- .prepare(
4289
- `INSERT INTO bootstrap_handoffs(
4290
- handoff_id, owner_run_id, receipt_hash, state, facts_json
4291
- ) VALUES (?, ?, ?, 'prepared', ?)`,
4292
- )
4293
- .run(
4294
- handoffId,
4295
- String(input.runId),
4296
- String(input.receiptHash),
4297
- JSON.stringify(outcome),
4298
- );
4299
- });
4300
- return structuredClone(outcome);
4301
- }
4302
-
4303
- inspectBootstrapHandoff(receiptHash: string): Record<string, unknown> {
4304
- this.#assertOpen();
4305
- if (!SHA256.test(receiptHash)) throw new Error("bootstrap-handoff-invalid");
4306
- const row = this.#database
4307
- .prepare(
4308
- `SELECT facts_json FROM bootstrap_handoffs WHERE receipt_hash = ?`,
4309
- )
4310
- .get(receiptHash) as { facts_json: string } | undefined;
4311
- if (!row) throw new Error("bootstrap-handoff-not-found");
4312
- return parseJsonRecord(row.facts_json);
4313
- }
4314
-
4315
- close(): Promise<void> {
4316
- if (this.#closing) return this.#closing;
4317
- if (this.#closed) return Promise.resolve();
4318
- this.#closing = Promise.resolve().then(async () => {
4319
- if (this.#orphanRecoveryTimer) {
4320
- clearTimeout(this.#orphanRecoveryTimer);
4321
- this.#orphanRecoveryTimer = undefined;
4322
- }
4323
- const reason = new Error("workflow-engine-closed");
4324
- const commands = [...this.#commands];
4325
- for (const [controller] of commands) controller.abort(reason);
4326
- const active = [...this.#active.values()];
4327
- for (const operation of active) operation.controller.abort(reason);
4328
- await Promise.allSettled(commands.map(([, entry]) => entry.settled));
4329
- await Promise.allSettled(active.map((operation) => operation.settled));
4330
- await Promise.allSettled([...this.#amendments.values()]);
4331
- for (const held of this.#operationLeases.values())
4332
- clearInterval(held.timer);
4333
- this.#operationLeases.clear();
4334
- this.#lifecycle?.close?.();
4335
- this.#database.close();
4336
- this.#runStore.close();
4337
- this.#closed = true;
4338
- });
4339
- return this.#closing;
4340
- }
4341
- }