@abelxiaoxing/cadence 1.8.3 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/README.md +324 -111
  2. package/THIRD_PARTY_NOTICES.md +15 -0
  3. package/agents/design-explorer.md +14 -26
  4. package/agents/diagnosis-worker.md +14 -25
  5. package/agents/implementation-worker.md +16 -35
  6. package/config/.env.example +2 -5
  7. package/config/workflow-task.example.json +7 -0
  8. package/docs/remote-access.md +60 -0
  9. package/docs/reuse-and-distribution.md +44 -0
  10. package/docs/service-management.md +108 -0
  11. package/licenses/bundled-web.txt +6245 -0
  12. package/licenses/pi-web-ui-MIT.txt +21 -0
  13. package/licenses/pi-web-ui-reuse.md +22 -0
  14. package/package.json +94 -21
  15. package/src/index.ts +22 -1807
  16. package/src/native-build.ts +6 -0
  17. package/src/native-client.ts +103 -0
  18. package/src/native-export.ts +58 -0
  19. package/src/native-history.ts +359 -0
  20. package/src/operator-cli.mjs +453 -1937
  21. package/src/plan-proposal.ts +87 -0
  22. package/src/sdk-worker.mjs +549 -0
  23. package/src/service/contracts/active.ts +82 -0
  24. package/src/service/contracts/budget.ts +148 -0
  25. package/src/service/contracts/build.ts +67 -0
  26. package/src/service/contracts/capture.ts +48 -0
  27. package/src/service/contracts/core.ts +33 -0
  28. package/src/service/contracts/current.ts +261 -0
  29. package/src/service/contracts/domain.ts +68 -0
  30. package/src/service/contracts/history.ts +2245 -0
  31. package/src/service/contracts/models.ts +139 -0
  32. package/src/service/coordination.ts +862 -0
  33. package/src/service/private-files.ts +204 -0
  34. package/src/service-cli.mjs +5592 -0
  35. package/src/service-client.ts +747 -0
  36. package/src/service-contract.ts +245 -0
  37. package/src/service-host.mjs +22382 -0
  38. package/src/service-http-client.ts +425 -0
  39. package/src/service-wire.ts +372 -0
  40. package/src/task-shell.ts +1045 -0
  41. package/src/task-store.ts +479 -0
  42. package/src/task-types.ts +184 -0
  43. package/src/verification-diagnostics.ts +29 -72
  44. package/src/worker-capture.mjs +1197 -0
  45. package/src/worker-public.mjs +1753 -0
  46. package/src/worker-redaction.mjs +608 -0
  47. package/src/worker-tools.mjs +202 -0
  48. package/web/dist/assets/KaTeX_AMS-Regular.ttf +0 -0
  49. package/web/dist/assets/KaTeX_AMS-Regular.woff +0 -0
  50. package/web/dist/assets/KaTeX_AMS-Regular.woff2 +0 -0
  51. package/web/dist/assets/KaTeX_Caligraphic-Bold.ttf +0 -0
  52. package/web/dist/assets/KaTeX_Caligraphic-Bold.woff +0 -0
  53. package/web/dist/assets/KaTeX_Caligraphic-Bold.woff2 +0 -0
  54. package/web/dist/assets/KaTeX_Caligraphic-Regular.ttf +0 -0
  55. package/web/dist/assets/KaTeX_Caligraphic-Regular.woff +0 -0
  56. package/web/dist/assets/KaTeX_Caligraphic-Regular.woff2 +0 -0
  57. package/web/dist/assets/KaTeX_Fraktur-Bold.ttf +0 -0
  58. package/web/dist/assets/KaTeX_Fraktur-Bold.woff +0 -0
  59. package/web/dist/assets/KaTeX_Fraktur-Bold.woff2 +0 -0
  60. package/web/dist/assets/KaTeX_Fraktur-Regular.ttf +0 -0
  61. package/web/dist/assets/KaTeX_Fraktur-Regular.woff +0 -0
  62. package/web/dist/assets/KaTeX_Fraktur-Regular.woff2 +0 -0
  63. package/web/dist/assets/KaTeX_Main-Bold.ttf +0 -0
  64. package/web/dist/assets/KaTeX_Main-Bold.woff +0 -0
  65. package/web/dist/assets/KaTeX_Main-Bold.woff2 +0 -0
  66. package/web/dist/assets/KaTeX_Main-BoldItalic.ttf +0 -0
  67. package/web/dist/assets/KaTeX_Main-BoldItalic.woff +0 -0
  68. package/web/dist/assets/KaTeX_Main-BoldItalic.woff2 +0 -0
  69. package/web/dist/assets/KaTeX_Main-Italic.ttf +0 -0
  70. package/web/dist/assets/KaTeX_Main-Italic.woff +0 -0
  71. package/web/dist/assets/KaTeX_Main-Italic.woff2 +0 -0
  72. package/web/dist/assets/KaTeX_Main-Regular.ttf +0 -0
  73. package/web/dist/assets/KaTeX_Main-Regular.woff +0 -0
  74. package/web/dist/assets/KaTeX_Main-Regular.woff2 +0 -0
  75. package/web/dist/assets/KaTeX_Math-BoldItalic.ttf +0 -0
  76. package/web/dist/assets/KaTeX_Math-BoldItalic.woff +0 -0
  77. package/web/dist/assets/KaTeX_Math-BoldItalic.woff2 +0 -0
  78. package/web/dist/assets/KaTeX_Math-Italic.ttf +0 -0
  79. package/web/dist/assets/KaTeX_Math-Italic.woff +0 -0
  80. package/web/dist/assets/KaTeX_Math-Italic.woff2 +0 -0
  81. package/web/dist/assets/KaTeX_SansSerif-Bold.ttf +0 -0
  82. package/web/dist/assets/KaTeX_SansSerif-Bold.woff +0 -0
  83. package/web/dist/assets/KaTeX_SansSerif-Bold.woff2 +0 -0
  84. package/web/dist/assets/KaTeX_SansSerif-Italic.ttf +0 -0
  85. package/web/dist/assets/KaTeX_SansSerif-Italic.woff +0 -0
  86. package/web/dist/assets/KaTeX_SansSerif-Italic.woff2 +0 -0
  87. package/web/dist/assets/KaTeX_SansSerif-Regular.ttf +0 -0
  88. package/web/dist/assets/KaTeX_SansSerif-Regular.woff +0 -0
  89. package/web/dist/assets/KaTeX_SansSerif-Regular.woff2 +0 -0
  90. package/web/dist/assets/KaTeX_Script-Regular.ttf +0 -0
  91. package/web/dist/assets/KaTeX_Script-Regular.woff +0 -0
  92. package/web/dist/assets/KaTeX_Script-Regular.woff2 +0 -0
  93. package/web/dist/assets/KaTeX_Size1-Regular.ttf +0 -0
  94. package/web/dist/assets/KaTeX_Size1-Regular.woff +0 -0
  95. package/web/dist/assets/KaTeX_Size1-Regular.woff2 +0 -0
  96. package/web/dist/assets/KaTeX_Size2-Regular.ttf +0 -0
  97. package/web/dist/assets/KaTeX_Size2-Regular.woff +0 -0
  98. package/web/dist/assets/KaTeX_Size2-Regular.woff2 +0 -0
  99. package/web/dist/assets/KaTeX_Size3-Regular.ttf +0 -0
  100. package/web/dist/assets/KaTeX_Size3-Regular.woff +0 -0
  101. package/web/dist/assets/KaTeX_Size3-Regular.woff2 +0 -0
  102. package/web/dist/assets/KaTeX_Size4-Regular.ttf +0 -0
  103. package/web/dist/assets/KaTeX_Size4-Regular.woff +0 -0
  104. package/web/dist/assets/KaTeX_Size4-Regular.woff2 +0 -0
  105. package/web/dist/assets/KaTeX_Typewriter-Regular.ttf +0 -0
  106. package/web/dist/assets/KaTeX_Typewriter-Regular.woff +0 -0
  107. package/web/dist/assets/KaTeX_Typewriter-Regular.woff2 +0 -0
  108. package/web/dist/assets/cadence.css +2 -0
  109. package/web/dist/assets/cadence.js +7 -0
  110. package/web/dist/assets/markdown.js +296 -0
  111. package/web/dist/assets/react.js +9 -0
  112. package/web/dist/assets/rolldown-runtime.js +1 -0
  113. package/web/dist/assets/timeline-client.js +41 -0
  114. package/web/dist/assets/vis-timeline.css +1274 -0
  115. package/web/dist/assets/vis-timeline.js +32 -0
  116. package/web/dist/index.html +18 -0
  117. package/web/dist/manifest.json +505 -0
  118. package/config/plan-draft.example.json +0 -125
  119. package/config/plan-draft.multiple-tasks.example.json +0 -375
  120. package/config/plan-draft.quick.example.json +0 -78
  121. package/config/routes.example.json +0 -39
  122. package/prompts/abel-design.md +0 -382
  123. package/prompts/abel-diagnose.md +0 -71
  124. package/prompts/abel-implement.md +0 -284
  125. package/prompts/abel-init.md +0 -52
  126. package/src/activation.ts +0 -44
  127. package/src/activity-contracts.ts +0 -82
  128. package/src/agent-registry.ts +0 -53
  129. package/src/apply-transaction.ts +0 -1437
  130. package/src/artifact-store.ts +0 -213
  131. package/src/build-windows-job.mjs +0 -95
  132. package/src/candidate-artifact.ts +0 -143
  133. package/src/candidate-patch.ts +0 -271
  134. package/src/candidate-workspace.ts +0 -80
  135. package/src/canonical.ts +0 -16
  136. package/src/change-contract-schema.ts +0 -191
  137. package/src/change-contract.ts +0 -324
  138. package/src/change-verification.ts +0 -1617
  139. package/src/child-budget.ts +0 -58
  140. package/src/child-model.ts +0 -142
  141. package/src/child-session.ts +0 -640
  142. package/src/contracts.ts +0 -2476
  143. package/src/control-contracts.ts +0 -382
  144. package/src/delivery-compiler.ts +0 -1805
  145. package/src/delivery-revision.ts +0 -42
  146. package/src/design-control.ts +0 -1299
  147. package/src/design-diagnostics.ts +0 -181
  148. package/src/design-journal.ts +0 -1773
  149. package/src/directory-sync.ts +0 -36
  150. package/src/durable-contracts.ts +0 -231
  151. package/src/durable-workflow.ts +0 -978
  152. package/src/evidence-draft.ts +0 -122
  153. package/src/execution-profile.ts +0 -64
  154. package/src/execution-retention.ts +0 -77
  155. package/src/file-snapshot.ts +0 -163
  156. package/src/implement-continuation.ts +0 -559
  157. package/src/implement-graph.ts +0 -690
  158. package/src/implement-plan.ts +0 -72
  159. package/src/install-config.d.mts +0 -6
  160. package/src/isolation-backend.ts +0 -353
  161. package/src/model-source.ts +0 -22
  162. package/src/openspec-cli.ts +0 -494
  163. package/src/operator-cli-entry.mjs +0 -24
  164. package/src/operator-tools.ts +0 -309
  165. package/src/package-candidate.ts +0 -227
  166. package/src/package-delivery.ts +0 -486
  167. package/src/package-state.ts +0 -320
  168. package/src/package-verification.ts +0 -635
  169. package/src/package-workflow.ts +0 -342
  170. package/src/packet-runtime.ts +0 -570
  171. package/src/parent-provider.ts +0 -375
  172. package/src/phase-execution.ts +0 -2259
  173. package/src/pi-adapter.ts +0 -19
  174. package/src/plan-draft-summary.ts +0 -527
  175. package/src/plan-draft.ts +0 -502
  176. package/src/remove-path.ts +0 -32
  177. package/src/route-policy.ts +0 -545
  178. package/src/run-state.ts +0 -420
  179. package/src/run-store.ts +0 -1179
  180. package/src/safe-path.ts +0 -77
  181. package/src/scoped-grep-worker.mjs +0 -23
  182. package/src/scoped-tools.ts +0 -775
  183. package/src/single-task-draft.ts +0 -142
  184. package/src/sqlite-schema.ts +0 -171
  185. package/src/state-root.ts +0 -215
  186. package/src/storage-schema.ts +0 -360
  187. package/src/subagent-activity.ts +0 -1417
  188. package/src/submit-tool.ts +0 -950
  189. package/src/task-ledger.ts +0 -1338
  190. package/src/transport-budget.ts +0 -52
  191. package/src/verification-capability.ts +0 -1191
  192. package/src/verification-environment-io.ts +0 -389
  193. package/src/verification-environment.ts +0 -162
  194. package/src/verification-identity.ts +0 -77
  195. package/src/verification-scan.ts +0 -64
  196. package/src/windows-job-backend.ts +0 -419
  197. package/src/windows-job.c +0 -149
  198. package/src/worker-broker.ts +0 -806
  199. package/src/workflow-engine.ts +0 -28
  200. package/src/workflow-policy.ts +0 -1679
  201. package/src/workflow-recovery-policy.ts +0 -97
  202. package/src/workflow-scheduling.ts +0 -105
  203. package/src/workflow-state-machine.ts +0 -4355
  204. package/src/workflow-status.ts +0 -576
  205. package/src/workspace-io-entry.mjs +0 -76
  206. package/src/workspace-io-worker.mjs +0 -1312
  207. package/src/workspace-io.ts +0 -89
  208. package/src/workspace-store.ts +0 -634
@@ -1,4355 +0,0 @@
1
- import { readFileSync } from "node:fs";
2
- import path from "node:path";
3
- import { DatabaseSync } from "node:sqlite";
4
- import type { WorkflowActivityUpdate } from "./activity-contracts.ts";
5
- import {
6
- assertPlanWithinChangeContract,
7
- normalizeChangeContract,
8
- retainedChangeContract,
9
- } from "./change-contract.ts";
10
- import { LIMITS, verificationInputPaths } from "./contracts.ts";
11
- import {
12
- assertControlCommand,
13
- type ControlCommand,
14
- type ControlStage,
15
- } from "./control-contracts.ts";
16
- import {
17
- compileImplementPlan,
18
- DeliveryValidationError,
19
- type ImplementPlan,
20
- type PlanTaskDraft,
21
- } from "./delivery-compiler.ts";
22
- import { compareDeliveryRevision } from "./delivery-revision.ts";
23
- import { snapshotFiles } from "./file-snapshot.ts";
24
- import type { RoutePolicy } from "./route-policy.ts";
25
- import {
26
- canonicalJson,
27
- type RunProjection,
28
- type RunState,
29
- } from "./run-state.ts";
30
- import { type OperationLease, RunStore } from "./run-store.ts";
31
- import { isSafeRegularFile } from "./safe-path.ts";
32
- import { configureSqlite, ensureSqliteSchema } from "./sqlite-schema.ts";
33
- import type { ResolvedStateRoot } from "./state-root.ts";
34
- import {
35
- AMENDMENT_SCHEMA,
36
- ENGINE_ADDITIONS,
37
- ENGINE_SCHEMA,
38
- RECOVERY_ADDITIONS,
39
- RECOVERY_GRANT_SCHEMA,
40
- RECOVERY_SCHEMA,
41
- REJECTED_DELIVERY_SCHEMA,
42
- } from "./storage-schema.ts";
43
- import { permitsContextRead } from "./submit-tool.ts";
44
- import {
45
- type ActiveOperation,
46
- assertDelivery,
47
- type BootstrapRow,
48
- baselineRecoveryKey,
49
- bootstrapAcceptanceHash,
50
- CHANGE_NAME,
51
- cancellableRead,
52
- classifyPersistedContextRequest,
53
- decideRecoveryAction,
54
- type EngineOperationRow,
55
- type EngineRunRow,
56
- type EngineTaskRow,
57
- emitWorkflowActivity,
58
- expectedClassification,
59
- type HeldOperationLease,
60
- hash,
61
- IDENTIFIER,
62
- isCancellationException,
63
- isRecord,
64
- isWorkflowApprovalCode,
65
- normalizeBootstrapAcceptanceFacts,
66
- normalizeVerificationPrerequisite,
67
- normalizeWorkflowContextRequest,
68
- normalizeWorkflowVerificationStatus,
69
- parseJsonRecord,
70
- parsePlan,
71
- phases,
72
- recoveryKey,
73
- type SafeAttemptDiagnostic,
74
- SHA256,
75
- type WorkflowApplication,
76
- type WorkflowApplicationContext,
77
- type WorkflowAttemptOutcome,
78
- type WorkflowAvailableDelivery,
79
- type WorkflowChangeVerifier,
80
- type WorkflowContextRequest,
81
- type WorkflowDelivery,
82
- type WorkflowDeliverySource,
83
- type WorkflowEngineOptions,
84
- type WorkflowRecoveryFact,
85
- type WorkflowRunLifecycle,
86
- type WorkflowVerificationStatus,
87
- type WorkflowWorker,
88
- } from "./workflow-policy.ts";
89
- import {
90
- assessRecoveryGrant,
91
- recoveryExhausted,
92
- } from "./workflow-recovery-policy.ts";
93
- import { selectRunnableTasks } from "./workflow-scheduling.ts";
94
- import {
95
- firstPausedTask,
96
- projectWorkflowStatus,
97
- type WorkflowResourceBudget,
98
- } from "./workflow-status.ts";
99
- export class WorkflowEngine {
100
- readonly #consumerRoot: string;
101
- readonly #stateRoot: ResolvedStateRoot;
102
- readonly #runStore: RunStore;
103
- readonly #database: DatabaseSync;
104
- readonly #deliverySource: WorkflowDeliverySource;
105
- readonly #worker: WorkflowWorker;
106
- readonly #changeVerifier: WorkflowChangeVerifier;
107
- readonly #application?: WorkflowApplication;
108
- readonly #lifecycle?: WorkflowRunLifecycle;
109
- readonly #now: () => number;
110
- readonly #leaseTtlMs: number;
111
- readonly #workHardLimit: number;
112
- #activeTasks = 0;
113
- readonly #capacityWaiters = new Set<() => void>();
114
- readonly #active = new Map<string, ActiveOperation>();
115
- readonly #operationLeases = new Map<string, HeldOperationLease>();
116
- #orphanRecoveryTimer?: ReturnType<typeof setTimeout>;
117
- readonly #commands = new Map<
118
- AbortController,
119
- { command: ControlCommand; settled: Promise<Record<string, unknown>> }
120
- >();
121
- #closing?: Promise<void>;
122
- readonly #amendments = new Map<string, Promise<Record<string, unknown>>>();
123
- #closed = false;
124
-
125
- private constructor(options: WorkflowEngineOptions) {
126
- this.#consumerRoot = path.resolve(options.consumerRoot);
127
- this.#stateRoot = options.stateRoot;
128
- this.#deliverySource = options.deliverySource;
129
- this.#worker = options.worker;
130
- this.#changeVerifier = options.changeVerifier;
131
- this.#application = options.application;
132
- this.#lifecycle = options.lifecycle;
133
- this.#now = options.now ?? Date.now;
134
- this.#leaseTtlMs = options.leaseTtlMs ?? 60_000;
135
- this.#workHardLimit = options.workHardLimit ?? 512;
136
- if (!Number.isSafeInteger(this.#workHardLimit) || this.#workHardLimit < 1)
137
- throw new Error("workflow-work-limit-invalid");
138
- this.#runStore = RunStore.open(options.stateRoot, { now: this.#now });
139
- this.#database = new DatabaseSync(options.stateRoot.databasePath);
140
- try {
141
- configureSqlite(this.#database);
142
- ensureSqliteSchema(this.#database, ENGINE_SCHEMA, ENGINE_ADDITIONS);
143
- ensureSqliteSchema(this.#database, RECOVERY_SCHEMA, RECOVERY_ADDITIONS);
144
- ensureSqliteSchema(this.#database, RECOVERY_GRANT_SCHEMA);
145
- ensureSqliteSchema(this.#database, AMENDMENT_SCHEMA);
146
- ensureSqliteSchema(this.#database, REJECTED_DELIVERY_SCHEMA);
147
- } catch (error) {
148
- this.#database.close();
149
- this.#runStore.close();
150
- throw error;
151
- }
152
- this.#transaction(() => {
153
- const budgets = this.#database
154
- .prepare(
155
- "SELECT run_id FROM workflow_work_budget WHERE recovery_policy = 0",
156
- )
157
- .all() as { run_id: string }[];
158
- for (const { run_id } of budgets) {
159
- const plans = this.#database
160
- .prepare(
161
- "SELECT plan_json FROM workflow_engine_deliveries WHERE run_id = ?",
162
- )
163
- .all(run_id) as { plan_json: string }[];
164
- const high = Math.max(
165
- 0,
166
- ...plans.map((row) => this.#phaseCount(JSON.parse(row.plan_json))),
167
- );
168
- this.#database
169
- .prepare(
170
- "UPDATE workflow_work_budget SET phase_high_water = ?, hard_limit = MAX(hard_limit, max_work, used), recovery_policy = 1 WHERE run_id = ?",
171
- )
172
- .run(high, run_id);
173
- }
174
- });
175
- this.#recoverInterruptedOperations();
176
- this.#recoverTerminalCleanup();
177
- }
178
-
179
- static open(options: WorkflowEngineOptions): WorkflowEngine {
180
- if (
181
- path.resolve(options.consumerRoot) !== options.stateRoot.consumerRoot ||
182
- options.stateRoot.consumerRootHash.length !== 64
183
- ) {
184
- throw new Error("workflow-engine-root-mismatch");
185
- }
186
- if (
187
- !options.deliverySource ||
188
- typeof options.deliverySource.load !== "function" ||
189
- !options.worker ||
190
- typeof options.worker.runAttempt !== "function" ||
191
- typeof options.worker.rebind !== "function" ||
192
- !options.changeVerifier ||
193
- typeof options.changeVerifier.verify !== "function" ||
194
- (options.application !== undefined &&
195
- (typeof options.application.prepare === "function") !==
196
- (typeof options.application.applyPrepared === "function"))
197
- ) {
198
- throw new Error("workflow-engine-service-invalid");
199
- }
200
- if (
201
- options.leaseTtlMs !== undefined &&
202
- (!Number.isSafeInteger(options.leaseTtlMs) || options.leaseTtlMs < 1)
203
- ) {
204
- throw new Error("workflow-engine-lease-invalid");
205
- }
206
- return new WorkflowEngine(options);
207
- }
208
-
209
- updateRoutePolicy(policy: RoutePolicy): void {
210
- this.#assertOpen();
211
- if (!this.#worker.updateRoutePolicy) {
212
- throw new Error("route-policy-update-unavailable");
213
- }
214
- this.#worker.updateRoutePolicy(policy);
215
- }
216
-
217
- routePolicyStatus(): Record<string, unknown> | undefined {
218
- this.#assertOpen();
219
- return this.#worker.routePolicyStatus?.();
220
- }
221
-
222
- #assertOpen(): void {
223
- if (this.#closed) throw new Error("workflow-engine-closed");
224
- }
225
-
226
- #transaction<T>(work: () => T): T {
227
- this.#database.exec("BEGIN IMMEDIATE");
228
- try {
229
- const result = work();
230
- this.#database.exec("COMMIT");
231
- return result;
232
- } catch (error) {
233
- this.#database.exec("ROLLBACK");
234
- throw error;
235
- }
236
- }
237
-
238
- #operationKey(runId: string, operationId: string): string {
239
- return `${runId}\0${operationId}`;
240
- }
241
-
242
- #leaseOperationId(runId: string, _operationId: string): string {
243
- return `engine-run-${hash(runId).slice(0, 40)}`;
244
- }
245
-
246
- #holdOperationLease(lease: OperationLease, commandOperationId: string): void {
247
- const key = this.#operationKey(lease.runId, commandOperationId);
248
- const existing = this.#operationLeases.get(key);
249
- if (existing) clearInterval(existing.timer);
250
- const intervalMs = Math.max(1, Math.floor(this.#leaseTtlMs / 3));
251
- const held: HeldOperationLease = {
252
- lease,
253
- timer: setInterval(() => {
254
- if (this.#operationLeases.get(key) !== held) return;
255
- try {
256
- held.lease = this.#runStore.renewLease(held.lease, this.#leaseTtlMs);
257
- this.#database
258
- .prepare(
259
- `UPDATE workflow_engine_operations
260
- SET lease_expires_at = ?
261
- WHERE run_id = ? AND operation_id = ? AND state = 'running'
262
- AND lease_token = ?`,
263
- )
264
- .run(
265
- held.lease.expiresAt,
266
- lease.runId,
267
- commandOperationId,
268
- lease.token,
269
- );
270
- } catch {
271
- held.error = new Error("lease-fenced");
272
- clearInterval(held.timer);
273
- const active = this.#active.get(lease.runId);
274
- if (active?.operationId === commandOperationId) {
275
- active.controller.abort(held.error);
276
- }
277
- }
278
- }, intervalMs),
279
- };
280
- held.timer.unref?.();
281
- this.#operationLeases.set(key, held);
282
- }
283
-
284
- #operationLease(runId: string, operationId: string): OperationLease {
285
- const held = this.#operationLeases.get(
286
- this.#operationKey(runId, operationId),
287
- );
288
- if (!held || held.error) throw held?.error ?? new Error("lease-fenced");
289
- this.#runStore.assertLease(held.lease);
290
- return held.lease;
291
- }
292
-
293
- #releaseOperationLease(runId: string, operationId: string): void {
294
- const key = this.#operationKey(runId, operationId);
295
- const held = this.#operationLeases.get(key);
296
- if (!held) return;
297
- clearInterval(held.timer);
298
- this.#operationLeases.delete(key);
299
- }
300
-
301
- #assertLeaseInTransaction(lease: OperationLease): void {
302
- const row = this.#database
303
- .prepare(
304
- `SELECT state, lease_token, lease_expires_at
305
- FROM operations WHERE run_id = ? AND operation_id = ?`,
306
- )
307
- .get(lease.runId, lease.operationId) as
308
- | {
309
- state: string;
310
- lease_token: string | null;
311
- lease_expires_at: number | null;
312
- }
313
- | undefined;
314
- if (
315
- row?.state !== "running" ||
316
- row.lease_token !== lease.token ||
317
- row.lease_expires_at === null ||
318
- row.lease_expires_at < this.#now()
319
- ) {
320
- throw new Error("lease-fenced");
321
- }
322
- }
323
-
324
- #leasedTransaction<T>(lease: OperationLease, work: () => T): T {
325
- return this.#transaction(() => {
326
- this.#assertLeaseInTransaction(lease);
327
- return work();
328
- });
329
- }
330
-
331
- amend(
332
- change: string,
333
- batchId: string,
334
- request: unknown,
335
- execute: (assertAuthority: () => void) => Promise<Record<string, unknown>>,
336
- ): Promise<Record<string, unknown>> {
337
- this.#assertOpen();
338
- if (this.#closing) throw new Error("workflow-engine-closed");
339
- const runId = this.#lookupRun("abel-implement", change);
340
- if (!runId) throw new Error("run-not-found");
341
- if (this.#amendments.has(runId)) throw new Error("amendment-busy");
342
- const operationId = `amend-${hash(batchId, canonicalJson(request)).slice(0, 48)}`;
343
- const pending = Promise.resolve()
344
- .then(async () => {
345
- const replay = this.#beginOperation(runId, operationId, "amend");
346
- if (replay) return replay;
347
- const lease = this.#operationLease(runId, operationId);
348
- const assertAuthority = () => {
349
- this.#assertOpen();
350
- if (this.#closing) throw new Error("workflow-engine-closed");
351
- this.#runStore.assertLease(lease);
352
- const status = this.#statusByRun(runId);
353
- if (
354
- !["approval-needed", "paused"].includes(String(status.state)) ||
355
- (status.decisionBatch as { id?: string } | undefined)?.id !==
356
- batchId
357
- )
358
- throw new Error("amendment-batch-stale");
359
- };
360
- try {
361
- assertAuthority();
362
- if (
363
- !isRecord(request) ||
364
- !["status", "validate-plan-draft"].includes(
365
- String(request.operation),
366
- )
367
- ) {
368
- this.#leasedTransaction(lease, () => {
369
- this.#database
370
- .prepare(
371
- "INSERT INTO workflow_amendment_budget(run_id, used) VALUES (?, 0) ON CONFLICT DO NOTHING",
372
- )
373
- .run(runId);
374
- const reserved = this.#database
375
- .prepare(
376
- "UPDATE workflow_amendment_budget SET used = used + 1 WHERE run_id = ? AND used < 64",
377
- )
378
- .run(runId);
379
- if (reserved.changes !== 1)
380
- throw new Error("amendment-budget-exhausted");
381
- });
382
- }
383
- this.#assertAmendmentContract(runId, change, request);
384
- const outcome = await execute(assertAuthority);
385
- assertAuthority();
386
- return this.#commitOperation(runId, operationId, outcome);
387
- } catch (error) {
388
- this.#interruptOperation(runId, operationId);
389
- throw error;
390
- }
391
- })
392
- .finally(() => {
393
- if (this.#amendments.get(runId) === pending)
394
- this.#amendments.delete(runId);
395
- });
396
- this.#amendments.set(runId, pending);
397
- return pending;
398
- }
399
-
400
- #assertAmendmentContract(
401
- runId: string,
402
- change: string,
403
- request: unknown,
404
- ): void {
405
- if (!isRecord(request)) throw new Error("amendment-contract-invalid");
406
- if (
407
- (request.operation === "approve-gate" && request.gate === "gate-a") ||
408
- (request.operation === "record-decision" &&
409
- request.category === "behavior")
410
- )
411
- throw new Error("amendment-changes-accepted-behavior");
412
- if (
413
- ![
414
- "write-artifact",
415
- "delete-artifact",
416
- "compile-plan",
417
- "validate-plan-draft",
418
- "finalize-delivery",
419
- ].includes(String(request.operation))
420
- )
421
- return;
422
- const relative = String(request.path);
423
- if (relative === "proposal.md" || relative.startsWith("specs/")) {
424
- const file = `openspec/changes/${change}/${relative}`;
425
- if (
426
- request.operation !== "write-artifact" ||
427
- !isSafeRegularFile(this.#consumerRoot, file) ||
428
- readFileSync(path.join(this.#consumerRoot, file), "utf8") !==
429
- request.content
430
- )
431
- throw new Error("amendment-changes-accepted-behavior");
432
- }
433
- const compile = [
434
- "compile-plan",
435
- "validate-plan-draft",
436
- "finalize-delivery",
437
- ].includes(String(request.operation));
438
- if (!compile) return;
439
- const draftPath = `openspec/changes/${change}/plan-draft.json`;
440
- if (!isSafeRegularFile(this.#consumerRoot, draftPath)) return;
441
- const run = this.#engineRun(runId);
442
- if (run.current_revision === null) return;
443
- const prior = this.#planForRevision(runId, run.current_revision);
444
- const contract = retainedChangeContract(prior);
445
- const draft = JSON.parse(
446
- readFileSync(path.join(this.#consumerRoot, draftPath), "utf8"),
447
- );
448
- if (
449
- draft.changeContract !== undefined &&
450
- canonicalJson(normalizeChangeContract(draft.changeContract)) !==
451
- canonicalJson(contract)
452
- )
453
- throw new Error("change-contract-mismatch");
454
- // Match DesignController's inheritance before validating authorized modes.
455
- if (draft.changeContract === undefined && prior.changeContract)
456
- draft.changeContract = contract;
457
- const next = compileImplementPlan(draft, {
458
- consumerRoot: this.#consumerRoot,
459
- bindExecutionInputs: true,
460
- }).plan;
461
- assertPlanWithinChangeContract(contract, next.tasks, [
462
- next.verification.change.fullSuite,
463
- next.verification.change.postApply,
464
- ]);
465
- }
466
-
467
- #lookupRun(stage: ControlStage, change: string): string | undefined {
468
- return (
469
- this.#database
470
- .prepare(
471
- `SELECT run_id FROM runs
472
- WHERE root_hash = ? AND stage = ? AND lookup_key = ?`,
473
- )
474
- .get(this.#stateRoot.consumerRootHash, stage, `change:${change}`) as
475
- | { run_id: string }
476
- | undefined
477
- )?.run_id;
478
- }
479
-
480
- #ensureEngineRun(runId: string): void {
481
- this.#database
482
- .prepare(
483
- `INSERT INTO workflow_engine_runs(
484
- run_id, current_revision, baseline_workspace_revision,
485
- current_workspace_revision, cleanup_state, next_queue_position
486
- ) VALUES (?, NULL, NULL, NULL, 'none', 1)
487
- ON CONFLICT(run_id) DO NOTHING`,
488
- )
489
- .run(runId);
490
- }
491
-
492
- #engineRun(runId: string): EngineRunRow {
493
- const row = this.#database
494
- .prepare(
495
- `SELECT current_revision, baseline_workspace_revision,
496
- current_workspace_revision, cleanup_state, route_id,
497
- route_fingerprint,
498
- transaction_id, verification_json, delivery_diagnostics_json,
499
- next_queue_position
500
- FROM workflow_engine_runs WHERE run_id = ?`,
501
- )
502
- .get(runId) as EngineRunRow | undefined;
503
- if (!row) throw new Error("workflow-engine-run-unavailable");
504
- return row;
505
- }
506
-
507
- #operation(
508
- runId: string,
509
- operationId: string,
510
- ): EngineOperationRow | undefined {
511
- return this.#database
512
- .prepare(
513
- `SELECT command, state, outcome_json FROM workflow_engine_operations
514
- WHERE run_id = ? AND operation_id = ?`,
515
- )
516
- .get(runId, operationId) as EngineOperationRow | undefined;
517
- }
518
-
519
- #operationReplay(
520
- runId: string,
521
- operationId: string,
522
- command: string,
523
- ): Record<string, unknown> | undefined {
524
- const row = this.#operation(runId, operationId);
525
- if (row && row.command !== command) {
526
- throw new Error("operation-command-mismatch");
527
- }
528
- return row?.state === "committed" && row.outcome_json
529
- ? parseJsonRecord(row.outcome_json)
530
- : undefined;
531
- }
532
-
533
- #beginOperation(
534
- runId: string,
535
- operationId: string,
536
- command: string,
537
- ): Record<string, unknown> | undefined {
538
- if (!IDENTIFIER.test(operationId)) throw new Error("invalid-operation-id");
539
- const replay = this.#operationReplay(runId, operationId, command);
540
- if (replay) return replay;
541
- const existing = this.#operation(runId, operationId);
542
- if (existing?.state === "running") {
543
- const leaseStatus = this.#runStore.inspectOperation(
544
- runId,
545
- this.#leaseOperationId(runId, operationId),
546
- );
547
- if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
548
- this.#database
549
- .prepare(
550
- `UPDATE workflow_engine_operations
551
- SET state = 'committed', lease_token = NULL,
552
- lease_expires_at = NULL, outcome_json = ?
553
- WHERE run_id = ? AND operation_id = ?`,
554
- )
555
- .run(JSON.stringify(leaseStatus.outcome), runId, operationId);
556
- return structuredClone(leaseStatus.outcome);
557
- }
558
- if (leaseStatus?.state === "running") {
559
- throw new Error("operation-already-running");
560
- }
561
- this.#database
562
- .prepare(
563
- `UPDATE workflow_engine_operations
564
- SET state = 'interrupted', lease_token = NULL,
565
- lease_expires_at = NULL
566
- WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
567
- )
568
- .run(runId, operationId);
569
- }
570
- const leaseOperationId = this.#leaseOperationId(runId, operationId);
571
- const priorLease = this.#runStore.inspectOperation(runId, leaseOperationId);
572
- const preempt = command === "cancel" || command === "discard";
573
- if (priorLease?.state === "running" && !preempt) {
574
- throw new Error("operation-already-running");
575
- }
576
- const lease = this.#runStore.acquireLease({
577
- runId,
578
- operationId: leaseOperationId,
579
- ttlMs: this.#leaseTtlMs,
580
- exclusive: true,
581
- ...(preempt ? { preempt: true } : {}),
582
- });
583
- if (preempt) {
584
- this.#database
585
- .prepare(
586
- `UPDATE workflow_engine_operations
587
- SET state = 'interrupted', lease_token = NULL,
588
- lease_expires_at = NULL
589
- WHERE run_id = ? AND operation_id <> ? AND state = 'running'`,
590
- )
591
- .run(runId, operationId);
592
- }
593
- this.#database
594
- .prepare(
595
- `INSERT INTO workflow_engine_operations(
596
- run_id, operation_id, command, state, lease_token,
597
- lease_expires_at, outcome_json
598
- ) VALUES (?, ?, ?, 'running', ?, ?, NULL)
599
- ON CONFLICT(run_id, operation_id) DO UPDATE SET
600
- command = excluded.command, state = 'running',
601
- lease_token = excluded.lease_token,
602
- lease_expires_at = excluded.lease_expires_at,
603
- outcome_json = NULL`,
604
- )
605
- .run(runId, operationId, command, lease.token, lease.expiresAt);
606
- this.#holdOperationLease(lease, operationId);
607
- return undefined;
608
- }
609
-
610
- #commitOperation(
611
- runId: string,
612
- operationId: string,
613
- outcome: Record<string, unknown>,
614
- ): Record<string, unknown> {
615
- const lease = this.#operationLease(runId, operationId);
616
- try {
617
- const committed = this.#runStore.commitLease(lease, outcome);
618
- const result = this.#database
619
- .prepare(
620
- `UPDATE workflow_engine_operations
621
- SET state = 'committed', lease_token = NULL,
622
- lease_expires_at = NULL, outcome_json = ?
623
- WHERE run_id = ? AND operation_id = ? AND state = 'running'
624
- AND lease_token = ?`,
625
- )
626
- .run(JSON.stringify(committed), runId, operationId, lease.token);
627
- if (Number(result.changes) !== 1) {
628
- throw new Error("operation-journal-fenced");
629
- }
630
- return structuredClone(committed);
631
- } finally {
632
- this.#releaseOperationLease(runId, operationId);
633
- }
634
- }
635
-
636
- #interruptOperation(runId: string, operationId: string): void {
637
- let lease: OperationLease;
638
- try {
639
- lease = this.#operationLease(runId, operationId);
640
- if (this.#operation(runId, operationId)?.command !== "amend")
641
- this.#pauseInterruptedRun(runId, operationId, lease);
642
- this.#runStore.interruptLease(lease);
643
- this.#database
644
- .prepare(
645
- `UPDATE workflow_engine_operations
646
- SET state = 'interrupted', lease_token = NULL,
647
- lease_expires_at = NULL
648
- WHERE run_id = ? AND operation_id = ? AND state = 'running'
649
- AND lease_token = ?`,
650
- )
651
- .run(runId, operationId, lease.token);
652
- } catch (error) {
653
- if (!(error instanceof Error) || error.message !== "lease-fenced") {
654
- throw error;
655
- }
656
- } finally {
657
- this.#releaseOperationLease(runId, operationId);
658
- }
659
- }
660
-
661
- #transition(
662
- runId: string,
663
- to: RunState,
664
- identity: string,
665
- code?: string,
666
- lease?: OperationLease,
667
- ): RunProjection {
668
- if (lease) this.#runStore.assertLease(lease);
669
- const current = this.#runStore.status(runId);
670
- if (
671
- current.state === to &&
672
- (to !== "paused" || code === undefined || current.pauseCode === code)
673
- ) {
674
- return current;
675
- }
676
- return this.#runStore.transition({
677
- runId,
678
- to,
679
- operationId: `eng-${hash(runId, identity, String(current.sequence), to).slice(0, 40)}`,
680
- ...(code ? { code } : {}),
681
- ...(lease ? { lease } : {}),
682
- });
683
- }
684
-
685
- #pauseInterruptedRun(
686
- runId: string,
687
- operationId: string,
688
- lease: OperationLease,
689
- ): void {
690
- const projection = this.#runStore.status(runId);
691
- if (["completed", "discarded", "rejected"].includes(projection.state))
692
- return;
693
- const write = (database: DatabaseSync) => {
694
- database
695
- .prepare(`UPDATE workflow_engine_tasks
696
- SET state = 'paused',
697
- pause_code = CASE WHEN pause_code = 'red-artifact-constraint'
698
- THEN pause_code ELSE 'operation-interrupted' END,
699
- context_request_json = CASE WHEN pause_code = 'red-artifact-constraint'
700
- THEN context_request_json ELSE NULL END,
701
- queue_position = NULL
702
- WHERE run_id = ? AND state IN ('phase-running', 'validating')`)
703
- .run(runId);
704
- };
705
- const active = [
706
- "created",
707
- "validating-delivery",
708
- "ready",
709
- "queued",
710
- "connecting",
711
- "running",
712
- "validating",
713
- "verifying",
714
- "retryable",
715
- "approval-needed",
716
- "change-verifying",
717
- "ready-to-apply",
718
- ];
719
- const to =
720
- projection.state === "applying"
721
- ? "recovering"
722
- : active.includes(projection.state)
723
- ? "paused"
724
- : undefined;
725
- if (to) {
726
- // Keep task and run projections atomic; retained evidence and budgets are
727
- // deliberately untouched. The original exception still reaches the caller.
728
- this.#runStore.transitionAtomically(
729
- {
730
- runId,
731
- to,
732
- lease,
733
- code: "operation-interrupted",
734
- operationId: `eng-${hash(runId, operationId, String(projection.sequence), "interrupted").slice(0, 40)}`,
735
- },
736
- write,
737
- );
738
- } else {
739
- this.#leasedTransaction(lease, () => write(this.#database));
740
- }
741
- }
742
-
743
- #recoverInterruptedOperations(): void {
744
- if (this.#orphanRecoveryTimer) {
745
- clearTimeout(this.#orphanRecoveryTimer);
746
- this.#orphanRecoveryTimer = undefined;
747
- }
748
- let nextExpiry: number | undefined;
749
- const interrupted = this.#database
750
- .prepare(
751
- `SELECT run_id, operation_id FROM workflow_engine_operations
752
- WHERE state = 'running' OR (state = 'interrupted' AND command <> 'amend'
753
- AND (run_id IN (SELECT run_id FROM workflow_engine_tasks
754
- WHERE state IN ('phase-running', 'validating'))
755
- OR run_id IN (SELECT run_id FROM runs WHERE state IN
756
- ('running', 'validating', 'verifying', 'change-verifying', 'applying'))))`,
757
- )
758
- .all() as unknown as Array<{ run_id: string; operation_id: string }>;
759
- for (const row of interrupted) {
760
- const leaseStatus = this.#runStore.inspectOperation(
761
- row.run_id,
762
- this.#leaseOperationId(row.run_id, row.operation_id),
763
- );
764
- if (leaseStatus?.state === "committed" && leaseStatus.outcome) {
765
- this.#database
766
- .prepare(
767
- `UPDATE workflow_engine_operations
768
- SET state = 'committed', lease_token = NULL,
769
- lease_expires_at = NULL, outcome_json = ?
770
- WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
771
- )
772
- .run(
773
- JSON.stringify(leaseStatus.outcome),
774
- row.run_id,
775
- row.operation_id,
776
- );
777
- continue;
778
- }
779
- if (leaseStatus?.state === "running") {
780
- if (leaseStatus.expiresAt !== undefined) {
781
- nextExpiry = Math.min(nextExpiry ?? Infinity, leaseStatus.expiresAt);
782
- }
783
- continue;
784
- }
785
- this.#database
786
- .prepare(
787
- `UPDATE workflow_engine_operations
788
- SET state = 'interrupted', lease_token = NULL,
789
- lease_expires_at = NULL
790
- WHERE run_id = ? AND operation_id = ? AND state = 'running'`,
791
- )
792
- .run(row.run_id, row.operation_id);
793
- // Claim the same run-exclusive lease before repairing projections. A live
794
- // owner (including a different host) must never be mistaken for an orphan.
795
- let recoveryLease: OperationLease;
796
- try {
797
- recoveryLease = this.#runStore.acquireLease({
798
- runId: row.run_id,
799
- operationId: this.#leaseOperationId(row.run_id, row.operation_id),
800
- ttlMs: this.#leaseTtlMs,
801
- exclusive: true,
802
- });
803
- } catch (error) {
804
- if (
805
- error instanceof Error &&
806
- error.message === "operation-already-running"
807
- )
808
- continue;
809
- throw error;
810
- }
811
- try {
812
- this.#pauseInterruptedRun(row.run_id, row.operation_id, recoveryLease);
813
- } finally {
814
- this.#runStore.interruptLease(recoveryLease);
815
- }
816
- }
817
- if (nextExpiry !== undefined && !this.#closed) {
818
- const delay = Math.max(1, nextExpiry - this.#now() + 1);
819
- this.#orphanRecoveryTimer = setTimeout(() => {
820
- this.#orphanRecoveryTimer = undefined;
821
- if (this.#closed) return;
822
- try {
823
- this.#recoverInterruptedOperations();
824
- } catch {
825
- // A later status/restart will surface durable journal corruption.
826
- }
827
- }, delay);
828
- this.#orphanRecoveryTimer.unref?.();
829
- }
830
- }
831
-
832
- #recoverTerminalCleanup(): void {
833
- const rows = this.#database
834
- .prepare(
835
- `SELECT workflow_engine_runs.run_id
836
- FROM workflow_engine_runs
837
- JOIN runs ON runs.run_id = workflow_engine_runs.run_id
838
- WHERE runs.state IN ('completed', 'discarded')
839
- AND workflow_engine_runs.cleanup_state = 'retained'`,
840
- )
841
- .all() as unknown as Array<{ run_id: string }>;
842
- for (const row of rows) {
843
- try {
844
- this.#cleanupTerminal(row.run_id, "restart-cleanup");
845
- } catch {
846
- // Terminal status remains truthful and a later reload retries cleanup.
847
- }
848
- }
849
- }
850
-
851
- #tasks(runId: string): EngineTaskRow[] {
852
- return this.#database
853
- .prepare(
854
- `SELECT task_id, task_order, delivery_revision, plan_json, state,
855
- phase, pause_code, context_request_json, attempt_diagnostic_json,
856
- route_id, route_fingerprint, queue_position
857
- FROM workflow_engine_tasks WHERE run_id = ? ORDER BY task_order`,
858
- )
859
- .all(runId) as unknown as EngineTaskRow[];
860
- }
861
-
862
- #taskPlan(row: EngineTaskRow): PlanTaskDraft {
863
- return JSON.parse(row.plan_json) as PlanTaskDraft;
864
- }
865
-
866
- #currentPlan(runId: string): ImplementPlan {
867
- const revision = this.#engineRun(runId).current_revision;
868
- if (revision === null) throw new Error("delivery-not-bound");
869
- return this.#planForRevision(runId, revision);
870
- }
871
-
872
- #planForRevision(runId: string, revision: number): ImplementPlan {
873
- const row = this.#database
874
- .prepare(
875
- `SELECT plan_json FROM workflow_engine_deliveries
876
- WHERE run_id = ? AND revision = ?`,
877
- )
878
- .get(runId, revision) as { plan_json: string } | undefined;
879
- if (!row) throw new Error("delivery-not-bound");
880
- return parsePlan(row.plan_json);
881
- }
882
-
883
- async #loadDelivery(
884
- runId: string,
885
- lease: OperationLease,
886
- stage: ControlStage,
887
- change: string,
888
- deliveryRevision?: number,
889
- receiptHash?: string,
890
- signal?: AbortSignal,
891
- ): Promise<WorkflowDelivery> {
892
- if (deliveryRevision === undefined && this.#deliverySource.discoverLatest) {
893
- try {
894
- const discover = this.#deliverySource.discoverLatest;
895
- const available = await cancellableRead(
896
- () => discover.call(this.#deliverySource, { stage, change, signal }),
897
- signal,
898
- );
899
- if (
900
- available &&
901
- Number.isSafeInteger(available.deliveryRevision) &&
902
- available.deliveryRevision > 0 &&
903
- SHA256.test(available.receiptHash)
904
- ) {
905
- deliveryRevision = available.deliveryRevision;
906
- receiptHash = available.receiptHash;
907
- }
908
- } catch {
909
- // Discovery is optional; authoritative loading still reports diagnostics.
910
- signal?.throwIfAborted();
911
- }
912
- }
913
- try {
914
- const delivery = await cancellableRead(
915
- () =>
916
- this.#deliverySource.load({
917
- stage,
918
- change,
919
- ...(deliveryRevision === undefined ? {} : { deliveryRevision }),
920
- ...(receiptHash === undefined ? {} : { receiptHash }),
921
- ...(signal ? { signal } : {}),
922
- }),
923
- signal,
924
- );
925
- signal?.throwIfAborted();
926
- assertDelivery(delivery, stage, change, deliveryRevision, receiptHash);
927
- return structuredClone(delivery);
928
- } catch (error) {
929
- if (
930
- error instanceof DeliveryValidationError &&
931
- deliveryRevision !== undefined &&
932
- receiptHash !== undefined
933
- ) {
934
- this.#leasedTransaction(lease, () => {
935
- this.#database
936
- .prepare(
937
- "INSERT INTO workflow_rejected_deliveries(run_id, delivery_revision, receipt_hash) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
938
- )
939
- .run(runId, deliveryRevision, receiptHash);
940
- });
941
- }
942
- throw error;
943
- }
944
- }
945
-
946
- #admitDelivery(
947
- runId: string,
948
- delivery: WorkflowDelivery,
949
- lease: OperationLease,
950
- ): void {
951
- this.#runStore.assertLease(lease);
952
- const existing = this.#database
953
- .prepare(
954
- `SELECT receipt_hash, plan_json FROM workflow_engine_deliveries
955
- WHERE run_id = ? AND revision = ?`,
956
- )
957
- .get(runId, delivery.revision) as
958
- | { receipt_hash: string; plan_json: string }
959
- | undefined;
960
- const capturedBudget = this.#database
961
- .prepare("SELECT hard_limit FROM workflow_work_budget WHERE run_id = ?")
962
- .get(runId) as { hard_limit: number } | undefined;
963
- if (
964
- this.#phaseCount(delivery.plan) >
965
- (capturedBudget?.hard_limit ?? this.#workHardLimit) &&
966
- !existing
967
- )
968
- throw new DeliveryValidationError([
969
- "verification-work-budget-insufficient",
970
- ]);
971
- const serializedPlan = JSON.stringify(delivery.plan);
972
- if (
973
- existing &&
974
- (existing.receipt_hash !== delivery.receiptHash ||
975
- existing.plan_json !== serializedPlan)
976
- ) {
977
- throw new Error("delivery-revision-conflict");
978
- }
979
- const engineRun = this.#engineRun(runId);
980
- if (
981
- engineRun.current_revision !== null &&
982
- delivery.revision < engineRun.current_revision
983
- ) {
984
- throw new Error("delivery-revision-stale");
985
- }
986
- const priorRows = this.#tasks(runId);
987
- const priorById = new Map(priorRows.map((row) => [row.task_id, row]));
988
- const priorPlan =
989
- engineRun.current_revision === null
990
- ? undefined
991
- : this.#planForRevision(runId, engineRun.current_revision);
992
- const comparison = compareDeliveryRevision(
993
- priorPlan,
994
- delivery.plan,
995
- priorRows,
996
- );
997
- const { compatible, boundaryExpanded } = comparison;
998
- let invalidated = comparison.invalidated;
999
- let revalidatedWorkspace:
1000
- | {
1001
- baselineRevisionId: string;
1002
- currentWorkspaceRevisionId: string;
1003
- }
1004
- | undefined;
1005
- if (
1006
- priorPlan &&
1007
- (invalidated.size > 0 || boundaryExpanded) &&
1008
- engineRun.baseline_workspace_revision &&
1009
- engineRun.current_workspace_revision
1010
- ) {
1011
- if (!this.#worker.revalidateDelivery) {
1012
- if (boundaryExpanded) {
1013
- throw new Error("workflow-delivery-revalidation-unavailable");
1014
- }
1015
- if (
1016
- engineRun.current_workspace_revision !==
1017
- engineRun.baseline_workspace_revision
1018
- ) {
1019
- throw new Error("workflow-delivery-revalidation-unavailable");
1020
- }
1021
- } else {
1022
- const result = this.#worker.revalidateDelivery({
1023
- runId,
1024
- deliveryRevision: delivery.revision,
1025
- plan: structuredClone(delivery.plan),
1026
- tasks: priorRows
1027
- .filter((row) => compatible.has(row.task_id))
1028
- .map((row) => ({
1029
- taskId: row.task_id,
1030
- deliveryRevision: row.delivery_revision,
1031
- state: row.state,
1032
- phase: row.phase,
1033
- contextReadPaths: this.#contextReads(runId, row.task_id),
1034
- })),
1035
- invalidatedTaskIds: [...invalidated].sort(),
1036
- baselineRevisionId: engineRun.baseline_workspace_revision,
1037
- currentWorkspaceRevisionId: engineRun.current_workspace_revision,
1038
- });
1039
- const resultIds = new Set(result.invalidatedTaskIds);
1040
- if (
1041
- !Array.isArray(result.invalidatedTaskIds) ||
1042
- resultIds.size !== result.invalidatedTaskIds.length ||
1043
- [...invalidated].some((taskId) => !resultIds.has(taskId)) ||
1044
- [...resultIds].some((taskId) => !priorById.has(taskId)) ||
1045
- !SHA256.test(result.baselineRevisionId) ||
1046
- !SHA256.test(result.currentWorkspaceRevisionId)
1047
- ) {
1048
- throw new Error("workflow-delivery-revalidation-invalid");
1049
- }
1050
- invalidated = resultIds;
1051
- revalidatedWorkspace = {
1052
- baselineRevisionId: result.baselineRevisionId,
1053
- currentWorkspaceRevisionId: result.currentWorkspaceRevisionId,
1054
- };
1055
- }
1056
- }
1057
- const deliveryBindings = delivery.approvalProofs
1058
- ? ([
1059
- {
1060
- runId,
1061
- gate: "gate-a" as const,
1062
- revision: delivery.revision,
1063
- receiptHash: delivery.receiptHash,
1064
- approvalProof: structuredClone(delivery.approvalProofs.gateA),
1065
- operationId: `bind-a-${hash(
1066
- runId,
1067
- String(delivery.revision),
1068
- delivery.approvalProofs.gateA.recordHash,
1069
- ).slice(0, 40)}`,
1070
- lease,
1071
- },
1072
- {
1073
- runId,
1074
- gate: "gate-b" as const,
1075
- revision: delivery.revision,
1076
- receiptHash: delivery.receiptHash,
1077
- approvalProof: structuredClone(delivery.approvalProofs.gateB),
1078
- operationId: `bind-b-${hash(
1079
- runId,
1080
- String(delivery.revision),
1081
- delivery.approvalProofs.gateB.recordHash,
1082
- ).slice(0, 40)}`,
1083
- lease,
1084
- },
1085
- ] as const)
1086
- : ([
1087
- {
1088
- runId,
1089
- gate: delivery.gate,
1090
- revision: delivery.revision,
1091
- receiptHash: delivery.receiptHash,
1092
- operationId: `bind-${hash(
1093
- runId,
1094
- String(delivery.revision),
1095
- delivery.receiptHash,
1096
- ).slice(0, 40)}`,
1097
- lease,
1098
- },
1099
- ] as const);
1100
- this.#runStore.bindDeliveriesAtomically(deliveryBindings, (database) => {
1101
- const engineLease = database
1102
- .prepare(
1103
- `SELECT state, lease_expires_at
1104
- FROM workflow_engine_operations
1105
- WHERE run_id = ? AND lease_token = ?`,
1106
- )
1107
- .get(runId, lease.token) as
1108
- | { state: string; lease_expires_at: number | null }
1109
- | undefined;
1110
- if (
1111
- engineLease?.state !== "running" ||
1112
- engineLease.lease_expires_at === null ||
1113
- engineLease.lease_expires_at < this.#now()
1114
- ) {
1115
- throw new Error("lease-fenced");
1116
- }
1117
- database
1118
- .prepare(
1119
- `INSERT INTO workflow_engine_runs(
1120
- run_id, current_revision, delivery_diagnostics_json,
1121
- next_queue_position
1122
- ) VALUES (?, ?, NULL, 1)
1123
- ON CONFLICT(run_id) DO UPDATE SET
1124
- current_revision = excluded.current_revision,
1125
- verification_json = NULL,
1126
- delivery_diagnostics_json = NULL`,
1127
- )
1128
- .run(runId, delivery.revision);
1129
- if (revalidatedWorkspace) {
1130
- database
1131
- .prepare(
1132
- `UPDATE workflow_engine_runs
1133
- SET baseline_workspace_revision = ?,
1134
- current_workspace_revision = ?, cleanup_state = 'retained'
1135
- WHERE run_id = ?`,
1136
- )
1137
- .run(
1138
- revalidatedWorkspace.baselineRevisionId,
1139
- revalidatedWorkspace.currentWorkspaceRevisionId,
1140
- runId,
1141
- );
1142
- }
1143
- database
1144
- .prepare(
1145
- `INSERT INTO workflow_engine_deliveries(
1146
- run_id, revision, gate, receipt_hash, plan_json
1147
- ) VALUES (?, ?, ?, ?, ?)
1148
- ON CONFLICT(run_id, revision) DO NOTHING`,
1149
- )
1150
- .run(
1151
- runId,
1152
- delivery.revision,
1153
- delivery.gate,
1154
- delivery.receiptHash,
1155
- serializedPlan,
1156
- );
1157
-
1158
- this.#growWorkBudget(database, runId, delivery.plan);
1159
- const admitted = new Set<string>();
1160
- delivery.plan.tasks.forEach((task, taskOrder) => {
1161
- admitted.add(task.taskId);
1162
- const taskJson = JSON.stringify(task);
1163
- const prior = priorById.get(task.taskId);
1164
- if (!prior) {
1165
- database
1166
- .prepare(
1167
- `INSERT INTO workflow_engine_tasks(
1168
- run_id, task_id, task_order, delivery_revision, plan_json,
1169
- state, phase
1170
- ) VALUES (?, ?, ?, ?, ?, 'pending', ?)`,
1171
- )
1172
- .run(
1173
- runId,
1174
- task.taskId,
1175
- taskOrder,
1176
- delivery.revision,
1177
- taskJson,
1178
- phases(task)[0],
1179
- );
1180
- return;
1181
- }
1182
- if (compatible.has(task.taskId) && !invalidated.has(task.taskId)) {
1183
- database
1184
- .prepare(
1185
- `UPDATE workflow_engine_tasks
1186
- SET task_order = ?, plan_json = ?
1187
- WHERE run_id = ? AND task_id = ?`,
1188
- )
1189
- .run(taskOrder, taskJson, runId, task.taskId);
1190
- return;
1191
- }
1192
- database
1193
- .prepare(
1194
- "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1195
- )
1196
- .run(runId, task.taskId);
1197
- database
1198
- .prepare(
1199
- `UPDATE workflow_engine_tasks
1200
- SET task_order = ?, delivery_revision = ?, plan_json = ?,
1201
- state = 'pending', phase = ?, pause_code = NULL,
1202
- context_request_json = NULL, attempt_diagnostic_json = NULL,
1203
- queue_position = NULL
1204
- WHERE run_id = ? AND task_id = ?`,
1205
- )
1206
- .run(
1207
- taskOrder,
1208
- delivery.revision,
1209
- taskJson,
1210
- phases(task)[0],
1211
- runId,
1212
- task.taskId,
1213
- );
1214
- });
1215
- for (const row of priorRows) {
1216
- if (!admitted.has(row.task_id)) {
1217
- database
1218
- .prepare(
1219
- "DELETE FROM workflow_context_reads WHERE run_id = ? AND task_id = ?",
1220
- )
1221
- .run(runId, row.task_id);
1222
- database
1223
- .prepare(
1224
- `DELETE FROM workflow_engine_tasks
1225
- WHERE run_id = ? AND task_id = ?`,
1226
- )
1227
- .run(runId, row.task_id);
1228
- }
1229
- }
1230
- });
1231
- }
1232
-
1233
- #cancelledDelivery(
1234
- runId: string,
1235
- operationId: string,
1236
- error: unknown,
1237
- signal: AbortSignal | undefined,
1238
- lease: OperationLease,
1239
- ): Record<string, unknown> | undefined {
1240
- if (!signal || !isCancellationException(error, signal)) return undefined;
1241
- try {
1242
- this.#runStore.assertLease(lease);
1243
- if (
1244
- ["validating-delivery", "ready"].includes(
1245
- this.#runStore.status(runId).state,
1246
- )
1247
- ) {
1248
- this.#transition(
1249
- runId,
1250
- "paused",
1251
- `${operationId}:delivery-cancelled`,
1252
- "operation-cancelled",
1253
- lease,
1254
- );
1255
- }
1256
- } catch (failure) {
1257
- if (!(failure instanceof Error) || failure.message !== "lease-fenced")
1258
- throw failure;
1259
- } finally {
1260
- this.#interruptOperation(runId, operationId);
1261
- }
1262
- return this.#statusByRun(runId);
1263
- }
1264
-
1265
- #deliveryFailure(
1266
- runId: string,
1267
- operationId: string,
1268
- error: unknown,
1269
- lease: OperationLease,
1270
- ): Record<string, unknown> | undefined {
1271
- if (!(error instanceof DeliveryValidationError)) return undefined;
1272
- this.#leasedTransaction(lease, () => {
1273
- this.#database
1274
- .prepare(
1275
- `UPDATE workflow_engine_runs
1276
- SET delivery_diagnostics_json = ? WHERE run_id = ?`,
1277
- )
1278
- .run(JSON.stringify(error.diagnostics), runId);
1279
- });
1280
- this.#transition(
1281
- runId,
1282
- "paused",
1283
- `${operationId}:delivery-invalid`,
1284
- "delivery-invalid",
1285
- lease,
1286
- );
1287
- return {
1288
- ...this.#statusByRun(runId),
1289
- delivery: {
1290
- code: "delivery-invalid",
1291
- diagnostics: [...error.diagnostics],
1292
- },
1293
- };
1294
- }
1295
-
1296
- #setTask(
1297
- runId: string,
1298
- taskId: string,
1299
- values: {
1300
- state?: string;
1301
- phase?: "red" | "green" | "refactor";
1302
- pauseCode?: string | null;
1303
- contextRequest?: WorkflowContextRequest | null;
1304
- attemptDiagnostic?: SafeAttemptDiagnostic | null;
1305
- routeId?: string | null;
1306
- routeFingerprint?: string | null;
1307
- queuePosition?: number | null;
1308
- },
1309
- lease?: OperationLease,
1310
- ): void {
1311
- if (values.attemptDiagnostic?.recovery && lease) {
1312
- const fact = values.attemptDiagnostic.recovery;
1313
- const prior = this.#recovery(runId, fact.key);
1314
- if (
1315
- !prior ||
1316
- fact.failures > prior.failures ||
1317
- fact.feedback.maxAttempts < prior.feedback.maxAttempts
1318
- )
1319
- this.#recordRecovery(runId, fact, lease);
1320
- }
1321
- const assignments: string[] = [];
1322
- const parameters: Array<string | number | null> = [];
1323
- if (values.state !== undefined) {
1324
- assignments.push("state = ?");
1325
- parameters.push(values.state);
1326
- }
1327
- if (values.phase !== undefined) {
1328
- assignments.push("phase = ?");
1329
- parameters.push(values.phase);
1330
- }
1331
- if (values.pauseCode !== undefined) {
1332
- assignments.push("pause_code = ?");
1333
- parameters.push(values.pauseCode);
1334
- }
1335
- if (values.contextRequest !== undefined) {
1336
- assignments.push("context_request_json = ?");
1337
- parameters.push(
1338
- values.contextRequest === null
1339
- ? null
1340
- : JSON.stringify(
1341
- normalizeWorkflowContextRequest(values.contextRequest),
1342
- ),
1343
- );
1344
- }
1345
- if (values.attemptDiagnostic !== undefined) {
1346
- assignments.push("attempt_diagnostic_json = ?");
1347
- parameters.push(
1348
- values.attemptDiagnostic === null
1349
- ? null
1350
- : JSON.stringify(
1351
- Object.fromEntries(
1352
- Object.entries(values.attemptDiagnostic).filter(
1353
- ([key]) => key !== "recovery",
1354
- ),
1355
- ),
1356
- ),
1357
- );
1358
- }
1359
- if (values.routeId !== undefined) {
1360
- assignments.push("route_id = ?");
1361
- parameters.push(values.routeId);
1362
- }
1363
- if (values.routeFingerprint !== undefined) {
1364
- assignments.push("route_fingerprint = ?");
1365
- parameters.push(values.routeFingerprint);
1366
- }
1367
- if (values.queuePosition !== undefined) {
1368
- assignments.push("queue_position = ?");
1369
- parameters.push(values.queuePosition);
1370
- }
1371
- if (assignments.length === 0) return;
1372
- const write = () => {
1373
- this.#database
1374
- .prepare(
1375
- `UPDATE workflow_engine_tasks SET ${assignments.join(", ")}
1376
- WHERE run_id = ? AND task_id = ?`,
1377
- )
1378
- .run(...parameters, runId, taskId);
1379
- };
1380
- if (lease) this.#leasedTransaction(lease, write);
1381
- else write();
1382
- }
1383
-
1384
- #setVerificationStatus(
1385
- runId: string,
1386
- status: WorkflowVerificationStatus | null,
1387
- lease: OperationLease,
1388
- ): void {
1389
- const serialized =
1390
- status === null
1391
- ? null
1392
- : JSON.stringify(normalizeWorkflowVerificationStatus(status));
1393
- this.#leasedTransaction(lease, () => {
1394
- this.#database
1395
- .prepare(
1396
- `UPDATE workflow_engine_runs SET verification_json = ?
1397
- WHERE run_id = ?`,
1398
- )
1399
- .run(serialized, runId);
1400
- });
1401
- }
1402
-
1403
- #queueTask(
1404
- runId: string,
1405
- row: EngineTaskRow,
1406
- lease: OperationLease,
1407
- reason: "conflict" | "capacity" = "conflict",
1408
- ): void {
1409
- const pauseCode = reason === "capacity" ? "worker-capacity" : null;
1410
- if (row.state === "queued" && row.queue_position !== null) {
1411
- if (row.pause_code !== pauseCode)
1412
- this.#setTask(runId, row.task_id, { pauseCode }, lease);
1413
- return;
1414
- }
1415
- const engineRun = this.#engineRun(runId);
1416
- this.#leasedTransaction(lease, () => {
1417
- this.#setTask(runId, row.task_id, {
1418
- state: "queued",
1419
- pauseCode,
1420
- queuePosition: engineRun.next_queue_position,
1421
- });
1422
- this.#database
1423
- .prepare(
1424
- `UPDATE workflow_engine_runs
1425
- SET next_queue_position = next_queue_position + 1
1426
- WHERE run_id = ?`,
1427
- )
1428
- .run(runId);
1429
- });
1430
- }
1431
-
1432
- #recordWorkspaceFacts(
1433
- runId: string,
1434
- outcome: WorkflowAttemptOutcome,
1435
- lease: OperationLease,
1436
- ): void {
1437
- const baselineRevisionId = outcome.baselineRevisionId;
1438
- const currentWorkspaceRevisionId =
1439
- outcome.kind === "phase-committed"
1440
- ? outcome.isolatedRevisionId
1441
- : outcome.currentWorkspaceRevisionId;
1442
- if (
1443
- (baselineRevisionId !== undefined && !SHA256.test(baselineRevisionId)) ||
1444
- (currentWorkspaceRevisionId !== undefined &&
1445
- !SHA256.test(currentWorkspaceRevisionId)) ||
1446
- (outcome.kind !== "phase-committed" &&
1447
- (baselineRevisionId === undefined) !==
1448
- (currentWorkspaceRevisionId === undefined))
1449
- ) {
1450
- throw new Error("workflow-attempt-workspace-fact-invalid");
1451
- }
1452
- if (currentWorkspaceRevisionId === undefined) return;
1453
- this.#leasedTransaction(lease, () => {
1454
- this.#database
1455
- .prepare(
1456
- `UPDATE workflow_engine_runs
1457
- SET baseline_workspace_revision = COALESCE(
1458
- baseline_workspace_revision, ?
1459
- ),
1460
- current_workspace_revision = ?,
1461
- cleanup_state = CASE
1462
- WHEN ? IS NULL THEN cleanup_state ELSE 'retained'
1463
- END
1464
- WHERE run_id = ?`,
1465
- )
1466
- .run(
1467
- baselineRevisionId ?? null,
1468
- currentWorkspaceRevisionId,
1469
- baselineRevisionId ?? null,
1470
- runId,
1471
- );
1472
- });
1473
- }
1474
-
1475
- #recordRouteFacts(
1476
- runId: string,
1477
- taskId: string,
1478
- outcome: WorkflowAttemptOutcome,
1479
- lease: OperationLease,
1480
- ): void {
1481
- if (
1482
- (outcome.routeId === undefined) !==
1483
- (outcome.routeFingerprint === undefined) ||
1484
- (outcome.routeId !== undefined && !IDENTIFIER.test(outcome.routeId)) ||
1485
- (outcome.routeFingerprint !== undefined &&
1486
- !SHA256.test(outcome.routeFingerprint))
1487
- ) {
1488
- throw new Error("workflow-attempt-route-fact-invalid");
1489
- }
1490
- if (!outcome.routeId || !outcome.routeFingerprint) return;
1491
- const routeId = outcome.routeId;
1492
- const routeFingerprint = outcome.routeFingerprint;
1493
- this.#leasedTransaction(lease, () => {
1494
- this.#setTask(runId, taskId, {
1495
- routeId,
1496
- routeFingerprint,
1497
- });
1498
- this.#database
1499
- .prepare(
1500
- `UPDATE workflow_engine_runs
1501
- SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
1502
- )
1503
- .run(routeId, routeFingerprint, runId);
1504
- });
1505
- }
1506
-
1507
- async #runTask(
1508
- runId: string,
1509
- row: EngineTaskRow,
1510
- operationId: string,
1511
- signal: AbortSignal,
1512
- lease: OperationLease,
1513
- onActivity?: (event: WorkflowActivityUpdate) => void,
1514
- ): Promise<void> {
1515
- const task = this.#taskPlan(row);
1516
- const orderedPhases = phases(task);
1517
- let phaseIndex = orderedPhases.indexOf(row.phase);
1518
- let artifactAttempts = 0;
1519
- const persistedContextRequest = row.context_request_json
1520
- ? normalizeWorkflowContextRequest(
1521
- JSON.parse(row.context_request_json) as unknown,
1522
- )
1523
- : undefined;
1524
- const persistedRedArtifactCorrection =
1525
- row.phase === "green" &&
1526
- persistedContextRequest !== undefined &&
1527
- classifyPersistedContextRequest(persistedContextRequest, task, "green")
1528
- .code === "red-artifact-constraint";
1529
- let redArtifactCorrection =
1530
- row.phase === "green" &&
1531
- (row.pause_code === "red-artifact-constraint" ||
1532
- persistedRedArtifactCorrection);
1533
- let requestedContext = persistedContextRequest;
1534
- let correctionContextRequest = redArtifactCorrection
1535
- ? persistedContextRequest
1536
- : undefined;
1537
- if (phaseIndex < 0) throw new Error("workflow-task-phase-invalid");
1538
- while (phaseIndex < orderedPhases.length) {
1539
- const phase = orderedPhases[phaseIndex];
1540
- row.phase = phase;
1541
- if (signal.aborted) {
1542
- this.#setTask(
1543
- runId,
1544
- row.task_id,
1545
- {
1546
- state: "paused",
1547
- phase,
1548
- pauseCode: "operation-cancelled",
1549
- contextRequest: null,
1550
- queuePosition: null,
1551
- },
1552
- lease,
1553
- );
1554
- return;
1555
- }
1556
- const current = this.#engineRun(runId);
1557
- if (current.current_revision === null)
1558
- throw new Error("delivery-not-bound");
1559
- const deliveryRevision = current.current_revision;
1560
- const plan = this.#planForRevision(runId, deliveryRevision);
1561
- this.#leasedTransaction(lease, () => {
1562
- for (const ref of requestedContext?.refs ?? []) {
1563
- if (
1564
- ref.kind === "requested-path" &&
1565
- ref.access === "read" &&
1566
- permitsContextRead(ref.path, task.roots) &&
1567
- (current.current_workspace_revision
1568
- ? current.baseline_workspace_revision &&
1569
- this.#worker.isContextReadAvailable?.({
1570
- runId,
1571
- deliveryRevision,
1572
- plan,
1573
- baselineRevisionId: current.baseline_workspace_revision,
1574
- currentWorkspaceRevisionId:
1575
- current.current_workspace_revision,
1576
- path: ref.path,
1577
- })
1578
- : isSafeRegularFile(this.#consumerRoot, ref.path))
1579
- ) {
1580
- this.#database
1581
- .prepare(
1582
- "INSERT INTO workflow_context_reads(run_id, task_id, path) VALUES (?, ?, ?) ON CONFLICT DO NOTHING",
1583
- )
1584
- .run(runId, row.task_id, ref.path);
1585
- }
1586
- }
1587
- });
1588
- const verificationStatus = current.verification_json
1589
- ? normalizeWorkflowVerificationStatus(
1590
- JSON.parse(current.verification_json) as unknown,
1591
- )
1592
- : undefined;
1593
- const priorDiagnostic = this.#safeAttemptDiagnostic(runId, row);
1594
- const priorRecovery = priorDiagnostic?.recovery;
1595
- const preparationKey = baselineRecoveryKey(task, phase);
1596
- const activeRecovery =
1597
- priorRecovery &&
1598
- [recoveryKey(task, phase, row, current), preparationKey].includes(
1599
- priorRecovery.key,
1600
- )
1601
- ? priorRecovery
1602
- : undefined;
1603
- const maxRecoveryAttempts = Math.min(
1604
- plan.verification.artifactCorrection.maxAttempts,
1605
- activeRecovery?.feedback.maxAttempts ?? Infinity,
1606
- );
1607
- if (
1608
- activeRecovery &&
1609
- activeRecovery.feedback.maxAttempts !== maxRecoveryAttempts
1610
- ) {
1611
- this.#leasedTransaction(lease, () => {
1612
- this.#database
1613
- .prepare(
1614
- "UPDATE workflow_recovery_incidents SET max_attempts = MIN(max_attempts, ?) WHERE run_id = ? AND incident_key = ?",
1615
- )
1616
- .run(maxRecoveryAttempts, runId, activeRecovery.key);
1617
- });
1618
- activeRecovery.feedback.maxAttempts = maxRecoveryAttempts;
1619
- }
1620
- const granted = this.#hasRecoveryGrant(
1621
- runId,
1622
- operationId,
1623
- activeRecovery?.key ?? recoveryKey(task, phase, row, current),
1624
- );
1625
- const verificationOnly =
1626
- activeRecovery?.key !== preparationKey &&
1627
- recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted) &&
1628
- (await this.#worker.hasPendingVerification?.({
1629
- runId,
1630
- operationId,
1631
- deliveryRevision: row.delivery_revision,
1632
- taskId: row.task_id,
1633
- phase,
1634
- task,
1635
- plan,
1636
- signal,
1637
- ...(current.baseline_workspace_revision
1638
- ? { baselineRevisionId: current.baseline_workspace_revision }
1639
- : {}),
1640
- ...(current.current_workspace_revision
1641
- ? { currentWorkspaceRevisionId: current.current_workspace_revision }
1642
- : {}),
1643
- })) === true;
1644
- if (
1645
- !verificationOnly &&
1646
- activeRecovery &&
1647
- recoveryExhausted(activeRecovery, maxRecoveryAttempts, granted)
1648
- ) {
1649
- this.#setTask(
1650
- runId,
1651
- row.task_id,
1652
- {
1653
- state: "paused",
1654
- phase,
1655
- pauseCode: activeRecovery.feedback.code,
1656
- attemptDiagnostic: priorDiagnostic ?? null,
1657
- queuePosition: null,
1658
- },
1659
- lease,
1660
- );
1661
- return;
1662
- }
1663
- if (activeRecovery && activeRecovery.key !== preparationKey)
1664
- artifactAttempts = Math.max(artifactAttempts, activeRecovery.failures);
1665
- const repair =
1666
- verificationStatus?.scope === "change-task-affected" &&
1667
- verificationStatus.attribution === "introduced" &&
1668
- verificationStatus.taskId === row.task_id &&
1669
- verificationStatus.failureIdentities &&
1670
- verificationStatus.failureIdentities.length > 0
1671
- ? {
1672
- attribution: "introduced" as const,
1673
- failureIdentities: [...verificationStatus.failureIdentities],
1674
- }
1675
- : undefined;
1676
- const artifactCorrection =
1677
- redArtifactCorrection && phase === "green"
1678
- ? {
1679
- code: "red-artifact-constraint" as const,
1680
- attempt: artifactAttempts + 1,
1681
- maxAttempts: maxRecoveryAttempts,
1682
- ...(correctionContextRequest
1683
- ? {
1684
- contextRequest: structuredClone(correctionContextRequest),
1685
- }
1686
- : {}),
1687
- }
1688
- : undefined;
1689
- if (this.#worker.prepareTask) {
1690
- // Reserve this selected task's conflicts while asynchronous baseline
1691
- // preparation runs, without reserving a Worker attempt.
1692
- this.#setTask(
1693
- runId,
1694
- row.task_id,
1695
- { state: "phase-ready", phase, queuePosition: null },
1696
- lease,
1697
- );
1698
- if (granted && activeRecovery?.key === preparationKey) {
1699
- this.#leasedTransaction(lease, () => {
1700
- const consumed = this.#database
1701
- .prepare(
1702
- "UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
1703
- )
1704
- .run(runId, operationId, preparationKey);
1705
- if (consumed.changes !== 1)
1706
- throw new Error("recovery-request-stale");
1707
- this.#database
1708
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
1709
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
1710
- .run(runId, preparationKey, runId);
1711
- });
1712
- }
1713
- const prepared = await this.#worker.prepareTask({
1714
- runId,
1715
- operationId,
1716
- deliveryRevision: row.delivery_revision,
1717
- taskId: row.task_id,
1718
- phase,
1719
- task: structuredClone(task),
1720
- plan: structuredClone(plan),
1721
- ...(current.baseline_workspace_revision
1722
- ? { baselineRevisionId: current.baseline_workspace_revision }
1723
- : {}),
1724
- ...(current.current_workspace_revision
1725
- ? { currentWorkspaceRevisionId: current.current_workspace_revision }
1726
- : {}),
1727
- signal,
1728
- });
1729
- this.#runStore.assertLease(lease);
1730
- if (prepared.kind === "phase-committed")
1731
- throw new Error("workflow-preparation-outcome-invalid");
1732
- this.#recordWorkspaceFacts(
1733
- runId,
1734
- {
1735
- ...prepared,
1736
- kind: "paused",
1737
- code: "task-prepared",
1738
- // Preparation has no merge authority over a sibling's newer commit.
1739
- ...(prepared.currentWorkspaceRevisionId
1740
- ? {
1741
- currentWorkspaceRevisionId:
1742
- this.#engineRun(runId).current_workspace_revision ??
1743
- prepared.currentWorkspaceRevisionId,
1744
- }
1745
- : {}),
1746
- },
1747
- lease,
1748
- );
1749
- // Preparation may finish successfully while close is settling it.
1750
- // Re-enter the cancellation guard before granting any Worker work.
1751
- if (signal.aborted) continue;
1752
- if (prepared.kind !== "prepared") {
1753
- const prerequisite =
1754
- prepared.kind !== "operation-cancelled"
1755
- ? normalizeVerificationPrerequisite(prepared.prerequisite)
1756
- : undefined;
1757
- if (prerequisite) {
1758
- const verification =
1759
- prerequisite.scope === "baseline-full-suite"
1760
- ? plan.verification.baseline?.fullSuite
1761
- : (task.baselineVerification ?? task.affectedVerification);
1762
- if (
1763
- !verification ||
1764
- verification.id !== prerequisite.verificationId ||
1765
- (prerequisite.scope === "baseline-task-affected" &&
1766
- prerequisite.taskId !== row.task_id) ||
1767
- (prerequisite.input &&
1768
- !verificationInputPaths(verification).includes(
1769
- prerequisite.input.path,
1770
- )) ||
1771
- (prerequisite.producer &&
1772
- !plan.outputs.some(
1773
- (output) =>
1774
- output.path === prerequisite.input?.path &&
1775
- output.producer.taskId === prerequisite.producer?.taskId &&
1776
- output.producer.phase === prerequisite.producer?.phase,
1777
- ))
1778
- )
1779
- throw new Error("workflow-prerequisite-evidence-invalid");
1780
- }
1781
- if (prerequisite && prepared.kind === "paused") {
1782
- const identity = hash(
1783
- canonicalJson({
1784
- ...prerequisite,
1785
- verificationId: undefined,
1786
- taskId: undefined,
1787
- producer: prerequisite.producer
1788
- ? { phase: prerequisite.producer.phase }
1789
- : undefined,
1790
- }),
1791
- );
1792
- const priorPreparation = this.#recovery(runId, preparationKey);
1793
- if (priorPreparation?.feedback.prerequisiteIdentity !== identity) {
1794
- const failures = (priorPreparation?.failures ?? 0) + 1;
1795
- this.#recordRecovery(
1796
- runId,
1797
- {
1798
- key: preparationKey,
1799
- failures,
1800
- feedback: {
1801
- code: prepared.code,
1802
- attempt: failures + 1,
1803
- maxAttempts: maxRecoveryAttempts,
1804
- strategy: "repair-verification",
1805
- prerequisiteIdentity: identity,
1806
- },
1807
- },
1808
- lease,
1809
- );
1810
- }
1811
- }
1812
- if (prepared.kind !== "operation-cancelled" && prepared.verification)
1813
- this.#setVerificationStatus(runId, prepared.verification, lease);
1814
- this.#setTask(
1815
- runId,
1816
- row.task_id,
1817
- {
1818
- state:
1819
- prepared.kind === "approval-needed"
1820
- ? "approval-needed"
1821
- : "paused",
1822
- phase,
1823
- pauseCode:
1824
- prepared.kind === "operation-cancelled"
1825
- ? "operation-cancelled"
1826
- : prepared.code,
1827
- attemptDiagnostic: {
1828
- ...priorDiagnostic,
1829
- prerequisite,
1830
- },
1831
- queuePosition: null,
1832
- },
1833
- lease,
1834
- );
1835
- return;
1836
- }
1837
- if (priorDiagnostic) delete priorDiagnostic.prerequisite;
1838
- if (activeRecovery?.key === preparationKey) {
1839
- // A successful original-baseline observation resolves this incident.
1840
- // Keep its events and immutable failure evidence, then re-enter the
1841
- // ordinary phase recovery checks before reserving Worker work.
1842
- this.#leasedTransaction(lease, () => {
1843
- this.#database
1844
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
1845
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
1846
- .run(runId, preparationKey, runId);
1847
- this.#database
1848
- .prepare(
1849
- "DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
1850
- )
1851
- .run(runId, preparationKey);
1852
- });
1853
- if (priorDiagnostic) delete priorDiagnostic.recovery;
1854
- const retainedDiagnostic =
1855
- priorDiagnostic && Object.keys(priorDiagnostic).length
1856
- ? priorDiagnostic
1857
- : null;
1858
- this.#setTask(
1859
- runId,
1860
- row.task_id,
1861
- { attemptDiagnostic: retainedDiagnostic },
1862
- lease,
1863
- );
1864
- row.attempt_diagnostic_json = retainedDiagnostic
1865
- ? JSON.stringify(retainedDiagnostic)
1866
- : null;
1867
- continue;
1868
- }
1869
- }
1870
- this.#setTask(
1871
- runId,
1872
- row.task_id,
1873
- {
1874
- state: "phase-running",
1875
- phase,
1876
- pauseCode: artifactCorrection?.code ?? null,
1877
- contextRequest: requestedContext ?? null,
1878
- attemptDiagnostic: priorDiagnostic ?? null,
1879
- queuePosition: null,
1880
- },
1881
- lease,
1882
- );
1883
- if (
1884
- !this.#reserveWork(
1885
- runId,
1886
- plan,
1887
- recoveryKey(task, phase, row, current),
1888
- lease,
1889
- granted ? operationId : undefined,
1890
- )
1891
- ) {
1892
- this.#setTask(
1893
- runId,
1894
- row.task_id,
1895
- {
1896
- state: "paused",
1897
- phase,
1898
- pauseCode: "change-work-budget-exhausted",
1899
- queuePosition: null,
1900
- },
1901
- lease,
1902
- );
1903
- return;
1904
- }
1905
- let initialReservation = true;
1906
- let outcome: WorkflowAttemptOutcome;
1907
- try {
1908
- outcome = await this.#worker.runAttempt({
1909
- runId,
1910
- operationId,
1911
- deliveryRevision: row.delivery_revision,
1912
- taskId: row.task_id,
1913
- phase,
1914
- task: structuredClone(task),
1915
- contextReadPaths: this.#contextReads(runId, row.task_id),
1916
- plan: structuredClone(plan),
1917
- ...(current.baseline_workspace_revision
1918
- ? { baselineRevisionId: current.baseline_workspace_revision }
1919
- : {}),
1920
- ...(current.current_workspace_revision
1921
- ? {
1922
- currentWorkspaceRevisionId: current.current_workspace_revision,
1923
- }
1924
- : {}),
1925
- ...(row.route_id || current.route_id
1926
- ? { routeId: row.route_id ?? current.route_id ?? undefined }
1927
- : {}),
1928
- ...(row.route_fingerprint || current.route_fingerprint
1929
- ? {
1930
- routeFingerprint:
1931
- row.route_fingerprint ??
1932
- current.route_fingerprint ??
1933
- undefined,
1934
- }
1935
- : {}),
1936
- ...(repair ? { repair } : {}),
1937
- ...(artifactCorrection ? { artifactCorrection } : {}),
1938
- ...(requestedContext ? { contextRequest: requestedContext } : {}),
1939
- ...(granted ? { additionalAttempt: true } : {}),
1940
- ...(verificationOnly ? { verificationOnly: true } : {}),
1941
- ...(activeRecovery
1942
- ? { recoveryFeedback: activeRecovery.feedback }
1943
- : {}),
1944
- signal,
1945
- recoveryDecision: (request) => {
1946
- signal.throwIfAborted();
1947
- this.#runStore.assertLease(lease);
1948
- return decideRecoveryAction(plan, request, {
1949
- additionalAttempt: granted,
1950
- verificationOnly,
1951
- });
1952
- },
1953
- reserveCandidate: () => {
1954
- signal.throwIfAborted();
1955
- if (verificationOnly) return false;
1956
- if (initialReservation) {
1957
- initialReservation = false;
1958
- return true;
1959
- }
1960
- if (granted) return false;
1961
- return this.#reserveWork(
1962
- runId,
1963
- plan,
1964
- recoveryKey(task, phase, row, current),
1965
- lease,
1966
- );
1967
- },
1968
- onActivity: (event) => {
1969
- const projection = this.#runStore.status(runId);
1970
- emitWorkflowActivity(onActivity, {
1971
- ...event,
1972
- stage: projection.stage,
1973
- runId,
1974
- ...(projection.change ? { change: projection.change } : {}),
1975
- taskId: row.task_id,
1976
- phase,
1977
- objective: task.objective,
1978
- legalCommands: ["status", "cancel", "discard"],
1979
- });
1980
- },
1981
- });
1982
- } catch (error) {
1983
- if (isCancellationException(error, signal)) {
1984
- outcome = { kind: "operation-cancelled", code: "cancelled" };
1985
- } else {
1986
- throw error;
1987
- }
1988
- }
1989
- if (!isRecord(outcome) || typeof outcome.kind !== "string") {
1990
- throw new Error("workflow-attempt-outcome-invalid");
1991
- }
1992
- this.#recordWorkspaceFacts(runId, outcome, lease);
1993
- this.#recordRouteFacts(runId, row.task_id, outcome, lease);
1994
- if (outcome.routeId && outcome.routeFingerprint) {
1995
- row.route_id = outcome.routeId;
1996
- row.route_fingerprint = outcome.routeFingerprint;
1997
- }
1998
- if (
1999
- outcome.kind !== "phase-committed" &&
2000
- outcome.kind !== "operation-cancelled" &&
2001
- outcome.verification
2002
- ) {
2003
- this.#setVerificationStatus(runId, outcome.verification, lease);
2004
- }
2005
- if (
2006
- outcome.kind !== "phase-committed" &&
2007
- outcome.kind !== "operation-cancelled" &&
2008
- outcome.contextRequest
2009
- ) {
2010
- requestedContext = normalizeWorkflowContextRequest(
2011
- outcome.contextRequest,
2012
- );
2013
- }
2014
- if (
2015
- outcome.kind !== "phase-committed" &&
2016
- outcome.kind !== "operation-cancelled" &&
2017
- outcome.attemptDiagnostic
2018
- ) {
2019
- const previous = row.attempt_diagnostic_json
2020
- ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
2021
- : undefined;
2022
- const fingerprint = hash(
2023
- outcome.code,
2024
- outcome.attemptDiagnostic.finalCategory ?? "unknown",
2025
- outcome.attemptDiagnostic.schema ?? "unknown",
2026
- JSON.stringify(outcome.attemptDiagnostic.identityMismatch ?? []),
2027
- ).slice(0, 16);
2028
- const sameFailure = previous?.fingerprint === fingerprint;
2029
- const sameFailureCount = sameFailure
2030
- ? (previous?.sameFailureCount ?? 1) + 1
2031
- : 1;
2032
- const attemptDiagnostic = {
2033
- ...outcome.attemptDiagnostic,
2034
- ...(priorDiagnostic?.recovery
2035
- ? { recovery: priorDiagnostic.recovery }
2036
- : {}),
2037
- fingerprint,
2038
- sameFailureCount,
2039
- ...(sameFailureCount > 1
2040
- ? { action: "rebind-or-revise-delivery" as const }
2041
- : {}),
2042
- };
2043
- // Only parent preparation can contribute original-baseline authority.
2044
- delete attemptDiagnostic.prerequisite;
2045
- this.#setTask(runId, row.task_id, { attemptDiagnostic }, lease);
2046
- row.attempt_diagnostic_json = JSON.stringify(attemptDiagnostic);
2047
- }
2048
- if (
2049
- outcome.kind === "paused" &&
2050
- outcome.code === "repair-attempts-exhausted"
2051
- ) {
2052
- const maxAttempts = plan.verification.repair.maxAttempts;
2053
- const failures = Math.max(
2054
- maxAttempts,
2055
- (activeRecovery?.failures ?? 0) +
2056
- (granted || verificationOnly ? 1 : 0),
2057
- );
2058
- const diagnostic: SafeAttemptDiagnostic = {
2059
- recovery: {
2060
- key: recoveryKey(task, phase, row, this.#engineRun(runId)),
2061
- failures,
2062
- feedback: {
2063
- code: outcome.code,
2064
- attempt: failures + 1,
2065
- maxAttempts,
2066
- strategy: "repair-verification",
2067
- },
2068
- },
2069
- };
2070
- this.#setTask(
2071
- runId,
2072
- row.task_id,
2073
- { attemptDiagnostic: diagnostic },
2074
- lease,
2075
- );
2076
- row.attempt_diagnostic_json = JSON.stringify(diagnostic);
2077
- }
2078
- if (
2079
- outcome.kind === "retryable" &&
2080
- ["artifact", "stale", "verification"].includes(
2081
- outcome.retryPolicy ?? "",
2082
- )
2083
- ) {
2084
- const key = recoveryKey(task, phase, row, this.#engineRun(runId));
2085
- const failures =
2086
- (activeRecovery?.key === key ? activeRecovery.failures : 0) + 1;
2087
- const strategy =
2088
- outcome.code === "needs-task-split" ||
2089
- outcome.code === "task-split-needed"
2090
- ? ("compact-patch" as const)
2091
- : outcome.retryPolicy === "stale"
2092
- ? ("refresh-candidate" as const)
2093
- : outcome.retryPolicy === "verification"
2094
- ? ("repair-verification" as const)
2095
- : ("revise-candidate" as const);
2096
- const diagnostic: SafeAttemptDiagnostic = {
2097
- ...this.#safeAttemptDiagnostic(runId, row),
2098
- recovery: {
2099
- key,
2100
- failures,
2101
- feedback: {
2102
- code: outcome.code,
2103
- attempt: failures + 1,
2104
- maxAttempts: maxRecoveryAttempts,
2105
- strategy,
2106
- ...(outcome.verification?.failureIdentities
2107
- ? { failureIdentities: outcome.verification.failureIdentities }
2108
- : {}),
2109
- },
2110
- },
2111
- };
2112
- this.#setTask(
2113
- runId,
2114
- row.task_id,
2115
- { attemptDiagnostic: diagnostic },
2116
- lease,
2117
- );
2118
- row.attempt_diagnostic_json = JSON.stringify(diagnostic);
2119
- if (outcome.code === "red-artifact-constraint") {
2120
- redArtifactCorrection = true;
2121
- correctionContextRequest = outcome.contextRequest
2122
- ? normalizeWorkflowContextRequest(outcome.contextRequest)
2123
- : correctionContextRequest;
2124
- requestedContext = correctionContextRequest ?? requestedContext;
2125
- }
2126
- artifactAttempts = Math.max(artifactAttempts + 1, failures);
2127
- const maxAttempts = maxRecoveryAttempts;
2128
- if (artifactAttempts < maxAttempts) {
2129
- this.#setTask(
2130
- runId,
2131
- row.task_id,
2132
- {
2133
- state: "retryable",
2134
- phase,
2135
- pauseCode: redArtifactCorrection
2136
- ? "red-artifact-constraint"
2137
- : outcome.code,
2138
- contextRequest: requestedContext ?? null,
2139
- queuePosition: null,
2140
- },
2141
- lease,
2142
- );
2143
- emitWorkflowActivity(onActivity, {
2144
- state: "retrying",
2145
- stage: this.#runStore.status(runId).stage,
2146
- runId,
2147
- taskId: row.task_id,
2148
- phase,
2149
- objective: task.objective,
2150
- legalCommands: ["status", "cancel", "discard"],
2151
- code: outcome.code,
2152
- attempt: artifactAttempts,
2153
- maxAttempts,
2154
- wait: "artifact-correction",
2155
- });
2156
- continue;
2157
- }
2158
- }
2159
- if (outcome.kind === "phase-committed") {
2160
- if (
2161
- !SHA256.test(outcome.artifactHash) ||
2162
- !SHA256.test(outcome.isolatedRevisionId) ||
2163
- (outcome.baselineRevisionId !== undefined &&
2164
- !SHA256.test(outcome.baselineRevisionId)) ||
2165
- outcome.classification !== expectedClassification(phase)
2166
- ) {
2167
- throw new Error("workflow-attempt-fact-invalid");
2168
- }
2169
- this.#leasedTransaction(lease, () => {
2170
- const resolvedKey = recoveryKey(task, phase, row, current);
2171
- if (this.#recovery(runId, resolvedKey)) {
2172
- this.#database
2173
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2174
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'resolved' FROM workflow_recovery_events WHERE run_id = ?`)
2175
- .run(runId, resolvedKey, runId);
2176
- }
2177
- this.#database
2178
- .prepare(
2179
- "DELETE FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
2180
- )
2181
- .run(runId, recoveryKey(task, phase, row, current));
2182
- });
2183
- if (repair) {
2184
- this.#setVerificationStatus(runId, null, lease);
2185
- this.#setTask(
2186
- runId,
2187
- row.task_id,
2188
- {
2189
- state: "verified",
2190
- phase,
2191
- pauseCode: null,
2192
- contextRequest: null,
2193
- attemptDiagnostic: null,
2194
- queuePosition: null,
2195
- },
2196
- lease,
2197
- );
2198
- return;
2199
- }
2200
- artifactAttempts = 0;
2201
- row.attempt_diagnostic_json = null;
2202
- redArtifactCorrection = false;
2203
- requestedContext = undefined;
2204
- correctionContextRequest = undefined;
2205
- phaseIndex += 1;
2206
- if (phaseIndex >= orderedPhases.length) {
2207
- this.#setTask(
2208
- runId,
2209
- row.task_id,
2210
- {
2211
- state: "verified",
2212
- phase,
2213
- pauseCode: null,
2214
- contextRequest: null,
2215
- attemptDiagnostic: null,
2216
- queuePosition: null,
2217
- },
2218
- lease,
2219
- );
2220
- return;
2221
- }
2222
- this.#setTask(
2223
- runId,
2224
- row.task_id,
2225
- {
2226
- state: "phase-ready",
2227
- phase: orderedPhases[phaseIndex],
2228
- pauseCode: null,
2229
- contextRequest: null,
2230
- attemptDiagnostic: null,
2231
- queuePosition: null,
2232
- },
2233
- lease,
2234
- );
2235
- continue;
2236
- }
2237
- if (outcome.kind === "operation-cancelled") {
2238
- this.#setTask(
2239
- runId,
2240
- row.task_id,
2241
- {
2242
- state: "paused",
2243
- phase,
2244
- pauseCode: "operation-cancelled",
2245
- contextRequest: null,
2246
- queuePosition: null,
2247
- },
2248
- lease,
2249
- );
2250
- return;
2251
- }
2252
- if (
2253
- outcome.kind === "paused" ||
2254
- outcome.kind === "retryable" ||
2255
- outcome.kind === "approval-needed"
2256
- ) {
2257
- if (typeof outcome.code !== "string" || outcome.code.length === 0) {
2258
- throw new Error("workflow-attempt-outcome-invalid");
2259
- }
2260
- const approvalCodeInvalid =
2261
- outcome.kind === "approval-needed" &&
2262
- !isWorkflowApprovalCode(outcome.code);
2263
- this.#setTask(
2264
- runId,
2265
- row.task_id,
2266
- {
2267
- state: approvalCodeInvalid ? "paused" : outcome.kind,
2268
- phase,
2269
- pauseCode: approvalCodeInvalid
2270
- ? "approval-code-invalid"
2271
- : outcome.code,
2272
- contextRequest: approvalCodeInvalid
2273
- ? null
2274
- : (requestedContext ?? null),
2275
- queuePosition: null,
2276
- },
2277
- lease,
2278
- );
2279
- return;
2280
- }
2281
- throw new Error("workflow-attempt-outcome-invalid");
2282
- }
2283
- }
2284
-
2285
- #firstPaused(rows: EngineTaskRow[]): EngineTaskRow | undefined {
2286
- return firstPausedTask(rows);
2287
- }
2288
-
2289
- #recoverLegacyContextApproval(runId: string): boolean {
2290
- const projection = this.#runStore.status(runId);
2291
- if (!["approval-needed", "paused"].includes(projection.state)) return false;
2292
- const rows = this.#tasks(runId);
2293
- const approvalRows = rows.filter((row) => row.state === "approval-needed");
2294
- if (approvalRows.length === 0) return false;
2295
- const reclassified = rows.flatMap((row) => {
2296
- if (
2297
- row.state !== "approval-needed" ||
2298
- row.pause_code !== "boundary-review-needed" ||
2299
- row.context_request_json === null
2300
- ) {
2301
- return [];
2302
- }
2303
- try {
2304
- const request = normalizeWorkflowContextRequest(
2305
- JSON.parse(row.context_request_json) as unknown,
2306
- );
2307
- const classified = classifyPersistedContextRequest(
2308
- request,
2309
- this.#taskPlan(row),
2310
- row.phase,
2311
- );
2312
- if (classified.kind === "approval-needed") return [];
2313
- return [
2314
- {
2315
- row,
2316
- state: classified.kind === "retryable" ? "retryable" : "paused",
2317
- code: classified.code,
2318
- contextRequest: JSON.stringify(
2319
- normalizeWorkflowContextRequest(classified.contextRequest),
2320
- ),
2321
- },
2322
- ];
2323
- } catch {
2324
- return [];
2325
- }
2326
- });
2327
- const reclassifiedTaskIds = new Set(
2328
- reclassified.map((entry) => entry.row.task_id),
2329
- );
2330
- const retainedApprovals = approvalRows.filter(
2331
- (row) => !reclassifiedTaskIds.has(row.task_id),
2332
- );
2333
- const updateTasks = (database: DatabaseSync) => {
2334
- for (const entry of reclassified) {
2335
- const updated = database
2336
- .prepare(
2337
- `UPDATE workflow_engine_tasks
2338
- SET state = ?, phase = ?, pause_code = ?, context_request_json = ?,
2339
- queue_position = NULL
2340
- WHERE run_id = ? AND task_id = ?
2341
- AND state = 'approval-needed'
2342
- AND pause_code = 'boundary-review-needed'`,
2343
- )
2344
- .run(
2345
- entry.state,
2346
- entry.row.phase,
2347
- entry.code,
2348
- entry.contextRequest,
2349
- runId,
2350
- entry.row.task_id,
2351
- );
2352
- if (Number(updated.changes) !== 1) {
2353
- throw new Error("workflow-legacy-context-race");
2354
- }
2355
- }
2356
- };
2357
- const transitionWithUpdates = (
2358
- to: "paused" | "approval-needed",
2359
- code: string,
2360
- ) =>
2361
- this.#runStore.transitionAtomically(
2362
- {
2363
- runId,
2364
- to,
2365
- operationId: `legacy-context-${hash(
2366
- runId,
2367
- String(projection.sequence),
2368
- to,
2369
- code,
2370
- JSON.stringify(
2371
- reclassified.map((entry) => [entry.row.task_id, entry.code]),
2372
- ),
2373
- ).slice(0, 40)}`,
2374
- code,
2375
- },
2376
- updateTasks,
2377
- );
2378
- if (retainedApprovals.length > 0) {
2379
- if (projection.state === "approval-needed") {
2380
- if (reclassified.length === 0) return false;
2381
- this.#transaction(() => updateTasks(this.#database));
2382
- } else {
2383
- transitionWithUpdates(
2384
- "approval-needed",
2385
- retainedApprovals[0].pause_code ?? "approval-code-invalid",
2386
- );
2387
- }
2388
- return true;
2389
- }
2390
- const first = reclassified[0];
2391
- if (!first) return false;
2392
- transitionWithUpdates("paused", first.code);
2393
- return true;
2394
- }
2395
-
2396
- async #finishChange(
2397
- runId: string,
2398
- operationId: string,
2399
- signal: AbortSignal,
2400
- lease: OperationLease,
2401
- onActivity?: (event: WorkflowActivityUpdate) => void,
2402
- ): Promise<Record<string, unknown>> {
2403
- const plan = this.#currentPlan(runId);
2404
- let engineRun = this.#engineRun(runId);
2405
- if (engineRun.current_revision === null)
2406
- throw new Error("delivery-not-bound");
2407
- const deliveryRevision = engineRun.current_revision;
2408
- this.#transition(
2409
- runId,
2410
- "change-verifying",
2411
- `${operationId}:change-verifying`,
2412
- undefined,
2413
- lease,
2414
- );
2415
- const projection = this.#runStore.status(runId);
2416
- emitWorkflowActivity(onActivity, {
2417
- state: "verifying",
2418
- stage: projection.stage,
2419
- runId,
2420
- ...(projection.change ? { change: projection.change } : {}),
2421
- objective: projection.change
2422
- ? `verify change ${projection.change}`
2423
- : "verify workflow change",
2424
- legalCommands: ["status", "cancel", "discard"],
2425
- });
2426
- const verification = await this.#changeVerifier.verify({
2427
- taskEvidence: this.#tasks(runId).map((row) => ({
2428
- taskId: row.task_id,
2429
- deliveryRevision: row.delivery_revision,
2430
- })),
2431
- runId,
2432
- deliveryRevision,
2433
- plan: structuredClone(plan),
2434
- ...(engineRun.baseline_workspace_revision
2435
- ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2436
- : {}),
2437
- ...(engineRun.current_workspace_revision
2438
- ? {
2439
- currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2440
- }
2441
- : {}),
2442
- signal,
2443
- });
2444
- this.#runStore.assertLease(lease);
2445
- if (verification.kind !== "verified") {
2446
- if (verification.verification) {
2447
- this.#setVerificationStatus(runId, verification.verification, lease);
2448
- }
2449
- if (
2450
- verification.code === "verification-repair-required" &&
2451
- verification.verification?.scope === "change-task-affected" &&
2452
- verification.verification.attribution === "introduced" &&
2453
- verification.verification.taskId
2454
- ) {
2455
- const task = this.#tasks(runId).find(
2456
- (candidate) =>
2457
- candidate.task_id === verification.verification?.taskId,
2458
- );
2459
- if (task) {
2460
- const repairKey = recoveryKey(
2461
- this.#taskPlan(task),
2462
- task.phase,
2463
- task,
2464
- this.#engineRun(runId),
2465
- );
2466
- const used = (
2467
- this.#database
2468
- .prepare(
2469
- "SELECT COUNT(*) AS count FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ? AND kind = 'repair'",
2470
- )
2471
- .get(runId, repairKey) as { count: number }
2472
- ).count;
2473
- if (used >= plan.verification.repair.maxAttempts) {
2474
- this.#setTask(
2475
- runId,
2476
- task.task_id,
2477
- {
2478
- state: "paused",
2479
- phase: task.phase,
2480
- pauseCode: "repair-attempts-exhausted",
2481
- attemptDiagnostic: {
2482
- recovery: {
2483
- key: recoveryKey(
2484
- this.#taskPlan(task),
2485
- task.phase,
2486
- task,
2487
- this.#engineRun(runId),
2488
- ),
2489
- failures: used,
2490
- feedback: {
2491
- code: "repair-attempts-exhausted",
2492
- attempt: used + 1,
2493
- maxAttempts: plan.verification.repair.maxAttempts,
2494
- strategy: "repair-verification",
2495
- failureIdentities:
2496
- verification.verification.failureIdentities,
2497
- },
2498
- },
2499
- },
2500
- queuePosition: null,
2501
- },
2502
- lease,
2503
- );
2504
- this.#transition(
2505
- runId,
2506
- "paused",
2507
- `${operationId}:repair-attempts-exhausted:${task.task_id}`,
2508
- "repair-attempts-exhausted",
2509
- lease,
2510
- );
2511
- return this.#statusByRun(runId);
2512
- }
2513
- this.#leasedTransaction(lease, () => {
2514
- this.#database
2515
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
2516
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'repair' FROM workflow_recovery_events WHERE run_id = ?`)
2517
- .run(runId, repairKey, runId);
2518
- });
2519
- const taskPlan = this.#taskPlan(task);
2520
- this.#setTask(
2521
- runId,
2522
- task.task_id,
2523
- {
2524
- state: "repairable",
2525
- phase: taskPlan.phases.refactor ? "refactor" : "green",
2526
- pauseCode: verification.code,
2527
- queuePosition: null,
2528
- },
2529
- lease,
2530
- );
2531
- this.#transition(
2532
- runId,
2533
- "running",
2534
- `${operationId}:repairable:${task.task_id}`,
2535
- undefined,
2536
- lease,
2537
- );
2538
- return this.#advance(runId, operationId, signal, lease, onActivity);
2539
- }
2540
- }
2541
- const approvalCodeInvalid =
2542
- verification.kind === "approval-needed" &&
2543
- !isWorkflowApprovalCode(verification.code);
2544
- const state =
2545
- verification.kind === "approval-needed" && !approvalCodeInvalid
2546
- ? "approval-needed"
2547
- : "paused";
2548
- const code = approvalCodeInvalid
2549
- ? "approval-code-invalid"
2550
- : verification.code;
2551
- this.#transition(
2552
- runId,
2553
- state,
2554
- `${operationId}:change-verification-${state}`,
2555
- code,
2556
- lease,
2557
- );
2558
- return this.#statusByRun(runId);
2559
- }
2560
- if (verification.currentWorkspaceRevisionId !== undefined) {
2561
- const currentWorkspaceRevisionId =
2562
- verification.currentWorkspaceRevisionId;
2563
- if (!SHA256.test(currentWorkspaceRevisionId)) {
2564
- throw new Error("workflow-verification-workspace-fact-invalid");
2565
- }
2566
- this.#leasedTransaction(lease, () => {
2567
- this.#database
2568
- .prepare(
2569
- `UPDATE workflow_engine_runs
2570
- SET current_workspace_revision = ?, cleanup_state = 'retained'
2571
- WHERE run_id = ?`,
2572
- )
2573
- .run(currentWorkspaceRevisionId, runId);
2574
- });
2575
- engineRun = this.#engineRun(runId);
2576
- }
2577
- this.#transition(
2578
- runId,
2579
- "ready-to-apply",
2580
- `${operationId}:ready-to-apply`,
2581
- undefined,
2582
- lease,
2583
- );
2584
- if (!this.#application) {
2585
- this.#transition(
2586
- runId,
2587
- "paused",
2588
- `${operationId}:application-unavailable`,
2589
- "application-unavailable",
2590
- lease,
2591
- );
2592
- return this.#statusByRun(runId);
2593
- }
2594
- const transactionId = `apply-${hash(
2595
- runId,
2596
- String(deliveryRevision),
2597
- operationId,
2598
- ).slice(0, 40)}`;
2599
- this.#leasedTransaction(lease, () => {
2600
- this.#database
2601
- .prepare(
2602
- `UPDATE workflow_engine_runs SET transaction_id = ? WHERE run_id = ?`,
2603
- )
2604
- .run(transactionId, runId);
2605
- });
2606
- const applicationInput = {
2607
- transactionId,
2608
- runId,
2609
- consumerRoot: this.#consumerRoot,
2610
- deliveryRevision,
2611
- plan: structuredClone(plan),
2612
- contextReadPaths: this.#contextReads(runId),
2613
- ...(engineRun.baseline_workspace_revision
2614
- ? { baselineRevisionId: engineRun.baseline_workspace_revision }
2615
- : {}),
2616
- ...(engineRun.current_workspace_revision
2617
- ? {
2618
- currentWorkspaceRevisionId: engineRun.current_workspace_revision,
2619
- }
2620
- : {}),
2621
- signal,
2622
- };
2623
- let application: Record<string, unknown>;
2624
- if (this.#application.prepare && this.#application.applyPrepared) {
2625
- const prepared = await this.#application.prepare(applicationInput);
2626
- this.#runStore.assertLease(lease);
2627
- if (prepared.state !== "prepared") {
2628
- return this.#settleApplication(
2629
- runId,
2630
- operationId,
2631
- prepared,
2632
- lease,
2633
- onActivity,
2634
- );
2635
- }
2636
- this.#transition(
2637
- runId,
2638
- "applying",
2639
- `${operationId}:applying`,
2640
- undefined,
2641
- lease,
2642
- );
2643
- emitWorkflowActivity(onActivity, {
2644
- state: "applying",
2645
- stage: projection.stage,
2646
- runId,
2647
- ...(projection.change ? { change: projection.change } : {}),
2648
- objective: projection.change
2649
- ? `apply change ${projection.change}`
2650
- : "apply workflow change",
2651
- legalCommands: ["status", "cancel", "discard"],
2652
- });
2653
- const active = this.#active.get(runId);
2654
- if (active) active.kind = "applying";
2655
- try {
2656
- application = await this.#application.applyPrepared(applicationInput);
2657
- } catch (error) {
2658
- this.#runStore.assertLease(lease);
2659
- if (this.#runStore.status(runId).state === "applying")
2660
- this.#transition(
2661
- runId,
2662
- "recovering",
2663
- `${operationId}:application-interrupted`,
2664
- "operation-interrupted",
2665
- lease,
2666
- );
2667
- throw error;
2668
- }
2669
- } else {
2670
- this.#transition(
2671
- runId,
2672
- "applying",
2673
- `${operationId}:applying`,
2674
- undefined,
2675
- lease,
2676
- );
2677
- emitWorkflowActivity(onActivity, {
2678
- state: "applying",
2679
- stage: projection.stage,
2680
- runId,
2681
- ...(projection.change ? { change: projection.change } : {}),
2682
- objective: projection.change
2683
- ? `apply change ${projection.change}`
2684
- : "apply workflow change",
2685
- legalCommands: ["status", "cancel", "discard"],
2686
- });
2687
- const active = this.#active.get(runId);
2688
- if (active) active.kind = "applying";
2689
- application = await this.#application.begin(applicationInput);
2690
- }
2691
- this.#runStore.assertLease(lease);
2692
- return this.#settleApplication(
2693
- runId,
2694
- operationId,
2695
- application,
2696
- lease,
2697
- onActivity,
2698
- );
2699
- }
2700
-
2701
- #settleApplication(
2702
- runId: string,
2703
- operationId: string,
2704
- outcome: Record<string, unknown>,
2705
- lease: OperationLease,
2706
- onActivity?: (event: WorkflowActivityUpdate) => void,
2707
- ): Record<string, unknown> {
2708
- const state = outcome.state;
2709
- if (state === "completed") {
2710
- const current = this.#runStore.status(runId);
2711
- if (current.state === "recovering") {
2712
- this.#transition(
2713
- runId,
2714
- "completed",
2715
- `${operationId}:recovered-complete`,
2716
- undefined,
2717
- lease,
2718
- );
2719
- } else if (
2720
- current.state === "applying" ||
2721
- current.state === "ready-to-apply"
2722
- ) {
2723
- this.#transition(
2724
- runId,
2725
- "completed",
2726
- `${operationId}:apply-complete`,
2727
- undefined,
2728
- lease,
2729
- );
2730
- }
2731
- this.#cleanupTerminal(runId, operationId, lease);
2732
- } else if (state === "paused") {
2733
- if (this.#runStore.status(runId).state === "applying") {
2734
- this.#transition(
2735
- runId,
2736
- "recovering",
2737
- `${operationId}:apply-recovering`,
2738
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2739
- lease,
2740
- );
2741
- const recovering = this.#runStore.status(runId);
2742
- emitWorkflowActivity(onActivity, {
2743
- state: "recovering",
2744
- stage: recovering.stage,
2745
- runId,
2746
- ...(recovering.change ? { change: recovering.change } : {}),
2747
- objective: recovering.change
2748
- ? `recover change ${recovering.change}`
2749
- : "recover workflow application",
2750
- code:
2751
- typeof outcome.code === "string"
2752
- ? outcome.code
2753
- : "recovery-required",
2754
- legalCommands: ["status", "cancel", "discard"],
2755
- });
2756
- }
2757
- this.#transition(
2758
- runId,
2759
- "paused",
2760
- `${operationId}:apply-paused`,
2761
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2762
- lease,
2763
- );
2764
- } else if (state === "discarded") {
2765
- if (this.#runStore.status(runId).state === "applying") {
2766
- this.#transition(
2767
- runId,
2768
- "recovering",
2769
- `${operationId}:discard-recovering`,
2770
- "run-discarded",
2771
- lease,
2772
- );
2773
- const recovering = this.#runStore.status(runId);
2774
- emitWorkflowActivity(onActivity, {
2775
- state: "recovering",
2776
- stage: recovering.stage,
2777
- runId,
2778
- ...(recovering.change ? { change: recovering.change } : {}),
2779
- objective: recovering.change
2780
- ? `recover change ${recovering.change}`
2781
- : "recover workflow application",
2782
- code: "run-discarded",
2783
- legalCommands: ["status"],
2784
- });
2785
- }
2786
- this.#transition(
2787
- runId,
2788
- "discarded",
2789
- `${operationId}:apply-discarded`,
2790
- undefined,
2791
- lease,
2792
- );
2793
- this.#cleanupTerminal(runId, operationId, lease);
2794
- } else if (state === "recovering") {
2795
- this.#transition(
2796
- runId,
2797
- "recovering",
2798
- `${operationId}:application-recovering`,
2799
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2800
- lease,
2801
- );
2802
- const recovering = this.#runStore.status(runId);
2803
- emitWorkflowActivity(onActivity, {
2804
- state: "recovering",
2805
- stage: recovering.stage,
2806
- runId,
2807
- ...(recovering.change ? { change: recovering.change } : {}),
2808
- objective: recovering.change
2809
- ? `recover change ${recovering.change}`
2810
- : "recover workflow application",
2811
- code:
2812
- typeof outcome.code === "string" ? outcome.code : "recovery-required",
2813
- legalCommands: ["status", "cancel", "discard"],
2814
- });
2815
- } else {
2816
- throw new Error("workflow-application-outcome-invalid");
2817
- }
2818
- return this.#statusByRun(runId);
2819
- }
2820
-
2821
- #applicationContext(runId: string): WorkflowApplicationContext {
2822
- const current = this.#engineRun(runId);
2823
- if (current.current_revision === null)
2824
- throw new Error("delivery-not-bound");
2825
- return {
2826
- runId,
2827
- deliveryRevision: current.current_revision,
2828
- plan: structuredClone(this.#currentPlan(runId)),
2829
- contextReadPaths: this.#contextReads(runId),
2830
- ...(current.baseline_workspace_revision
2831
- ? { baselineRevisionId: current.baseline_workspace_revision }
2832
- : {}),
2833
- ...(current.current_workspace_revision
2834
- ? { currentWorkspaceRevisionId: current.current_workspace_revision }
2835
- : {}),
2836
- };
2837
- }
2838
-
2839
- #cleanupTerminal(
2840
- runId: string,
2841
- _currentOperationId: string,
2842
- lease?: OperationLease,
2843
- ): void {
2844
- if (lease) this.#runStore.assertLease(lease);
2845
- this.#lifecycle?.cleanup(runId);
2846
- const cleanup = () => {
2847
- this.#database
2848
- .prepare("DELETE FROM workflow_engine_tasks WHERE run_id = ?")
2849
- .run(runId);
2850
- this.#database
2851
- .prepare("DELETE FROM workflow_engine_deliveries WHERE run_id = ?")
2852
- .run(runId);
2853
- this.#database
2854
- .prepare(`DELETE FROM workspace_revisions WHERE owner_run_id = ?`)
2855
- .run(runId);
2856
- this.#database
2857
- .prepare(`DELETE FROM apply_transactions WHERE owner_run_id = ?`)
2858
- .run(runId);
2859
- this.#database
2860
- .prepare(
2861
- `UPDATE workflow_engine_runs
2862
- SET baseline_workspace_revision = NULL,
2863
- current_workspace_revision = NULL,
2864
- transaction_id = NULL, verification_json = NULL,
2865
- cleanup_state = 'complete'
2866
- WHERE run_id = ?`,
2867
- )
2868
- .run(runId);
2869
- };
2870
- if (lease) this.#leasedTransaction(lease, cleanup);
2871
- else this.#transaction(cleanup);
2872
- }
2873
-
2874
- #waitForCapacity(signal: AbortSignal): Promise<void> {
2875
- if (signal.aborted || this.#activeTasks < LIMITS.maxActiveChildSessions)
2876
- return Promise.resolve();
2877
- return new Promise((resolve) => {
2878
- const wake = () => {
2879
- this.#capacityWaiters.delete(wake);
2880
- signal.removeEventListener("abort", wake);
2881
- resolve();
2882
- };
2883
- this.#capacityWaiters.add(wake);
2884
- signal.addEventListener("abort", wake, { once: true });
2885
- });
2886
- }
2887
-
2888
- async #advance(
2889
- runId: string,
2890
- operationId: string,
2891
- signal: AbortSignal,
2892
- lease: OperationLease,
2893
- onActivity?: (event: WorkflowActivityUpdate) => void,
2894
- ): Promise<Record<string, unknown>> {
2895
- this.#transition(
2896
- runId,
2897
- "running",
2898
- `${operationId}:running`,
2899
- undefined,
2900
- lease,
2901
- );
2902
- const run = this.#runStore.status(runId);
2903
- emitWorkflowActivity(onActivity, {
2904
- state: "running",
2905
- stage: run.stage,
2906
- runId,
2907
- ...(run.change ? { change: run.change } : {}),
2908
- objective: run.change ? `change ${run.change}` : "workflow run",
2909
- legalCommands: ["status", "cancel", "discard"],
2910
- });
2911
- const existingVerification = this.#engineRun(runId).verification_json;
2912
- const repairStatus = existingVerification
2913
- ? normalizeWorkflowVerificationStatus(
2914
- JSON.parse(existingVerification) as unknown,
2915
- )
2916
- : undefined;
2917
- const retainsRepairContext =
2918
- repairStatus?.scope === "change-task-affected" &&
2919
- repairStatus.attribution === "introduced" &&
2920
- repairStatus.taskId !== undefined &&
2921
- this.#tasks(runId).some(
2922
- (task) =>
2923
- task.task_id === repairStatus.taskId && task.state !== "verified",
2924
- );
2925
- if (!retainsRepairContext) {
2926
- this.#setVerificationStatus(runId, null, lease);
2927
- }
2928
- const attempted = new Set<string>();
2929
- // Keep every launched operation owned until it settles, including failures.
2930
- // A fulfilled envelope avoids unhandled rejections while another task wins
2931
- // the race; the finally barrier also protects cancellation and storage close.
2932
- type Settlement =
2933
- | { taskId: string; status: "fulfilled" }
2934
- | { taskId: string; status: "rejected"; reason: unknown };
2935
- const pending = new Map<string, Promise<Settlement>>();
2936
- let firstFailure: { reason: unknown } | undefined;
2937
- try {
2938
- while (!signal.aborted) {
2939
- if (firstFailure) throw firstFailure.reason;
2940
- const rows = this.#tasks(runId);
2941
- const selected = selectRunnableTasks({
2942
- rows,
2943
- attempted,
2944
- activeTasks: this.#activeTasks,
2945
- capacity: LIMITS.maxActiveChildSessions,
2946
- nextQueuePosition: this.#engineRun(runId).next_queue_position,
2947
- });
2948
- const runnable = selected.runnable.map((taskId) => {
2949
- const row = rows.find((candidate) => candidate.task_id === taskId);
2950
- if (!row) throw new Error("scheduler-task-unavailable");
2951
- attempted.add(taskId);
2952
- return row;
2953
- });
2954
- for (const update of selected.updates) {
2955
- const row = rows.find(
2956
- (candidate) => candidate.task_id === update.taskId,
2957
- );
2958
- if (!row) throw new Error("scheduler-task-unavailable");
2959
- if (update.kind === "pending")
2960
- this.#setTask(
2961
- runId,
2962
- row.task_id,
2963
- { state: "pending", pauseCode: null, queuePosition: null },
2964
- lease,
2965
- );
2966
- else this.#queueTask(runId, row, lease, update.reason);
2967
- }
2968
- this.#activeTasks += runnable.length;
2969
- for (const row of runnable) {
2970
- const operation = this.#runTask(
2971
- runId,
2972
- row,
2973
- operationId,
2974
- signal,
2975
- lease,
2976
- onActivity,
2977
- )
2978
- .then<Settlement, Settlement>(
2979
- () => ({ taskId: row.task_id, status: "fulfilled" }),
2980
- (reason: unknown) => {
2981
- firstFailure ??= { reason };
2982
- return { taskId: row.task_id, status: "rejected", reason };
2983
- },
2984
- )
2985
- .finally(() => {
2986
- this.#activeTasks--;
2987
- for (const wake of this.#capacityWaiters) wake();
2988
- });
2989
- pending.set(row.task_id, operation);
2990
- }
2991
- if (pending.size > 0) {
2992
- const settled = await Promise.race([
2993
- ...pending.values(),
2994
- ...(selected.capacityBlocked
2995
- ? [this.#waitForCapacity(signal).then(() => null)]
2996
- : []),
2997
- ]);
2998
- if (settled === null) continue;
2999
- pending.delete(settled.taskId);
3000
- if (settled.status === "rejected") throw settled.reason;
3001
- continue;
3002
- }
3003
- if (selected.capacityBlocked) {
3004
- await this.#waitForCapacity(signal);
3005
- continue;
3006
- }
3007
- break;
3008
- }
3009
- } finally {
3010
- await Promise.all(pending.values());
3011
- }
3012
- if (firstFailure) throw firstFailure.reason;
3013
-
3014
- if (signal.aborted) {
3015
- const current = this.#runStore.status(runId);
3016
- if (!["paused", "discarded", "completed"].includes(current.state)) {
3017
- this.#transition(
3018
- runId,
3019
- "paused",
3020
- `${operationId}:cancelled`,
3021
- "operation-cancelled",
3022
- lease,
3023
- );
3024
- }
3025
- return this.#statusByRun(runId);
3026
- }
3027
- const rows = this.#tasks(runId);
3028
- const paused = this.#firstPaused(rows);
3029
- if (paused) {
3030
- this.#transition(
3031
- runId,
3032
- paused.state === "approval-needed" ? "approval-needed" : "paused",
3033
- `${operationId}:task-paused:${paused.task_id}`,
3034
- paused.pause_code ?? "task-paused",
3035
- lease,
3036
- );
3037
- return this.#statusByRun(runId);
3038
- }
3039
- if (rows.every((row) => row.state === "verified")) {
3040
- return this.#finishChange(runId, operationId, signal, lease, onActivity);
3041
- }
3042
- this.#transition(
3043
- runId,
3044
- "paused",
3045
- `${operationId}:scheduler-stalled`,
3046
- "scheduler-integrity-paused",
3047
- lease,
3048
- );
3049
- return this.#statusByRun(runId);
3050
- }
3051
-
3052
- #contextReads(runId: string, taskId?: string): string[] {
3053
- const rows =
3054
- taskId === undefined
3055
- ? this.#database
3056
- .prepare(
3057
- "SELECT DISTINCT path FROM workflow_context_reads WHERE run_id = ? ORDER BY path",
3058
- )
3059
- .all(runId)
3060
- : this.#database
3061
- .prepare(
3062
- "SELECT path FROM workflow_context_reads WHERE run_id = ? AND task_id = ? ORDER BY path",
3063
- )
3064
- .all(runId, taskId);
3065
- return rows.map((row) => String(row.path));
3066
- }
3067
-
3068
- #phaseCount(plan: ImplementPlan): number {
3069
- return plan.tasks.reduce((sum, task) => sum + phases(task).length, 0);
3070
- }
3071
-
3072
- #growWorkBudget(
3073
- database: DatabaseSync,
3074
- runId: string,
3075
- plan: ImplementPlan,
3076
- ): void {
3077
- const count = this.#phaseCount(plan);
3078
- database
3079
- .prepare(`INSERT INTO workflow_work_budget(run_id, used, max_work, phase_high_water, hard_limit, recovery_policy)
3080
- VALUES (?, 0, ?, ?, ?, 1) ON CONFLICT(run_id) DO UPDATE SET
3081
- phase_high_water = MAX(phase_high_water, excluded.phase_high_water),
3082
- max_work = MIN(hard_limit, MAX(max_work, 24 + 3 * MAX(phase_high_water, excluded.phase_high_water)))`)
3083
- .run(
3084
- runId,
3085
- Math.min(this.#workHardLimit, 24 + 3 * count),
3086
- count,
3087
- this.#workHardLimit,
3088
- );
3089
- }
3090
-
3091
- #hasRecoveryGrant(runId: string, operationId: string, key: string): boolean {
3092
- return Boolean(
3093
- this.#database
3094
- .prepare(
3095
- "SELECT 1 FROM workflow_recovery_grants WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
3096
- )
3097
- .get(runId, operationId, key),
3098
- );
3099
- }
3100
-
3101
- #recoveryConditions(runId: string, key: string) {
3102
- const run = this.#engineRun(runId);
3103
- const task = this.#tasks(runId).find((row) =>
3104
- [
3105
- recoveryKey(this.#taskPlan(row), row.phase, row, run),
3106
- baselineRecoveryKey(this.#taskPlan(row), row.phase),
3107
- ].includes(key),
3108
- );
3109
- return {
3110
- route: task?.route_fingerprint ?? run.route_fingerprint,
3111
- context: task
3112
- ? hash(
3113
- canonicalJson(
3114
- snapshotFiles(
3115
- this.#consumerRoot,
3116
- this.#contextReads(runId, task.task_id),
3117
- ),
3118
- ),
3119
- )
3120
- : null,
3121
- };
3122
- }
3123
-
3124
- #failureSequence(runId: string, key: string): number {
3125
- return (
3126
- this.#database
3127
- .prepare(
3128
- "SELECT COALESCE(MAX(sequence), 0) AS sequence FROM workflow_recovery_events WHERE run_id = ? AND incident_key = ?",
3129
- )
3130
- .get(runId, key) as { sequence: number }
3131
- ).sequence;
3132
- }
3133
-
3134
- #grantRecovery(
3135
- runId: string,
3136
- operationId: string,
3137
- request: NonNullable<
3138
- Extract<ControlCommand, { command: "resume" }>["recovery"]
3139
- >,
3140
- lease: OperationLease,
3141
- ): void {
3142
- this.#leasedTransaction(lease, () => {
3143
- const incident = this.#recovery(runId, request.incidentKey);
3144
- const current = this.#tasks(runId).some(
3145
- (row) =>
3146
- ["paused", "retryable"].includes(row.state) &&
3147
- [
3148
- recoveryKey(
3149
- this.#taskPlan(row),
3150
- row.phase,
3151
- row,
3152
- this.#engineRun(runId),
3153
- ),
3154
- baselineRecoveryKey(this.#taskPlan(row), row.phase),
3155
- ].includes(request.incidentKey),
3156
- );
3157
- if (
3158
- !current ||
3159
- !incident ||
3160
- incident.failures < incident.feedback.maxAttempts ||
3161
- this.#failureSequence(runId, request.incidentKey) !==
3162
- request.failureSequence
3163
- )
3164
- throw new Error("recovery-request-stale");
3165
- const conditions = this.#database
3166
- .prepare(
3167
- "SELECT conditions_json FROM workflow_recovery_incidents WHERE run_id = ? AND incident_key = ?",
3168
- )
3169
- .get(runId, request.incidentKey) as { conditions_json: string };
3170
- const previous = JSON.parse(conditions.conditions_json);
3171
- const observed = this.#recoveryConditions(runId, request.incidentKey);
3172
- const budget = this.#database
3173
- .prepare(
3174
- "SELECT used, max_work FROM workflow_work_budget WHERE run_id = ?",
3175
- )
3176
- .get(runId) as { used: number; max_work: number };
3177
- const rejected = assessRecoveryGrant({
3178
- current,
3179
- incident,
3180
- failureSequence: this.#failureSequence(runId, request.incidentKey),
3181
- requestedSequence: request.failureSequence,
3182
- reason: request.reason,
3183
- previous,
3184
- observed,
3185
- budget,
3186
- });
3187
- if (rejected) throw new Error(rejected);
3188
- this.#database
3189
- .prepare(
3190
- "INSERT INTO workflow_recovery_grants(run_id, operation_id, incident_key, failure_sequence, reason) VALUES (?, ?, ?, ?, ?)",
3191
- )
3192
- .run(
3193
- runId,
3194
- operationId,
3195
- request.incidentKey,
3196
- request.failureSequence,
3197
- request.reason,
3198
- );
3199
- });
3200
- }
3201
-
3202
- #reserveWork(
3203
- runId: string,
3204
- plan: ImplementPlan,
3205
- key: string,
3206
- lease: OperationLease,
3207
- grantOperationId?: string,
3208
- ): boolean {
3209
- return this.#leasedTransaction(lease, () => {
3210
- this.#growWorkBudget(this.#database, runId, plan);
3211
- const reserved = this.#database
3212
- .prepare(`UPDATE workflow_work_budget SET used = used + 1
3213
- WHERE run_id = ? AND used < max_work`)
3214
- .run(runId);
3215
- if (reserved.changes !== 1) return false;
3216
- if (grantOperationId) {
3217
- const consumed = this.#database
3218
- .prepare(
3219
- "UPDATE workflow_recovery_grants SET consumed = 1 WHERE run_id = ? AND operation_id = ? AND incident_key = ? AND consumed = 0",
3220
- )
3221
- .run(runId, grantOperationId, key);
3222
- if (consumed.changes !== 1) throw new Error("recovery-request-stale");
3223
- }
3224
- this.#database
3225
- .prepare(`INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
3226
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'launch' FROM workflow_recovery_events WHERE run_id = ?`)
3227
- .run(runId, key, runId);
3228
- return true;
3229
- });
3230
- }
3231
-
3232
- #recovery(runId: string, key: string): WorkflowRecoveryFact | undefined {
3233
- const row = this.#database
3234
- .prepare(
3235
- `SELECT failures, max_attempts, feedback_json FROM workflow_recovery_incidents
3236
- WHERE run_id = ? AND incident_key = ?`,
3237
- )
3238
- .get(runId, key) as
3239
- | { failures: number; max_attempts: number; feedback_json: string }
3240
- | undefined;
3241
- return row
3242
- ? {
3243
- key,
3244
- failures: row.failures,
3245
- feedback: {
3246
- ...JSON.parse(row.feedback_json),
3247
- maxAttempts: row.max_attempts,
3248
- },
3249
- }
3250
- : undefined;
3251
- }
3252
-
3253
- #recordRecovery(
3254
- runId: string,
3255
- fact: WorkflowRecoveryFact,
3256
- lease: OperationLease,
3257
- ): void {
3258
- this.#leasedTransaction(lease, () => {
3259
- this.#database
3260
- .prepare(
3261
- `INSERT INTO workflow_recovery_incidents(run_id, incident_key, failures, max_attempts, feedback_json, conditions_json)
3262
- VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(run_id, incident_key) DO UPDATE SET
3263
- failures = MAX(failures, excluded.failures), max_attempts = MIN(max_attempts, excluded.max_attempts), feedback_json = excluded.feedback_json, conditions_json = excluded.conditions_json`,
3264
- )
3265
- .run(
3266
- runId,
3267
- fact.key,
3268
- fact.failures,
3269
- fact.feedback.maxAttempts,
3270
- JSON.stringify(fact.feedback),
3271
- JSON.stringify(this.#recoveryConditions(runId, fact.key)),
3272
- );
3273
- this.#database
3274
- .prepare(
3275
- `INSERT INTO workflow_recovery_events(run_id, sequence, incident_key, kind)
3276
- SELECT ?, COALESCE(MAX(sequence), 0) + 1, ?, 'failure' FROM workflow_recovery_events WHERE run_id = ?`,
3277
- )
3278
- .run(runId, fact.key, runId);
3279
- });
3280
- }
3281
-
3282
- #safeAttemptDiagnostic(
3283
- runId: string,
3284
- row: EngineTaskRow | undefined,
3285
- ): SafeAttemptDiagnostic | undefined {
3286
- if (!row) return undefined;
3287
- const diagnostic: SafeAttemptDiagnostic = row.attempt_diagnostic_json
3288
- ? (JSON.parse(row.attempt_diagnostic_json) as SafeAttemptDiagnostic)
3289
- : {};
3290
- delete diagnostic.recovery;
3291
- const recovery =
3292
- this.#recovery(
3293
- runId,
3294
- baselineRecoveryKey(this.#taskPlan(row), row.phase),
3295
- ) ??
3296
- this.#recovery(
3297
- runId,
3298
- recoveryKey(
3299
- this.#taskPlan(row),
3300
- row.phase,
3301
- row,
3302
- this.#engineRun(runId),
3303
- ),
3304
- );
3305
- if (recovery) diagnostic.recovery = recovery;
3306
- return Object.keys(diagnostic).length ? diagnostic : undefined;
3307
- }
3308
-
3309
- #statusByRun(runId: string): Record<string, unknown> {
3310
- const projection = this.#runStore.status(runId);
3311
- const rows = this.#tasks(runId);
3312
- let engineRun: EngineRunRow | undefined;
3313
- try {
3314
- engineRun = this.#engineRun(runId);
3315
- } catch {
3316
- engineRun = undefined;
3317
- }
3318
- const attemptDiagnostics = new Map<string, SafeAttemptDiagnostic>();
3319
- const failureSequences = new Map<string, number>();
3320
- for (const row of rows) {
3321
- const diagnostic = this.#safeAttemptDiagnostic(runId, row);
3322
- if (!diagnostic) continue;
3323
- attemptDiagnostics.set(row.task_id, diagnostic);
3324
- if (diagnostic.recovery) {
3325
- const key = diagnostic.recovery.key;
3326
- failureSequences.set(key, this.#failureSequence(runId, key));
3327
- }
3328
- }
3329
- const amendmentUsed =
3330
- (
3331
- this.#database
3332
- .prepare(
3333
- "SELECT used FROM workflow_amendment_budget WHERE run_id = ?",
3334
- )
3335
- .get(runId) as { used: number } | undefined
3336
- )?.used ?? 0;
3337
- const resourceBudget = this.#database
3338
- .prepare(
3339
- "SELECT used, max_work AS maximum, phase_high_water AS phaseHighWater, hard_limit AS hardLimit FROM workflow_work_budget WHERE run_id = ?",
3340
- )
3341
- .get(runId) as WorkflowResourceBudget | undefined;
3342
- return projectWorkflowStatus({
3343
- projection,
3344
- rows,
3345
- engineRun,
3346
- attemptDiagnostics,
3347
- failureSequences,
3348
- amendmentUsed,
3349
- resourceBudget,
3350
- });
3351
- }
3352
-
3353
- async #withAvailableDelivery(
3354
- outcome: Record<string, unknown>,
3355
- signal?: AbortSignal,
3356
- ): Promise<Record<string, unknown>> {
3357
- if (
3358
- outcome.stage !== "abel-implement" ||
3359
- (isRecord(outcome.pause) &&
3360
- outcome.pause.code === "operation-cancelled") ||
3361
- (outcome.state !== "approval-needed" &&
3362
- !(
3363
- outcome.state === "paused" &&
3364
- (isRecord(outcome.decisionBatch) ||
3365
- (isRecord(outcome.recovery) && outcome.recovery.exhausted === true))
3366
- )) ||
3367
- typeof outcome.change !== "string" ||
3368
- !this.#deliverySource.discoverLatest
3369
- ) {
3370
- return outcome;
3371
- }
3372
- let available: WorkflowAvailableDelivery | undefined;
3373
- try {
3374
- const discover = this.#deliverySource.discoverLatest;
3375
- const change = outcome.change;
3376
- available = await cancellableRead(
3377
- () =>
3378
- discover.call(this.#deliverySource, {
3379
- stage: "abel-implement",
3380
- change,
3381
- ...(signal ? { signal } : {}),
3382
- }),
3383
- signal,
3384
- );
3385
- } catch {
3386
- return outcome;
3387
- }
3388
- const currentRevision =
3389
- typeof outcome.deliveryRevision === "number"
3390
- ? outcome.deliveryRevision
3391
- : 0;
3392
- if (
3393
- !available ||
3394
- !Number.isSafeInteger(available.deliveryRevision) ||
3395
- available.deliveryRevision <= currentRevision ||
3396
- !SHA256.test(available.receiptHash)
3397
- ) {
3398
- return outcome;
3399
- }
3400
- if (
3401
- this.#database
3402
- .prepare(
3403
- "SELECT 1 FROM workflow_rejected_deliveries WHERE run_id = ? AND delivery_revision = ? AND receipt_hash = ?",
3404
- )
3405
- .get(
3406
- String(outcome.runId),
3407
- available.deliveryRevision,
3408
- available.receiptHash,
3409
- )
3410
- ) {
3411
- return outcome;
3412
- }
3413
- const conditionalCommands = (
3414
- Array.isArray(outcome.conditionalCommands)
3415
- ? outcome.conditionalCommands
3416
- : [{ command: "resume", stage: outcome.stage, change: outcome.change }]
3417
- ).map((entry) =>
3418
- isRecord(entry) && entry.command === "resume"
3419
- ? {
3420
- ...entry,
3421
- satisfiedBy: {
3422
- deliveryRevision: available.deliveryRevision,
3423
- receiptHash: available.receiptHash,
3424
- },
3425
- }
3426
- : entry,
3427
- );
3428
- return {
3429
- ...outcome,
3430
- legalCommands:
3431
- outcome.state === "approval-needed"
3432
- ? ["status", "resume", "discard"]
3433
- : ["status", "resume", "rebind", "discard"],
3434
- availableDelivery: structuredClone(available),
3435
- continuation:
3436
- (isRecord(outcome.resourceBudget) &&
3437
- outcome.resourceBudget.remaining === 0) ||
3438
- (isRecord(outcome.recovery) &&
3439
- outcome.recovery.code === "change-recovery-budget-exhausted")
3440
- ? undefined
3441
- : {
3442
- owner: "parent",
3443
- automatic: true,
3444
- command: "resume",
3445
- stage: "abel-implement",
3446
- change: outcome.change,
3447
- },
3448
- ...(conditionalCommands === undefined ? {} : { conditionalCommands }),
3449
- };
3450
- }
3451
-
3452
- #launchAdvance(
3453
- runId: string,
3454
- operationId: string,
3455
- externalSignal?: AbortSignal,
3456
- onActivity?: (event: WorkflowActivityUpdate) => void,
3457
- ): Promise<Record<string, unknown>> {
3458
- const lease = this.#operationLease(runId, operationId);
3459
- const controller = new AbortController();
3460
- const abortFromExternal = () => {
3461
- controller.abort(
3462
- externalSignal?.reason ?? new Error("operation-cancelled"),
3463
- );
3464
- };
3465
- if (externalSignal?.aborted) abortFromExternal();
3466
- else
3467
- externalSignal?.addEventListener("abort", abortFromExternal, {
3468
- once: true,
3469
- });
3470
- const active: ActiveOperation = {
3471
- operationId,
3472
- kind: "work",
3473
- controller,
3474
- settled: Promise.resolve({}),
3475
- };
3476
- this.#active.set(runId, active);
3477
- const settled = (async () => {
3478
- try {
3479
- const outcome = await this.#advance(
3480
- runId,
3481
- operationId,
3482
- controller.signal,
3483
- lease,
3484
- onActivity,
3485
- );
3486
- return this.#commitOperation(runId, operationId, outcome);
3487
- } catch (error) {
3488
- this.#interruptOperation(runId, operationId);
3489
- const controlSettled =
3490
- controller.signal.aborted &&
3491
- controller.signal.reason instanceof Error &&
3492
- ["operation-cancelled", "run-discarded"].includes(
3493
- controller.signal.reason.message,
3494
- ) &&
3495
- (isCancellationException(error, controller.signal) ||
3496
- (error instanceof Error &&
3497
- ["lease-fenced", "operation-journal-fenced"].includes(
3498
- error.message,
3499
- )));
3500
- if (controlSettled) return this.#statusByRun(runId);
3501
- throw error;
3502
- } finally {
3503
- externalSignal?.removeEventListener("abort", abortFromExternal);
3504
- if (this.#active.get(runId) === active) this.#active.delete(runId);
3505
- }
3506
- })();
3507
- active.settled = settled;
3508
- return settled;
3509
- }
3510
-
3511
- async #start(
3512
- command: Extract<ControlCommand, { command: "start" }>,
3513
- signal?: AbortSignal,
3514
- onActivity?: (event: WorkflowActivityUpdate) => void,
3515
- ) {
3516
- const existingRun = this.#lookupRun(command.stage, command.change);
3517
- if (existingRun) {
3518
- this.#ensureEngineRun(existingRun);
3519
- const replay = this.#operationReplay(
3520
- existingRun,
3521
- command.operationId,
3522
- command.command,
3523
- );
3524
- if (replay) return replay;
3525
- const restarted = this.#runStore.startRun({
3526
- stage: command.stage,
3527
- change: command.change,
3528
- operationId: command.operationId,
3529
- });
3530
- if (restarted.state !== "created") {
3531
- const begun = this.#beginOperation(
3532
- existingRun,
3533
- command.operationId,
3534
- command.command,
3535
- );
3536
- if (begun) return begun;
3537
- return this.#commitOperation(
3538
- existingRun,
3539
- command.operationId,
3540
- this.#statusByRun(existingRun),
3541
- );
3542
- }
3543
- }
3544
-
3545
- const runId =
3546
- existingRun ??
3547
- this.#runStore.startRun({
3548
- stage: command.stage,
3549
- change: command.change,
3550
- operationId: command.operationId,
3551
- }).runId;
3552
- this.#ensureEngineRun(runId);
3553
- const begun = this.#beginOperation(
3554
- runId,
3555
- command.operationId,
3556
- command.command,
3557
- );
3558
- if (begun) return begun;
3559
- const lease = this.#operationLease(runId, command.operationId);
3560
- this.#transition(
3561
- runId,
3562
- "validating-delivery",
3563
- `${command.operationId}:validating-delivery`,
3564
- undefined,
3565
- lease,
3566
- );
3567
- emitWorkflowActivity(onActivity, {
3568
- state: "validating",
3569
- stage: command.stage,
3570
- runId,
3571
- change: command.change,
3572
- objective: `validate delivery ${command.change}`,
3573
- legalCommands: ["status", "cancel", "discard"],
3574
- });
3575
- try {
3576
- const delivery = await this.#loadDelivery(
3577
- runId,
3578
- lease,
3579
- command.stage,
3580
- command.change,
3581
- undefined,
3582
- undefined,
3583
- signal,
3584
- );
3585
- this.#admitDelivery(runId, delivery, lease);
3586
- this.#transition(
3587
- runId,
3588
- "ready",
3589
- `${command.operationId}:delivery-ready`,
3590
- undefined,
3591
- lease,
3592
- );
3593
- return this.#launchAdvance(
3594
- runId,
3595
- command.operationId,
3596
- signal,
3597
- onActivity,
3598
- );
3599
- } catch (error) {
3600
- const cancelled = this.#cancelledDelivery(
3601
- runId,
3602
- command.operationId,
3603
- error,
3604
- signal,
3605
- lease,
3606
- );
3607
- if (cancelled) return cancelled;
3608
- const failure = this.#deliveryFailure(
3609
- runId,
3610
- command.operationId,
3611
- error,
3612
- lease,
3613
- );
3614
- if (failure) {
3615
- return this.#commitOperation(runId, command.operationId, failure);
3616
- }
3617
- this.#interruptOperation(runId, command.operationId);
3618
- throw error;
3619
- }
3620
- }
3621
-
3622
- async #resume(
3623
- command: Extract<ControlCommand, { command: "resume" }>,
3624
- signal?: AbortSignal,
3625
- onActivity?: (event: WorkflowActivityUpdate) => void,
3626
- ) {
3627
- const runId = this.#lookupRun(command.stage, command.change);
3628
- if (!runId) throw new Error("run-not-found");
3629
- this.#recoverLegacyContextApproval(runId);
3630
- const replay = this.#beginOperation(
3631
- runId,
3632
- command.operationId,
3633
- command.command,
3634
- );
3635
- if (replay) return replay;
3636
- const lease = this.#operationLease(runId, command.operationId);
3637
- const current = this.#runStore.status(runId);
3638
- if (current.state === "recovering") {
3639
- emitWorkflowActivity(onActivity, {
3640
- state: "recovering",
3641
- stage: current.stage,
3642
- runId,
3643
- ...(current.change ? { change: current.change } : {}),
3644
- objective: current.change
3645
- ? `recover change ${current.change}`
3646
- : "recover workflow application",
3647
- legalCommands: ["status", "cancel", "discard"],
3648
- });
3649
- const transactionId = this.#engineRun(runId).transaction_id;
3650
- if (!transactionId || !this.#application) {
3651
- this.#interruptOperation(runId, command.operationId);
3652
- throw new Error("apply-recovery-unavailable");
3653
- }
3654
- const recovered = await this.#application.recover(
3655
- transactionId,
3656
- this.#applicationContext(runId),
3657
- );
3658
- this.#runStore.assertLease(lease);
3659
- const outcome = this.#settleApplication(
3660
- runId,
3661
- command.operationId,
3662
- recovered,
3663
- lease,
3664
- onActivity,
3665
- );
3666
- return this.#commitOperation(runId, command.operationId, outcome);
3667
- }
3668
- if (
3669
- !["paused", "retryable", "approval-needed", "ready"].includes(
3670
- current.state,
3671
- )
3672
- ) {
3673
- this.#interruptOperation(runId, command.operationId);
3674
- throw new Error("resume-not-allowed");
3675
- }
3676
- try {
3677
- const engineRevision = this.#engineRun(runId).current_revision;
3678
- const projectedRevision = current.deliveryRevision;
3679
- if (
3680
- command.deliveryRevision === undefined &&
3681
- (current.state === "approval-needed" ||
3682
- this.#statusByRun(runId).decisionBatch !== undefined ||
3683
- this.#statusByRun(runId).recovery !== undefined) &&
3684
- this.#deliverySource.discoverLatest
3685
- ) {
3686
- const available = await this.#deliverySource.discoverLatest({
3687
- stage: command.stage,
3688
- change: command.change,
3689
- signal,
3690
- });
3691
- this.#runStore.assertLease(lease);
3692
- if (available && available.deliveryRevision > (engineRevision ?? 0)) {
3693
- command = {
3694
- ...command,
3695
- deliveryRevision: available.deliveryRevision,
3696
- receiptHash: available.receiptHash,
3697
- };
3698
- }
3699
- }
3700
- const deliveryOutOfSync =
3701
- projectedRevision !== undefined && projectedRevision !== engineRevision;
3702
- if (
3703
- current.state === "approval-needed" &&
3704
- (command.deliveryRevision === undefined ||
3705
- command.receiptHash === undefined ||
3706
- command.deliveryRevision <=
3707
- (engineRevision ?? projectedRevision ?? 0))
3708
- ) {
3709
- this.#interruptOperation(runId, command.operationId);
3710
- throw new Error("approval-receipt-required");
3711
- }
3712
- if (
3713
- command.deliveryRevision !== undefined ||
3714
- engineRevision === null ||
3715
- deliveryOutOfSync
3716
- ) {
3717
- this.#transition(
3718
- runId,
3719
- "validating-delivery",
3720
- `${command.operationId}:validating-delivery`,
3721
- undefined,
3722
- lease,
3723
- );
3724
- emitWorkflowActivity(onActivity, {
3725
- state: "validating",
3726
- stage: command.stage,
3727
- runId,
3728
- change: command.change,
3729
- objective: `validate delivery ${command.change}`,
3730
- legalCommands: ["status", "cancel", "discard"],
3731
- });
3732
- const requestedRevision =
3733
- command.deliveryRevision ??
3734
- (deliveryOutOfSync ? projectedRevision : undefined);
3735
- const projectedBinding = current.deliveryBindings.find(
3736
- (binding) =>
3737
- binding.gate === "gate-b" && binding.revision === requestedRevision,
3738
- );
3739
- const delivery = await this.#loadDelivery(
3740
- runId,
3741
- lease,
3742
- command.stage,
3743
- command.change,
3744
- requestedRevision,
3745
- command.receiptHash ?? projectedBinding?.receiptHash,
3746
- signal,
3747
- );
3748
- this.#admitDelivery(runId, delivery, lease);
3749
- }
3750
- if (command.recovery)
3751
- this.#grantRecovery(
3752
- runId,
3753
- command.operationId,
3754
- command.recovery,
3755
- lease,
3756
- );
3757
- this.#transition(
3758
- runId,
3759
- "ready",
3760
- `${command.operationId}:resume-ready`,
3761
- undefined,
3762
- lease,
3763
- );
3764
- return this.#launchAdvance(
3765
- runId,
3766
- command.operationId,
3767
- signal,
3768
- onActivity,
3769
- );
3770
- } catch (error) {
3771
- const cancelled = this.#cancelledDelivery(
3772
- runId,
3773
- command.operationId,
3774
- error,
3775
- signal,
3776
- lease,
3777
- );
3778
- if (cancelled) return cancelled;
3779
- const failure = this.#deliveryFailure(
3780
- runId,
3781
- command.operationId,
3782
- error,
3783
- lease,
3784
- );
3785
- if (failure) {
3786
- return this.#commitOperation(runId, command.operationId, failure);
3787
- }
3788
- // A revised delivery may invalidate a grant. Retain its admitted facts,
3789
- // but never leave a rejected resume looking ready or still validating.
3790
- if (
3791
- error instanceof Error &&
3792
- [
3793
- "recovery-request-stale",
3794
- "recovery-evidence-unavailable",
3795
- "change-work-budget-exhausted",
3796
- ].includes(error.message) &&
3797
- this.#runStore.status(runId).state === "validating-delivery"
3798
- ) {
3799
- this.#transition(
3800
- runId,
3801
- "paused",
3802
- `${command.operationId}:recovery-rejected`,
3803
- error.message,
3804
- lease,
3805
- );
3806
- }
3807
- this.#interruptOperation(runId, command.operationId);
3808
- throw error;
3809
- }
3810
- }
3811
-
3812
- #rebind(command: Extract<ControlCommand, { command: "rebind" }>) {
3813
- const runId = this.#lookupRun(command.stage, command.change);
3814
- if (!runId) throw new Error("run-not-found");
3815
- const replay = this.#beginOperation(
3816
- runId,
3817
- command.operationId,
3818
- command.command,
3819
- );
3820
- if (replay) return replay;
3821
- const lease = this.#operationLease(runId, command.operationId);
3822
- const state = this.#runStore.status(runId).state;
3823
- if (!["paused", "retryable"].includes(state)) {
3824
- this.#interruptOperation(runId, command.operationId);
3825
- throw new Error("rebind-not-allowed");
3826
- }
3827
- const task = this.#tasks(runId).find((row) =>
3828
- ["paused", "retryable"].includes(row.state),
3829
- );
3830
- if (!task) {
3831
- this.#interruptOperation(runId, command.operationId);
3832
- throw new Error("rebind-task-unavailable");
3833
- }
3834
- const rebound = this.#worker.rebind({
3835
- runId,
3836
- taskId: task.task_id,
3837
- task: structuredClone(this.#taskPlan(task)),
3838
- role: "implementation-worker",
3839
- routeId: command.routeId,
3840
- });
3841
- if (!rebound.ok) {
3842
- this.#interruptOperation(runId, command.operationId);
3843
- throw new Error(rebound.code);
3844
- }
3845
- const routeId = rebound.routeId ?? rebound.route?.id ?? command.routeId;
3846
- const routeFingerprint =
3847
- rebound.routeFingerprint ?? rebound.route?.fingerprint;
3848
- if (
3849
- !IDENTIFIER.test(routeId) ||
3850
- (routeFingerprint !== undefined && !SHA256.test(routeFingerprint))
3851
- ) {
3852
- this.#interruptOperation(runId, command.operationId);
3853
- throw new Error("route-rebind-result-invalid");
3854
- }
3855
- const priorRouteId = task.route_id ?? this.#engineRun(runId).route_id;
3856
- const priorRouteFingerprint =
3857
- task.route_fingerprint ?? this.#engineRun(runId).route_fingerprint;
3858
- this.#leasedTransaction(lease, () => {
3859
- this.#setTask(runId, task.task_id, {
3860
- routeId,
3861
- routeFingerprint: routeFingerprint ?? null,
3862
- });
3863
- this.#database
3864
- .prepare(
3865
- `UPDATE workflow_engine_runs
3866
- SET route_id = ?, route_fingerprint = ? WHERE run_id = ?`,
3867
- )
3868
- .run(routeId, routeFingerprint ?? null, runId);
3869
- });
3870
- const outcome = {
3871
- ...this.#statusByRun(runId),
3872
- routeBinding: {
3873
- routeId,
3874
- ...(routeFingerprint ? { routeFingerprint } : {}),
3875
- },
3876
- operation: {
3877
- kind: "route-rebound",
3878
- ...(priorRouteId ? { priorRouteId } : {}),
3879
- ...(priorRouteFingerprint ? { priorRouteFingerprint } : {}),
3880
- routeId,
3881
- ...(routeFingerprint ? { routeFingerprint } : {}),
3882
- },
3883
- };
3884
- return this.#commitOperation(runId, command.operationId, outcome);
3885
- }
3886
-
3887
- async #controlApply(
3888
- runId: string,
3889
- operationId: string,
3890
- intent: "cancel" | "discard",
3891
- lease: OperationLease,
3892
- onActivity?: (event: WorkflowActivityUpdate) => void,
3893
- ): Promise<Record<string, unknown>> {
3894
- const transactionId = this.#engineRun(runId).transaction_id;
3895
- if (!transactionId || !this.#application) {
3896
- throw new Error("apply-recovery-unavailable");
3897
- }
3898
- const context = this.#applicationContext(runId);
3899
- this.#application.requestControl(transactionId, intent, context);
3900
- const active = this.#active.get(runId);
3901
- if (active && active.operationId !== operationId) {
3902
- active.controller.abort(
3903
- new Error(
3904
- intent === "cancel" ? "operation-cancelled" : "run-discarded",
3905
- ),
3906
- );
3907
- }
3908
- if (this.#runStore.status(runId).state === "applying") {
3909
- this.#transition(
3910
- runId,
3911
- "recovering",
3912
- `${operationId}:${intent}-recovering`,
3913
- intent === "cancel" ? "operation-cancelled" : "run-discarded",
3914
- lease,
3915
- );
3916
- const recovering = this.#runStore.status(runId);
3917
- emitWorkflowActivity(onActivity, {
3918
- state: "recovering",
3919
- stage: recovering.stage,
3920
- runId,
3921
- ...(recovering.change ? { change: recovering.change } : {}),
3922
- objective: recovering.change
3923
- ? `recover change ${recovering.change}`
3924
- : "recover workflow application",
3925
- code: intent === "cancel" ? "operation-cancelled" : "run-discarded",
3926
- legalCommands: ["status"],
3927
- });
3928
- }
3929
- const recovered = await this.#application.recover(transactionId, context);
3930
- this.#runStore.assertLease(lease);
3931
- return this.#settleApplication(
3932
- runId,
3933
- operationId,
3934
- recovered,
3935
- lease,
3936
- onActivity,
3937
- );
3938
- }
3939
-
3940
- async #abortActiveOperation(runId: string, reason: Error): Promise<void> {
3941
- const pending: Promise<unknown>[] = [];
3942
- const amendment = this.#amendments.get(runId);
3943
- if (amendment) pending.push(amendment);
3944
- for (const [controller, entry] of this.#commands) {
3945
- if (
3946
- !["start", "resume"].includes(entry.command.command) ||
3947
- this.#lookupRun(entry.command.stage, entry.command.change) !== runId
3948
- )
3949
- continue;
3950
- controller.abort(reason);
3951
- pending.push(entry.settled);
3952
- }
3953
- const active = this.#active.get(runId);
3954
- if (active) {
3955
- active.controller.abort(reason);
3956
- pending.push(active.settled);
3957
- }
3958
- const results = await Promise.allSettled(pending);
3959
- for (const result of results) {
3960
- if (result.status !== "rejected") continue;
3961
- const error = result.reason;
3962
- if (
3963
- error !== reason &&
3964
- !(
3965
- error instanceof Error &&
3966
- ["lease-fenced", "operation-journal-fenced"].includes(error.message)
3967
- )
3968
- )
3969
- throw error;
3970
- }
3971
- }
3972
-
3973
- async #cancel(
3974
- command: Extract<ControlCommand, { command: "cancel" }>,
3975
- onActivity?: (event: WorkflowActivityUpdate) => void,
3976
- ) {
3977
- const runId = this.#lookupRun(command.stage, command.change);
3978
- if (!runId) throw new Error("run-not-found");
3979
- const replay = this.#beginOperation(
3980
- runId,
3981
- command.operationId,
3982
- command.command,
3983
- );
3984
- if (replay) return replay;
3985
- const lease = this.#operationLease(runId, command.operationId);
3986
- const state = this.#runStore.status(runId).state;
3987
- let outcome: Record<string, unknown>;
3988
- if (state === "applying" || state === "recovering") {
3989
- outcome = await this.#controlApply(
3990
- runId,
3991
- command.operationId,
3992
- "cancel",
3993
- lease,
3994
- onActivity,
3995
- );
3996
- } else {
3997
- const activeSettlement = this.#abortActiveOperation(
3998
- runId,
3999
- new Error("operation-cancelled"),
4000
- );
4001
- const current = this.#runStore.status(runId);
4002
- if (
4003
- !["paused", "discarded", "completed", "rejected"].includes(
4004
- current.state,
4005
- )
4006
- ) {
4007
- this.#transition(
4008
- runId,
4009
- "paused",
4010
- `${command.operationId}:cancelled`,
4011
- "operation-cancelled",
4012
- lease,
4013
- );
4014
- }
4015
- await activeSettlement;
4016
- for (const task of this.#tasks(runId)) {
4017
- if (
4018
- ["phase-ready", "phase-running", "validating"].includes(task.state)
4019
- ) {
4020
- this.#setTask(
4021
- runId,
4022
- task.task_id,
4023
- {
4024
- state: "paused",
4025
- pauseCode: "operation-cancelled",
4026
- queuePosition: null,
4027
- },
4028
- lease,
4029
- );
4030
- }
4031
- }
4032
- outcome = this.#statusByRun(runId);
4033
- }
4034
- outcome = {
4035
- ...outcome,
4036
- operation: { kind: "operation-cancelled" },
4037
- };
4038
- return this.#commitOperation(runId, command.operationId, outcome);
4039
- }
4040
-
4041
- async #discard(
4042
- command: Extract<ControlCommand, { command: "discard" }>,
4043
- onActivity?: (event: WorkflowActivityUpdate) => void,
4044
- ) {
4045
- const runId = this.#lookupRun(command.stage, command.change);
4046
- if (!runId) throw new Error("run-not-found");
4047
- const replay = this.#beginOperation(
4048
- runId,
4049
- command.operationId,
4050
- command.command,
4051
- );
4052
- if (replay) return replay;
4053
- const lease = this.#operationLease(runId, command.operationId);
4054
- const state = this.#runStore.status(runId).state;
4055
- let outcome: Record<string, unknown>;
4056
- if (state === "applying" || state === "recovering") {
4057
- outcome = await this.#controlApply(
4058
- runId,
4059
- command.operationId,
4060
- "discard",
4061
- lease,
4062
- onActivity,
4063
- );
4064
- } else {
4065
- const activeSettlement = this.#abortActiveOperation(
4066
- runId,
4067
- new Error("run-discarded"),
4068
- );
4069
- const current = this.#runStore.status(runId);
4070
- if (current.state !== "discarded") {
4071
- if (["completed", "rejected"].includes(current.state)) {
4072
- this.#interruptOperation(runId, command.operationId);
4073
- throw new Error("discard-not-allowed");
4074
- }
4075
- this.#transition(
4076
- runId,
4077
- "discarded",
4078
- `${command.operationId}:discarded`,
4079
- undefined,
4080
- lease,
4081
- );
4082
- await activeSettlement;
4083
- this.#cleanupTerminal(runId, command.operationId, lease);
4084
- } else {
4085
- await activeSettlement;
4086
- }
4087
- outcome = this.#statusByRun(runId);
4088
- }
4089
- return this.#commitOperation(runId, command.operationId, outcome);
4090
- }
4091
-
4092
- execute(
4093
- value: unknown,
4094
- signal?: AbortSignal,
4095
- onActivity?: (event: WorkflowActivityUpdate) => void,
4096
- ): Promise<Record<string, unknown>> {
4097
- let command: ControlCommand;
4098
- try {
4099
- this.#assertOpen();
4100
- if (this.#closing) throw new Error("workflow-engine-closed");
4101
- command = assertControlCommand(value);
4102
- } catch (error) {
4103
- return Promise.reject(error);
4104
- }
4105
- const controller = new AbortController();
4106
- const onAbort = () => controller.abort(signal?.reason);
4107
- if (signal?.aborted) onAbort();
4108
- else signal?.addEventListener("abort", onAbort, { once: true });
4109
- // Register before execution can yield or invoke external callbacks.
4110
- const settled = Promise.resolve()
4111
- .then(() => this.#executeCommand(command, controller.signal, onActivity))
4112
- .finally(() => {
4113
- signal?.removeEventListener("abort", onAbort);
4114
- this.#commands.delete(controller);
4115
- });
4116
- this.#commands.set(controller, { command, settled });
4117
- return settled;
4118
- }
4119
-
4120
- async #executeCommand(
4121
- command: ControlCommand,
4122
- signal?: AbortSignal,
4123
- onActivity?: (event: WorkflowActivityUpdate) => void,
4124
- ): Promise<Record<string, unknown>> {
4125
- switch (command.command) {
4126
- case "start":
4127
- return this.#withAvailableDelivery(
4128
- await this.#start(command, signal, onActivity),
4129
- signal,
4130
- );
4131
- case "status": {
4132
- const runId = this.#lookupRun(command.stage, command.change);
4133
- if (!runId) {
4134
- return {
4135
- stage: command.stage,
4136
- change: command.change,
4137
- state: "not-started",
4138
- durable: true,
4139
- completed: false,
4140
- legalCommands: ["start"],
4141
- tasks: [],
4142
- queue: [],
4143
- };
4144
- }
4145
- this.#recoverLegacyContextApproval(runId);
4146
- return this.#withAvailableDelivery(this.#statusByRun(runId), signal);
4147
- }
4148
- case "resume":
4149
- return this.#withAvailableDelivery(
4150
- await this.#resume(command, signal, onActivity),
4151
- signal,
4152
- );
4153
- case "rebind":
4154
- return this.#rebind(command);
4155
- case "cancel":
4156
- return this.#cancel(command, onActivity);
4157
- case "discard":
4158
- return this.#discard(command, onActivity);
4159
- }
4160
- }
4161
-
4162
- prepareBootstrapHandoff(
4163
- input: Record<string, unknown>,
4164
- ): Record<string, unknown> {
4165
- this.#assertOpen();
4166
- const acceptanceFacts = normalizeBootstrapAcceptanceFacts(
4167
- input.acceptanceFacts,
4168
- );
4169
- const workspaceManifestHash =
4170
- typeof input.workspaceManifestHash === "string" &&
4171
- SHA256.test(input.workspaceManifestHash)
4172
- ? input.workspaceManifestHash
4173
- : undefined;
4174
- if (
4175
- typeof input.operationId !== "string" ||
4176
- !IDENTIFIER.test(input.operationId) ||
4177
- typeof input.runId !== "string" ||
4178
- typeof input.change !== "string" ||
4179
- !CHANGE_NAME.test(input.change) ||
4180
- typeof input.receiptHash !== "string" ||
4181
- !SHA256.test(input.receiptHash) ||
4182
- typeof input.graphHash !== "string" ||
4183
- !SHA256.test(input.graphHash) ||
4184
- typeof input.acceptanceHash !== "string" ||
4185
- !SHA256.test(input.acceptanceHash) ||
4186
- (acceptanceFacts !== undefined) !==
4187
- (workspaceManifestHash !== undefined) ||
4188
- (acceptanceFacts !== undefined &&
4189
- workspaceManifestHash !== undefined &&
4190
- bootstrapAcceptanceHash(acceptanceFacts, workspaceManifestHash) !==
4191
- input.acceptanceHash)
4192
- ) {
4193
- throw new Error("bootstrap-handoff-invalid");
4194
- }
4195
- this.#runStore.status(input.runId);
4196
- const existing = this.#database
4197
- .prepare(
4198
- `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
4199
- FROM bootstrap_handoffs WHERE receipt_hash = ?`,
4200
- )
4201
- .get(input.receiptHash) as BootstrapRow | undefined;
4202
- if (existing) {
4203
- const outcome = parseJsonRecord(existing.facts_json);
4204
- const existingAcceptanceFacts = normalizeBootstrapAcceptanceFacts(
4205
- outcome.acceptanceFacts,
4206
- );
4207
- const storedBindingValid =
4208
- outcome.handoffId === existing.handoff_id &&
4209
- outcome.runId === existing.owner_run_id &&
4210
- outcome.receiptHash === existing.receipt_hash &&
4211
- outcome.selectorBeforeCutover === "bootstrap" &&
4212
- existing.state === "prepared" &&
4213
- outcome.state === "prepared" &&
4214
- outcome.selectorCasPending === true;
4215
- if (!storedBindingValid) throw new Error("bootstrap-handoff-invalid");
4216
- const stableBindingMatches =
4217
- existing.owner_run_id === input.runId &&
4218
- outcome.change === input.change &&
4219
- outcome.graphHash === input.graphHash;
4220
- if (!stableBindingMatches) {
4221
- throw new Error("bootstrap-handoff-conflict");
4222
- }
4223
- const acceptanceMatches =
4224
- outcome.acceptanceHash === input.acceptanceHash &&
4225
- canonicalJson(existingAcceptanceFacts ?? null) ===
4226
- canonicalJson(acceptanceFacts ?? null) &&
4227
- outcome.workspaceManifestHash === workspaceManifestHash;
4228
- if (acceptanceMatches) return structuredClone(outcome);
4229
- if (
4230
- existing.state !== "prepared" ||
4231
- outcome.state !== "prepared" ||
4232
- outcome.selectorBeforeCutover !== "bootstrap" ||
4233
- outcome.selectorCasPending !== true ||
4234
- acceptanceFacts === undefined ||
4235
- workspaceManifestHash === undefined
4236
- ) {
4237
- throw new Error("bootstrap-handoff-conflict");
4238
- }
4239
- const refreshedHandoffId = `handoff-${hash(
4240
- input.runId,
4241
- input.receiptHash,
4242
- input.graphHash,
4243
- input.acceptanceHash,
4244
- workspaceManifestHash,
4245
- ).slice(0, 40)}`;
4246
- const refreshed = {
4247
- ...outcome,
4248
- handoffId: refreshedHandoffId,
4249
- acceptanceHash: input.acceptanceHash,
4250
- acceptanceFacts,
4251
- workspaceManifestHash,
4252
- };
4253
- this.#transaction(() => {
4254
- const result = this.#database
4255
- .prepare(
4256
- `UPDATE bootstrap_handoffs
4257
- SET handoff_id = ?, facts_json = ?
4258
- WHERE handoff_id = ? AND state = 'prepared' AND facts_json = ?`,
4259
- )
4260
- .run(
4261
- refreshedHandoffId,
4262
- JSON.stringify(refreshed),
4263
- existing.handoff_id,
4264
- existing.facts_json,
4265
- );
4266
- if (Number(result.changes) !== 1) {
4267
- throw new Error("bootstrap-handoff-conflict");
4268
- }
4269
- });
4270
- return structuredClone(refreshed);
4271
- }
4272
- const handoffId = `handoff-${hash(
4273
- input.runId,
4274
- input.receiptHash,
4275
- input.graphHash,
4276
- input.acceptanceHash,
4277
- workspaceManifestHash ?? "acceptance-pending",
4278
- ).slice(0, 40)}`;
4279
- const outcome = {
4280
- handoffId,
4281
- state: "prepared",
4282
- runId: input.runId,
4283
- change: input.change,
4284
- receiptHash: input.receiptHash,
4285
- graphHash: input.graphHash,
4286
- acceptanceHash: input.acceptanceHash,
4287
- ...(acceptanceFacts ? { acceptanceFacts } : {}),
4288
- ...(workspaceManifestHash ? { workspaceManifestHash } : {}),
4289
- selectorBeforeCutover: "bootstrap",
4290
- selectorCasPending: true,
4291
- };
4292
- const preparedRows = this.#database
4293
- .prepare(
4294
- `SELECT handoff_id, owner_run_id, receipt_hash, state, facts_json
4295
- FROM bootstrap_handoffs WHERE state = 'prepared'
4296
- ORDER BY handoff_id`,
4297
- )
4298
- .all() as unknown as BootstrapRow[];
4299
- if (preparedRows.length > 0) throw new Error("bootstrap-handoff-conflict");
4300
- this.#transaction(() => {
4301
- this.#database
4302
- .prepare(
4303
- `INSERT INTO bootstrap_handoffs(
4304
- handoff_id, owner_run_id, receipt_hash, state, facts_json
4305
- ) VALUES (?, ?, ?, 'prepared', ?)`,
4306
- )
4307
- .run(
4308
- handoffId,
4309
- String(input.runId),
4310
- String(input.receiptHash),
4311
- JSON.stringify(outcome),
4312
- );
4313
- });
4314
- return structuredClone(outcome);
4315
- }
4316
-
4317
- inspectBootstrapHandoff(receiptHash: string): Record<string, unknown> {
4318
- this.#assertOpen();
4319
- if (!SHA256.test(receiptHash)) throw new Error("bootstrap-handoff-invalid");
4320
- const row = this.#database
4321
- .prepare(
4322
- `SELECT facts_json FROM bootstrap_handoffs WHERE receipt_hash = ?`,
4323
- )
4324
- .get(receiptHash) as { facts_json: string } | undefined;
4325
- if (!row) throw new Error("bootstrap-handoff-not-found");
4326
- return parseJsonRecord(row.facts_json);
4327
- }
4328
-
4329
- close(): Promise<void> {
4330
- if (this.#closing) return this.#closing;
4331
- if (this.#closed) return Promise.resolve();
4332
- this.#closing = Promise.resolve().then(async () => {
4333
- if (this.#orphanRecoveryTimer) {
4334
- clearTimeout(this.#orphanRecoveryTimer);
4335
- this.#orphanRecoveryTimer = undefined;
4336
- }
4337
- const reason = new Error("workflow-engine-closed");
4338
- const commands = [...this.#commands];
4339
- for (const [controller] of commands) controller.abort(reason);
4340
- const active = [...this.#active.values()];
4341
- for (const operation of active) operation.controller.abort(reason);
4342
- await Promise.allSettled(commands.map(([, entry]) => entry.settled));
4343
- await Promise.allSettled(active.map((operation) => operation.settled));
4344
- await Promise.allSettled([...this.#amendments.values()]);
4345
- for (const held of this.#operationLeases.values())
4346
- clearInterval(held.timer);
4347
- this.#operationLeases.clear();
4348
- this.#lifecycle?.close?.();
4349
- this.#database.close();
4350
- this.#runStore.close();
4351
- this.#closed = true;
4352
- });
4353
- return this.#closing;
4354
- }
4355
- }