@osolmaz/pi-workflows 0.16.4 → 0.16.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/README.md +5 -5
  2. package/dist/builtins/autoimplement.workflow.d.ts +5 -1
  3. package/dist/builtins/autoimplement.workflow.js +29 -10
  4. package/dist/builtins/autoimplement.workflow.js.map +1 -1
  5. package/dist/client/client.js +3 -1
  6. package/dist/client/client.js.map +1 -1
  7. package/dist/client/protocol.d.ts +1 -1
  8. package/dist/client/protocol.js +18 -1
  9. package/dist/client/protocol.js.map +1 -1
  10. package/dist/client/view.d.ts +7 -5
  11. package/dist/extension/index.d.ts +2 -1
  12. package/dist/extension/index.js +115 -165
  13. package/dist/extension/index.js.map +1 -1
  14. package/dist/extension/recorder.d.ts +1 -1
  15. package/dist/extension/recorder.js +2 -2
  16. package/dist/extension/recorder.js.map +1 -1
  17. package/dist/extension/step-message.d.ts +1 -1
  18. package/dist/extension/step-message.js +4 -4
  19. package/dist/extension/step-message.js.map +1 -1
  20. package/dist/extension/workflow-message-coordinator.d.ts +12 -1
  21. package/dist/extension/workflow-message-coordinator.js +65 -54
  22. package/dist/extension/workflow-message-coordinator.js.map +1 -1
  23. package/dist/resource-managers/index.d.ts +1 -2
  24. package/dist/resource-managers/index.js +1 -2
  25. package/dist/resource-managers/index.js.map +1 -1
  26. package/dist/resource-managers/sqlite.d.ts +2 -295
  27. package/dist/resource-managers/sqlite.js +71 -1528
  28. package/dist/resource-managers/sqlite.js.map +1 -1
  29. package/dist/resource-managers/types.d.ts +2 -7
  30. package/dist/server/resource-runner-entry.js +2 -6
  31. package/dist/server/resource-runner-entry.js.map +1 -1
  32. package/dist/server/rpc-bridge.js +1 -1
  33. package/dist/server/rpc-bridge.js.map +1 -1
  34. package/dist/server/rpc-executor.js +2 -4
  35. package/dist/server/rpc-executor.js.map +1 -1
  36. package/dist/server/server-entry.js +3 -0
  37. package/dist/server/server-entry.js.map +1 -1
  38. package/dist/server/server.d.ts +22 -8
  39. package/dist/server/server.js +498 -543
  40. package/dist/server/server.js.map +1 -1
  41. package/dist/server/state.d.ts +16 -29
  42. package/dist/server/state.js +105 -319
  43. package/dist/server/state.js.map +1 -1
  44. package/dist/server/view.d.ts +4 -5
  45. package/dist/server/view.js +51 -51
  46. package/dist/server/view.js.map +1 -1
  47. package/dist/server/workflow-runner-entry.d.ts +1 -1
  48. package/dist/server/workflow-runner-entry.js +1 -49
  49. package/dist/server/workflow-runner-entry.js.map +1 -1
  50. package/dist/server/workflow-runner-protocol.d.ts +2 -7
  51. package/dist/server/workflow-runner-protocol.js +5 -10
  52. package/dist/server/workflow-runner-protocol.js.map +1 -1
  53. package/dist/server/workflow-runner-store.d.ts +5 -6
  54. package/dist/server/workflow-runner-store.js +7 -14
  55. package/dist/server/workflow-runner-store.js.map +1 -1
  56. package/dist/state/attempt-time.d.ts +21 -0
  57. package/dist/state/attempt-time.js +95 -0
  58. package/dist/state/attempt-time.js.map +1 -0
  59. package/dist/state/json.d.ts +2 -1
  60. package/dist/state/json.js.map +1 -1
  61. package/dist/state/project-store.d.ts +39 -0
  62. package/dist/state/project-store.js +154 -0
  63. package/dist/state/project-store.js.map +1 -0
  64. package/dist/state/prune.d.ts +28 -0
  65. package/dist/state/prune.js +278 -83
  66. package/dist/state/prune.js.map +1 -1
  67. package/dist/state/schema.js +23 -23
  68. package/dist/state/workflow-messages.d.ts +1 -1
  69. package/dist/state/workflow-messages.js +3 -1
  70. package/dist/state/workflow-messages.js.map +1 -1
  71. package/dist/viewer/backup.js +5 -0
  72. package/dist/viewer/backup.js.map +1 -1
  73. package/dist/workflows/diagnostics.d.ts +8 -0
  74. package/dist/workflows/diagnostics.js +61 -0
  75. package/dist/workflows/diagnostics.js.map +1 -0
  76. package/dist/workflows/engine.d.ts +2 -15
  77. package/dist/workflows/engine.js +125 -245
  78. package/dist/workflows/engine.js.map +1 -1
  79. package/dist/workflows/human-decision.d.ts +3 -7
  80. package/dist/workflows/human-decision.js +2 -56
  81. package/dist/workflows/human-decision.js.map +1 -1
  82. package/dist/workflows/index.d.ts +2 -1
  83. package/dist/workflows/index.js +1 -0
  84. package/dist/workflows/index.js.map +1 -1
  85. package/dist/workflows/queue.d.ts +267 -0
  86. package/dist/workflows/queue.js +1248 -0
  87. package/dist/workflows/queue.js.map +1 -0
  88. package/dist/workflows/requests.d.ts +47 -0
  89. package/dist/workflows/requests.js +158 -0
  90. package/dist/workflows/requests.js.map +1 -0
  91. package/dist/workflows/schema.js +2 -4
  92. package/dist/workflows/schema.js.map +1 -1
  93. package/dist/workflows/store.d.ts +11 -16
  94. package/dist/workflows/store.js +174 -169
  95. package/dist/workflows/store.js.map +1 -1
  96. package/dist/workflows/tool-input.d.ts +6 -9
  97. package/dist/workflows/tool-input.js +15 -12
  98. package/dist/workflows/tool-input.js.map +1 -1
  99. package/dist/workflows/transitions.d.ts +38 -0
  100. package/dist/workflows/transitions.js +276 -0
  101. package/dist/workflows/transitions.js.map +1 -0
  102. package/dist/workflows/types.d.ts +9 -42
  103. package/dist/workflows/workflow-message-content.d.ts +7 -1
  104. package/dist/workflows/workflow-message-content.js +22 -4
  105. package/dist/workflows/workflow-message-content.js.map +1 -1
  106. package/docs/2026-08-25-workflow-follow-ups.md +5 -5
  107. package/docs/2026-09-02-unify-workflow-messages-plan.md +2 -0
  108. package/docs/2026-09-04-workflow-run-state-plan.md +2 -0
  109. package/docs/2026-09-06-durable-execution-plan.md +176 -0
  110. package/docs/DEFERRED_TURNS.md +84 -105
  111. package/docs/HUMAN_DECISIONS.md +17 -19
  112. package/docs/HUMAN_DECISION_PRESENTATIONS.md +1 -1
  113. package/docs/SQLITE_STATE.md +87 -30
  114. package/docs/WORKFLOW_SERVER.md +50 -11
  115. package/docs/WORKFLOW_STEP_MESSAGES.md +72 -54
  116. package/docs/plans/2026-08-04-controller-runtime-plan.md +2 -0
  117. package/docs/plans/2026-08-21-deferred-turn-intents-plan.md +2 -0
  118. package/docs/plans/2026-08-27-workflow-terminal-restart-plan.md +2 -0
  119. package/docs/plans/2026-09-04-automatic-state-retention-plan.md +283 -0
  120. package/docs/workflows.md +109 -44
  121. package/examples/workflows/autoresearch.workflow.ts +0 -2
  122. package/examples/workflows/branch.workflow.ts +0 -4
  123. package/examples/workflows/echo.workflow.ts +0 -2
  124. package/examples/workflows/live-settings.workflow.ts +0 -2
  125. package/examples/workflows/two-turn.workflow.ts +0 -2
  126. package/herdr-plugin.toml +1 -1
  127. package/package.json +1 -1
  128. package/protocol/fixtures/client-v1.json +1 -1
  129. package/skills/pi-workflows/SKILL.md +8 -7
  130. package/src/builtins/autoimplement.workflow.ts +31 -10
  131. package/src/client/client.ts +3 -1
  132. package/src/client/protocol.ts +21 -1
  133. package/src/client/view.ts +9 -5
  134. package/src/extension/index.ts +117 -189
  135. package/src/extension/recorder.ts +2 -2
  136. package/src/extension/step-message.ts +4 -4
  137. package/src/extension/workflow-message-coordinator.ts +78 -50
  138. package/src/resource-managers/index.ts +1 -9
  139. package/src/resource-managers/sqlite.ts +330 -2655
  140. package/src/resource-managers/types.ts +2 -3
  141. package/src/server/resource-runner-entry.ts +2 -7
  142. package/src/server/rpc-bridge.ts +1 -1
  143. package/src/server/rpc-executor.ts +4 -9
  144. package/src/server/server-entry.ts +3 -0
  145. package/src/server/server.ts +568 -644
  146. package/src/server/state.ts +122 -466
  147. package/src/server/view.ts +56 -60
  148. package/src/server/workflow-runner-entry.ts +2 -55
  149. package/src/server/workflow-runner-protocol.ts +5 -15
  150. package/src/server/workflow-runner-store.ts +12 -27
  151. package/src/state/attempt-time.ts +122 -0
  152. package/src/state/json.ts +2 -1
  153. package/src/state/project-store.ts +218 -0
  154. package/src/state/prune.ts +408 -95
  155. package/src/state/schema.ts +23 -23
  156. package/src/state/workflow-messages.ts +4 -2
  157. package/src/viewer/backup.ts +7 -0
  158. package/src/workflows/diagnostics.ts +78 -0
  159. package/src/workflows/engine.ts +133 -291
  160. package/src/workflows/human-decision.ts +3 -81
  161. package/src/workflows/index.ts +8 -2
  162. package/src/workflows/queue.ts +2091 -0
  163. package/src/workflows/requests.ts +283 -0
  164. package/src/workflows/schema.ts +2 -6
  165. package/src/workflows/store.ts +223 -231
  166. package/src/workflows/tool-input.ts +20 -12
  167. package/src/workflows/transitions.ts +330 -0
  168. package/src/workflows/types.ts +9 -46
  169. package/src/workflows/workflow-message-content.ts +28 -4
  170. package/dist/resource-managers/runtime.d.ts +0 -59
  171. package/dist/resource-managers/runtime.js +0 -423
  172. package/dist/resource-managers/runtime.js.map +0 -1
  173. package/src/resource-managers/runtime.ts +0 -576
@@ -1,11 +1,12 @@
1
1
  import { createHash, randomBytes, randomUUID } from "node:crypto";
2
- import fs from "node:fs";
3
- import path from "node:path";
4
- import { StateDatabase, workflowStatePath } from "../state/database.js";
5
- import { canonicalJson } from "../state/json.js";
6
2
  import { resourceIdFor, tokenHash } from "../state/mutation.js";
7
- import { initializeViewerRun, recordViewerDeltas } from "../state/viewer.js";
8
- import { WorkflowMessageStore } from "../state/workflow-messages.js";
3
+ import {
4
+ ProjectStore,
5
+ isRecord,
6
+ validTimestamp,
7
+ epoch,
8
+ isSequenceRow,
9
+ } from "../state/project-store.js";
9
10
  import {
10
11
  EffectRequestConflictError,
11
12
  ManagedResourceConflictError,
@@ -31,114 +32,6 @@ import type {
31
32
  JsonObject,
32
33
  } from "./types.js";
33
34
 
34
- export type WorkflowRunLaunchStatus =
35
- | "queued"
36
- | "starting"
37
- | "running"
38
- | "parked"
39
- | "done"
40
- | "failed"
41
- | "cancelled";
42
-
43
- export type WorkflowRunReservationOptions = {
44
- runId: string;
45
- workflowName: string;
46
- workflowSourceRef: string;
47
- workflowSource: unknown;
48
- definitionDigest: string;
49
- definitionSnapshot: unknown;
50
- input: unknown;
51
- launchOptions?: unknown;
52
- runnerId: string;
53
- originSessionId: string;
54
- executionMode?: "interactive" | "headless";
55
- parentRunId?: string;
56
- lineageKind?: "continuation" | "restart";
57
- restartNumber?: number;
58
- parentTerminalFingerprint?: string;
59
- now?: string;
60
- };
61
-
62
- export type WorkflowRunClaimOptions = WorkflowRunReservationOptions & {
63
- claimToken: string;
64
- leaseMs: number;
65
- };
66
-
67
- export type WorkflowRunQueueViewRecord = {
68
- runId: string;
69
- workflowName: string;
70
- workflowSourceRef: string;
71
- workflowSource: unknown;
72
- initialized: boolean;
73
- definitionDigest: string;
74
- runStateStatus: string;
75
- paused: boolean;
76
- status: WorkflowRunLaunchStatus;
77
- originSessionId: string | null;
78
- executionMode: "interactive" | "headless";
79
- parentRunId: string | null;
80
- rootRunId: string;
81
- lineageKind: "continuation" | "restart" | null;
82
- restartNumber: number;
83
- parentTerminalFingerprint: string | null;
84
- errorCode: string | null;
85
- errorMessage: string | null;
86
- createdAt: string;
87
- updatedAt: string;
88
- startedAt: string | null;
89
- finishedAt: string | null;
90
- };
91
-
92
- export type WorkflowRunQueueRecord = {
93
- runId: string;
94
- workflowName: string;
95
- workflowSourceRef: string;
96
- workflowSource: unknown;
97
- initialized: boolean;
98
- definitionDigest: string;
99
- input: unknown;
100
- launchOptions: unknown;
101
- status: WorkflowRunLaunchStatus;
102
- runnerId: string | null;
103
- claimToken: string | null;
104
- claimGeneration: number | null;
105
- claimExpiresAt: string | null;
106
- affinityRunnerId: string | null;
107
- originSessionId: string | null;
108
- executionMode: "interactive" | "headless";
109
- parentRunId: string | null;
110
- rootRunId: string;
111
- lineageKind: "continuation" | "restart" | null;
112
- restartNumber: number;
113
- parentTerminalFingerprint: string | null;
114
- errorCode: string | null;
115
- errorMessage: string | null;
116
- createdAt: string;
117
- updatedAt: string;
118
- startedAt: string | null;
119
- finishedAt: string | null;
120
- };
121
-
122
- export type WorkflowRunPreparationResult =
123
- | {
124
- state: "claimed";
125
- run: WorkflowRunQueueRecord & { claimToken: string };
126
- }
127
- | {
128
- state: "adopted";
129
- run: WorkflowRunQueueRecord;
130
- };
131
-
132
- export type RunEventRecord = {
133
- seq: number;
134
- recordedAt: string;
135
- runId: string;
136
- workflowRef: string;
137
- type: string;
138
- runnerId: string | null;
139
- payload: JsonObject;
140
- };
141
-
142
35
  type ResourceManagerRow = {
143
36
  managedResourceId: string;
144
37
  resourceId: string;
@@ -152,85 +45,6 @@ type ResourceManagerRow = {
152
45
  deletionRequestedAt: number | null;
153
46
  };
154
47
 
155
- type LeaseRow = {
156
- generation: number;
157
- ownerType: string | null;
158
- ownerId: string | null;
159
- tokenHash: Buffer | null;
160
- expiresAt: number | null;
161
- };
162
-
163
- type RunRow = {
164
- runId: string;
165
- resourceId: string;
166
- workflowName: string;
167
- workflowRef: string;
168
- runStatus: string;
169
- paused: number;
170
- definitionDigest: Buffer;
171
- definitionHash: Buffer;
172
- inputHash: Buffer;
173
- launchOptionsHash: Buffer;
174
- status: WorkflowRunLaunchStatus;
175
- availableAt: number;
176
- affinityRunnerId: string | null;
177
- consecutiveErrors: number;
178
- errorCode: string | null;
179
- errorHash: Buffer | null;
180
- originSessionId: string | null;
181
- executionMode: "interactive" | "headless";
182
- parentRunId: string | null;
183
- rootRunId: string;
184
- lineageKind: "continuation" | "restart" | null;
185
- restartNumber: number;
186
- parentTerminalFingerprint: Buffer | null;
187
- createdAt: number;
188
- updatedAt: number;
189
- startedAt: number | null;
190
- finishedAt: number | null;
191
- leaseGeneration: number;
192
- ownerId: string | null;
193
- claimExpiresAt: number | null;
194
- };
195
- type WorkflowRunViewRow = {
196
- runId: string;
197
- workflowName: string;
198
- workflowRef: string;
199
- runStatus: string;
200
- paused: number;
201
- definitionDigest: Buffer;
202
- status: WorkflowRunLaunchStatus;
203
- originSessionId: string | null;
204
- executionMode: "interactive" | "headless";
205
- parentRunId: string | null;
206
- rootRunId: string;
207
- lineageKind: "continuation" | "restart" | null;
208
- restartNumber: number;
209
- parentTerminalFingerprint: Buffer | null;
210
- errorCode: string | null;
211
- errorMessage: string | null;
212
- createdAt: number;
213
- updatedAt: number;
214
- startedAt: number | null;
215
- finishedAt: number | null;
216
- sourceType: "builtin" | "file";
217
- sourceRef: string;
218
- sourceRevision: string;
219
- };
220
-
221
- type RunListRevisionRow = {
222
- count: number;
223
- revisionSum: number;
224
- updatedAt: number;
225
- };
226
-
227
- type RunSourceIdentityRow = {
228
- mountPath: string;
229
- sourceType: "builtin" | "file";
230
- sourceRef: string;
231
- sourceRevision: string;
232
- };
233
-
234
48
  type EffectRow = {
235
49
  effectId: string;
236
50
  resourceUid: string;
@@ -244,18 +58,6 @@ type EffectRow = {
244
58
  errorHash: Buffer | null;
245
59
  };
246
60
 
247
- type CancelledRunEffectRow = {
248
- effectId: string;
249
- resourceId: string;
250
- status: "pending" | "applying";
251
- attemptCount: number;
252
- };
253
-
254
- type ExpiredInteractionRow = {
255
- requestId: string;
256
- attemptId: string;
257
- };
258
-
259
61
  type WorkflowRow = {
260
62
  requestId: string;
261
63
  resourceUid: string;
@@ -268,49 +70,109 @@ type WorkflowRow = {
268
70
  errorHash: Buffer | null;
269
71
  };
270
72
 
271
- export class SqliteResourceManagerStore implements ResourceManagerStore {
272
- readonly filePath: string;
273
- readonly state: StateDatabase;
274
- private readonly workflowMessages: WorkflowMessageStore;
275
- private readonly ownsState: boolean;
276
- private readonly projectId: string | null;
277
- private closed = false;
278
-
279
- constructor(
280
- filePath: string = workflowStatePath(),
281
- options: {
282
- readOnly?: boolean;
283
- projectPath?: string;
284
- state?: StateDatabase;
285
- /** Server-only global view. Project-scoped mutations still require projectPath. */
286
- global?: boolean;
287
- } = {},
288
- ) {
289
- this.ownsState = options.state === undefined;
290
- this.state =
291
- options.state ??
292
- new StateDatabase({
293
- filePath,
294
- mode: options.readOnly === true ? "read-only" : "read-write",
295
- checkLegacyState: filePath === workflowStatePath(),
296
- });
297
- this.filePath = this.state.filePath;
298
- this.workflowMessages = new WorkflowMessageStore(this.state);
299
- if (options.global === true) {
300
- this.projectId = null;
301
- } else {
302
- const projectPath = options.projectPath === undefined ? process.cwd() : options.projectPath;
303
- this.projectId =
304
- options.readOnly === true ? this.findProject(projectPath) : this.ensureProject(projectPath);
305
- }
306
- }
73
+ function resourceManagerSelect(clause: string): string {
74
+ return `SELECT c.controller_resource_id AS managedResourceId,
75
+ c.resource_id AS resourceId, r.revision AS resourceVersion,
76
+ c.controller_name AS resourceManagerName, c.resource_key AS resourceKey,
77
+ c.uid, c.generation, c.spec_hash AS specHash, c.status_hash AS statusHash,
78
+ c.deletion_requested_at AS deletionRequestedAt
79
+ FROM controller_resources c JOIN resources r ON r.resource_id = c.resource_id ${clause}`;
80
+ }
307
81
 
308
- close(): void {
309
- if (this.closed) return;
310
- this.closed = true;
311
- if (this.ownsState) this.state.close();
312
- }
82
+ function effectSelect(clause: string): string {
83
+ return `SELECT e.effect_id AS effectId, c.uid AS resourceUid, c.generation,
84
+ e.effect_type AS kind, e.status AS state,
85
+ json_extract(CAST(b.content AS TEXT), '$.requestFingerprint') AS requestFingerprint,
86
+ e.created_at AS startedAt, e.settled_at AS completedAt,
87
+ e.external_ref AS externalRef, e.error_hash AS errorHash
88
+ FROM effects e
89
+ JOIN controller_resources c ON c.resource_id = e.source_resource_id
90
+ JOIN blobs b ON b.blob_hash = e.payload_hash ${clause}`;
91
+ }
92
+
93
+ function workflowSelect(clause: string): string {
94
+ return `SELECT w.request_id AS requestId, c.uid AS resourceUid,
95
+ w.request_key AS requestKey, w.input_fingerprint AS inputFingerprint,
96
+ w.workflow_name AS workflowName, COALESCE(w.run_id, w.reserved_run_id) AS runId, w.status,
97
+ w.attempt_count AS attemptCount, w.error_hash AS errorHash
98
+ FROM controller_workflows w
99
+ JOIN controller_resources c ON c.controller_resource_id = w.controller_resource_id ${clause}`;
100
+ }
101
+
102
+ type QueueListRow = {
103
+ resourceManager: string;
104
+ resourceKey: string;
105
+ availableAt: number;
106
+ consecutiveErrors: number;
107
+ claimExpiresAt: number | null;
108
+ };
109
+
110
+ type ResourceManagerEventRow = {
111
+ seq: number;
112
+ recordedAt: number;
113
+ resourceManager: string;
114
+ resourceKey: string;
115
+ eventType: string;
116
+ payloadHash: Buffer | null;
117
+ };
118
+
119
+ function isResourceManagerRow(value: unknown): value is ResourceManagerRow {
120
+ return isRecord(value);
121
+ }
122
+
123
+ function isEffectRow(value: unknown): value is EffectRow {
124
+ return isRecord(value);
125
+ }
126
+
127
+ function isWorkflowRow(value: unknown): value is WorkflowRow {
128
+ return isRecord(value);
129
+ }
130
+
131
+ function isQueueListRow(value: unknown): value is QueueListRow {
132
+ return isRecord(value);
133
+ }
134
+
135
+ function isResourceManagerEventRow(value: unknown): value is ResourceManagerEventRow {
136
+ return isRecord(value);
137
+ }
138
+
139
+ function isFinalizerRow(value: unknown): value is { finalizer: string } {
140
+ return isRecord(value);
141
+ }
142
+
143
+ function isErrorCountRow(value: unknown): value is { consecutiveErrors: number } {
144
+ return isRecord(value);
145
+ }
146
+
147
+ function isQueueVersionRow(value: unknown): value is { queueVersion: number } {
148
+ return isRecord(value);
149
+ }
150
+
151
+ function isKeyRow(value: unknown): value is { key: string } {
152
+ return isRecord(value);
153
+ }
154
+
155
+ function effectIdFor(sourceResourceId: string, key: string): string {
156
+ return `effect-${createHash("sha256").update(`${sourceResourceId}\0${key}`).digest("hex").slice(0, 40)}`;
157
+ }
158
+
159
+ function effectStatus(value: EffectRecord["state"]): string {
160
+ return value === "indeterminate" ? "ambiguous" : value;
161
+ }
162
+
163
+ function resourceManagerEffectState(value: string): EffectRecord["state"] {
164
+ return value === "ambiguous" ? "indeterminate" : (value as EffectRecord["state"]);
165
+ }
166
+
167
+ function validateName(value: string, label: string): void {
168
+ validateKey(value, label);
169
+ }
170
+
171
+ function validateKey(value: string, label: string): void {
172
+ if (value.length === 0 || value.length > 512) throw new Error(`${label} is invalid`);
173
+ }
313
174
 
175
+ export class SqliteResourceManagerStore extends ProjectStore implements ResourceManagerStore {
314
176
  putResource<TSpec, TStatus>(options: {
315
177
  resourceManager: string;
316
178
  key: string;
@@ -1082,2472 +944,285 @@ export class SqliteResourceManagerStore implements ResourceManagerStore {
1082
944
  }));
1083
945
  }
1084
946
 
1085
- reserveWorkflowRun(options: WorkflowRunReservationOptions): WorkflowRunQueueRecord {
1086
- validateRunId(options.runId);
1087
- const now = epoch(validTimestamp(options.now));
1088
- const definitionDigest = digestBuffer(options.definitionDigest);
1089
- return this.state.transaction(() =>
1090
- this.reserveWorkflowRunInTransaction(options, now, definitionDigest),
1091
- );
1092
- }
1093
-
1094
- private reserveWorkflowRunInTransaction(
1095
- options: WorkflowRunReservationOptions,
1096
- now: number,
1097
- definitionDigest: Buffer,
1098
- ): WorkflowRunQueueRecord {
1099
- if (this.getWorkflowRun(options.runId) !== undefined) {
1100
- throw new Error(`Workflow run already reserved: ${options.runId}`);
1101
- }
1102
- const lineageKind =
1103
- options.parentRunId === undefined ? null : (options.lineageKind ?? "continuation");
1104
- if (options.parentRunId === undefined && options.lineageKind !== undefined) {
1105
- throw new Error("A root workflow run cannot declare lineage");
1106
- }
1107
- let rootRunId = options.runId;
1108
- let restartNumber = 0;
1109
- let parentTerminalFingerprint: Buffer | null = null;
1110
- if (options.parentRunId !== undefined) {
1111
- const parent = this.requireWorkflowRunRow(options.parentRunId);
1112
- if (parent.originSessionId !== options.originSessionId) {
1113
- throw new Error("Workflow parent belongs to another Pi session");
1114
- }
1115
- rootRunId = parent.rootRunId;
1116
- if (lineageKind === "restart") {
1117
- if (
1118
- !["completed", "failed", "timed_out", "cancelled"].includes(parent.runStatus) ||
1119
- !["done", "failed", "cancelled"].includes(parent.status)
1120
- ) {
1121
- throw new Error("Restart parent is not terminal");
1122
- }
1123
- if (options.parentTerminalFingerprint === undefined) {
1124
- throw new Error("Restart requires the parent terminal fingerprint");
1125
- }
1126
- parentTerminalFingerprint = terminalFingerprintBuffer(options.parentTerminalFingerprint);
1127
- restartNumber = options.restartNumber ?? parent.restartNumber + 1;
1128
- if (restartNumber !== parent.restartNumber + 1) {
1129
- throw new Error("Restart number does not follow its parent");
1130
- }
1131
- } else {
1132
- if (
1133
- options.parentTerminalFingerprint !== undefined ||
1134
- (options.restartNumber !== undefined && options.restartNumber !== parent.restartNumber)
1135
- ) {
1136
- throw new Error("Continuation lineage conflicts with restart metadata");
1137
- }
1138
- restartNumber = parent.restartNumber;
1139
- if (parent.status === "parked") {
1140
- const parentLease = this.requireLease(parent.resourceId);
1141
- if (parentLease.ownerId !== null) {
1142
- throw new Error("Continuation parent still has an active owner");
1143
- }
1144
- this.state.connection
1145
- .prepare(
1146
- `UPDATE run_queue
1147
- SET status = 'done', updated_at = ?, finished_at = ?
1148
- WHERE run_id = ? AND status = 'parked'`,
1149
- )
1150
- .run(now, now, parent.runId);
1151
- const parentRevision = this.resourceRevision(parent.resourceId);
1152
- this.bumpResource(parent.resourceId, parentRevision, now);
1153
- this.insertEvent(
1154
- parent.resourceId,
1155
- parentRevision + 1,
1156
- "run.queue_done_for_continuation",
1157
- "session",
1158
- options.originSessionId,
1159
- { continuationRunId: options.runId },
1160
- now,
1161
- );
1162
- } else if (parent.status === "done") {
1163
- throw new Error("Continuation parent already has a reserved continuation");
1164
- } else {
1165
- throw new Error(`Continuation parent queue is ${parent.status}`);
1166
- }
1167
- }
1168
- } else if (
1169
- options.restartNumber !== undefined ||
1170
- options.parentTerminalFingerprint !== undefined
1171
- ) {
1172
- throw new Error("A root workflow run cannot declare restart metadata");
1173
- }
1174
- const resourceId = resourceIdFor("run", options.runId);
1175
- const definitionHash = this.state.putJson(options.definitionSnapshot, now);
1176
- const queuedSource = queuedWorkflowSource(options.workflowSource);
1177
- const inputHash = this.state.putJson(options.input ?? null, now);
1178
- const launchHash = this.state.putJson(options.launchOptions ?? {}, now);
1179
- this.state.connection
1180
- .prepare(
1181
- `INSERT INTO workflow_definitions(
1182
- definition_digest, workflow_name, definition_hash, created_at
1183
- ) VALUES (?, ?, ?, ?)
1184
- ON CONFLICT(definition_digest) DO NOTHING`,
1185
- )
1186
- .run(definitionDigest, options.workflowName, definitionHash, now);
1187
- this.state.connection
1188
- .prepare(
1189
- `INSERT INTO resources(resource_id, resource_type, aggregate_key, revision, created_at, updated_at)
1190
- VALUES (?, 'run', ?, 1, ?, ?)`,
1191
- )
1192
- .run(resourceId, options.runId, now, now);
1193
- this.state.connection
1194
- .prepare("INSERT INTO leases(resource_id, generation) VALUES (?, 0)")
1195
- .run(resourceId);
1196
- this.state.connection
1197
- .prepare(
1198
- `INSERT INTO runs(
1199
- run_id, resource_id, project_id, parent_run_id, root_run_id, lineage_kind,
1200
- restart_number, parent_terminal_fingerprint, definition_digest,
1201
- workflow_ref, launch_options_hash, status, paused,
1202
- input_hash, created_at, updated_at
1203
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`,
1204
- )
1205
- .run(
1206
- options.runId,
1207
- resourceId,
1208
- this.requireProjectId(),
1209
- options.parentRunId ?? null,
1210
- rootRunId,
1211
- lineageKind,
1212
- restartNumber,
1213
- parentTerminalFingerprint,
1214
- definitionDigest,
1215
- options.workflowSourceRef,
1216
- launchHash,
1217
- inputHash,
1218
- now,
1219
- now,
1220
- );
1221
- initializeViewerRun(this.state, options.runId, now);
1222
- insertQueuedRunSources(this.state, options.runId, queuedSource);
1223
- this.state.connection
1224
- .prepare(
1225
- `INSERT INTO run_bindings(run_id, origin_session_id, execution_mode, created_at)
1226
- VALUES (?, ?, ?, ?)`,
1227
- )
1228
- .run(options.runId, options.originSessionId, options.executionMode ?? "interactive", now);
1229
- this.state.connection
947
+ private resourceManagerRow(ref: ManagedResourceRef): ResourceManagerRow | undefined {
948
+ if (this.projectId === null) return undefined;
949
+ const row = this.state.connection
1230
950
  .prepare(
1231
- `INSERT INTO run_queue(
1232
- run_id, status, available_at, affinity_runner_id, origin_session_id,
1233
- consecutive_errors, created_at, updated_at
1234
- ) VALUES (?, 'queued', ?, ?, ?, 0, ?, ?)`,
951
+ resourceManagerSelect(
952
+ "WHERE c.project_id = ? AND c.controller_name = ? AND c.resource_key = ?",
953
+ ),
1235
954
  )
1236
- .run(options.runId, now, options.runnerId, options.originSessionId, now, now);
1237
- this.insertEvent(resourceId, 1, "run.queued", "session", options.originSessionId, {}, now);
1238
- return this.requireWorkflowRun(options.runId);
1239
- }
1240
-
1241
- prepareOrAdoptWorkflowRun(options: WorkflowRunClaimOptions): WorkflowRunPreparationResult {
1242
- validateRunId(options.runId);
1243
- const now = epoch(validTimestamp(options.now));
1244
- const definitionDigest = digestBuffer(options.definitionDigest);
1245
- return this.state.transaction(() => {
1246
- const existing = this.workflowRunRow(options.runId);
1247
- if (existing !== undefined) {
1248
- this.assertWorkflowRunPreparationCompatible(existing, options, definitionDigest);
1249
- return { state: "adopted", run: this.mapWorkflowRun(existing) };
1250
- }
1251
- this.reserveWorkflowRunInTransaction(options, now, definitionDigest);
1252
- const claimed = this.claimRunInTransaction(
1253
- options.runId,
1254
- options.runnerId,
1255
- options.claimToken,
1256
- options.leaseMs,
1257
- now,
1258
- );
1259
- if (claimed === undefined) {
1260
- throw new Error(`Workflow run could not be claimed: ${options.runId}`);
1261
- }
1262
- return {
1263
- state: "claimed",
1264
- run: claimed as WorkflowRunQueueRecord & { claimToken: string },
1265
- };
1266
- });
955
+ .get(this.projectId, ref.resourceManager, ref.key);
956
+ return isResourceManagerRow(row) ? row : undefined;
1267
957
  }
1268
958
 
1269
- enqueueWorkflowRun(options: WorkflowRunClaimOptions): WorkflowRunQueueRecord {
1270
- if (this.getWorkflowRun(options.runId) === undefined) {
1271
- this.reserveWorkflowRun({
1272
- runId: options.runId,
1273
- workflowName: options.workflowName,
1274
- workflowSourceRef: options.workflowSourceRef,
1275
- workflowSource: options.workflowSource,
1276
- definitionDigest: options.definitionDigest,
1277
- definitionSnapshot: options.definitionSnapshot,
1278
- input: options.input,
1279
- launchOptions: options.launchOptions ?? {},
1280
- runnerId: options.runnerId,
1281
- originSessionId: options.originSessionId,
1282
- ...(options.parentRunId === undefined ? {} : { parentRunId: options.parentRunId }),
1283
- ...(options.lineageKind === undefined ? {} : { lineageKind: options.lineageKind }),
1284
- ...(options.restartNumber === undefined ? {} : { restartNumber: options.restartNumber }),
1285
- ...(options.parentTerminalFingerprint === undefined
1286
- ? {}
1287
- : { parentTerminalFingerprint: options.parentTerminalFingerprint }),
1288
- ...(options.now === undefined ? {} : { now: options.now }),
1289
- });
1290
- }
1291
- const claimed = this.claimWorkflowRun({
1292
- runId: options.runId,
1293
- runnerId: options.runnerId,
1294
- claimToken: options.claimToken,
1295
- leaseMs: options.leaseMs,
1296
- ...(options.now === undefined ? {} : { now: options.now }),
1297
- });
1298
- if (claimed === undefined) {
1299
- throw new Error(`Workflow run could not be claimed: ${options.runId}`);
1300
- }
1301
- return claimed;
959
+ private requireResourceManagerRow(ref: ManagedResourceRef): ResourceManagerRow {
960
+ const row = this.resourceManagerRow(ref);
961
+ if (row === undefined) throw new ManagedResourceNotFoundError(ref.resourceManager, ref.key);
962
+ return row;
1302
963
  }
1303
964
 
1304
- getWorkflowRun(runId: string): WorkflowRunQueueRecord | undefined {
1305
- const row = this.workflowRunRow(runId);
1306
- return row === undefined ? undefined : this.mapWorkflowRun(row);
965
+ private requireResourceManagerRowByUid(uid: string): ResourceManagerRow {
966
+ const row = this.state.connection.prepare(resourceManagerSelect("WHERE c.uid = ?")).get(uid);
967
+ /* istanbul ignore if -- impossible after exact schema and transaction checks */
968
+ if (!isResourceManagerRow(row)) throw new Error(`Managed resource not found: ${uid}`);
969
+ return row;
1307
970
  }
1308
971
 
1309
- getWorkflowRunView(runId: string): WorkflowRunQueueViewRecord | undefined {
1310
- const row = this.workflowRunViewRows("WHERE r.run_id = ?", [runId])[0];
1311
- return row === undefined ? undefined : workflowRunViewRecord(row);
972
+ private requireResource(ref: ManagedResourceRef): ManagedResource {
973
+ const resource = this.getResource(ref);
974
+ if (resource === undefined)
975
+ throw new ManagedResourceNotFoundError(ref.resourceManager, ref.key);
976
+ return resource;
1312
977
  }
1313
978
 
1314
- workflowRunListRevision(): { total: number; revision: string } {
1315
- const row = this.state.connection
1316
- .prepare(
1317
- `SELECT count(*) AS count, COALESCE(sum(res.revision), 0) AS revisionSum,
1318
- COALESCE(max(q.updated_at), 0) AS updatedAt
1319
- FROM runs r JOIN run_queue q ON q.run_id = r.run_id
1320
- JOIN resources res ON res.resource_id = r.resource_id`,
1321
- )
1322
- .get();
1323
- if (!isRunListRevisionRow(row)) throw new Error("Workflow run list revision is invalid");
979
+ /* istanbul ignore next -- pure projection covered by integration tests */
980
+ private mapManagedResource(row: ResourceManagerRow): ManagedResource {
981
+ const spec = this.state.readJson(row.specHash);
982
+ const status = this.state.readJson(row.statusHash) as ManagedResourceStatus<unknown>;
1324
983
  return {
1325
- total: row.count,
1326
- revision: `${row.count}:${row.revisionSum}:${row.updatedAt}`,
984
+ metadata: {
985
+ uid: row.uid,
986
+ resourceManager: row.resourceManagerName,
987
+ key: row.resourceKey,
988
+ resourceVersion: row.resourceVersion,
989
+ generation: row.generation,
990
+ ...(row.deletionRequestedAt === null
991
+ ? {}
992
+ : { deletionTimestamp: new Date(row.deletionRequestedAt).toISOString() }),
993
+ finalizers: this.finalizers(row.managedResourceId),
994
+ },
995
+ spec,
996
+ status,
1327
997
  };
1328
998
  }
1329
999
 
1330
- listWorkflowRunViews(options: { offset: number; limit: number }): {
1331
- total: number;
1332
- revision: string;
1333
- runs: WorkflowRunQueueViewRecord[];
1334
- } {
1335
- const current = this.workflowRunListRevision();
1336
- const rows = this.workflowRunViewRows("ORDER BY q.created_at DESC LIMIT ? OFFSET ?", [
1337
- options.limit,
1338
- options.offset,
1339
- ]);
1340
- return { ...current, runs: rows.map(workflowRunViewRecord) };
1341
- }
1342
-
1343
- workflowRunProjectPath(runId: string): string | undefined {
1344
- const row = this.state.connection
1000
+ private finalizers(managedResourceId: string): string[] {
1001
+ const rows = this.state.connection
1345
1002
  .prepare(
1346
- `SELECT p.canonical_path AS canonicalPath
1347
- FROM runs r JOIN projects p ON p.project_id = r.project_id WHERE r.run_id = ?`,
1003
+ "SELECT finalizer FROM controller_finalizers WHERE controller_resource_id = ? ORDER BY position",
1348
1004
  )
1349
- .get(runId);
1350
- return isCanonicalPathRow(row) ? row.canonicalPath : undefined;
1351
- }
1005
+ .all(managedResourceId);
1006
+ return rows.flatMap((row) => (isFinalizerRow(row) ? [row.finalizer] : []));
1007
+ }
1352
1008
 
1353
- listWorkflowRuns(
1354
- options: {
1355
- statuses?: WorkflowRunLaunchStatus[];
1356
- excludeRunIds?: string[];
1357
- limit?: number;
1358
- } = {},
1359
- ): WorkflowRunQueueRecord[] {
1360
- const clauses: string[] = [];
1361
- const params: unknown[] = [];
1362
- if (this.projectId !== null) {
1363
- clauses.push("r.project_id = ?");
1364
- params.push(this.projectId);
1365
- }
1366
- if (options.statuses !== undefined && options.statuses.length > 0) {
1367
- clauses.push(`q.status IN (${options.statuses.map(() => "?").join(", ")})`);
1368
- params.push(...options.statuses);
1369
- }
1370
- if (options.excludeRunIds !== undefined && options.excludeRunIds.length > 0) {
1371
- clauses.push(`r.run_id NOT IN (${options.excludeRunIds.map(() => "?").join(", ")})`);
1372
- params.push(...options.excludeRunIds);
1373
- }
1374
- if (options.limit !== undefined) params.push(options.limit);
1375
- const rows = this.state.connection
1009
+ private replaceFinalizers(managedResourceId: string, finalizers: string[]): void {
1010
+ const unique = [...new Set(finalizers)];
1011
+ if (unique.length !== finalizers.length)
1012
+ throw new Error("ResourceManager finalizers must be unique");
1013
+ this.state.connection
1014
+ .prepare("DELETE FROM controller_finalizers WHERE controller_resource_id = ?")
1015
+ .run(managedResourceId);
1016
+ const insert = this.state.connection.prepare(
1017
+ "INSERT INTO controller_finalizers(controller_resource_id, finalizer, position) VALUES (?, ?, ?)",
1018
+ );
1019
+ unique.forEach((finalizer, index) => insert.run(managedResourceId, finalizer, index));
1020
+ }
1021
+
1022
+ private enqueueResourceManagerRow(
1023
+ managedResourceId: string,
1024
+ availableAt: number,
1025
+ now: number,
1026
+ ): void {
1027
+ this.state.connection
1376
1028
  .prepare(
1377
- workflowRunSelect(
1378
- `${clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`} ORDER BY q.created_at DESC${options.limit === undefined ? "" : " LIMIT ?"}`,
1379
- ),
1029
+ `INSERT INTO controller_queue(
1030
+ controller_resource_id, available_at, queue_version,
1031
+ consecutive_errors, created_at, updated_at
1032
+ ) VALUES (?, ?, 1, 0, ?, ?)
1033
+ ON CONFLICT(controller_resource_id) DO UPDATE SET
1034
+ available_at = MIN(controller_queue.available_at, excluded.available_at),
1035
+ queue_version = controller_queue.queue_version + 1,
1036
+ updated_at = excluded.updated_at`,
1380
1037
  )
1381
- .all(...params);
1382
- return rows.filter(isRunRow).map((row) => this.mapWorkflowRun(row));
1038
+ .run(managedResourceId, availableAt, now, now);
1383
1039
  }
1384
1040
 
1385
- findSessionReservation(sessionId: string): WorkflowRunQueueRecord | undefined {
1041
+ private queueErrors(managedResourceId: string): number {
1386
1042
  const row = this.state.connection
1387
1043
  .prepare(
1388
- workflowRunSelect(
1389
- "WHERE b.origin_session_id = ? AND q.status NOT IN ('done', 'failed', 'cancelled') ORDER BY q.created_at DESC LIMIT 1",
1390
- ),
1044
+ "SELECT consecutive_errors AS consecutiveErrors FROM controller_queue WHERE controller_resource_id = ?",
1391
1045
  )
1392
- .get(sessionId);
1393
- return isRunRow(row) ? this.mapWorkflowRun(row) : undefined;
1394
- }
1395
-
1396
- findSessionReservationView(sessionId: string): WorkflowRunQueueViewRecord | undefined {
1397
- const row = this.workflowRunViewRows(
1398
- "WHERE b.origin_session_id = ? AND q.status NOT IN ('done', 'failed', 'cancelled') ORDER BY q.created_at DESC LIMIT 1",
1399
- [sessionId],
1400
- )[0];
1401
- return row === undefined ? undefined : workflowRunViewRecord(row);
1402
- }
1403
-
1404
- claimWorkflowRun(options: {
1405
- runId: string;
1406
- runnerId: string;
1407
- claimToken: string;
1408
- leaseMs: number;
1409
- now?: string;
1410
- }): WorkflowRunQueueRecord | undefined {
1411
- const now = epoch(validTimestamp(options.now));
1412
- return this.claimRun(options.runId, options.runnerId, options.claimToken, options.leaseMs, now);
1413
- }
1414
-
1415
- /** Schedule a runner that validates one pending interactive submission. */
1416
- claimWorkflowRunForInteractionValidation(options: {
1417
- runId: string;
1418
- runnerId: string;
1419
- claimToken: string;
1420
- leaseMs: number;
1421
- now?: string;
1422
- }): WorkflowRunQueueRecord | undefined {
1423
- const now = epoch(validTimestamp(options.now));
1424
- return this.claimRun(
1425
- options.runId,
1426
- options.runnerId,
1427
- options.claimToken,
1428
- options.leaseMs,
1429
- now,
1430
- {
1431
- allowPendingInteraction: true,
1432
- },
1433
- );
1046
+ .get(managedResourceId);
1047
+ return isErrorCountRow(row) ? row.consecutiveErrors : 0;
1434
1048
  }
1435
1049
 
1436
- /** Take a short server claim without scheduling a parked interactive run. */
1437
- claimWorkflowRunForControl(options: {
1438
- runId: string;
1439
- runnerId: string;
1440
- claimToken: string;
1441
- leaseMs: number;
1442
- now?: string;
1443
- }): WorkflowRunQueueRecord | undefined {
1444
- const now = epoch(validTimestamp(options.now));
1445
- return this.claimRun(
1446
- options.runId,
1447
- options.runnerId,
1448
- options.claimToken,
1449
- options.leaseMs,
1450
- now,
1451
- { allowPendingInteraction: true, preserveQueueStatus: true },
1452
- );
1050
+ private queueVersion(managedResourceId: string): number {
1051
+ const row = this.state.connection
1052
+ .prepare(
1053
+ "SELECT queue_version AS queueVersion FROM controller_queue WHERE controller_resource_id = ?",
1054
+ )
1055
+ .get(managedResourceId);
1056
+ /* istanbul ignore if -- exact schema and internal query shape */
1057
+ /* istanbul ignore if -- impossible after exact schema and transaction checks */
1058
+ if (!isQueueVersionRow(row)) throw new Error("ResourceManager queue item is missing");
1059
+ return row.queueVersion;
1453
1060
  }
1454
1061
 
1455
- beginWorkflowRunInteractionTimeout(options: {
1456
- runId: string;
1457
- targetSessionId: string;
1458
- claimToken: string;
1459
- now?: string;
1460
- }): boolean {
1461
- const now = epoch(validTimestamp(options.now));
1062
+ private settleResourceManagerClaim(
1063
+ claim: ResourceManagerQueueClaim,
1064
+ requeue: QueueRequeueOptions | undefined,
1065
+ nowValue?: string,
1066
+ ): boolean {
1067
+ const now = epoch(validTimestamp(nowValue));
1462
1068
  return this.state.transaction(() => {
1463
- if (
1464
- !this.verifyWorkflowRunClaim({
1465
- runId: options.runId,
1466
- claimToken: options.claimToken,
1467
- now: new Date(now).toISOString(),
1468
- })
1469
- ) {
1069
+ const row = this.resourceManagerRow({
1070
+ resourceManager: claim.resourceManager,
1071
+ key: claim.key,
1072
+ });
1073
+ if (row === undefined) return false;
1074
+ try {
1075
+ this.assertResourceManagerClaim(row, claim, now, false);
1076
+ } catch {
1470
1077
  return false;
1471
1078
  }
1472
- const row = this.requireWorkflowRunRow(options.runId);
1473
- const interaction = this.state.connection
1474
- .prepare(
1475
- `SELECT i.request_id AS requestId, i.attempt_id AS attemptId
1476
- FROM interactive_requests i
1477
- JOIN node_attempts a ON a.attempt_id = i.attempt_id
1478
- JOIN runs r ON r.run_id = i.run_id
1479
- WHERE i.run_id = ? AND i.target_session_id = ?
1480
- AND i.status = 'pending' AND r.paused = 0
1481
- AND a.deadline_at IS NOT NULL AND a.deadline_at <= ?
1482
- AND EXISTS (
1483
- SELECT 1 FROM workflow_messages m
1484
- JOIN workflow_turns t ON t.workflow_message_id = m.workflow_message_id
1485
- WHERE m.source_id = i.request_id AND m.kind = 'step' AND t.state = 'started'
1486
- )
1487
- ORDER BY a.deadline_at, i.request_id LIMIT 1`,
1488
- )
1489
- .get(options.runId, options.targetSessionId, now);
1490
- if (!isExpiredInteractionRow(interaction)) return false;
1491
- const request = this.state.connection
1492
- .prepare(
1493
- `UPDATE interactive_requests
1494
- SET status = 'cancelled', revision = revision + 1, updated_at = ?
1495
- WHERE request_id = ? AND run_id = ? AND status = 'pending'`,
1496
- )
1497
- .run(now, interaction.requestId, options.runId);
1498
- const run = this.state.connection
1499
- .prepare(
1500
- `UPDATE runs
1501
- SET status = 'running', status_detail = 'finishing expired interaction deadline',
1502
- updated_at = ?, finished_at = NULL
1503
- WHERE run_id = ? AND status = 'waiting'`,
1504
- )
1505
- .run(now, options.runId);
1506
- const queue = this.state.connection
1507
- .prepare(
1508
- `UPDATE run_queue SET status = 'starting', error_code = NULL, error_hash = NULL,
1509
- updated_at = ?, finished_at = NULL
1510
- WHERE run_id = ? AND status = 'parked'`,
1511
- )
1512
- .run(now, options.runId);
1513
- /* istanbul ignore if -- the expired parked interaction was selected above */
1514
- if (request.changes !== 1 || run.changes !== 1 || queue.changes !== 1) {
1515
- throw new Error(`Workflow run ${options.runId} changed during interaction timeout`);
1079
+ const currentQueueVersion = this.queueVersion(row.managedResourceId);
1080
+ if (requeue === undefined) {
1081
+ if (currentQueueVersion === claim.queueVersion) {
1082
+ this.state.connection
1083
+ .prepare("DELETE FROM controller_queue WHERE controller_resource_id = ?")
1084
+ .run(row.managedResourceId);
1085
+ }
1086
+ } else {
1087
+ const errorHash =
1088
+ requeue.error === undefined ? null : this.state.putText(requeue.error, now);
1089
+ this.state.connection
1090
+ .prepare(
1091
+ `UPDATE controller_queue
1092
+ SET available_at = ?, queue_version = queue_version + 1,
1093
+ consecutive_errors = consecutive_errors + 1,
1094
+ last_error_hash = ?, updated_at = ? WHERE controller_resource_id = ?`,
1095
+ )
1096
+ .run(epoch(requeue.availableAt), errorHash, now, row.managedResourceId);
1516
1097
  }
1517
- const error = "Workflow node model-turn deadline expired";
1518
- const receiptHash = this.state.putJson({ status: "rejected", error }, now);
1519
- this.state.connection
1520
- .prepare(
1521
- `UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
1522
- WHERE request_id = ? AND outcome = 'validating'`,
1523
- )
1524
- .run(receiptHash, interaction.requestId);
1525
- const revision = this.resourceRevision(row.resourceId);
1526
- const lease = this.requireLease(row.resourceId);
1527
- this.bumpResource(row.resourceId, revision, now);
1528
- this.insertEvent(
1529
- row.resourceId,
1530
- revision + 1,
1531
- "run.interaction_timeout_started",
1532
- lease.ownerType ?? "system",
1533
- lease.ownerId,
1534
- { requestId: interaction.requestId, attemptId: interaction.attemptId },
1535
- now,
1536
- lease.generation || undefined,
1537
- );
1538
- recordViewerDeltas(
1539
- this.state,
1540
- options.runId,
1541
- [{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
1542
- now,
1543
- );
1098
+ this.releaseLease(row.resourceId, claim, now);
1544
1099
  return true;
1545
1100
  });
1546
1101
  }
1547
1102
 
1548
- markWorkflowRunRunning(options: { runId: string; claimToken: string; now?: string }): boolean {
1549
- return this.updateClaimedRunStatus(options.runId, options.claimToken, "running", options.now);
1103
+ private assertResourceManagerClaim(
1104
+ row: ResourceManagerRow,
1105
+ claim: ResourceManagerQueueClaim,
1106
+ now: number,
1107
+ requireUnexpired = true,
1108
+ ): void {
1109
+ if (row.resourceManagerName !== claim.resourceManager || row.resourceKey !== claim.key)
1110
+ throw new Error("ResourceManager claim targets another resource");
1111
+ const lease = this.requireLease(row.resourceId);
1112
+ if (
1113
+ lease.ownerType !== "controller" ||
1114
+ lease.ownerId !== claim.ownerId ||
1115
+ lease.generation !== claim.generation ||
1116
+ lease.tokenHash === null ||
1117
+ !lease.tokenHash.equals(tokenHash(claim.token)) ||
1118
+ (requireUnexpired && (lease.expiresAt === null || lease.expiresAt <= now))
1119
+ ) {
1120
+ throw new Error("ResourceManager claim is stale");
1121
+ }
1550
1122
  }
1551
1123
 
1552
- claimNextWorkflowRun(options: {
1553
- runnerId: string;
1554
- sessionId?: string;
1555
- claimToken: string;
1556
- leaseMs: number;
1557
- now?: string;
1558
- excludeRunIds?: string[];
1559
- }): WorkflowRunQueueRecord | undefined {
1560
- const now = epoch(validTimestamp(options.now));
1561
- const clauses = [
1562
- "q.status IN ('queued', 'parked', 'starting', 'running')",
1563
- "q.available_at <= ?",
1564
- "(l.owner_id IS NULL OR l.expires_at <= ? OR l.owner_id = ?)",
1565
- "(q.affinity_runner_id IS NULL OR q.affinity_runner_id = ? OR q.status IN ('parked', 'starting', 'running'))",
1566
- "NOT (q.status = 'parked' AND (r.status = 'waiting' OR r.paused = 1))",
1567
- "NOT EXISTS (SELECT 1 FROM interactive_requests i WHERE i.run_id = r.run_id AND i.status = 'pending')",
1568
- "(q.error_code IS NULL OR q.error_code NOT IN ('workflowSourceChanged', 'runnerNoProgress'))",
1569
- ];
1570
- const params: unknown[] = [now, now, options.runnerId, options.runnerId];
1571
- if (this.projectId !== null) {
1572
- clauses.unshift("r.project_id = ?");
1573
- params.unshift(this.projectId);
1574
- }
1575
- if (options.sessionId !== undefined) {
1576
- clauses.push("b.origin_session_id = ?");
1577
- params.push(options.sessionId);
1578
- } else {
1579
- clauses.push("(b.execution_mode = 'headless' OR q.status <> 'queued')");
1580
- }
1581
- if (options.excludeRunIds !== undefined && options.excludeRunIds.length > 0) {
1582
- clauses.push(`r.run_id NOT IN (${options.excludeRunIds.map(() => "?").join(", ")})`);
1583
- params.push(...options.excludeRunIds);
1584
- }
1585
- const row = this.state.connection
1124
+ private releaseLease(resourceId: string, claim: ResourceManagerQueueClaim, now: number): void {
1125
+ const row = this.resourceManagerRow({ resourceManager: claim.resourceManager, key: claim.key });
1126
+ if (row === undefined) return;
1127
+ const revision = this.resourceRevision(resourceId);
1128
+ const result = this.state.connection
1586
1129
  .prepare(
1587
- workflowRunSelect(
1588
- `WHERE ${clauses.join(" AND ")} ORDER BY q.available_at, q.created_at LIMIT 1`,
1589
- ),
1130
+ `UPDATE leases SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
1131
+ acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
1132
+ WHERE resource_id = ? AND owner_id = ? AND token_hash = ? AND generation = ?`,
1590
1133
  )
1591
- .get(...params);
1592
- if (!isRunRow(row)) return undefined;
1593
- return this.claimRun(row.runId, options.runnerId, options.claimToken, options.leaseMs, now);
1134
+ .run(resourceId, claim.ownerId, tokenHash(claim.token), claim.generation);
1135
+ if (result.changes === 1) {
1136
+ this.bumpResource(resourceId, revision, now);
1137
+ this.insertEvent(
1138
+ resourceId,
1139
+ revision + 1,
1140
+ "lease.released",
1141
+ "controller",
1142
+ claim.ownerId,
1143
+ {},
1144
+ now,
1145
+ claim.generation,
1146
+ );
1147
+ claim.resourceVersion = revision + 1;
1148
+ }
1594
1149
  }
1595
1150
 
1596
- renewWorkflowRunClaim(options: {
1597
- runId: string;
1598
- claimToken: string;
1599
- leaseMs: number;
1600
- now?: string;
1601
- }): boolean {
1602
- const now = epoch(validTimestamp(options.now));
1603
- return this.state.transaction(() => {
1604
- const row = this.workflowRunRow(options.runId);
1605
- if (row === undefined || row.ownerId === null) return false;
1606
- const lease = this.requireLease(row.resourceId);
1607
- const result = this.state.connection
1608
- .prepare(
1609
- `UPDATE leases SET heartbeat_at = ?, expires_at = ?
1610
- WHERE resource_id = ? AND owner_type = ? AND owner_id = ?
1611
- AND token_hash = ? AND generation = ? AND expires_at > ?`,
1612
- )
1613
- .run(
1614
- now,
1615
- now + options.leaseMs,
1616
- row.resourceId,
1617
- lease.ownerType,
1618
- row.ownerId,
1619
- tokenHash(options.claimToken),
1620
- lease.generation,
1621
- now,
1622
- );
1623
- if (result.changes === 1) {
1624
- recordViewerDeltas(
1625
- this.state,
1626
- options.runId,
1627
- [{ targetType: "summary" }, { targetType: "replay" }],
1628
- now,
1629
- );
1630
- return true;
1631
- }
1632
- return false;
1633
- });
1151
+ private effectRow(resourceUid: string, key: string): EffectRow | undefined {
1152
+ const row = this.state.connection
1153
+ .prepare(effectSelect("WHERE c.uid = ? AND e.idempotency_key = ?"))
1154
+ .get(resourceUid, key);
1155
+ return isEffectRow(row) ? row : undefined;
1634
1156
  }
1635
1157
 
1636
- verifyWorkflowRunClaim(options: { runId: string; claimToken: string; now?: string }): boolean {
1637
- const now = epoch(validTimestamp(options.now));
1638
- const row = this.workflowRunRow(options.runId);
1639
- if (row === undefined || row.ownerId === null || row.claimExpiresAt === null) return false;
1640
- const lease = this.requireLease(row.resourceId);
1641
- return (
1642
- lease.tokenHash !== null &&
1643
- lease.tokenHash.equals(tokenHash(options.claimToken)) &&
1644
- row.claimExpiresAt > now
1645
- );
1158
+ private requireEffectRow(resourceUid: string, key: string): EffectRow {
1159
+ const row = this.effectRow(resourceUid, key);
1160
+ if (row === undefined) throw new Error(`Effect record not found: ${resourceUid}/${key}`);
1161
+ return row;
1646
1162
  }
1647
1163
 
1648
- workflowRunAuthority(
1649
- runId: string,
1650
- claimToken: string,
1651
- ):
1652
- | {
1653
- actor: { type: "session" | "host"; id: string };
1654
- ownerType: "session" | "host";
1655
- ownerId: string;
1656
- token: string;
1657
- generation: number;
1658
- leaseMs: number;
1659
- }
1660
- | undefined {
1661
- const row = this.workflowRunRow(runId);
1662
- if (row === undefined || row.ownerId === null || row.claimExpiresAt === null) return undefined;
1663
- const lease = this.requireLease(row.resourceId);
1664
- if (lease.tokenHash === null || !lease.tokenHash.equals(tokenHash(claimToken)))
1665
- return undefined;
1666
- const ownerType = row.ownerId.startsWith("host-") ? "host" : "session";
1164
+ /* istanbul ignore next -- pure projection covered by integration tests */
1165
+ private mapEffect(row: EffectRow): EffectRecord {
1667
1166
  return {
1668
- actor: { type: ownerType, id: row.ownerId },
1669
- ownerType,
1670
- ownerId: row.ownerId,
1671
- token: claimToken,
1672
- generation: row.leaseGeneration,
1673
- leaseMs: 30_000,
1167
+ key: this.effectKey(row.effectId),
1168
+ resourceUid: row.resourceUid,
1169
+ generation: row.generation,
1170
+ kind: row.kind,
1171
+ state: resourceManagerEffectState(row.state),
1172
+ requestFingerprint: row.requestFingerprint,
1173
+ startedAt: new Date(row.startedAt).toISOString(),
1174
+ ...(row.completedAt === null ? {} : { completedAt: new Date(row.completedAt).toISOString() }),
1175
+ ...(row.externalRef === null ? {} : { externalRef: row.externalRef }),
1176
+ ...(row.errorHash === null
1177
+ ? {}
1178
+ : { error: this.state.readBlob(row.errorHash)?.content.toString("utf8") ?? "" }),
1674
1179
  };
1675
1180
  }
1676
1181
 
1677
- parkWorkflowRun(options: { runId: string; claimToken: string; now?: string }): boolean {
1678
- return this.releaseRunClaim(options.runId, options.claimToken, "parked", options.now);
1182
+ private effectKey(effectId: string): string {
1183
+ const row = this.state.connection
1184
+ .prepare("SELECT idempotency_key AS key FROM effects WHERE effect_id = ?")
1185
+ .get(effectId);
1186
+ /* istanbul ignore if -- exact schema and internal query shape */
1187
+ /* istanbul ignore if -- impossible after exact schema and transaction checks */
1188
+ if (!isKeyRow(row)) throw new Error(`Effect is missing: ${effectId}`);
1189
+ return row.key;
1679
1190
  }
1680
1191
 
1681
- parkWorkflowRunForRunnerNoProgress(options: {
1682
- runId: string;
1683
- claimToken: string;
1684
- detail: string;
1685
- now?: string;
1686
- }): boolean {
1687
- const now = epoch(validTimestamp(options.now));
1688
- return this.state.transaction(() => {
1689
- if (
1690
- !this.verifyWorkflowRunClaim({
1691
- runId: options.runId,
1692
- claimToken: options.claimToken,
1693
- now: new Date(now).toISOString(),
1694
- })
1695
- ) {
1696
- return false;
1697
- }
1698
- const row = this.requireWorkflowRunRow(options.runId);
1699
- const lease = this.requireLease(row.resourceId);
1700
- const detail = options.detail.slice(0, 8_192);
1701
- const errorHash = this.state.putText(detail, now);
1702
- const run = this.state.connection
1703
- .prepare(
1704
- `UPDATE runs SET status_detail = ?, updated_at = ?, finished_at = NULL
1705
- WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
1706
- )
1707
- .run(detail, now, options.runId);
1708
- const queue = this.state.connection
1709
- .prepare(
1710
- `UPDATE run_queue
1711
- SET status = 'parked', error_code = 'runnerNoProgress', error_hash = ?,
1712
- available_at = ?, updated_at = ?, finished_at = NULL
1713
- WHERE run_id = ? AND status IN ('queued', 'starting', 'running', 'parked')`,
1714
- )
1715
- .run(errorHash, now, now, options.runId);
1716
- if (run.changes !== 1 || queue.changes !== 1) {
1717
- throw new Error(`Workflow run ${options.runId} changed during runner recovery`);
1718
- }
1719
- const released = this.state.connection
1720
- .prepare(
1721
- `UPDATE leases
1722
- SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
1723
- acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
1724
- WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
1725
- )
1726
- .run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
1727
- /* istanbul ignore if -- the exact live claim is stable in this transaction */
1728
- if (released.changes !== 1) {
1729
- throw new Error(`Workflow run ${options.runId} claim changed during runner recovery`);
1730
- }
1731
- const revision = this.resourceRevision(row.resourceId);
1732
- this.bumpResource(row.resourceId, revision, now);
1733
- this.insertEvent(
1734
- row.resourceId,
1735
- revision + 1,
1736
- "run.worker_no_progress",
1737
- lease.ownerType ?? "system",
1738
- lease.ownerId,
1739
- { status: "parked", code: "runnerNoProgress" },
1740
- now,
1741
- lease.generation || undefined,
1742
- );
1743
- recordViewerDeltas(
1744
- this.state,
1745
- options.runId,
1746
- [{ targetType: "summary" }, { targetType: "replay" }],
1747
- now,
1748
- );
1749
- return true;
1750
- });
1751
- }
1752
-
1753
- pauseParkedWorkflowRun(options: { runId: string; now?: string }): boolean {
1754
- const now = epoch(validTimestamp(options.now));
1755
- return this.state.transaction(() => {
1756
- const row = this.workflowRunRow(options.runId);
1757
- if (row === undefined || row.status !== "parked") return false;
1758
- const lease = this.requireLease(row.resourceId);
1759
- if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
1760
- const pending = this.state.connection
1761
- .prepare(
1762
- `SELECT 1 FROM interactive_requests
1763
- WHERE run_id = ? AND status = 'pending' LIMIT 1`,
1764
- )
1765
- .get(options.runId);
1766
- if (pending === undefined) return false;
1767
- if (row.paused === 1) return true;
1768
- if (!["running", "waiting"].includes(row.runStatus)) return false;
1769
- const changed = this.state.connection
1770
- .prepare(
1771
- `UPDATE runs SET paused = 1, status_detail = 'paused', updated_at = ?
1772
- WHERE run_id = ? AND paused = 0 AND status IN ('running', 'waiting')`,
1773
- )
1774
- .run(now, options.runId);
1775
- if (changed.changes !== 1) return false;
1776
- const revision = this.resourceRevision(row.resourceId);
1777
- this.bumpResource(row.resourceId, revision, now);
1778
- this.insertEvent(
1779
- row.resourceId,
1780
- revision + 1,
1781
- "run.paused",
1782
- "control",
1783
- null,
1784
- { status: "parked" },
1785
- now,
1786
- );
1787
- recordViewerDeltas(
1788
- this.state,
1789
- options.runId,
1790
- [{ targetType: "summary" }, { targetType: "replay" }],
1791
- now,
1792
- );
1793
- return true;
1794
- });
1795
- }
1796
-
1797
- resumePausedInteraction(options: { runId: string; now?: string }): boolean {
1798
- const now = epoch(validTimestamp(options.now));
1799
- return this.state.transaction(() => {
1800
- const row = this.workflowRunRow(options.runId);
1801
- if (
1802
- row === undefined ||
1803
- row.status !== "parked" ||
1804
- row.runStatus !== "waiting" ||
1805
- row.paused !== 1
1806
- ) {
1807
- return false;
1808
- }
1809
- const lease = this.requireLease(row.resourceId);
1810
- if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
1811
- const pending = this.state.connection
1812
- .prepare(
1813
- `SELECT 1 FROM interactive_requests
1814
- WHERE run_id = ? AND status = 'pending' LIMIT 1`,
1815
- )
1816
- .get(options.runId);
1817
- if (pending === undefined) return false;
1818
- const changed = this.state.connection
1819
- .prepare(
1820
- `UPDATE runs
1821
- SET paused = 0, status_detail = 'waiting for origin-session input', updated_at = ?
1822
- WHERE run_id = ? AND status = 'waiting' AND paused = 1`,
1823
- )
1824
- .run(now, options.runId);
1825
- if (changed.changes !== 1) return false;
1826
- const revision = this.resourceRevision(row.resourceId);
1827
- this.bumpResource(row.resourceId, revision, now);
1828
- this.insertEvent(
1829
- row.resourceId,
1830
- revision + 1,
1831
- "run.resumed",
1832
- "control",
1833
- null,
1834
- { status: "waiting" },
1835
- now,
1836
- );
1837
- recordViewerDeltas(
1838
- this.state,
1839
- options.runId,
1840
- [{ targetType: "summary" }, { targetType: "replay" }],
1841
- now,
1842
- );
1843
- return true;
1844
- });
1845
- }
1846
-
1847
- isWorkflowRunPaused(runId: string): boolean {
1848
- return this.workflowRunRow(runId)?.paused === 1;
1849
- }
1850
-
1851
- parkWorkflowRunForSourceChange(options: {
1852
- runId: string;
1853
- claimToken: string;
1854
- detail: string;
1855
- now?: string;
1856
- }): boolean {
1857
- const now = epoch(validTimestamp(options.now));
1858
- return this.state.transaction(() => {
1859
- const row = this.workflowRunRow(options.runId);
1860
- if (
1861
- row === undefined ||
1862
- !this.verifyWorkflowRunClaim({
1863
- runId: options.runId,
1864
- claimToken: options.claimToken,
1865
- now: new Date(now).toISOString(),
1866
- })
1867
- ) {
1868
- return false;
1869
- }
1870
- const lease = this.requireLease(row.resourceId);
1871
- const detail = options.detail.slice(0, 8_192);
1872
- const errorHash = this.state.putText(detail, now);
1873
- const run = this.state.connection
1874
- .prepare(
1875
- `UPDATE runs SET status_detail = ?, updated_at = ?, finished_at = NULL
1876
- WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
1877
- )
1878
- .run(detail, now, options.runId);
1879
- const queue = this.state.connection
1880
- .prepare(
1881
- `UPDATE run_queue
1882
- SET status = 'parked', error_code = 'workflowSourceChanged', error_hash = ?,
1883
- available_at = ?, updated_at = ?, finished_at = NULL
1884
- WHERE run_id = ? AND status IN ('queued', 'starting', 'running', 'parked')`,
1885
- )
1886
- .run(errorHash, now, now, options.runId);
1887
- /* istanbul ignore if -- exact live claim and nonterminal checks make both updates mandatory */
1888
- if (run.changes !== 1 || queue.changes !== 1) {
1889
- throw new Error(`Workflow run ${options.runId} has inconsistent source-change state`);
1890
- }
1891
- const released = this.state.connection
1892
- .prepare(
1893
- `UPDATE leases
1894
- SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
1895
- acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
1896
- WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
1897
- )
1898
- .run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
1899
- /* istanbul ignore if -- the exact live claim is stable in this transaction */
1900
- if (released.changes !== 1) {
1901
- throw new Error(`Workflow run ${options.runId} claim changed during source recovery`);
1902
- }
1903
- const revision = this.resourceRevision(row.resourceId);
1904
- this.bumpResource(row.resourceId, revision, now);
1905
- this.insertEvent(
1906
- row.resourceId,
1907
- revision + 1,
1908
- "run.source_changed",
1909
- lease.ownerType ?? "system",
1910
- lease.ownerId,
1911
- { status: "parked", code: "workflowSourceChanged" },
1912
- now,
1913
- lease.generation || undefined,
1914
- );
1915
- recordViewerDeltas(
1916
- this.state,
1917
- options.runId,
1918
- [{ targetType: "summary" }, { targetType: "replay" }],
1919
- now,
1920
- );
1921
- return true;
1922
- });
1923
- }
1924
-
1925
- parkWorkflowRunForAmbiguousEffect(options: {
1926
- runId: string;
1927
- claimToken: string;
1928
- now?: string;
1929
- }): boolean {
1930
- const now = epoch(validTimestamp(options.now));
1931
- return this.state.transaction(() => {
1932
- const row = this.workflowRunRow(options.runId);
1933
- if (
1934
- row === undefined ||
1935
- !this.verifyWorkflowRunClaim({
1936
- runId: options.runId,
1937
- claimToken: options.claimToken,
1938
- now: new Date(now).toISOString(),
1939
- })
1940
- ) {
1941
- return false;
1942
- }
1943
- const lease = this.requireLease(row.resourceId);
1944
- const detail = "effect outcome is ambiguous; explicit recovery is required";
1945
- const errorHash = this.state.putText(detail, now);
1946
- this.state.connection
1947
- .prepare(
1948
- `UPDATE node_attempts SET status = 'waiting', updated_at = ?
1949
- WHERE run_id = ? AND status IN ('pending', 'running')`,
1950
- )
1951
- .run(now, options.runId);
1952
- const run = this.state.connection
1953
- .prepare(
1954
- `UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
1955
- WHERE run_id = ? AND status = 'running'`,
1956
- )
1957
- .run(detail, now, now, options.runId);
1958
- const queue = this.state.connection
1959
- .prepare(
1960
- `UPDATE run_queue
1961
- SET status = 'parked', error_code = 'effectAmbiguous', error_hash = ?, updated_at = ?
1962
- WHERE run_id = ? AND status IN ('starting', 'running', 'parked')`,
1963
- )
1964
- .run(errorHash, now, options.runId);
1965
- /* istanbul ignore if -- exact live claim and run checks make both updates mandatory */
1966
- if (run.changes !== 1 || queue.changes !== 1) {
1967
- throw new Error(`Workflow run ${options.runId} has inconsistent ambiguous-effect state`);
1968
- }
1969
- const released = this.state.connection
1970
- .prepare(
1971
- `UPDATE leases
1972
- SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
1973
- acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
1974
- WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
1975
- )
1976
- .run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
1977
- /* istanbul ignore if -- the exact live claim is stable in this transaction */
1978
- if (released.changes !== 1) {
1979
- throw new Error(`Workflow run ${options.runId} claim changed during effect recovery`);
1980
- }
1981
- const revision = this.resourceRevision(row.resourceId);
1982
- this.bumpResource(row.resourceId, revision, now);
1983
- this.insertEvent(
1984
- row.resourceId,
1985
- revision + 1,
1986
- "run.effect_ambiguous",
1987
- lease.ownerType ?? "system",
1988
- lease.ownerId,
1989
- { status: "waiting", code: "effectAmbiguous" },
1990
- now,
1991
- lease.generation || undefined,
1992
- );
1993
- recordViewerDeltas(
1994
- this.state,
1995
- options.runId,
1996
- [{ targetType: "summary" }, { targetType: "replay" }],
1997
- now,
1998
- );
1999
- return true;
2000
- });
2001
- }
2002
-
2003
- failWorkflowRun(options: {
2004
- runId: string;
2005
- claimToken?: string;
2006
- errorCode: string;
2007
- errorMessage: string;
2008
- now?: string;
2009
- }): boolean {
2010
- return this.terminalRun(
2011
- options.runId,
2012
- options.claimToken,
2013
- "failed",
2014
- options.errorCode,
2015
- options.errorMessage,
2016
- options.now,
2017
- );
2018
- }
2019
-
2020
- cancelWorkflowRun(options: { runId: string; claimToken?: string; now?: string }): boolean {
2021
- const cancelled = this.terminalRun(
2022
- options.runId,
2023
- options.claimToken,
2024
- "cancelled",
2025
- "cancelled",
2026
- "Workflow run cancelled",
2027
- options.now,
2028
- );
2029
- if (cancelled || options.claimToken !== undefined) return cancelled;
2030
- return this.cancelStaleWorkflowRun({
2031
- runId: options.runId,
2032
- ...(options.now === undefined ? {} : { now: options.now }),
2033
- });
2034
- }
2035
-
2036
- /** Cancel a nonterminal run only when its claim is absent or expired. */
2037
- cancelStaleWorkflowRun(options: {
2038
- runId: string;
2039
- controlId?: string;
2040
- claimToken?: string;
2041
- now?: string;
2042
- }): boolean {
2043
- const now = epoch(validTimestamp(options.now));
2044
- const controlId = options.controlId ?? "workflow-control";
2045
- const claimToken = options.claimToken ?? randomUUID();
2046
- return this.state.transaction(() => {
2047
- const row = this.workflowRunRow(options.runId);
2048
- if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return false;
2049
- const lease = this.requireLease(row.resourceId);
2050
- if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
2051
-
2052
- const generation = lease.generation + 1;
2053
- const claimed = this.state.connection
2054
- .prepare(
2055
- `UPDATE leases
2056
- SET generation = ?, owner_type = 'system', owner_id = ?, token_hash = ?,
2057
- acquired_at = ?, heartbeat_at = ?, expires_at = ?
2058
- WHERE resource_id = ? AND generation = ?
2059
- AND (owner_id IS NULL OR expires_at IS NULL OR expires_at <= ?)`,
2060
- )
2061
- .run(
2062
- generation,
2063
- controlId,
2064
- tokenHash(claimToken),
2065
- now,
2066
- now,
2067
- now + 30_000,
2068
- row.resourceId,
2069
- lease.generation,
2070
- now,
2071
- );
2072
- if (claimed.changes !== 1) return false;
2073
-
2074
- const errorHash = this.state.putText("Workflow run cancelled", now);
2075
- const queue = this.state.connection
2076
- .prepare(
2077
- `UPDATE run_queue
2078
- SET status = 'cancelled', error_code = 'cancelled', error_hash = ?,
2079
- updated_at = ?, finished_at = ?
2080
- WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
2081
- )
2082
- .run(errorHash, now, now, options.runId);
2083
- /* istanbul ignore if -- the control claim selects one nonterminal queue row */
2084
- if (queue.changes !== 1) {
2085
- throw new Error(`Workflow run ${options.runId} has inconsistent queue state`);
2086
- }
2087
- const run = this.state.connection
2088
- .prepare(
2089
- `UPDATE runs
2090
- SET status = 'cancelled', paused = 0, status_detail = NULL, error_hash = ?,
2091
- updated_at = ?, finished_at = ?
2092
- WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
2093
- )
2094
- .run(errorHash, now, now, options.runId);
2095
- /* istanbul ignore if -- the control claim selects one nonterminal durable run */
2096
- if (run.changes !== 1) {
2097
- throw new Error(`Workflow run ${options.runId} has inconsistent durable state`);
2098
- }
2099
- this.settleWorkflowRunMessages(options.runId, now);
2100
- this.cancelWorkflowRunDependents(options.runId, controlId, errorHash, now);
2101
-
2102
- const released = this.state.connection
2103
- .prepare(
2104
- `UPDATE leases
2105
- SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
2106
- acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
2107
- WHERE resource_id = ? AND owner_type = 'system' AND owner_id = ?
2108
- AND token_hash = ? AND generation = ? AND expires_at > ?`,
2109
- )
2110
- .run(row.resourceId, controlId, tokenHash(claimToken), generation, now);
2111
- /* istanbul ignore if -- the exact control claim was written in this transaction */
2112
- if (released.changes !== 1) {
2113
- throw new Error(`Workflow run ${options.runId} control claim changed during cancellation`);
2114
- }
2115
- const revision = this.resourceRevision(row.resourceId);
2116
- this.bumpResource(row.resourceId, revision, now);
2117
- this.insertEvent(
2118
- row.resourceId,
2119
- revision + 1,
2120
- "run.queue_cancelled",
2121
- "control",
2122
- controlId,
2123
- { status: "cancelled", code: "cancelled", staleControl: true },
2124
- now,
2125
- generation,
2126
- );
2127
- recordViewerDeltas(
2128
- this.state,
2129
- options.runId,
2130
- [{ targetType: "summary" }, { targetType: "replay" }],
2131
- now,
2132
- );
2133
- return true;
2134
- });
2135
- }
2136
-
2137
- deleteWorkflowRun(options: { runId: string; claimToken: string }): boolean {
2138
- const now = Date.now();
2139
- return this.state.transaction(() => {
2140
- const row = this.workflowRunRow(options.runId);
2141
- if (row === undefined) return false;
2142
- const lease = this.requireLease(row.resourceId);
2143
- if (
2144
- lease.ownerId === null ||
2145
- lease.tokenHash === null ||
2146
- lease.expiresAt === null ||
2147
- lease.expiresAt <= now ||
2148
- !lease.tokenHash.equals(tokenHash(options.claimToken))
2149
- ) {
2150
- return false;
2151
- }
2152
- return (
2153
- this.state.connection
2154
- .prepare("DELETE FROM resources WHERE resource_id = ?")
2155
- .run(row.resourceId).changes === 1
2156
- );
2157
- });
2158
- }
2159
-
2160
- completeWorkflowRun(options: { runId: string; claimToken: string; now?: string }): boolean {
2161
- return this.releaseRunClaim(options.runId, options.claimToken, "done", options.now);
2162
- }
2163
-
2164
- repairCanonicalWorkflowSourceRun(): never {
2165
- throw new Error("Legacy workflow source repair is not available after the SQLite cutover");
2166
- }
2167
-
2168
- claimLegacyWorkflowSourceRun(): never {
2169
- throw new Error("Legacy workflow source claims are not available after the SQLite cutover");
2170
- }
2171
-
2172
- recordRunEvent(options: {
2173
- runId: string;
2174
- workflowRef: string;
2175
- type: string;
2176
- payload?: JsonObject;
2177
- runnerId?: string;
2178
- now?: string;
2179
- }): RunEventRecord {
2180
- return this.state.transaction(() => {
2181
- const row = this.requireWorkflowRunRow(options.runId);
2182
- const now = epoch(validTimestamp(options.now));
2183
- const revision = this.resourceRevision(row.resourceId) + 1;
2184
- this.bumpResource(row.resourceId, revision - 1, now);
2185
- const eventId = this.insertEvent(
2186
- row.resourceId,
2187
- revision,
2188
- options.type,
2189
- options.runnerId?.startsWith("host-") ? "host" : "session",
2190
- options.runnerId ?? null,
2191
- options.payload ?? {},
2192
- now,
2193
- row.leaseGeneration || undefined,
2194
- );
2195
- const seq = this.state.connection
2196
- .prepare("SELECT event_seq AS seq FROM events WHERE event_id = ?")
2197
- .get(eventId);
2198
- /* istanbul ignore if -- exact schema and internal query shape */
2199
- if (!isSequenceRow(seq)) throw new Error("Run event was not recorded");
2200
- return {
2201
- seq: seq.seq,
2202
- recordedAt: new Date(now).toISOString(),
2203
- runId: options.runId,
2204
- workflowRef: options.workflowRef,
2205
- type: options.type,
2206
- runnerId: options.runnerId ?? null,
2207
- payload: options.payload ?? {},
2208
- };
2209
- });
2210
- }
2211
-
2212
- listRunEventsAfter(seq: number, options: { limit?: number } = {}): RunEventRecord[] {
2213
- const rows = this.state.connection
2214
- .prepare(
2215
- `SELECT e.event_seq AS seq, e.recorded_at AS recordedAt, r.run_id AS runId,
2216
- r.workflow_ref AS workflowRef, e.event_type AS eventType,
2217
- e.actor_id AS runnerId, e.payload_hash AS payloadHash
2218
- FROM events e JOIN runs r ON r.resource_id = e.resource_id
2219
- WHERE e.event_seq > ? ORDER BY e.event_seq LIMIT ?`,
2220
- )
2221
- .all(seq, options.limit ?? 100);
2222
- return rows.filter(isRunEventRow).map((row) => ({
2223
- seq: row.seq,
2224
- recordedAt: new Date(row.recordedAt).toISOString(),
2225
- runId: row.runId,
2226
- workflowRef: row.workflowRef,
2227
- type: row.eventType,
2228
- runnerId: row.runnerId,
2229
- payload: row.payloadHash === null ? {} : (this.state.readJson(row.payloadHash) as JsonObject),
2230
- }));
2231
- }
2232
-
2233
- settleRunEffect(runId: string, effectType: "run.park_queue" | "run.settle_queue"): void {
2234
- const rows = this.state.connection
2235
- .prepare(
2236
- `SELECT e.effect_id AS effectId, e.resource_id AS resourceId
2237
- FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
2238
- WHERE r.run_id = ? AND e.effect_type = ? AND e.status = 'pending'`,
2239
- )
2240
- .all(runId, effectType);
2241
- for (const row of rows) {
2242
- /* istanbul ignore if -- exact schema and internal query shape */
2243
- if (!isEffectIdentityRow(row)) continue;
2244
- this.state.transaction(() => {
2245
- const now = Date.now();
2246
- const revision = this.resourceRevision(row.resourceId);
2247
- this.state.connection
2248
- .prepare(
2249
- `UPDATE effects SET status = 'applied', updated_at = ?, settled_at = ?
2250
- WHERE effect_id = ? AND status = 'pending'`,
2251
- )
2252
- .run(now, now, row.effectId);
2253
- this.bumpResource(row.resourceId, revision, now);
2254
- this.insertEvent(
2255
- row.resourceId,
2256
- revision + 1,
2257
- "effect.applied",
2258
- "system",
2259
- null,
2260
- { runId, effectType },
2261
- now,
2262
- );
2263
- });
2264
- }
2265
- }
2266
-
2267
- setWorkflowRunOriginSession(runId: string, originSessionId: string): boolean {
2268
- const now = Date.now();
2269
- return (
2270
- this.state.connection
2271
- .prepare(
2272
- `INSERT INTO run_bindings(run_id, origin_session_id, execution_mode, created_at)
2273
- VALUES (?, ?, 'interactive', ?)
2274
- ON CONFLICT(run_id) DO UPDATE SET origin_session_id = excluded.origin_session_id`,
2275
- )
2276
- .run(runId, originSessionId, now).changes === 1
2277
- );
2278
- }
2279
-
2280
- private ensureProject(projectPath: string): string {
2281
- const canonicalPath = canonicalProjectPath(projectPath);
2282
- const projectId = projectIdFor(canonicalPath);
2283
- this.state.connection
2284
- .prepare(
2285
- `INSERT INTO projects(project_id, canonical_path, created_at)
2286
- VALUES (?, ?, ?) ON CONFLICT(project_id) DO NOTHING`,
2287
- )
2288
- .run(projectId, canonicalPath, Date.now());
2289
- return projectId;
2290
- }
2291
-
2292
- private findProject(projectPath: string): string | null {
2293
- const row = this.state.connection
2294
- .prepare("SELECT project_id AS projectId FROM projects WHERE canonical_path = ?")
2295
- .get(canonicalProjectPath(projectPath));
2296
- return isProjectRow(row) ? row.projectId : null;
2297
- }
2298
-
2299
- private requireProjectId(): string {
2300
- if (this.projectId === null)
2301
- throw new Error("ResourceManager project is not registered in the state database");
2302
- return this.projectId;
2303
- }
2304
-
2305
- private resourceManagerRow(ref: ManagedResourceRef): ResourceManagerRow | undefined {
2306
- if (this.projectId === null) return undefined;
2307
- const row = this.state.connection
2308
- .prepare(
2309
- resourceManagerSelect(
2310
- "WHERE c.project_id = ? AND c.controller_name = ? AND c.resource_key = ?",
2311
- ),
2312
- )
2313
- .get(this.projectId, ref.resourceManager, ref.key);
2314
- return isResourceManagerRow(row) ? row : undefined;
2315
- }
2316
-
2317
- private requireResourceManagerRow(ref: ManagedResourceRef): ResourceManagerRow {
2318
- const row = this.resourceManagerRow(ref);
2319
- if (row === undefined) throw new ManagedResourceNotFoundError(ref.resourceManager, ref.key);
2320
- return row;
2321
- }
2322
-
2323
- private requireResourceManagerRowByUid(uid: string): ResourceManagerRow {
2324
- const row = this.state.connection.prepare(resourceManagerSelect("WHERE c.uid = ?")).get(uid);
2325
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2326
- if (!isResourceManagerRow(row)) throw new Error(`Managed resource not found: ${uid}`);
2327
- return row;
2328
- }
2329
-
2330
- private requireResource(ref: ManagedResourceRef): ManagedResource {
2331
- const resource = this.getResource(ref);
2332
- if (resource === undefined)
2333
- throw new ManagedResourceNotFoundError(ref.resourceManager, ref.key);
2334
- return resource;
2335
- }
2336
-
2337
- /* istanbul ignore next -- pure projection covered by integration tests */
2338
- private mapManagedResource(row: ResourceManagerRow): ManagedResource {
2339
- const spec = this.state.readJson(row.specHash);
2340
- const status = this.state.readJson(row.statusHash) as ManagedResourceStatus<unknown>;
2341
- return {
2342
- metadata: {
2343
- uid: row.uid,
2344
- resourceManager: row.resourceManagerName,
2345
- key: row.resourceKey,
2346
- resourceVersion: row.resourceVersion,
2347
- generation: row.generation,
2348
- ...(row.deletionRequestedAt === null
2349
- ? {}
2350
- : { deletionTimestamp: new Date(row.deletionRequestedAt).toISOString() }),
2351
- finalizers: this.finalizers(row.managedResourceId),
2352
- },
2353
- spec,
2354
- status,
2355
- };
2356
- }
2357
-
2358
- private finalizers(managedResourceId: string): string[] {
2359
- const rows = this.state.connection
2360
- .prepare(
2361
- "SELECT finalizer FROM controller_finalizers WHERE controller_resource_id = ? ORDER BY position",
2362
- )
2363
- .all(managedResourceId);
2364
- return rows.flatMap((row) => (isFinalizerRow(row) ? [row.finalizer] : []));
2365
- }
2366
-
2367
- private replaceFinalizers(managedResourceId: string, finalizers: string[]): void {
2368
- const unique = [...new Set(finalizers)];
2369
- if (unique.length !== finalizers.length)
2370
- throw new Error("ResourceManager finalizers must be unique");
2371
- this.state.connection
2372
- .prepare("DELETE FROM controller_finalizers WHERE controller_resource_id = ?")
2373
- .run(managedResourceId);
2374
- const insert = this.state.connection.prepare(
2375
- "INSERT INTO controller_finalizers(controller_resource_id, finalizer, position) VALUES (?, ?, ?)",
2376
- );
2377
- unique.forEach((finalizer, index) => insert.run(managedResourceId, finalizer, index));
2378
- }
2379
-
2380
- private enqueueResourceManagerRow(
2381
- managedResourceId: string,
2382
- availableAt: number,
2383
- now: number,
2384
- ): void {
2385
- this.state.connection
2386
- .prepare(
2387
- `INSERT INTO controller_queue(
2388
- controller_resource_id, available_at, queue_version,
2389
- consecutive_errors, created_at, updated_at
2390
- ) VALUES (?, ?, 1, 0, ?, ?)
2391
- ON CONFLICT(controller_resource_id) DO UPDATE SET
2392
- available_at = MIN(controller_queue.available_at, excluded.available_at),
2393
- queue_version = controller_queue.queue_version + 1,
2394
- updated_at = excluded.updated_at`,
2395
- )
2396
- .run(managedResourceId, availableAt, now, now);
2397
- }
2398
-
2399
- private queueErrors(managedResourceId: string): number {
2400
- const row = this.state.connection
2401
- .prepare(
2402
- "SELECT consecutive_errors AS consecutiveErrors FROM controller_queue WHERE controller_resource_id = ?",
2403
- )
2404
- .get(managedResourceId);
2405
- return isErrorCountRow(row) ? row.consecutiveErrors : 0;
2406
- }
2407
-
2408
- private queueVersion(managedResourceId: string): number {
2409
- const row = this.state.connection
2410
- .prepare(
2411
- "SELECT queue_version AS queueVersion FROM controller_queue WHERE controller_resource_id = ?",
2412
- )
2413
- .get(managedResourceId);
2414
- /* istanbul ignore if -- exact schema and internal query shape */
2415
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2416
- if (!isQueueVersionRow(row)) throw new Error("ResourceManager queue item is missing");
2417
- return row.queueVersion;
2418
- }
2419
-
2420
- private settleResourceManagerClaim(
2421
- claim: ResourceManagerQueueClaim,
2422
- requeue: QueueRequeueOptions | undefined,
2423
- nowValue?: string,
2424
- ): boolean {
2425
- const now = epoch(validTimestamp(nowValue));
2426
- return this.state.transaction(() => {
2427
- const row = this.resourceManagerRow({
2428
- resourceManager: claim.resourceManager,
2429
- key: claim.key,
2430
- });
2431
- if (row === undefined) return false;
2432
- try {
2433
- this.assertResourceManagerClaim(row, claim, now, false);
2434
- } catch {
2435
- return false;
2436
- }
2437
- const currentQueueVersion = this.queueVersion(row.managedResourceId);
2438
- if (requeue === undefined) {
2439
- if (currentQueueVersion === claim.queueVersion) {
2440
- this.state.connection
2441
- .prepare("DELETE FROM controller_queue WHERE controller_resource_id = ?")
2442
- .run(row.managedResourceId);
2443
- }
2444
- } else {
2445
- const errorHash =
2446
- requeue.error === undefined ? null : this.state.putText(requeue.error, now);
2447
- this.state.connection
2448
- .prepare(
2449
- `UPDATE controller_queue
2450
- SET available_at = ?, queue_version = queue_version + 1,
2451
- consecutive_errors = consecutive_errors + 1,
2452
- last_error_hash = ?, updated_at = ? WHERE controller_resource_id = ?`,
2453
- )
2454
- .run(epoch(requeue.availableAt), errorHash, now, row.managedResourceId);
2455
- }
2456
- this.releaseLease(row.resourceId, claim, now);
2457
- return true;
2458
- });
2459
- }
2460
-
2461
- private assertResourceManagerClaim(
2462
- row: ResourceManagerRow,
2463
- claim: ResourceManagerQueueClaim,
2464
- now: number,
2465
- requireUnexpired = true,
2466
- ): void {
2467
- if (row.resourceManagerName !== claim.resourceManager || row.resourceKey !== claim.key)
2468
- throw new Error("ResourceManager claim targets another resource");
2469
- const lease = this.requireLease(row.resourceId);
2470
- if (
2471
- lease.ownerType !== "controller" ||
2472
- lease.ownerId !== claim.ownerId ||
2473
- lease.generation !== claim.generation ||
2474
- lease.tokenHash === null ||
2475
- !lease.tokenHash.equals(tokenHash(claim.token)) ||
2476
- (requireUnexpired && (lease.expiresAt === null || lease.expiresAt <= now))
2477
- ) {
2478
- throw new Error("ResourceManager claim is stale");
2479
- }
2480
- }
2481
-
2482
- private requireLease(resourceId: string): LeaseRow {
2483
- const row = this.state.connection
2484
- .prepare(
2485
- `SELECT generation, owner_type AS ownerType, owner_id AS ownerId,
2486
- token_hash AS tokenHash, expires_at AS expiresAt
2487
- FROM leases WHERE resource_id = ?`,
2488
- )
2489
- .get(resourceId);
2490
- /* istanbul ignore if -- exact schema and internal query shape */
2491
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2492
- if (!isLeaseRow(row)) throw new Error(`Lease is missing: ${resourceId}`);
2493
- return row;
2494
- }
2495
-
2496
- private releaseLease(resourceId: string, claim: ResourceManagerQueueClaim, now: number): void {
2497
- const row = this.resourceManagerRow({ resourceManager: claim.resourceManager, key: claim.key });
2498
- if (row === undefined) return;
2499
- const revision = this.resourceRevision(resourceId);
2500
- const result = this.state.connection
2501
- .prepare(
2502
- `UPDATE leases SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
2503
- acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
2504
- WHERE resource_id = ? AND owner_id = ? AND token_hash = ? AND generation = ?`,
2505
- )
2506
- .run(resourceId, claim.ownerId, tokenHash(claim.token), claim.generation);
2507
- if (result.changes === 1) {
2508
- this.bumpResource(resourceId, revision, now);
2509
- this.insertEvent(
2510
- resourceId,
2511
- revision + 1,
2512
- "lease.released",
2513
- "controller",
2514
- claim.ownerId,
2515
- {},
2516
- now,
2517
- claim.generation,
2518
- );
2519
- claim.resourceVersion = revision + 1;
2520
- }
2521
- }
2522
-
2523
- private resourceRevision(resourceId: string): number {
2524
- const row = this.state.connection
2525
- .prepare("SELECT revision FROM resources WHERE resource_id = ?")
2526
- .get(resourceId);
2527
- /* istanbul ignore if -- exact schema and internal query shape */
2528
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2529
- if (!isRevisionRow(row)) throw new Error(`Resource is missing: ${resourceId}`);
2530
- return row.revision;
2531
- }
2532
-
2533
- private bumpResource(resourceId: string, expectedRevision: number, now: number): void {
2534
- const result = this.state.connection
2535
- .prepare(
2536
- "UPDATE resources SET revision = revision + 1, updated_at = ? WHERE resource_id = ? AND revision = ?",
2537
- )
2538
- .run(now, resourceId, expectedRevision);
2539
- if (result.changes !== 1) throw new Error("Resource revision conflict");
2540
- const run = this.state.connection
2541
- .prepare(
2542
- `SELECT r.run_id AS runId
2543
- FROM runs r JOIN viewer_runs v ON v.run_id = r.run_id
2544
- WHERE r.resource_id = ?`,
2545
- )
2546
- .get(resourceId);
2547
- if (isRecord(run) && typeof run.runId === "string") {
2548
- recordViewerDeltas(
2549
- this.state,
2550
- run.runId,
2551
- [
2552
- { targetType: "summary" },
2553
- { targetType: "graph" },
2554
- { targetType: "replay" },
2555
- { targetType: "inspector", targetKey: "run" },
2556
- ],
2557
- now,
2558
- );
2559
- }
2560
- }
2561
-
2562
- private insertEvent(
2563
- resourceId: string,
2564
- revision: number,
2565
- type: string,
2566
- actorType: string,
2567
- actorId: string | null,
2568
- payload: unknown,
2569
- now: number,
2570
- leaseGeneration?: number,
2571
- ): string {
2572
- const eventId = `event-${randomUUID()}`;
2573
- const payloadHash = this.state.putJson(payload, now);
2574
- this.state.connection
2575
- .prepare(
2576
- `INSERT INTO events(
2577
- event_id, resource_id, resource_revision, event_type, actor_type,
2578
- actor_id, lease_generation, payload_hash, recorded_at
2579
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
2580
- )
2581
- .run(
2582
- eventId,
2583
- resourceId,
2584
- revision,
2585
- type,
2586
- actorType,
2587
- actorId,
2588
- leaseGeneration ?? null,
2589
- payloadHash,
2590
- now,
2591
- );
2592
- return eventId;
2593
- }
2594
-
2595
- private effectRow(resourceUid: string, key: string): EffectRow | undefined {
2596
- const row = this.state.connection
2597
- .prepare(effectSelect("WHERE c.uid = ? AND e.idempotency_key = ?"))
2598
- .get(resourceUid, key);
2599
- return isEffectRow(row) ? row : undefined;
2600
- }
2601
-
2602
- private requireEffectRow(resourceUid: string, key: string): EffectRow {
2603
- const row = this.effectRow(resourceUid, key);
2604
- if (row === undefined) throw new Error(`Effect record not found: ${resourceUid}/${key}`);
2605
- return row;
2606
- }
2607
-
2608
- /* istanbul ignore next -- pure projection covered by integration tests */
2609
- private mapEffect(row: EffectRow): EffectRecord {
2610
- return {
2611
- key: this.effectKey(row.effectId),
2612
- resourceUid: row.resourceUid,
2613
- generation: row.generation,
2614
- kind: row.kind,
2615
- state: resourceManagerEffectState(row.state),
2616
- requestFingerprint: row.requestFingerprint,
2617
- startedAt: new Date(row.startedAt).toISOString(),
2618
- ...(row.completedAt === null ? {} : { completedAt: new Date(row.completedAt).toISOString() }),
2619
- ...(row.externalRef === null ? {} : { externalRef: row.externalRef }),
2620
- ...(row.errorHash === null
2621
- ? {}
2622
- : { error: this.state.readBlob(row.errorHash)?.content.toString("utf8") ?? "" }),
2623
- };
2624
- }
2625
-
2626
- private effectKey(effectId: string): string {
2627
- const row = this.state.connection
2628
- .prepare("SELECT idempotency_key AS key FROM effects WHERE effect_id = ?")
2629
- .get(effectId);
2630
- /* istanbul ignore if -- exact schema and internal query shape */
2631
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2632
- if (!isKeyRow(row)) throw new Error(`Effect is missing: ${effectId}`);
2633
- return row.key;
2634
- }
2635
-
2636
- private workflowRow(resourceUid: string, requestKey: string): WorkflowRow | undefined {
2637
- const row = this.state.connection
2638
- .prepare(workflowSelect("WHERE c.uid = ? AND w.request_key = ?"))
2639
- .get(resourceUid, requestKey);
2640
- return isWorkflowRow(row) ? row : undefined;
2641
- }
2642
-
2643
- private workflowRowById(requestId: string): WorkflowRow | undefined {
2644
- const row = this.state.connection
2645
- .prepare(workflowSelect("WHERE w.request_id = ?"))
2646
- .get(requestId);
2647
- return isWorkflowRow(row) ? row : undefined;
2648
- }
2649
-
2650
- private requireWorkflowRow(requestId: string): WorkflowRow {
2651
- const row = this.workflowRowById(requestId);
2652
- if (row === undefined) throw new Error(`Workflow request not found: ${requestId}`);
2653
- return row;
2654
- }
2655
-
2656
- /* istanbul ignore next -- pure projection covered by integration tests */
2657
- private mapWorkflow(row: WorkflowRow): ChildWorkflowRecord {
2658
- return {
2659
- requestId: row.requestId,
2660
- resourceUid: row.resourceUid,
2661
- requestKey: row.requestKey,
2662
- inputFingerprint: row.inputFingerprint.toString("hex"),
2663
- workflow: row.workflowName,
2664
- ...(row.runId === null ? {} : { runId: row.runId }),
2665
- state: row.status,
2666
- attempt: row.attemptCount,
2667
- ...(row.errorHash === null
2668
- ? {}
2669
- : { error: this.state.readBlob(row.errorHash)?.content.toString("utf8") ?? "" }),
2670
- };
2671
- }
2672
-
2673
- private workflowRunRow(runId: string): RunRow | undefined {
2674
- const row = this.state.connection.prepare(workflowRunSelect("WHERE r.run_id = ?")).get(runId);
2675
- return isRunRow(row) ? row : undefined;
2676
- }
2677
-
2678
- private requireWorkflowRunRow(runId: string): RunRow {
2679
- const row = this.workflowRunRow(runId);
2680
- if (row === undefined) throw new Error(`Workflow run not found: ${runId}`);
2681
- return row;
2682
- }
2683
-
2684
- private requireWorkflowRun(runId: string): WorkflowRunQueueRecord {
2685
- return this.mapWorkflowRun(this.requireWorkflowRunRow(runId));
2686
- }
2687
-
2688
- private assertWorkflowRunPreparationCompatible(
2689
- row: RunRow,
2690
- options: WorkflowRunReservationOptions,
2691
- definitionDigest: Buffer,
2692
- ): void {
2693
- const expectedLineageKind =
2694
- options.parentRunId === undefined ? null : (options.lineageKind ?? "continuation");
2695
- const parent =
2696
- options.parentRunId === undefined
2697
- ? undefined
2698
- : this.requireWorkflowRunRow(options.parentRunId);
2699
- const expectedRootRunId = parent?.rootRunId ?? options.runId;
2700
- const expectedRestartNumber =
2701
- expectedLineageKind === "restart"
2702
- ? (options.restartNumber ?? (parent?.restartNumber ?? -1) + 1)
2703
- : (parent?.restartNumber ?? 0);
2704
- const expectedParentTerminalFingerprint =
2705
- options.parentTerminalFingerprint === undefined
2706
- ? null
2707
- : terminalFingerprintBuffer(options.parentTerminalFingerprint);
2708
- const compatible =
2709
- row.workflowName === options.workflowName &&
2710
- row.workflowRef === options.workflowSourceRef &&
2711
- row.definitionDigest.equals(definitionDigest) &&
2712
- canonicalJson(readQueuedRunSources(this.state, row)) ===
2713
- canonicalJson(queuedWorkflowSource(options.workflowSource)) &&
2714
- canonicalJson(this.state.readJson(row.inputHash)) === canonicalJson(options.input ?? null) &&
2715
- canonicalJson(this.state.readJson(row.launchOptionsHash)) ===
2716
- canonicalJson(options.launchOptions ?? {}) &&
2717
- row.originSessionId === options.originSessionId &&
2718
- row.executionMode === (options.executionMode ?? "interactive") &&
2719
- row.parentRunId === (options.parentRunId ?? null) &&
2720
- row.rootRunId === expectedRootRunId &&
2721
- row.lineageKind === expectedLineageKind &&
2722
- row.restartNumber === expectedRestartNumber &&
2723
- ((row.parentTerminalFingerprint === null && expectedParentTerminalFingerprint === null) ||
2724
- (row.parentTerminalFingerprint !== null &&
2725
- expectedParentTerminalFingerprint !== null &&
2726
- row.parentTerminalFingerprint.equals(expectedParentTerminalFingerprint)));
2727
- if (!compatible) {
2728
- throw new Error(`Workflow run preparation conflicts: ${options.runId}`);
2729
- }
2730
- }
2731
-
2732
- private workflowRunViewRows(clause: string, params: unknown[]): WorkflowRunViewRow[] {
2733
- return this.state.connection
2734
- .prepare(workflowRunViewSelect(clause))
2735
- .all(...params)
2736
- .filter(isWorkflowRunViewRow);
2737
- }
2738
-
2739
- /* istanbul ignore next -- pure projection covered by integration tests */
2740
- private mapWorkflowRun(row: RunRow): WorkflowRunQueueRecord {
2741
- return {
2742
- runId: row.runId,
2743
- workflowName: row.workflowName,
2744
- workflowSourceRef: row.workflowRef,
2745
- workflowSource: readQueuedRunSources(this.state, row),
2746
- initialized: row.runStatus !== "queued",
2747
- definitionDigest: `sha256:${row.definitionDigest.toString("hex")}`,
2748
- input: this.state.readJson(row.inputHash),
2749
- launchOptions: this.state.readJson(row.launchOptionsHash),
2750
- status: row.status,
2751
- runnerId: row.ownerId,
2752
- claimToken: null,
2753
- claimGeneration: row.ownerId === null ? null : row.leaseGeneration,
2754
- claimExpiresAt:
2755
- row.claimExpiresAt === null ? null : new Date(row.claimExpiresAt).toISOString(),
2756
- affinityRunnerId: row.affinityRunnerId,
2757
- originSessionId: row.executionMode === "headless" ? null : row.originSessionId,
2758
- executionMode: row.executionMode,
2759
- parentRunId: row.parentRunId,
2760
- rootRunId: row.rootRunId,
2761
- lineageKind: row.lineageKind,
2762
- restartNumber: row.restartNumber,
2763
- parentTerminalFingerprint: row.parentTerminalFingerprint?.toString("hex") ?? null,
2764
- errorCode: row.errorCode,
2765
- errorMessage:
2766
- row.errorHash === null
2767
- ? null
2768
- : (this.state.readBlob(row.errorHash)?.content.toString("utf8") ?? null),
2769
- createdAt: new Date(row.createdAt).toISOString(),
2770
- updatedAt: new Date(row.updatedAt).toISOString(),
2771
- startedAt: row.startedAt === null ? null : new Date(row.startedAt).toISOString(),
2772
- finishedAt: row.finishedAt === null ? null : new Date(row.finishedAt).toISOString(),
2773
- };
2774
- }
2775
-
2776
- private claimRun(
2777
- runId: string,
2778
- runnerId: string,
2779
- claimToken: string,
2780
- leaseMs: number,
2781
- now: number,
2782
- options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
2783
- ): WorkflowRunQueueRecord | undefined {
2784
- return this.state.transaction(() =>
2785
- this.claimRunInTransaction(runId, runnerId, claimToken, leaseMs, now, options),
2786
- );
2787
- }
2788
-
2789
- private claimRunInTransaction(
2790
- runId: string,
2791
- runnerId: string,
2792
- claimToken: string,
2793
- leaseMs: number,
2794
- now: number,
2795
- options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
2796
- ): WorkflowRunQueueRecord | undefined {
2797
- const row = this.workflowRunRow(runId);
2798
- if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return undefined;
2799
- if (
2800
- options.allowPendingInteraction !== true &&
2801
- this.state.connection
2802
- .prepare(
2803
- `SELECT 1 FROM interactive_requests
2804
- WHERE run_id = ? AND status = 'pending' LIMIT 1`,
2805
- )
2806
- .get(runId) !== undefined
2807
- ) {
2808
- return undefined;
2809
- }
2810
- const lease = this.requireLease(row.resourceId);
2811
- if (
2812
- lease.ownerId !== null &&
2813
- lease.expiresAt !== null &&
2814
- lease.expiresAt > now &&
2815
- lease.ownerId !== runnerId
2816
- )
2817
- return undefined;
2818
- const generation = lease.generation + 1;
2819
- const expiresAt = now + leaseMs;
2820
- const result = this.state.connection
2821
- .prepare(
2822
- `UPDATE leases SET generation = ?, owner_type = ?, owner_id = ?, token_hash = ?,
2823
- acquired_at = ?, heartbeat_at = ?, expires_at = ?
2824
- WHERE resource_id = ? AND generation = ?`,
2825
- )
2826
- .run(
2827
- generation,
2828
- runnerId.startsWith("host-") ? "host" : "session",
2829
- runnerId,
2830
- tokenHash(claimToken),
2831
- now,
2832
- now,
2833
- expiresAt,
2834
- row.resourceId,
2835
- lease.generation,
2836
- );
2837
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2838
- if (result.changes !== 1) return undefined;
2839
- if (options.preserveQueueStatus !== true) {
2840
- this.state.connection
2841
- .prepare(
2842
- `UPDATE run_queue
2843
- SET status = 'starting', error_code = NULL, error_hash = NULL, updated_at = ?
2844
- WHERE run_id = ?`,
2845
- )
2846
- .run(now, runId);
2847
- }
2848
- const revision = this.resourceRevision(row.resourceId);
2849
- this.bumpResource(row.resourceId, revision, now);
2850
- this.insertEvent(
2851
- row.resourceId,
2852
- revision + 1,
2853
- "lease.claimed",
2854
- runnerId.startsWith("host-") ? "host" : "session",
2855
- runnerId,
2856
- { expiresAt },
2857
- now,
2858
- generation,
2859
- );
2860
- return { ...this.requireWorkflowRun(runId), claimToken };
2861
- }
2862
-
2863
- private updateClaimedRunStatus(
2864
- runId: string,
2865
- claimToken: string,
2866
- status: WorkflowRunLaunchStatus,
2867
- nowValue?: string,
2868
- ): boolean {
2869
- const now = epoch(validTimestamp(nowValue));
2870
- return this.state.transaction(() => {
2871
- if (
2872
- !this.verifyWorkflowRunClaim({
2873
- runId,
2874
- claimToken,
2875
- now: new Date(now).toISOString(),
2876
- })
2877
- ) {
2878
- return false;
2879
- }
2880
- const row = this.requireWorkflowRunRow(runId);
2881
- const changed =
2882
- this.state.connection
2883
- .prepare(
2884
- "UPDATE run_queue SET status = ?, started_at = COALESCE(started_at, ?), updated_at = ? WHERE run_id = ?",
2885
- )
2886
- .run(status, now, now, runId).changes === 1;
2887
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2888
- if (!changed) return false;
2889
- const revision = this.resourceRevision(row.resourceId);
2890
- const lease = this.requireLease(row.resourceId);
2891
- this.bumpResource(row.resourceId, revision, now);
2892
- this.insertEvent(
2893
- row.resourceId,
2894
- revision + 1,
2895
- `run.queue_${status}`,
2896
- lease.ownerType ?? "system",
2897
- lease.ownerId,
2898
- { status },
2899
- now,
2900
- lease.generation || undefined,
2901
- );
2902
- return true;
2903
- });
2904
- }
2905
-
2906
- private releaseRunClaim(
2907
- runId: string,
2908
- claimToken: string,
2909
- status: WorkflowRunLaunchStatus,
2910
- nowValue?: string,
2911
- ): boolean {
2912
- const now = epoch(validTimestamp(nowValue));
2913
- return this.state.transaction(() => {
2914
- const row = this.workflowRunRow(runId);
2915
- if (
2916
- row === undefined ||
2917
- !this.verifyWorkflowRunClaim({ runId, claimToken, now: new Date(now).toISOString() })
2918
- )
2919
- return false;
2920
- this.state.connection
2921
- .prepare(
2922
- "UPDATE run_queue SET status = ?, updated_at = ?, finished_at = ? WHERE run_id = ?",
2923
- )
2924
- .run(status, now, status === "done" ? now : null, runId);
2925
- const lease = this.requireLease(row.resourceId);
2926
- const update = this.state.connection
2927
- .prepare(
2928
- `UPDATE leases SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
2929
- acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
2930
- WHERE resource_id = ? AND token_hash = ? AND generation = ?`,
2931
- )
2932
- .run(row.resourceId, tokenHash(claimToken), lease.generation);
2933
- /* istanbul ignore if -- impossible after exact schema and transaction checks */
2934
- if (update.changes !== 1) return false;
2935
- const revision = this.resourceRevision(row.resourceId);
2936
- this.bumpResource(row.resourceId, revision, now);
2937
- this.insertEvent(
2938
- row.resourceId,
2939
- revision + 1,
2940
- `run.queue_${status}`,
2941
- lease.ownerType ?? "system",
2942
- lease.ownerId,
2943
- { status },
2944
- now,
2945
- lease.generation || undefined,
2946
- );
2947
- recordViewerDeltas(
2948
- this.state,
2949
- runId,
2950
- [{ targetType: "summary" }, { targetType: "replay" }],
2951
- now,
2952
- );
2953
- return true;
2954
- });
2955
- }
2956
-
2957
- private terminalRun(
2958
- runId: string,
2959
- claimToken: string | undefined,
2960
- status: "failed" | "cancelled",
2961
- code: string,
2962
- error: string,
2963
- nowValue?: string,
2964
- ): boolean {
2965
- const now = epoch(validTimestamp(nowValue));
2966
- return this.state.transaction(() => {
2967
- const row = this.workflowRunRow(runId);
2968
- if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return false;
2969
- const lease = this.requireLease(row.resourceId);
2970
- if (claimToken === undefined) {
2971
- if (!(["queued", "starting"] as WorkflowRunLaunchStatus[]).includes(row.status)) {
2972
- return false;
2973
- }
2974
- if (lease.ownerId !== null) return false;
2975
- } else if (
2976
- lease.ownerId === null ||
2977
- lease.tokenHash === null ||
2978
- lease.expiresAt === null ||
2979
- lease.expiresAt <= now ||
2980
- !lease.tokenHash.equals(tokenHash(claimToken))
2981
- ) {
2982
- return false;
2983
- }
2984
- const revision = this.resourceRevision(row.resourceId);
2985
- const errorHash = this.state.putText(error, now);
2986
- const queueUpdate = this.state.connection
2987
- .prepare(
2988
- `UPDATE run_queue
2989
- SET status = ?, error_code = ?, error_hash = ?, updated_at = ?, finished_at = ?
2990
- WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
2991
- )
2992
- .run(status, code, errorHash, now, now, runId);
2993
- if (queueUpdate.changes !== 1) return false;
2994
- if (claimToken !== undefined) {
2995
- const leaseUpdate = this.state.connection
2996
- .prepare(
2997
- `UPDATE leases
2998
- SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
2999
- acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
3000
- WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
3001
- )
3002
- .run(row.resourceId, tokenHash(claimToken), lease.generation, now);
3003
- if (leaseUpdate.changes !== 1) {
3004
- throw new Error(`Workflow run ${runId} claim changed during terminal transition`);
3005
- }
3006
- }
3007
- this.state.connection
3008
- .prepare(
3009
- `UPDATE runs
3010
- SET status = ?, paused = 0, status_detail = NULL, error_hash = ?,
3011
- updated_at = ?, finished_at = ?
3012
- WHERE run_id = ?`,
3013
- )
3014
- .run(status, errorHash, now, now, runId);
3015
- this.settleWorkflowRunMessages(runId, now);
3016
- if (status === "cancelled") {
3017
- this.cancelWorkflowRunDependents(
3018
- runId,
3019
- lease.ownerId ?? "workflow-control",
3020
- errorHash,
3021
- now,
3022
- );
3023
- }
3024
- this.bumpResource(row.resourceId, revision, now);
3025
- this.insertEvent(
3026
- row.resourceId,
3027
- revision + 1,
3028
- `run.queue_${status}`,
3029
- lease.ownerType ?? "system",
3030
- lease.ownerId,
3031
- { status, code, error },
3032
- now,
3033
- lease.generation || undefined,
3034
- );
3035
- recordViewerDeltas(
3036
- this.state,
3037
- runId,
3038
- [{ targetType: "summary" }, { targetType: "replay" }],
3039
- now,
3040
- );
3041
- return true;
3042
- });
3043
- }
3044
-
3045
- private settleWorkflowRunMessages(runId: string, now: number): void {
3046
- this.workflowMessages.settleOpenTurnsForRun(runId, "lost", now);
3047
- this.workflowMessages.cancelPendingForRun(runId, now);
3048
- }
3049
-
3050
- private cancelWorkflowRunDependents(
3051
- runId: string,
3052
- actorId: string,
3053
- errorHash: Buffer,
3054
- now: number,
3055
- ): void {
3056
- this.state.connection
3057
- .prepare(
3058
- `UPDATE node_attempts
3059
- SET status = 'cancelled', error_hash = COALESCE(error_hash, ?),
3060
- updated_at = ?, finished_at = COALESCE(finished_at, ?)
3061
- WHERE run_id = ? AND status IN ('pending', 'running', 'waiting', 'interrupted')`,
3062
- )
3063
- .run(errorHash, now, now, runId);
3064
-
3065
- const effects = this.state.connection
3066
- .prepare(
3067
- `SELECT e.effect_id AS effectId, e.resource_id AS resourceId,
3068
- e.status, e.attempt_count AS attemptCount
3069
- FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
3070
- WHERE r.run_id = ? AND e.owner_scope = 'run' AND e.status IN ('pending', 'applying')
3071
- ORDER BY e.effect_id`,
3072
- )
3073
- .all(runId)
3074
- .filter(isCancelledRunEffectRow);
3075
- for (const effect of effects) {
3076
- const status = effect.status === "applying" ? "ambiguous" : "cancelled";
3077
- const revision = this.resourceRevision(effect.resourceId);
3078
- const changed = this.state.connection
3079
- .prepare(
3080
- `UPDATE effects
3081
- SET status = ?, next_attempt_at = NULL, error_hash = ?, updated_at = ?, settled_at = ?
3082
- WHERE effect_id = ? AND status = ? AND attempt_count = ?`,
3083
- )
3084
- .run(status, errorHash, now, now, effect.effectId, effect.status, effect.attemptCount);
3085
- /* istanbul ignore if -- cancellation serializes the selected effect transition */
3086
- if (changed.changes !== 1) {
3087
- throw new Error(`Workflow effect ${effect.effectId} changed during cancellation`);
3088
- }
3089
- if (effect.status === "applying") {
3090
- this.state.connection
3091
- .prepare(
3092
- `UPDATE effect_attempts
3093
- SET finished_at = ?, outcome = 'interrupted', error_hash = ?
3094
- WHERE effect_id = ? AND attempt_number = ? AND finished_at IS NULL`,
3095
- )
3096
- .run(now, errorHash, effect.effectId, effect.attemptCount);
3097
- }
3098
- this.bumpResource(effect.resourceId, revision, now);
3099
- this.insertEvent(
3100
- effect.resourceId,
3101
- revision + 1,
3102
- `effect.${status}`,
3103
- "control",
3104
- actorId,
3105
- { runId, reason: "workflowCancelled" },
3106
- now,
3107
- );
3108
- }
3109
-
3110
- this.state.connection
3111
- .prepare(
3112
- `INSERT INTO human_decision_resolutions(
3113
- decision_id, outcome, provenance, response_hash, reason, channel,
3114
- actor_id, request_digest, resolved_at
3115
- )
3116
- SELECT d.decision_id, 'cancelled', 'explicit_cancel', NULL,
3117
- 'Workflow run cancelled', NULL, ?, d.request_digest, ?
3118
- FROM human_decisions d
3119
- LEFT JOIN human_decision_resolutions r ON r.decision_id = d.decision_id
3120
- WHERE d.run_id = ? AND r.decision_id IS NULL`,
3121
- )
3122
- .run(actorId, now, runId);
3123
- const receiptHash = this.state.putJson(
3124
- { status: "rejected", error: "Workflow run cancelled" },
3125
- now,
3126
- );
3127
- this.state.connection
3128
- .prepare(
3129
- `UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
3130
- WHERE outcome = 'validating'
3131
- AND request_id IN (SELECT request_id FROM interactive_requests WHERE run_id = ?)`,
3132
- )
3133
- .run(receiptHash, runId);
3134
- this.state.connection
3135
- .prepare(
3136
- `UPDATE interactive_requests
3137
- SET status = 'cancelled', revision = revision + 1, updated_at = ?
3138
- WHERE run_id = ? AND status = 'pending'`,
3139
- )
3140
- .run(now, runId);
3141
- }
3142
-
3143
- private assertWritable(): void {
3144
- if (this.state.mode === "read-only") throw new Error("ResourceManager store is read-only");
1192
+ private workflowRow(resourceUid: string, requestKey: string): WorkflowRow | undefined {
1193
+ const row = this.state.connection
1194
+ .prepare(workflowSelect("WHERE c.uid = ? AND w.request_key = ?"))
1195
+ .get(resourceUid, requestKey);
1196
+ return isWorkflowRow(row) ? row : undefined;
3145
1197
  }
3146
- }
3147
-
3148
- function resourceManagerSelect(clause: string): string {
3149
- return `SELECT c.controller_resource_id AS managedResourceId,
3150
- c.resource_id AS resourceId, r.revision AS resourceVersion,
3151
- c.controller_name AS resourceManagerName, c.resource_key AS resourceKey,
3152
- c.uid, c.generation, c.spec_hash AS specHash, c.status_hash AS statusHash,
3153
- c.deletion_requested_at AS deletionRequestedAt
3154
- FROM controller_resources c JOIN resources r ON r.resource_id = c.resource_id ${clause}`;
3155
- }
3156
-
3157
- function effectSelect(clause: string): string {
3158
- return `SELECT e.effect_id AS effectId, c.uid AS resourceUid, c.generation,
3159
- e.effect_type AS kind, e.status AS state,
3160
- json_extract(CAST(b.content AS TEXT), '$.requestFingerprint') AS requestFingerprint,
3161
- e.created_at AS startedAt, e.settled_at AS completedAt,
3162
- e.external_ref AS externalRef, e.error_hash AS errorHash
3163
- FROM effects e
3164
- JOIN controller_resources c ON c.resource_id = e.source_resource_id
3165
- JOIN blobs b ON b.blob_hash = e.payload_hash ${clause}`;
3166
- }
3167
-
3168
- function workflowSelect(clause: string): string {
3169
- return `SELECT w.request_id AS requestId, c.uid AS resourceUid,
3170
- w.request_key AS requestKey, w.input_fingerprint AS inputFingerprint,
3171
- w.workflow_name AS workflowName, COALESCE(w.run_id, w.reserved_run_id) AS runId, w.status,
3172
- w.attempt_count AS attemptCount, w.error_hash AS errorHash
3173
- FROM controller_workflows w
3174
- JOIN controller_resources c ON c.controller_resource_id = w.controller_resource_id ${clause}`;
3175
- }
3176
-
3177
- function workflowRunSelect(clause: string): string {
3178
- return `SELECT r.run_id AS runId, r.resource_id AS resourceId,
3179
- d.workflow_name AS workflowName, r.workflow_ref AS workflowRef, r.status AS runStatus,
3180
- r.paused, r.definition_digest AS definitionDigest, d.definition_hash AS definitionHash,
3181
- r.input_hash AS inputHash,
3182
- r.launch_options_hash AS launchOptionsHash,
3183
- q.status, q.available_at AS availableAt, q.affinity_runner_id AS affinityRunnerId,
3184
- q.consecutive_errors AS consecutiveErrors, q.error_code AS errorCode,
3185
- q.error_hash AS errorHash, b.origin_session_id AS originSessionId,
3186
- b.execution_mode AS executionMode, r.parent_run_id AS parentRunId,
3187
- r.root_run_id AS rootRunId, r.lineage_kind AS lineageKind,
3188
- r.restart_number AS restartNumber,
3189
- r.parent_terminal_fingerprint AS parentTerminalFingerprint,
3190
- q.created_at AS createdAt, q.updated_at AS updatedAt,
3191
- q.started_at AS startedAt, q.finished_at AS finishedAt,
3192
- l.generation AS leaseGeneration, l.owner_id AS ownerId, l.expires_at AS claimExpiresAt
3193
- FROM runs r JOIN workflow_definitions d ON d.definition_digest = r.definition_digest
3194
- JOIN run_queue q ON q.run_id = r.run_id
3195
- LEFT JOIN run_bindings b ON b.run_id = r.run_id
3196
- JOIN leases l ON l.resource_id = r.resource_id ${clause}`;
3197
- }
3198
-
3199
- function workflowRunViewSelect(clause: string): string {
3200
- return `SELECT r.run_id AS runId, d.workflow_name AS workflowName,
3201
- r.workflow_ref AS workflowRef, r.status AS runStatus, r.paused,
3202
- r.definition_digest AS definitionDigest, q.status,
3203
- b.origin_session_id AS originSessionId, b.execution_mode AS executionMode,
3204
- r.parent_run_id AS parentRunId, r.root_run_id AS rootRunId,
3205
- r.lineage_kind AS lineageKind, r.restart_number AS restartNumber,
3206
- r.parent_terminal_fingerprint AS parentTerminalFingerprint,
3207
- q.error_code AS errorCode, CAST(error.content AS TEXT) AS errorMessage,
3208
- q.created_at AS createdAt, q.updated_at AS updatedAt,
3209
- q.started_at AS startedAt, q.finished_at AS finishedAt,
3210
- source.source_type AS sourceType, source.source_ref AS sourceRef,
3211
- source.source_revision AS sourceRevision
3212
- FROM runs r JOIN workflow_definitions d ON d.definition_digest = r.definition_digest
3213
- JOIN run_queue q ON q.run_id = r.run_id
3214
- LEFT JOIN run_bindings b ON b.run_id = r.run_id
3215
- LEFT JOIN run_sources source ON source.run_id = r.run_id AND source.mount_path = ''
3216
- LEFT JOIN blobs error ON error.blob_hash = q.error_hash ${clause}`;
3217
- }
3218
-
3219
- type QueueListRow = {
3220
- resourceManager: string;
3221
- resourceKey: string;
3222
- availableAt: number;
3223
- consecutiveErrors: number;
3224
- claimExpiresAt: number | null;
3225
- };
3226
- type ResourceManagerEventRow = {
3227
- seq: number;
3228
- recordedAt: number;
3229
- resourceManager: string;
3230
- resourceKey: string;
3231
- eventType: string;
3232
- payloadHash: Buffer | null;
3233
- };
3234
- type RunEventRow = {
3235
- seq: number;
3236
- recordedAt: number;
3237
- runId: string;
3238
- workflowRef: string;
3239
- eventType: string;
3240
- runnerId: string | null;
3241
- payloadHash: Buffer | null;
3242
- };
3243
- function isRecord(value: unknown): value is Record<string, unknown> {
3244
- return typeof value === "object" && value !== null && !Array.isArray(value);
3245
- }
3246
- function isResourceManagerRow(value: unknown): value is ResourceManagerRow {
3247
- return isRecord(value);
3248
- }
3249
- function isLeaseRow(value: unknown): value is LeaseRow {
3250
- return isRecord(value);
3251
- }
3252
- function workflowRunViewRecord(row: WorkflowRunViewRow): WorkflowRunQueueViewRecord {
3253
- const root =
3254
- row.sourceType === "builtin"
3255
- ? { kind: "builtin", id: row.sourceRef, revision: row.sourceRevision }
3256
- : { kind: "file", path: row.sourceRef, hash: row.sourceRevision };
3257
- return {
3258
- runId: row.runId,
3259
- workflowName: row.workflowName,
3260
- workflowSourceRef: row.workflowRef,
3261
- workflowSource: { root, mounted: [] },
3262
- initialized: row.runStatus !== "queued",
3263
- definitionDigest: `sha256:${row.definitionDigest.toString("hex")}`,
3264
- runStateStatus: row.runStatus,
3265
- paused: row.paused === 1,
3266
- status: row.status,
3267
- originSessionId: row.executionMode === "headless" ? null : row.originSessionId,
3268
- executionMode: row.executionMode,
3269
- parentRunId: row.parentRunId,
3270
- rootRunId: row.rootRunId,
3271
- lineageKind: row.lineageKind,
3272
- restartNumber: row.restartNumber,
3273
- parentTerminalFingerprint: row.parentTerminalFingerprint?.toString("hex") ?? null,
3274
- errorCode: row.errorCode,
3275
- errorMessage: row.errorMessage,
3276
- createdAt: new Date(row.createdAt).toISOString(),
3277
- updatedAt: new Date(row.updatedAt).toISOString(),
3278
- startedAt: row.startedAt === null ? null : new Date(row.startedAt).toISOString(),
3279
- finishedAt: row.finishedAt === null ? null : new Date(row.finishedAt).toISOString(),
3280
- };
3281
- }
3282
-
3283
- function isWorkflowRunViewRow(value: unknown): value is WorkflowRunViewRow {
3284
- return (
3285
- isRecord(value) &&
3286
- typeof value.runId === "string" &&
3287
- typeof value.workflowName === "string" &&
3288
- typeof value.workflowRef === "string" &&
3289
- typeof value.runStatus === "string" &&
3290
- typeof value.paused === "number" &&
3291
- Buffer.isBuffer(value.definitionDigest) &&
3292
- typeof value.status === "string" &&
3293
- (value.originSessionId === null || typeof value.originSessionId === "string") &&
3294
- (value.executionMode === "interactive" || value.executionMode === "headless") &&
3295
- (value.parentRunId === null || typeof value.parentRunId === "string") &&
3296
- typeof value.rootRunId === "string" &&
3297
- (value.lineageKind === null ||
3298
- value.lineageKind === "continuation" ||
3299
- value.lineageKind === "restart") &&
3300
- typeof value.restartNumber === "number" &&
3301
- (value.parentTerminalFingerprint === null ||
3302
- Buffer.isBuffer(value.parentTerminalFingerprint)) &&
3303
- (value.errorCode === null || typeof value.errorCode === "string") &&
3304
- (value.errorMessage === null || typeof value.errorMessage === "string") &&
3305
- typeof value.createdAt === "number" &&
3306
- typeof value.updatedAt === "number" &&
3307
- (value.startedAt === null || typeof value.startedAt === "number") &&
3308
- (value.finishedAt === null || typeof value.finishedAt === "number") &&
3309
- (value.sourceType === "builtin" || value.sourceType === "file") &&
3310
- typeof value.sourceRef === "string" &&
3311
- typeof value.sourceRevision === "string"
3312
- );
3313
- }
3314
-
3315
- function isRunListRevisionRow(value: unknown): value is RunListRevisionRow {
3316
- return (
3317
- isRecord(value) &&
3318
- typeof value.count === "number" &&
3319
- typeof value.revisionSum === "number" &&
3320
- typeof value.updatedAt === "number"
3321
- );
3322
- }
3323
-
3324
- function isRunRow(value: unknown): value is RunRow {
3325
- return isRecord(value);
3326
- }
3327
- function isRunSourceIdentityRow(value: unknown): value is RunSourceIdentityRow {
3328
- return isRecord(value);
3329
- }
3330
- function isEffectRow(value: unknown): value is EffectRow {
3331
- return isRecord(value);
3332
- }
3333
- function isCancelledRunEffectRow(value: unknown): value is CancelledRunEffectRow {
3334
- return (
3335
- isRecord(value) &&
3336
- typeof value.effectId === "string" &&
3337
- typeof value.resourceId === "string" &&
3338
- (value.status === "pending" || value.status === "applying") &&
3339
- typeof value.attemptCount === "number"
3340
- );
3341
- }
3342
- function isExpiredInteractionRow(value: unknown): value is ExpiredInteractionRow {
3343
- return (
3344
- isRecord(value) && typeof value.requestId === "string" && typeof value.attemptId === "string"
3345
- );
3346
- }
3347
- function isWorkflowRow(value: unknown): value is WorkflowRow {
3348
- return isRecord(value);
3349
- }
3350
- function isQueueListRow(value: unknown): value is QueueListRow {
3351
- return isRecord(value);
3352
- }
3353
- function isResourceManagerEventRow(value: unknown): value is ResourceManagerEventRow {
3354
- return isRecord(value);
3355
- }
3356
- function isRunEventRow(value: unknown): value is RunEventRow {
3357
- return isRecord(value);
3358
- }
3359
- function isProjectRow(value: unknown): value is { projectId: string } {
3360
- return isRecord(value);
3361
- }
3362
- function isCanonicalPathRow(value: unknown): value is { canonicalPath: string } {
3363
- return isRecord(value) && typeof value.canonicalPath === "string";
3364
- }
3365
- function isFinalizerRow(value: unknown): value is { finalizer: string } {
3366
- return isRecord(value);
3367
- }
3368
- function isErrorCountRow(value: unknown): value is { consecutiveErrors: number } {
3369
- return isRecord(value);
3370
- }
3371
- function isQueueVersionRow(value: unknown): value is { queueVersion: number } {
3372
- return isRecord(value);
3373
- }
3374
- function isRevisionRow(value: unknown): value is { revision: number } {
3375
- return isRecord(value);
3376
- }
3377
- function isSequenceRow(value: unknown): value is { seq: number } {
3378
- return isRecord(value);
3379
- }
3380
- function isKeyRow(value: unknown): value is { key: string } {
3381
- return isRecord(value);
3382
- }
3383
- function isEffectIdentityRow(value: unknown): value is { effectId: string; resourceId: string } {
3384
- return isRecord(value);
3385
- }
3386
1198
 
3387
- function effectIdFor(sourceResourceId: string, key: string): string {
3388
- return `effect-${createHash("sha256").update(`${sourceResourceId}\0${key}`).digest("hex").slice(0, 40)}`;
3389
- }
3390
- function projectIdFor(canonicalPath: string): string {
3391
- return `project-${createHash("sha256").update(canonicalPath).digest("hex").slice(0, 40)}`;
3392
- }
3393
- function canonicalProjectPath(value: string): string {
3394
- try {
3395
- return fs.realpathSync.native(value);
3396
- } catch {
3397
- return path.resolve(value);
1199
+ private workflowRowById(requestId: string): WorkflowRow | undefined {
1200
+ const row = this.state.connection
1201
+ .prepare(workflowSelect("WHERE w.request_id = ?"))
1202
+ .get(requestId);
1203
+ return isWorkflowRow(row) ? row : undefined;
3398
1204
  }
3399
- }
3400
- function digestBuffer(value: string): Buffer {
3401
- const hex = value.startsWith("sha256:") ? value.slice(7) : value;
3402
- if (!/^[a-f0-9]{64}$/i.test(hex)) throw new Error("Expected a SHA-256 digest");
3403
- return Buffer.from(hex, "hex");
3404
- }
3405
- function terminalFingerprintBuffer(value: string): Buffer {
3406
- const hex = value.startsWith("sha256:") ? value.slice(7) : value;
3407
- if (!/^[a-f0-9]{64}$/i.test(hex)) throw new Error("Expected a terminal fingerprint");
3408
- return Buffer.from(hex, "hex");
3409
- }
3410
- type QueuedSource = {
3411
- root:
3412
- | { kind: "builtin"; id: string; revision: string }
3413
- | { kind: "file"; path: string; hash: string };
3414
- mounted: Array<{
3415
- mountPath: string[];
3416
- workflowName: string;
3417
- source:
3418
- | { kind: "builtin"; id: string; revision: string }
3419
- | { kind: "file"; path: string; hash: string };
3420
- }>;
3421
- };
3422
1205
 
3423
- function queuedWorkflowSource(value: unknown): QueuedSource {
3424
- if (isWorkflowSource(value)) {
3425
- return { root: value, mounted: [] };
3426
- }
3427
- if (
3428
- isRecord(value) &&
3429
- isWorkflowSource(value.root) &&
3430
- Array.isArray(value.mounted) &&
3431
- value.mounted.every(isMountedWorkflowSource)
3432
- ) {
3433
- return { root: value.root, mounted: value.mounted };
1206
+ private requireWorkflowRow(requestId: string): WorkflowRow {
1207
+ const row = this.workflowRowById(requestId);
1208
+ if (row === undefined) throw new Error(`Workflow request not found: ${requestId}`);
1209
+ return row;
3434
1210
  }
3435
- throw new Error("Stored workflow source identity is invalid");
3436
- }
3437
-
3438
- function isWorkflowSource(value: unknown): value is QueuedSource["root"] {
3439
- return (
3440
- isRecord(value) &&
3441
- ((value.kind === "builtin" &&
3442
- typeof value.id === "string" &&
3443
- typeof value.revision === "string") ||
3444
- (value.kind === "file" && typeof value.path === "string" && typeof value.hash === "string"))
3445
- );
3446
- }
3447
1211
 
3448
- function isMountedWorkflowSource(value: unknown): value is QueuedSource["mounted"][number] {
3449
- return (
3450
- isRecord(value) &&
3451
- Array.isArray(value.mountPath) &&
3452
- value.mountPath.every((part) => typeof part === "string") &&
3453
- typeof value.workflowName === "string" &&
3454
- isWorkflowSource(value.source)
3455
- );
3456
- }
3457
-
3458
- function insertQueuedRunSources(state: StateDatabase, runId: string, value: QueuedSource): void {
3459
- const insert = state.connection.prepare(
3460
- `INSERT INTO run_sources(run_id, mount_path, source_type, source_ref, source_revision)
3461
- VALUES (?, ?, ?, ?, ?)`,
3462
- );
3463
- const root = queuedSourceParts(value.root);
3464
- insert.run(runId, "", root.type, root.ref, root.revision);
3465
- for (const mounted of value.mounted) {
3466
- const source = queuedSourceParts(mounted.source);
3467
- insert.run(runId, mounted.mountPath.join("/"), source.type, source.ref, source.revision);
1212
+ /* istanbul ignore next -- pure projection covered by integration tests */
1213
+ private mapWorkflow(row: WorkflowRow): ChildWorkflowRecord {
1214
+ return {
1215
+ requestId: row.requestId,
1216
+ resourceUid: row.resourceUid,
1217
+ requestKey: row.requestKey,
1218
+ inputFingerprint: row.inputFingerprint.toString("hex"),
1219
+ workflow: row.workflowName,
1220
+ ...(row.runId === null ? {} : { runId: row.runId }),
1221
+ state: row.status,
1222
+ attempt: row.attemptCount,
1223
+ ...(row.errorHash === null
1224
+ ? {}
1225
+ : { error: this.state.readBlob(row.errorHash)?.content.toString("utf8") ?? "" }),
1226
+ };
3468
1227
  }
3469
1228
  }
3470
-
3471
- function queuedSourceParts(source: QueuedSource["root"]): {
3472
- type: "builtin" | "file";
3473
- ref: string;
3474
- revision: string;
3475
- } {
3476
- return source.kind === "builtin"
3477
- ? { type: "builtin", ref: source.id, revision: source.revision }
3478
- : { type: "file", ref: source.path, revision: source.hash };
3479
- }
3480
-
3481
- function readQueuedRunSources(state: StateDatabase, run: RunRow): QueuedSource {
3482
- const rows = state.connection
3483
- .prepare(
3484
- `SELECT mount_path AS mountPath, source_type AS sourceType,
3485
- source_ref AS sourceRef, source_revision AS sourceRevision
3486
- FROM run_sources WHERE run_id = ? ORDER BY mount_path`,
3487
- )
3488
- .all(run.runId)
3489
- .filter(isRunSourceIdentityRow);
3490
- const rootRow = rows.find((row) => row.mountPath === "");
3491
- if (rootRow === undefined) throw new Error(`Workflow run source is missing: ${run.runId}`);
3492
- const snapshot = state.readJson(run.definitionHash);
3493
- const mounts =
3494
- isRecord(snapshot) &&
3495
- isRecord(snapshot.composition) &&
3496
- Array.isArray(snapshot.composition.mounts)
3497
- ? snapshot.composition.mounts
3498
- : [];
3499
- const names = new Map(
3500
- mounts.flatMap((mount) => {
3501
- if (
3502
- !isRecord(mount) ||
3503
- !Array.isArray(mount.mountPath) ||
3504
- !mount.mountPath.every((part) => typeof part === "string") ||
3505
- typeof mount.workflowName !== "string"
3506
- ) {
3507
- return [];
3508
- }
3509
- return [[mount.mountPath.join("/"), mount.workflowName] as const];
3510
- }),
3511
- );
3512
- return {
3513
- root: rowToSource(rootRow),
3514
- mounted: rows
3515
- .filter((row) => row.mountPath !== "")
3516
- .map((row) => ({
3517
- mountPath: row.mountPath.split("/"),
3518
- workflowName: names.get(row.mountPath) ?? row.mountPath,
3519
- source: rowToSource(row),
3520
- })),
3521
- };
3522
- }
3523
-
3524
- function rowToSource(row: RunSourceIdentityRow): QueuedSource["root"] {
3525
- return row.sourceType === "builtin"
3526
- ? { kind: "builtin", id: row.sourceRef, revision: row.sourceRevision }
3527
- : { kind: "file", path: row.sourceRef, hash: row.sourceRevision };
3528
- }
3529
-
3530
- function effectStatus(value: EffectRecord["state"]): string {
3531
- return value === "indeterminate" ? "ambiguous" : value;
3532
- }
3533
- function resourceManagerEffectState(value: string): EffectRecord["state"] {
3534
- return value === "ambiguous" ? "indeterminate" : (value as EffectRecord["state"]);
3535
- }
3536
- function validTimestamp(value?: string): string {
3537
- const date = value === undefined ? new Date() : new Date(value);
3538
- if (!Number.isFinite(date.getTime())) throw new Error("Invalid timestamp");
3539
- return date.toISOString();
3540
- }
3541
- function epoch(value: string): number {
3542
- return Date.parse(value);
3543
- }
3544
- function validateName(value: string, label: string): void {
3545
- validateKey(value, label);
3546
- }
3547
- function validateKey(value: string, label: string): void {
3548
- if (value.length === 0 || value.length > 512) throw new Error(`${label} is invalid`);
3549
- }
3550
- function validateRunId(value: string): void {
3551
- if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(value))
3552
- throw new Error(`Invalid workflow run id: ${value}`);
3553
- }