@osolmaz/pi-workflows 0.16.5 → 0.16.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/builtins/autoimplement.workflow.d.ts +5 -1
- package/dist/builtins/autoimplement.workflow.js +29 -10
- package/dist/builtins/autoimplement.workflow.js.map +1 -1
- package/dist/client/client.js +3 -1
- package/dist/client/client.js.map +1 -1
- package/dist/client/protocol.d.ts +1 -1
- package/dist/client/protocol.js +18 -1
- package/dist/client/protocol.js.map +1 -1
- package/dist/client/view.d.ts +7 -5
- package/dist/extension/index.d.ts +2 -1
- package/dist/extension/index.js +116 -166
- package/dist/extension/index.js.map +1 -1
- package/dist/extension/recorder.d.ts +1 -1
- package/dist/extension/recorder.js +2 -2
- package/dist/extension/recorder.js.map +1 -1
- package/dist/extension/step-message.d.ts +1 -1
- package/dist/extension/step-message.js +4 -4
- package/dist/extension/step-message.js.map +1 -1
- package/dist/extension/widget.js +1 -1
- package/dist/extension/widget.js.map +1 -1
- package/dist/extension/workflow-message-coordinator.d.ts +12 -3
- package/dist/extension/workflow-message-coordinator.js +108 -94
- package/dist/extension/workflow-message-coordinator.js.map +1 -1
- package/dist/render/graph-render.js +1 -1
- package/dist/resource-managers/index.d.ts +1 -2
- package/dist/resource-managers/index.js +1 -2
- package/dist/resource-managers/index.js.map +1 -1
- package/dist/resource-managers/sqlite.d.ts +2 -295
- package/dist/resource-managers/sqlite.js +71 -1528
- package/dist/resource-managers/sqlite.js.map +1 -1
- package/dist/resource-managers/types.d.ts +2 -7
- package/dist/server/resource-runner-entry.js +2 -6
- package/dist/server/resource-runner-entry.js.map +1 -1
- package/dist/server/rpc-bridge.js +1 -1
- package/dist/server/rpc-bridge.js.map +1 -1
- package/dist/server/rpc-executor.js +2 -4
- package/dist/server/rpc-executor.js.map +1 -1
- package/dist/server/server-entry.js +3 -0
- package/dist/server/server-entry.js.map +1 -1
- package/dist/server/server.d.ts +9 -8
- package/dist/server/server.js +364 -541
- package/dist/server/server.js.map +1 -1
- package/dist/server/state.d.ts +16 -29
- package/dist/server/state.js +105 -319
- package/dist/server/state.js.map +1 -1
- package/dist/server/view.d.ts +6 -5
- package/dist/server/view.js +68 -51
- package/dist/server/view.js.map +1 -1
- package/dist/server/workflow-runner-entry.d.ts +1 -1
- package/dist/server/workflow-runner-entry.js +1 -49
- package/dist/server/workflow-runner-entry.js.map +1 -1
- package/dist/server/workflow-runner-protocol.d.ts +2 -7
- package/dist/server/workflow-runner-protocol.js +5 -10
- package/dist/server/workflow-runner-protocol.js.map +1 -1
- package/dist/server/workflow-runner-store.d.ts +5 -6
- package/dist/server/workflow-runner-store.js +7 -14
- package/dist/server/workflow-runner-store.js.map +1 -1
- package/dist/state/attempt-time.d.ts +21 -0
- package/dist/state/attempt-time.js +95 -0
- package/dist/state/attempt-time.js.map +1 -0
- package/dist/state/json.d.ts +2 -1
- package/dist/state/json.js.map +1 -1
- package/dist/state/project-store.d.ts +39 -0
- package/dist/state/project-store.js +154 -0
- package/dist/state/project-store.js.map +1 -0
- package/dist/state/prune.js +0 -7
- package/dist/state/prune.js.map +1 -1
- package/dist/state/schema.js +23 -23
- package/dist/state/workflow-messages.d.ts +1 -1
- package/dist/state/workflow-messages.js +3 -1
- package/dist/state/workflow-messages.js.map +1 -1
- package/dist/viewer/backup.js +5 -0
- package/dist/viewer/backup.js.map +1 -1
- package/dist/workflows/diagnostics.d.ts +8 -0
- package/dist/workflows/diagnostics.js +61 -0
- package/dist/workflows/diagnostics.js.map +1 -0
- package/dist/workflows/engine.d.ts +2 -16
- package/dist/workflows/engine.js +128 -252
- package/dist/workflows/engine.js.map +1 -1
- package/dist/workflows/errors.d.ts +1 -1
- package/dist/workflows/errors.js +2 -2
- package/dist/workflows/errors.js.map +1 -1
- package/dist/workflows/human-decision.d.ts +3 -7
- package/dist/workflows/human-decision.js +2 -56
- package/dist/workflows/human-decision.js.map +1 -1
- package/dist/workflows/index.d.ts +2 -1
- package/dist/workflows/index.js +1 -0
- package/dist/workflows/index.js.map +1 -1
- package/dist/workflows/queue.d.ts +267 -0
- package/dist/workflows/queue.js +1248 -0
- package/dist/workflows/queue.js.map +1 -0
- package/dist/workflows/requests.d.ts +47 -0
- package/dist/workflows/requests.js +158 -0
- package/dist/workflows/requests.js.map +1 -0
- package/dist/workflows/schema.js +2 -4
- package/dist/workflows/schema.js.map +1 -1
- package/dist/workflows/store.d.ts +11 -16
- package/dist/workflows/store.js +177 -174
- package/dist/workflows/store.js.map +1 -1
- package/dist/workflows/tool-input.d.ts +6 -9
- package/dist/workflows/tool-input.js +15 -12
- package/dist/workflows/tool-input.js.map +1 -1
- package/dist/workflows/transitions.d.ts +38 -0
- package/dist/workflows/transitions.js +276 -0
- package/dist/workflows/transitions.js.map +1 -0
- package/dist/workflows/types.d.ts +9 -42
- package/dist/workflows/workflow-message-content.d.ts +7 -1
- package/dist/workflows/workflow-message-content.js +22 -4
- package/dist/workflows/workflow-message-content.js.map +1 -1
- package/docs/2026-08-25-workflow-follow-ups.md +5 -5
- package/docs/2026-09-02-unify-workflow-messages-plan.md +2 -0
- package/docs/2026-09-04-workflow-run-state-plan.md +2 -0
- package/docs/2026-09-06-durable-execution-plan.md +176 -0
- package/docs/2026-09-07-workflow-handoff-plan.md +84 -0
- package/docs/DEFERRED_TURNS.md +84 -105
- package/docs/HUMAN_DECISIONS.md +17 -19
- package/docs/HUMAN_DECISION_PRESENTATIONS.md +1 -1
- package/docs/SQLITE_STATE.md +68 -29
- package/docs/WORKFLOW_SERVER.md +21 -10
- package/docs/WORKFLOW_STEP_MESSAGES.md +83 -54
- package/docs/plans/2026-08-04-controller-runtime-plan.md +2 -0
- package/docs/plans/2026-08-21-deferred-turn-intents-plan.md +2 -0
- package/docs/plans/2026-08-27-workflow-terminal-restart-plan.md +2 -0
- package/docs/workflows.md +111 -45
- package/examples/workflows/autoresearch.workflow.ts +0 -2
- package/examples/workflows/branch.workflow.ts +0 -4
- package/examples/workflows/echo.workflow.ts +0 -2
- package/examples/workflows/live-settings.workflow.ts +0 -2
- package/examples/workflows/two-turn.workflow.ts +0 -2
- package/herdr-plugin.toml +1 -1
- package/package.json +1 -1
- package/protocol/fixtures/client-v1.json +1 -1
- package/skills/pi-workflows/SKILL.md +8 -7
- package/src/builtins/autoimplement.workflow.ts +31 -10
- package/src/client/client.ts +3 -1
- package/src/client/protocol.ts +21 -1
- package/src/client/view.ts +9 -5
- package/src/extension/index.ts +118 -190
- package/src/extension/recorder.ts +2 -2
- package/src/extension/step-message.ts +4 -4
- package/src/extension/widget.ts +1 -1
- package/src/extension/workflow-message-coordinator.ts +133 -95
- package/src/render/graph-render.ts +1 -1
- package/src/resource-managers/index.ts +1 -9
- package/src/resource-managers/sqlite.ts +330 -2655
- package/src/resource-managers/types.ts +2 -3
- package/src/server/resource-runner-entry.ts +2 -7
- package/src/server/rpc-bridge.ts +1 -1
- package/src/server/rpc-executor.ts +4 -9
- package/src/server/server-entry.ts +3 -0
- package/src/server/server.ts +416 -641
- package/src/server/state.ts +122 -466
- package/src/server/view.ts +73 -58
- package/src/server/workflow-runner-entry.ts +2 -55
- package/src/server/workflow-runner-protocol.ts +5 -15
- package/src/server/workflow-runner-store.ts +12 -27
- package/src/state/attempt-time.ts +122 -0
- package/src/state/json.ts +2 -1
- package/src/state/project-store.ts +218 -0
- package/src/state/prune.ts +0 -13
- package/src/state/schema.ts +23 -23
- package/src/state/workflow-messages.ts +4 -2
- package/src/viewer/backup.ts +7 -0
- package/src/workflows/diagnostics.ts +78 -0
- package/src/workflows/engine.ts +136 -299
- package/src/workflows/errors.ts +4 -2
- package/src/workflows/human-decision.ts +3 -81
- package/src/workflows/index.ts +8 -2
- package/src/workflows/queue.ts +2091 -0
- package/src/workflows/requests.ts +283 -0
- package/src/workflows/schema.ts +2 -6
- package/src/workflows/store.ts +226 -236
- package/src/workflows/tool-input.ts +20 -12
- package/src/workflows/transitions.ts +330 -0
- package/src/workflows/types.ts +9 -46
- package/src/workflows/workflow-message-content.ts +28 -4
- package/dist/resource-managers/runtime.d.ts +0 -59
- package/dist/resource-managers/runtime.js +0 -423
- package/dist/resource-managers/runtime.js.map +0 -1
- package/src/resource-managers/runtime.ts +0 -576
|
@@ -0,0 +1,2091 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { closeRunTime } from "../state/attempt-time.js";
|
|
3
|
+
import { StateDatabase } from "../state/database.js";
|
|
4
|
+
import { canonicalJson, type JsonObject } from "../state/json.js";
|
|
5
|
+
import { resourceIdFor, tokenHash } from "../state/mutation.js";
|
|
6
|
+
import {
|
|
7
|
+
ProjectStore,
|
|
8
|
+
isRecord,
|
|
9
|
+
validTimestamp,
|
|
10
|
+
epoch,
|
|
11
|
+
isSequenceRow,
|
|
12
|
+
} from "../state/project-store.js";
|
|
13
|
+
import { initializeViewerRun, recordViewerDeltas } from "../state/viewer.js";
|
|
14
|
+
import { WorkflowMessageStore } from "../state/workflow-messages.js";
|
|
15
|
+
|
|
16
|
+
export type WorkflowRunLaunchStatus =
|
|
17
|
+
| "queued"
|
|
18
|
+
| "starting"
|
|
19
|
+
| "running"
|
|
20
|
+
| "parked"
|
|
21
|
+
| "done"
|
|
22
|
+
| "failed"
|
|
23
|
+
| "cancelled";
|
|
24
|
+
|
|
25
|
+
export type WorkflowRunReservationOptions = {
|
|
26
|
+
runId: string;
|
|
27
|
+
workflowName: string;
|
|
28
|
+
workflowSourceRef: string;
|
|
29
|
+
workflowSource: unknown;
|
|
30
|
+
definitionDigest: string;
|
|
31
|
+
definitionSnapshot: unknown;
|
|
32
|
+
input: unknown;
|
|
33
|
+
launchOptions?: unknown;
|
|
34
|
+
originSessionId: string;
|
|
35
|
+
executionMode?: "interactive" | "headless";
|
|
36
|
+
parentRunId?: string;
|
|
37
|
+
lineageKind?: "restart";
|
|
38
|
+
restartNumber?: number;
|
|
39
|
+
parentRunRevision?: number;
|
|
40
|
+
now?: string;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export type WorkflowRunQueueViewRecord = {
|
|
44
|
+
runId: string;
|
|
45
|
+
workflowName: string;
|
|
46
|
+
workflowSourceRef: string;
|
|
47
|
+
workflowSource: unknown;
|
|
48
|
+
initialized: boolean;
|
|
49
|
+
definitionDigest: string;
|
|
50
|
+
runStateStatus: string;
|
|
51
|
+
paused: boolean;
|
|
52
|
+
status: WorkflowRunLaunchStatus;
|
|
53
|
+
originSessionId: string | null;
|
|
54
|
+
executionMode: "interactive" | "headless";
|
|
55
|
+
parentRunId: string | null;
|
|
56
|
+
rootRunId: string;
|
|
57
|
+
lineageKind: "restart" | null;
|
|
58
|
+
restartNumber: number;
|
|
59
|
+
parentRunRevision: number | null;
|
|
60
|
+
errorCode: string | null;
|
|
61
|
+
errorMessage: string | null;
|
|
62
|
+
createdAt: string;
|
|
63
|
+
updatedAt: string;
|
|
64
|
+
startedAt: string | null;
|
|
65
|
+
finishedAt: string | null;
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
export type WorkflowRunQueueRecord = {
|
|
69
|
+
runId: string;
|
|
70
|
+
workflowName: string;
|
|
71
|
+
workflowSourceRef: string;
|
|
72
|
+
workflowSource: unknown;
|
|
73
|
+
initialized: boolean;
|
|
74
|
+
definitionDigest: string;
|
|
75
|
+
input: unknown;
|
|
76
|
+
launchOptions: unknown;
|
|
77
|
+
status: WorkflowRunLaunchStatus;
|
|
78
|
+
runnerId: string | null;
|
|
79
|
+
claimToken: string | null;
|
|
80
|
+
claimGeneration: number | null;
|
|
81
|
+
claimExpiresAt: string | null;
|
|
82
|
+
originSessionId: string | null;
|
|
83
|
+
executionMode: "interactive" | "headless";
|
|
84
|
+
parentRunId: string | null;
|
|
85
|
+
rootRunId: string;
|
|
86
|
+
lineageKind: "restart" | null;
|
|
87
|
+
restartNumber: number;
|
|
88
|
+
parentRunRevision: number | null;
|
|
89
|
+
errorCode: string | null;
|
|
90
|
+
errorMessage: string | null;
|
|
91
|
+
createdAt: string;
|
|
92
|
+
updatedAt: string;
|
|
93
|
+
startedAt: string | null;
|
|
94
|
+
finishedAt: string | null;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
export type WorkflowRunPreparationResult =
|
|
98
|
+
| {
|
|
99
|
+
state: "reserved";
|
|
100
|
+
run: WorkflowRunQueueRecord;
|
|
101
|
+
}
|
|
102
|
+
| {
|
|
103
|
+
state: "adopted";
|
|
104
|
+
run: WorkflowRunQueueRecord;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
export type RunEventRecord = {
|
|
108
|
+
seq: number;
|
|
109
|
+
recordedAt: string;
|
|
110
|
+
runId: string;
|
|
111
|
+
workflowRef: string;
|
|
112
|
+
type: string;
|
|
113
|
+
runnerId: string | null;
|
|
114
|
+
payload: JsonObject;
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
type RunRow = {
|
|
118
|
+
runId: string;
|
|
119
|
+
resourceId: string;
|
|
120
|
+
workflowName: string;
|
|
121
|
+
workflowRef: string;
|
|
122
|
+
runStatus: string;
|
|
123
|
+
paused: number;
|
|
124
|
+
definitionDigest: Buffer;
|
|
125
|
+
definitionHash: Buffer;
|
|
126
|
+
inputHash: Buffer;
|
|
127
|
+
launchOptionsHash: Buffer;
|
|
128
|
+
status: WorkflowRunLaunchStatus;
|
|
129
|
+
availableAt: number;
|
|
130
|
+
consecutiveErrors: number;
|
|
131
|
+
errorCode: string | null;
|
|
132
|
+
errorHash: Buffer | null;
|
|
133
|
+
originSessionId: string | null;
|
|
134
|
+
executionMode: "interactive" | "headless";
|
|
135
|
+
parentRunId: string | null;
|
|
136
|
+
rootRunId: string;
|
|
137
|
+
lineageKind: "restart" | null;
|
|
138
|
+
restartNumber: number;
|
|
139
|
+
parentRunRevision: number | null;
|
|
140
|
+
createdAt: number;
|
|
141
|
+
updatedAt: number;
|
|
142
|
+
startedAt: number | null;
|
|
143
|
+
finishedAt: number | null;
|
|
144
|
+
leaseGeneration: number;
|
|
145
|
+
ownerId: string | null;
|
|
146
|
+
claimExpiresAt: number | null;
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
type WorkflowRunViewRow = {
|
|
150
|
+
runId: string;
|
|
151
|
+
workflowName: string;
|
|
152
|
+
workflowRef: string;
|
|
153
|
+
runStatus: string;
|
|
154
|
+
paused: number;
|
|
155
|
+
definitionDigest: Buffer;
|
|
156
|
+
status: WorkflowRunLaunchStatus;
|
|
157
|
+
originSessionId: string | null;
|
|
158
|
+
executionMode: "interactive" | "headless";
|
|
159
|
+
parentRunId: string | null;
|
|
160
|
+
rootRunId: string;
|
|
161
|
+
lineageKind: "restart" | null;
|
|
162
|
+
restartNumber: number;
|
|
163
|
+
parentRunRevision: number | null;
|
|
164
|
+
errorCode: string | null;
|
|
165
|
+
errorMessage: string | null;
|
|
166
|
+
createdAt: number;
|
|
167
|
+
updatedAt: number;
|
|
168
|
+
startedAt: number | null;
|
|
169
|
+
finishedAt: number | null;
|
|
170
|
+
sourceType: "builtin" | "file";
|
|
171
|
+
sourceRef: string;
|
|
172
|
+
sourceRevision: string;
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
type RunListRevisionRow = {
|
|
176
|
+
count: number;
|
|
177
|
+
revisionSum: number;
|
|
178
|
+
updatedAt: number;
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
type RunSourceIdentityRow = {
|
|
182
|
+
mountPath: string;
|
|
183
|
+
sourceType: "builtin" | "file";
|
|
184
|
+
sourceRef: string;
|
|
185
|
+
sourceRevision: string;
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
type CancelledRunEffectRow = {
|
|
189
|
+
effectId: string;
|
|
190
|
+
resourceId: string;
|
|
191
|
+
status: "pending" | "applying";
|
|
192
|
+
attemptCount: number;
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
type ExpiredInteractionRow = {
|
|
196
|
+
requestId: string;
|
|
197
|
+
attemptId: string;
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
function workflowRunSelect(clause: string): string {
|
|
201
|
+
return `SELECT r.run_id AS runId, r.resource_id AS resourceId,
|
|
202
|
+
d.workflow_name AS workflowName, r.workflow_ref AS workflowRef, r.status AS runStatus,
|
|
203
|
+
r.paused, r.definition_digest AS definitionDigest, d.definition_hash AS definitionHash,
|
|
204
|
+
r.input_hash AS inputHash,
|
|
205
|
+
r.launch_options_hash AS launchOptionsHash,
|
|
206
|
+
q.status, q.available_at AS availableAt,
|
|
207
|
+
q.consecutive_errors AS consecutiveErrors, q.error_code AS errorCode,
|
|
208
|
+
q.error_hash AS errorHash, b.origin_session_id AS originSessionId,
|
|
209
|
+
b.execution_mode AS executionMode, r.parent_run_id AS parentRunId,
|
|
210
|
+
r.root_run_id AS rootRunId, r.lineage_kind AS lineageKind,
|
|
211
|
+
r.restart_number AS restartNumber,
|
|
212
|
+
r.parent_run_revision AS parentRunRevision,
|
|
213
|
+
q.created_at AS createdAt, q.updated_at AS updatedAt,
|
|
214
|
+
q.started_at AS startedAt, q.finished_at AS finishedAt,
|
|
215
|
+
l.generation AS leaseGeneration, l.owner_id AS ownerId, l.expires_at AS claimExpiresAt
|
|
216
|
+
FROM runs r JOIN workflow_definitions d ON d.definition_digest = r.definition_digest
|
|
217
|
+
JOIN run_queue q ON q.run_id = r.run_id
|
|
218
|
+
LEFT JOIN run_bindings b ON b.run_id = r.run_id
|
|
219
|
+
JOIN leases l ON l.resource_id = r.resource_id ${clause}`;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function workflowRunViewSelect(clause: string): string {
|
|
223
|
+
return `SELECT r.run_id AS runId, d.workflow_name AS workflowName,
|
|
224
|
+
r.workflow_ref AS workflowRef, r.status AS runStatus, r.paused,
|
|
225
|
+
r.definition_digest AS definitionDigest, q.status,
|
|
226
|
+
b.origin_session_id AS originSessionId, b.execution_mode AS executionMode,
|
|
227
|
+
r.parent_run_id AS parentRunId, r.root_run_id AS rootRunId,
|
|
228
|
+
r.lineage_kind AS lineageKind, r.restart_number AS restartNumber,
|
|
229
|
+
r.parent_run_revision AS parentRunRevision,
|
|
230
|
+
q.error_code AS errorCode, CAST(error.content AS TEXT) AS errorMessage,
|
|
231
|
+
q.created_at AS createdAt, q.updated_at AS updatedAt,
|
|
232
|
+
q.started_at AS startedAt, q.finished_at AS finishedAt,
|
|
233
|
+
source.source_type AS sourceType, source.source_ref AS sourceRef,
|
|
234
|
+
source.source_revision AS sourceRevision
|
|
235
|
+
FROM runs r JOIN workflow_definitions d ON d.definition_digest = r.definition_digest
|
|
236
|
+
JOIN run_queue q ON q.run_id = r.run_id
|
|
237
|
+
LEFT JOIN run_bindings b ON b.run_id = r.run_id
|
|
238
|
+
LEFT JOIN run_sources source ON source.run_id = r.run_id AND source.mount_path = ''
|
|
239
|
+
LEFT JOIN blobs error ON error.blob_hash = q.error_hash ${clause}`;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
type RunEventRow = {
|
|
243
|
+
seq: number;
|
|
244
|
+
recordedAt: number;
|
|
245
|
+
runId: string;
|
|
246
|
+
workflowRef: string;
|
|
247
|
+
eventType: string;
|
|
248
|
+
runnerId: string | null;
|
|
249
|
+
payloadHash: Buffer | null;
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
function workflowRunViewRecord(row: WorkflowRunViewRow): WorkflowRunQueueViewRecord {
|
|
253
|
+
const root =
|
|
254
|
+
row.sourceType === "builtin"
|
|
255
|
+
? { kind: "builtin", id: row.sourceRef, revision: row.sourceRevision }
|
|
256
|
+
: { kind: "file", path: row.sourceRef, hash: row.sourceRevision };
|
|
257
|
+
return {
|
|
258
|
+
runId: row.runId,
|
|
259
|
+
workflowName: row.workflowName,
|
|
260
|
+
workflowSourceRef: row.workflowRef,
|
|
261
|
+
workflowSource: { root, mounted: [] },
|
|
262
|
+
initialized: row.runStatus !== "queued",
|
|
263
|
+
definitionDigest: `sha256:${row.definitionDigest.toString("hex")}`,
|
|
264
|
+
runStateStatus: row.runStatus,
|
|
265
|
+
paused: row.paused === 1,
|
|
266
|
+
status: row.status,
|
|
267
|
+
originSessionId: row.executionMode === "headless" ? null : row.originSessionId,
|
|
268
|
+
executionMode: row.executionMode,
|
|
269
|
+
parentRunId: row.parentRunId,
|
|
270
|
+
rootRunId: row.rootRunId,
|
|
271
|
+
lineageKind: row.lineageKind,
|
|
272
|
+
restartNumber: row.restartNumber,
|
|
273
|
+
parentRunRevision: row.parentRunRevision,
|
|
274
|
+
errorCode: row.errorCode,
|
|
275
|
+
errorMessage: row.errorMessage,
|
|
276
|
+
createdAt: new Date(row.createdAt).toISOString(),
|
|
277
|
+
updatedAt: new Date(row.updatedAt).toISOString(),
|
|
278
|
+
startedAt: row.startedAt === null ? null : new Date(row.startedAt).toISOString(),
|
|
279
|
+
finishedAt: row.finishedAt === null ? null : new Date(row.finishedAt).toISOString(),
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function isWorkflowRunViewRow(value: unknown): value is WorkflowRunViewRow {
|
|
284
|
+
return (
|
|
285
|
+
isRecord(value) &&
|
|
286
|
+
typeof value.runId === "string" &&
|
|
287
|
+
typeof value.workflowName === "string" &&
|
|
288
|
+
typeof value.workflowRef === "string" &&
|
|
289
|
+
typeof value.runStatus === "string" &&
|
|
290
|
+
typeof value.paused === "number" &&
|
|
291
|
+
Buffer.isBuffer(value.definitionDigest) &&
|
|
292
|
+
typeof value.status === "string" &&
|
|
293
|
+
(value.originSessionId === null || typeof value.originSessionId === "string") &&
|
|
294
|
+
(value.executionMode === "interactive" || value.executionMode === "headless") &&
|
|
295
|
+
(value.parentRunId === null || typeof value.parentRunId === "string") &&
|
|
296
|
+
typeof value.rootRunId === "string" &&
|
|
297
|
+
(value.lineageKind === null || value.lineageKind === "restart") &&
|
|
298
|
+
typeof value.restartNumber === "number" &&
|
|
299
|
+
(value.parentRunRevision === null || typeof value.parentRunRevision === "number") &&
|
|
300
|
+
(value.errorCode === null || typeof value.errorCode === "string") &&
|
|
301
|
+
(value.errorMessage === null || typeof value.errorMessage === "string") &&
|
|
302
|
+
typeof value.createdAt === "number" &&
|
|
303
|
+
typeof value.updatedAt === "number" &&
|
|
304
|
+
(value.startedAt === null || typeof value.startedAt === "number") &&
|
|
305
|
+
(value.finishedAt === null || typeof value.finishedAt === "number") &&
|
|
306
|
+
(value.sourceType === "builtin" || value.sourceType === "file") &&
|
|
307
|
+
typeof value.sourceRef === "string" &&
|
|
308
|
+
typeof value.sourceRevision === "string"
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function isRunListRevisionRow(value: unknown): value is RunListRevisionRow {
|
|
313
|
+
return (
|
|
314
|
+
isRecord(value) &&
|
|
315
|
+
typeof value.count === "number" &&
|
|
316
|
+
typeof value.revisionSum === "number" &&
|
|
317
|
+
typeof value.updatedAt === "number"
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function isRunRow(value: unknown): value is RunRow {
|
|
322
|
+
return isRecord(value);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function isRunSourceIdentityRow(value: unknown): value is RunSourceIdentityRow {
|
|
326
|
+
return isRecord(value);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function isCancelledRunEffectRow(value: unknown): value is CancelledRunEffectRow {
|
|
330
|
+
return (
|
|
331
|
+
isRecord(value) &&
|
|
332
|
+
typeof value.effectId === "string" &&
|
|
333
|
+
typeof value.resourceId === "string" &&
|
|
334
|
+
(value.status === "pending" || value.status === "applying") &&
|
|
335
|
+
typeof value.attemptCount === "number"
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function isExpiredInteractionRow(value: unknown): value is ExpiredInteractionRow {
|
|
340
|
+
return (
|
|
341
|
+
isRecord(value) && typeof value.requestId === "string" && typeof value.attemptId === "string"
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
function isRunEventRow(value: unknown): value is RunEventRow {
|
|
346
|
+
return isRecord(value);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function isCanonicalPathRow(value: unknown): value is { canonicalPath: string } {
|
|
350
|
+
return isRecord(value) && typeof value.canonicalPath === "string";
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function isEffectIdentityRow(value: unknown): value is { effectId: string; resourceId: string } {
|
|
354
|
+
return isRecord(value);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function digestBuffer(value: string): Buffer {
|
|
358
|
+
const hex = value.startsWith("sha256:") ? value.slice(7) : value;
|
|
359
|
+
if (!/^[a-f0-9]{64}$/i.test(hex)) throw new Error("Expected a SHA-256 digest");
|
|
360
|
+
return Buffer.from(hex, "hex");
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
type QueuedSource = {
|
|
364
|
+
root:
|
|
365
|
+
| { kind: "builtin"; id: string; revision: string }
|
|
366
|
+
| { kind: "file"; path: string; hash: string };
|
|
367
|
+
mounted: Array<{
|
|
368
|
+
mountPath: string[];
|
|
369
|
+
workflowName: string;
|
|
370
|
+
source:
|
|
371
|
+
| { kind: "builtin"; id: string; revision: string }
|
|
372
|
+
| { kind: "file"; path: string; hash: string };
|
|
373
|
+
}>;
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
function queuedWorkflowSource(value: unknown): QueuedSource {
|
|
377
|
+
if (isWorkflowSource(value)) {
|
|
378
|
+
return { root: value, mounted: [] };
|
|
379
|
+
}
|
|
380
|
+
if (
|
|
381
|
+
isRecord(value) &&
|
|
382
|
+
isWorkflowSource(value.root) &&
|
|
383
|
+
Array.isArray(value.mounted) &&
|
|
384
|
+
value.mounted.every(isMountedWorkflowSource)
|
|
385
|
+
) {
|
|
386
|
+
return { root: value.root, mounted: value.mounted };
|
|
387
|
+
}
|
|
388
|
+
throw new Error("Stored workflow source identity is invalid");
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function isWorkflowSource(value: unknown): value is QueuedSource["root"] {
|
|
392
|
+
return (
|
|
393
|
+
isRecord(value) &&
|
|
394
|
+
((value.kind === "builtin" &&
|
|
395
|
+
typeof value.id === "string" &&
|
|
396
|
+
typeof value.revision === "string") ||
|
|
397
|
+
(value.kind === "file" && typeof value.path === "string" && typeof value.hash === "string"))
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
function isMountedWorkflowSource(value: unknown): value is QueuedSource["mounted"][number] {
|
|
402
|
+
return (
|
|
403
|
+
isRecord(value) &&
|
|
404
|
+
Array.isArray(value.mountPath) &&
|
|
405
|
+
value.mountPath.every((part) => typeof part === "string") &&
|
|
406
|
+
typeof value.workflowName === "string" &&
|
|
407
|
+
isWorkflowSource(value.source)
|
|
408
|
+
);
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function insertQueuedRunSources(state: StateDatabase, runId: string, value: QueuedSource): void {
|
|
412
|
+
const insert = state.connection.prepare(
|
|
413
|
+
`INSERT INTO run_sources(run_id, mount_path, source_type, source_ref, source_revision)
|
|
414
|
+
VALUES (?, ?, ?, ?, ?)`,
|
|
415
|
+
);
|
|
416
|
+
const root = queuedSourceParts(value.root);
|
|
417
|
+
insert.run(runId, "", root.type, root.ref, root.revision);
|
|
418
|
+
for (const mounted of value.mounted) {
|
|
419
|
+
const source = queuedSourceParts(mounted.source);
|
|
420
|
+
insert.run(runId, mounted.mountPath.join("/"), source.type, source.ref, source.revision);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function queuedSourceParts(source: QueuedSource["root"]): {
|
|
425
|
+
type: "builtin" | "file";
|
|
426
|
+
ref: string;
|
|
427
|
+
revision: string;
|
|
428
|
+
} {
|
|
429
|
+
return source.kind === "builtin"
|
|
430
|
+
? { type: "builtin", ref: source.id, revision: source.revision }
|
|
431
|
+
: { type: "file", ref: source.path, revision: source.hash };
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function readQueuedRunSources(state: StateDatabase, run: RunRow): QueuedSource {
|
|
435
|
+
const rows = state.connection
|
|
436
|
+
.prepare(
|
|
437
|
+
`SELECT mount_path AS mountPath, source_type AS sourceType,
|
|
438
|
+
source_ref AS sourceRef, source_revision AS sourceRevision
|
|
439
|
+
FROM run_sources WHERE run_id = ? ORDER BY mount_path`,
|
|
440
|
+
)
|
|
441
|
+
.all(run.runId)
|
|
442
|
+
.filter(isRunSourceIdentityRow);
|
|
443
|
+
const rootRow = rows.find((row) => row.mountPath === "");
|
|
444
|
+
if (rootRow === undefined) throw new Error(`Workflow run source is missing: ${run.runId}`);
|
|
445
|
+
const snapshot = state.readJson(run.definitionHash);
|
|
446
|
+
const mounts =
|
|
447
|
+
isRecord(snapshot) &&
|
|
448
|
+
isRecord(snapshot.composition) &&
|
|
449
|
+
Array.isArray(snapshot.composition.mounts)
|
|
450
|
+
? snapshot.composition.mounts
|
|
451
|
+
: [];
|
|
452
|
+
const names = new Map(
|
|
453
|
+
mounts.flatMap((mount) => {
|
|
454
|
+
if (
|
|
455
|
+
!isRecord(mount) ||
|
|
456
|
+
!Array.isArray(mount.mountPath) ||
|
|
457
|
+
!mount.mountPath.every((part) => typeof part === "string") ||
|
|
458
|
+
typeof mount.workflowName !== "string"
|
|
459
|
+
) {
|
|
460
|
+
return [];
|
|
461
|
+
}
|
|
462
|
+
return [[mount.mountPath.join("/"), mount.workflowName] as const];
|
|
463
|
+
}),
|
|
464
|
+
);
|
|
465
|
+
return {
|
|
466
|
+
root: rowToSource(rootRow),
|
|
467
|
+
mounted: rows
|
|
468
|
+
.filter((row) => row.mountPath !== "")
|
|
469
|
+
.map((row) => ({
|
|
470
|
+
mountPath: row.mountPath.split("/"),
|
|
471
|
+
workflowName: names.get(row.mountPath) ?? row.mountPath,
|
|
472
|
+
source: rowToSource(row),
|
|
473
|
+
})),
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function rowToSource(row: RunSourceIdentityRow): QueuedSource["root"] {
|
|
478
|
+
return row.sourceType === "builtin"
|
|
479
|
+
? { kind: "builtin", id: row.sourceRef, revision: row.sourceRevision }
|
|
480
|
+
: { kind: "file", path: row.sourceRef, hash: row.sourceRevision };
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
function validateRunId(value: string): void {
|
|
484
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(value))
|
|
485
|
+
throw new Error(`Invalid workflow run id: ${value}`);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
export class WorkflowRunQueueStore extends ProjectStore {
|
|
489
|
+
private readonly workflowMessages = new WorkflowMessageStore(this.state);
|
|
490
|
+
reserveWorkflowRun(options: WorkflowRunReservationOptions): WorkflowRunQueueRecord {
|
|
491
|
+
validateRunId(options.runId);
|
|
492
|
+
const now = epoch(validTimestamp(options.now));
|
|
493
|
+
const definitionDigest = digestBuffer(options.definitionDigest);
|
|
494
|
+
return this.state.transaction(() =>
|
|
495
|
+
this.reserveWorkflowRunInTransaction(options, now, definitionDigest),
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
private reserveWorkflowRunInTransaction(
|
|
500
|
+
options: WorkflowRunReservationOptions,
|
|
501
|
+
now: number,
|
|
502
|
+
definitionDigest: Buffer,
|
|
503
|
+
): WorkflowRunQueueRecord {
|
|
504
|
+
if (this.getWorkflowRun(options.runId) !== undefined) {
|
|
505
|
+
throw new Error(`Workflow run already reserved: ${options.runId}`);
|
|
506
|
+
}
|
|
507
|
+
const lineageKind = options.parentRunId === undefined ? null : (options.lineageKind ?? null);
|
|
508
|
+
if (
|
|
509
|
+
(options.parentRunId === undefined && options.lineageKind !== undefined) ||
|
|
510
|
+
(options.parentRunId !== undefined && lineageKind !== "restart")
|
|
511
|
+
) {
|
|
512
|
+
throw new Error("Only an explicit restart can declare a parent run");
|
|
513
|
+
}
|
|
514
|
+
let rootRunId = options.runId;
|
|
515
|
+
let restartNumber = 0;
|
|
516
|
+
let parentRunRevision: number | null = null;
|
|
517
|
+
if (options.parentRunId !== undefined) {
|
|
518
|
+
const parent = this.requireWorkflowRunRow(options.parentRunId);
|
|
519
|
+
if (parent.originSessionId !== options.originSessionId) {
|
|
520
|
+
throw new Error("Workflow parent belongs to another Pi session");
|
|
521
|
+
}
|
|
522
|
+
rootRunId = parent.rootRunId;
|
|
523
|
+
if (lineageKind === "restart") {
|
|
524
|
+
if (
|
|
525
|
+
!["completed", "failed", "timed_out", "cancelled"].includes(parent.runStatus) ||
|
|
526
|
+
!["done", "failed", "cancelled"].includes(parent.status)
|
|
527
|
+
) {
|
|
528
|
+
throw new Error("Restart parent is not terminal");
|
|
529
|
+
}
|
|
530
|
+
if (
|
|
531
|
+
options.parentRunRevision === undefined ||
|
|
532
|
+
!Number.isSafeInteger(options.parentRunRevision) ||
|
|
533
|
+
options.parentRunRevision < 0
|
|
534
|
+
) {
|
|
535
|
+
throw new Error("Restart requires the exact parent run revision");
|
|
536
|
+
}
|
|
537
|
+
if (options.parentRunRevision !== this.resourceRevision(parent.resourceId)) {
|
|
538
|
+
throw new Error("Restart parent revision changed");
|
|
539
|
+
}
|
|
540
|
+
parentRunRevision = options.parentRunRevision;
|
|
541
|
+
restartNumber = options.restartNumber ?? parent.restartNumber + 1;
|
|
542
|
+
if (restartNumber !== parent.restartNumber + 1) {
|
|
543
|
+
throw new Error("Restart number does not follow its parent");
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
} else if (options.restartNumber !== undefined || options.parentRunRevision !== undefined) {
|
|
547
|
+
throw new Error("A root workflow run cannot declare restart metadata");
|
|
548
|
+
}
|
|
549
|
+
const resourceId = resourceIdFor("run", options.runId);
|
|
550
|
+
const definitionHash = this.state.putJson(options.definitionSnapshot, now);
|
|
551
|
+
const queuedSource = queuedWorkflowSource(options.workflowSource);
|
|
552
|
+
const inputHash = this.state.putJson(options.input ?? null, now);
|
|
553
|
+
const launchHash = this.state.putJson(options.launchOptions ?? {}, now);
|
|
554
|
+
this.state.connection
|
|
555
|
+
.prepare(
|
|
556
|
+
`INSERT INTO workflow_definitions(
|
|
557
|
+
definition_digest, workflow_name, definition_hash, created_at
|
|
558
|
+
) VALUES (?, ?, ?, ?)
|
|
559
|
+
ON CONFLICT(definition_digest) DO NOTHING`,
|
|
560
|
+
)
|
|
561
|
+
.run(definitionDigest, options.workflowName, definitionHash, now);
|
|
562
|
+
this.state.connection
|
|
563
|
+
.prepare(
|
|
564
|
+
`INSERT INTO resources(resource_id, resource_type, aggregate_key, revision, created_at, updated_at)
|
|
565
|
+
VALUES (?, 'run', ?, 1, ?, ?)`,
|
|
566
|
+
)
|
|
567
|
+
.run(resourceId, options.runId, now, now);
|
|
568
|
+
this.state.connection
|
|
569
|
+
.prepare("INSERT INTO leases(resource_id, generation) VALUES (?, 0)")
|
|
570
|
+
.run(resourceId);
|
|
571
|
+
this.state.connection
|
|
572
|
+
.prepare(
|
|
573
|
+
`INSERT INTO runs(
|
|
574
|
+
run_id, resource_id, project_id, parent_run_id, root_run_id, lineage_kind,
|
|
575
|
+
restart_number, parent_run_revision, definition_digest,
|
|
576
|
+
workflow_ref, launch_options_hash, status, paused,
|
|
577
|
+
input_hash, created_at, updated_at
|
|
578
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`,
|
|
579
|
+
)
|
|
580
|
+
.run(
|
|
581
|
+
options.runId,
|
|
582
|
+
resourceId,
|
|
583
|
+
this.requireProjectId(),
|
|
584
|
+
options.parentRunId ?? null,
|
|
585
|
+
rootRunId,
|
|
586
|
+
lineageKind,
|
|
587
|
+
restartNumber,
|
|
588
|
+
parentRunRevision,
|
|
589
|
+
definitionDigest,
|
|
590
|
+
options.workflowSourceRef,
|
|
591
|
+
launchHash,
|
|
592
|
+
inputHash,
|
|
593
|
+
now,
|
|
594
|
+
now,
|
|
595
|
+
);
|
|
596
|
+
initializeViewerRun(this.state, options.runId, now);
|
|
597
|
+
insertQueuedRunSources(this.state, options.runId, queuedSource);
|
|
598
|
+
this.state.connection
|
|
599
|
+
.prepare(
|
|
600
|
+
`INSERT INTO run_bindings(run_id, origin_session_id, execution_mode, created_at)
|
|
601
|
+
VALUES (?, ?, ?, ?)`,
|
|
602
|
+
)
|
|
603
|
+
.run(options.runId, options.originSessionId, options.executionMode ?? "interactive", now);
|
|
604
|
+
this.state.connection
|
|
605
|
+
.prepare(
|
|
606
|
+
`INSERT INTO run_queue(
|
|
607
|
+
run_id, status, available_at, origin_session_id,
|
|
608
|
+
consecutive_errors, created_at, updated_at
|
|
609
|
+
) VALUES (?, 'queued', ?, ?, 0, ?, ?)`,
|
|
610
|
+
)
|
|
611
|
+
.run(
|
|
612
|
+
options.runId,
|
|
613
|
+
now,
|
|
614
|
+
options.executionMode === "headless" ? null : options.originSessionId,
|
|
615
|
+
now,
|
|
616
|
+
now,
|
|
617
|
+
);
|
|
618
|
+
this.insertEvent(resourceId, 1, "run.queued", "session", options.originSessionId, {}, now);
|
|
619
|
+
return this.requireWorkflowRun(options.runId);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
reserveOrAdoptWorkflowRun(options: WorkflowRunReservationOptions): WorkflowRunPreparationResult {
|
|
623
|
+
validateRunId(options.runId);
|
|
624
|
+
const now = epoch(validTimestamp(options.now));
|
|
625
|
+
const definitionDigest = digestBuffer(options.definitionDigest);
|
|
626
|
+
return this.state.transaction(() => {
|
|
627
|
+
const existing = this.workflowRunRow(options.runId);
|
|
628
|
+
if (existing !== undefined) {
|
|
629
|
+
this.assertWorkflowRunPreparationCompatible(existing, options, definitionDigest);
|
|
630
|
+
return { state: "adopted", run: this.mapWorkflowRun(existing) };
|
|
631
|
+
}
|
|
632
|
+
return {
|
|
633
|
+
state: "reserved",
|
|
634
|
+
run: this.reserveWorkflowRunInTransaction(options, now, definitionDigest),
|
|
635
|
+
};
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
getWorkflowRun(runId: string): WorkflowRunQueueRecord | undefined {
|
|
640
|
+
const row = this.workflowRunRow(runId);
|
|
641
|
+
return row === undefined ? undefined : this.mapWorkflowRun(row);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
getWorkflowRunView(runId: string): WorkflowRunQueueViewRecord | undefined {
|
|
645
|
+
const row = this.workflowRunViewRows("WHERE r.run_id = ?", [runId])[0];
|
|
646
|
+
return row === undefined ? undefined : workflowRunViewRecord(row);
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
workflowRunListRevision(): { total: number; revision: string } {
|
|
650
|
+
const row = this.state.connection
|
|
651
|
+
.prepare(
|
|
652
|
+
`SELECT count(*) AS count, COALESCE(sum(res.revision), 0) AS revisionSum,
|
|
653
|
+
COALESCE(max(q.updated_at), 0) AS updatedAt
|
|
654
|
+
FROM runs r JOIN run_queue q ON q.run_id = r.run_id
|
|
655
|
+
JOIN resources res ON res.resource_id = r.resource_id`,
|
|
656
|
+
)
|
|
657
|
+
.get();
|
|
658
|
+
if (!isRunListRevisionRow(row)) throw new Error("Workflow run list revision is invalid");
|
|
659
|
+
return {
|
|
660
|
+
total: row.count,
|
|
661
|
+
revision: `${row.count}:${row.revisionSum}:${row.updatedAt}`,
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
listWorkflowRunViews(options: { offset: number; limit: number }): {
|
|
666
|
+
total: number;
|
|
667
|
+
revision: string;
|
|
668
|
+
runs: WorkflowRunQueueViewRecord[];
|
|
669
|
+
} {
|
|
670
|
+
const current = this.workflowRunListRevision();
|
|
671
|
+
const rows = this.workflowRunViewRows("ORDER BY q.created_at DESC LIMIT ? OFFSET ?", [
|
|
672
|
+
options.limit,
|
|
673
|
+
options.offset,
|
|
674
|
+
]);
|
|
675
|
+
return { ...current, runs: rows.map(workflowRunViewRecord) };
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
workflowRunProjectPath(runId: string): string | undefined {
|
|
679
|
+
const row = this.state.connection
|
|
680
|
+
.prepare(
|
|
681
|
+
`SELECT p.canonical_path AS canonicalPath
|
|
682
|
+
FROM runs r JOIN projects p ON p.project_id = r.project_id WHERE r.run_id = ?`,
|
|
683
|
+
)
|
|
684
|
+
.get(runId);
|
|
685
|
+
return isCanonicalPathRow(row) ? row.canonicalPath : undefined;
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
listWorkflowRuns(
|
|
689
|
+
options: {
|
|
690
|
+
statuses?: WorkflowRunLaunchStatus[];
|
|
691
|
+
excludeRunIds?: string[];
|
|
692
|
+
limit?: number;
|
|
693
|
+
} = {},
|
|
694
|
+
): WorkflowRunQueueRecord[] {
|
|
695
|
+
const clauses: string[] = [];
|
|
696
|
+
const params: unknown[] = [];
|
|
697
|
+
if (this.projectId !== null) {
|
|
698
|
+
clauses.push("r.project_id = ?");
|
|
699
|
+
params.push(this.projectId);
|
|
700
|
+
}
|
|
701
|
+
if (options.statuses !== undefined && options.statuses.length > 0) {
|
|
702
|
+
clauses.push(`q.status IN (${options.statuses.map(() => "?").join(", ")})`);
|
|
703
|
+
params.push(...options.statuses);
|
|
704
|
+
}
|
|
705
|
+
if (options.excludeRunIds !== undefined && options.excludeRunIds.length > 0) {
|
|
706
|
+
clauses.push(`r.run_id NOT IN (${options.excludeRunIds.map(() => "?").join(", ")})`);
|
|
707
|
+
params.push(...options.excludeRunIds);
|
|
708
|
+
}
|
|
709
|
+
if (options.limit !== undefined) params.push(options.limit);
|
|
710
|
+
const rows = this.state.connection
|
|
711
|
+
.prepare(
|
|
712
|
+
workflowRunSelect(
|
|
713
|
+
`${clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`} ORDER BY q.created_at DESC${options.limit === undefined ? "" : " LIMIT ?"}`,
|
|
714
|
+
),
|
|
715
|
+
)
|
|
716
|
+
.all(...params);
|
|
717
|
+
return rows.filter(isRunRow).map((row) => this.mapWorkflowRun(row));
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
findSessionReservation(sessionId: string): WorkflowRunQueueRecord | undefined {
|
|
721
|
+
const row = this.state.connection
|
|
722
|
+
.prepare(
|
|
723
|
+
workflowRunSelect(
|
|
724
|
+
"WHERE b.origin_session_id = ? AND b.execution_mode = 'interactive' AND q.status NOT IN ('done', 'failed', 'cancelled') ORDER BY q.created_at DESC LIMIT 1",
|
|
725
|
+
),
|
|
726
|
+
)
|
|
727
|
+
.get(sessionId);
|
|
728
|
+
return isRunRow(row) ? this.mapWorkflowRun(row) : undefined;
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
findSessionReservationView(sessionId: string): WorkflowRunQueueViewRecord | undefined {
|
|
732
|
+
const row = this.workflowRunViewRows(
|
|
733
|
+
"WHERE b.origin_session_id = ? AND b.execution_mode = 'interactive' AND q.status NOT IN ('done', 'failed', 'cancelled') ORDER BY q.created_at DESC LIMIT 1",
|
|
734
|
+
[sessionId],
|
|
735
|
+
)[0];
|
|
736
|
+
return row === undefined ? undefined : workflowRunViewRecord(row);
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
claimWorkflowRun(options: {
|
|
740
|
+
runId: string;
|
|
741
|
+
runnerId: string;
|
|
742
|
+
claimToken: string;
|
|
743
|
+
leaseMs: number;
|
|
744
|
+
now?: string;
|
|
745
|
+
}): WorkflowRunQueueRecord | undefined {
|
|
746
|
+
const now = epoch(validTimestamp(options.now));
|
|
747
|
+
return this.claimRun(options.runId, options.runnerId, options.claimToken, options.leaseMs, now);
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
/** Schedule a runner that validates one pending interactive submission. */
|
|
751
|
+
claimWorkflowRunForInteractionValidation(options: {
|
|
752
|
+
runId: string;
|
|
753
|
+
runnerId: string;
|
|
754
|
+
claimToken: string;
|
|
755
|
+
leaseMs: number;
|
|
756
|
+
now?: string;
|
|
757
|
+
}): WorkflowRunQueueRecord | undefined {
|
|
758
|
+
const now = epoch(validTimestamp(options.now));
|
|
759
|
+
return this.claimRun(
|
|
760
|
+
options.runId,
|
|
761
|
+
options.runnerId,
|
|
762
|
+
options.claimToken,
|
|
763
|
+
options.leaseMs,
|
|
764
|
+
now,
|
|
765
|
+
{
|
|
766
|
+
allowPendingInteraction: true,
|
|
767
|
+
},
|
|
768
|
+
);
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/** Take a short server claim without scheduling a parked interactive run. */
|
|
772
|
+
claimWorkflowRunForControl(options: {
|
|
773
|
+
runId: string;
|
|
774
|
+
runnerId: string;
|
|
775
|
+
claimToken: string;
|
|
776
|
+
leaseMs: number;
|
|
777
|
+
now?: string;
|
|
778
|
+
}): WorkflowRunQueueRecord | undefined {
|
|
779
|
+
const now = epoch(validTimestamp(options.now));
|
|
780
|
+
return this.claimRun(
|
|
781
|
+
options.runId,
|
|
782
|
+
options.runnerId,
|
|
783
|
+
options.claimToken,
|
|
784
|
+
options.leaseMs,
|
|
785
|
+
now,
|
|
786
|
+
{ allowPendingInteraction: true, preserveQueueStatus: true },
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
beginWorkflowRunInteractionTimeout(options: {
|
|
791
|
+
runId: string;
|
|
792
|
+
targetSessionId: string;
|
|
793
|
+
claimToken: string;
|
|
794
|
+
now?: string;
|
|
795
|
+
}): boolean {
|
|
796
|
+
const now = epoch(validTimestamp(options.now));
|
|
797
|
+
return this.state.transaction(() => {
|
|
798
|
+
if (
|
|
799
|
+
!this.verifyWorkflowRunClaim({
|
|
800
|
+
runId: options.runId,
|
|
801
|
+
claimToken: options.claimToken,
|
|
802
|
+
now: new Date(now).toISOString(),
|
|
803
|
+
})
|
|
804
|
+
) {
|
|
805
|
+
return false;
|
|
806
|
+
}
|
|
807
|
+
const row = this.requireWorkflowRunRow(options.runId);
|
|
808
|
+
const interaction = this.state.connection
|
|
809
|
+
.prepare(
|
|
810
|
+
`SELECT i.request_id AS requestId, i.attempt_id AS attemptId
|
|
811
|
+
FROM interactive_requests i
|
|
812
|
+
JOIN node_attempts a ON a.attempt_id = i.attempt_id
|
|
813
|
+
JOIN runs r ON r.run_id = i.run_id
|
|
814
|
+
WHERE i.run_id = ? AND i.target_session_id = ?
|
|
815
|
+
AND i.status = 'pending' AND r.paused = 0
|
|
816
|
+
AND a.timeout_ms > 0
|
|
817
|
+
AND (SELECT COALESCE(SUM(elapsed_ms), 0) FROM attempt_active_intervals t
|
|
818
|
+
WHERE t.attempt_id = a.attempt_id) >= a.timeout_ms
|
|
819
|
+
ORDER BY a.started_at, i.request_id LIMIT 1`,
|
|
820
|
+
)
|
|
821
|
+
.get(options.runId, options.targetSessionId);
|
|
822
|
+
if (!isExpiredInteractionRow(interaction)) return false;
|
|
823
|
+
closeRunTime(this.state, options.runId);
|
|
824
|
+
const request = this.state.connection
|
|
825
|
+
.prepare(
|
|
826
|
+
`UPDATE interactive_requests
|
|
827
|
+
SET status = 'cancelled', revision = revision + 1, updated_at = ?
|
|
828
|
+
WHERE request_id = ? AND run_id = ? AND status = 'pending'`,
|
|
829
|
+
)
|
|
830
|
+
.run(now, interaction.requestId, options.runId);
|
|
831
|
+
const run = this.state.connection
|
|
832
|
+
.prepare(
|
|
833
|
+
`UPDATE runs
|
|
834
|
+
SET status = 'running', status_detail = 'finishing expired interaction deadline',
|
|
835
|
+
updated_at = ?, finished_at = NULL
|
|
836
|
+
WHERE run_id = ? AND status = 'waiting'`,
|
|
837
|
+
)
|
|
838
|
+
.run(now, options.runId);
|
|
839
|
+
const queue = this.state.connection
|
|
840
|
+
.prepare(
|
|
841
|
+
`UPDATE run_queue SET status = 'starting', error_code = NULL, error_hash = NULL,
|
|
842
|
+
updated_at = ?, finished_at = NULL
|
|
843
|
+
WHERE run_id = ? AND status = 'parked'`,
|
|
844
|
+
)
|
|
845
|
+
.run(now, options.runId);
|
|
846
|
+
/* istanbul ignore if -- the expired parked interaction was selected above */
|
|
847
|
+
if (request.changes !== 1 || run.changes !== 1 || queue.changes !== 1) {
|
|
848
|
+
throw new Error(`Workflow run ${options.runId} changed during interaction timeout`);
|
|
849
|
+
}
|
|
850
|
+
const error = "Workflow node model-turn deadline expired";
|
|
851
|
+
const receiptHash = this.state.putJson({ status: "rejected", error }, now);
|
|
852
|
+
this.state.connection
|
|
853
|
+
.prepare(
|
|
854
|
+
`UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
|
|
855
|
+
WHERE request_id = ? AND outcome = 'validating'`,
|
|
856
|
+
)
|
|
857
|
+
.run(receiptHash, interaction.requestId);
|
|
858
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
859
|
+
const lease = this.requireLease(row.resourceId);
|
|
860
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
861
|
+
this.insertEvent(
|
|
862
|
+
row.resourceId,
|
|
863
|
+
revision + 1,
|
|
864
|
+
"run.interaction_timeout_started",
|
|
865
|
+
lease.ownerType ?? "system",
|
|
866
|
+
lease.ownerId,
|
|
867
|
+
{ requestId: interaction.requestId, attemptId: interaction.attemptId },
|
|
868
|
+
now,
|
|
869
|
+
lease.generation || undefined,
|
|
870
|
+
);
|
|
871
|
+
recordViewerDeltas(
|
|
872
|
+
this.state,
|
|
873
|
+
options.runId,
|
|
874
|
+
[{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
|
|
875
|
+
now,
|
|
876
|
+
);
|
|
877
|
+
return true;
|
|
878
|
+
});
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
markWorkflowRunRunning(options: { runId: string; claimToken: string; now?: string }): boolean {
|
|
882
|
+
return this.updateClaimedRunStatus(options.runId, options.claimToken, "running", options.now);
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
claimNextWorkflowRun(options: {
|
|
886
|
+
runnerId: string;
|
|
887
|
+
claimToken: string;
|
|
888
|
+
leaseMs: number;
|
|
889
|
+
now?: string;
|
|
890
|
+
excludeRunIds?: string[];
|
|
891
|
+
}): WorkflowRunQueueRecord | undefined {
|
|
892
|
+
const now = epoch(validTimestamp(options.now));
|
|
893
|
+
const clauses = [
|
|
894
|
+
"q.status IN ('queued', 'parked', 'starting', 'running')",
|
|
895
|
+
"q.available_at <= ?",
|
|
896
|
+
"(l.owner_id IS NULL OR l.expires_at <= ? OR l.owner_id = ?)",
|
|
897
|
+
"NOT (q.status = 'parked' AND (r.status = 'waiting' OR r.paused = 1))",
|
|
898
|
+
"NOT EXISTS (SELECT 1 FROM interactive_requests i WHERE i.run_id = r.run_id AND i.status = 'pending')",
|
|
899
|
+
"(q.error_code IS NULL OR q.error_code NOT IN ('workflowSourceChanged', 'runnerNoProgress'))",
|
|
900
|
+
];
|
|
901
|
+
const params: unknown[] = [now, now, options.runnerId];
|
|
902
|
+
if (this.projectId !== null) {
|
|
903
|
+
clauses.unshift("r.project_id = ?");
|
|
904
|
+
params.unshift(this.projectId);
|
|
905
|
+
}
|
|
906
|
+
if (options.excludeRunIds !== undefined && options.excludeRunIds.length > 0) {
|
|
907
|
+
clauses.push(`r.run_id NOT IN (${options.excludeRunIds.map(() => "?").join(", ")})`);
|
|
908
|
+
params.push(...options.excludeRunIds);
|
|
909
|
+
}
|
|
910
|
+
const row = this.state.connection
|
|
911
|
+
.prepare(
|
|
912
|
+
workflowRunSelect(
|
|
913
|
+
`WHERE ${clauses.join(" AND ")} ORDER BY q.available_at, q.created_at LIMIT 1`,
|
|
914
|
+
),
|
|
915
|
+
)
|
|
916
|
+
.get(...params);
|
|
917
|
+
if (!isRunRow(row)) return undefined;
|
|
918
|
+
return this.claimRun(row.runId, options.runnerId, options.claimToken, options.leaseMs, now);
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
renewWorkflowRunClaim(options: {
|
|
922
|
+
runId: string;
|
|
923
|
+
claimToken: string;
|
|
924
|
+
leaseMs: number;
|
|
925
|
+
now?: string;
|
|
926
|
+
}): boolean {
|
|
927
|
+
const now = epoch(validTimestamp(options.now));
|
|
928
|
+
return this.state.transaction(() => {
|
|
929
|
+
const row = this.workflowRunRow(options.runId);
|
|
930
|
+
if (row === undefined || row.ownerId === null) return false;
|
|
931
|
+
const lease = this.requireLease(row.resourceId);
|
|
932
|
+
const result = this.state.connection
|
|
933
|
+
.prepare(
|
|
934
|
+
`UPDATE leases SET heartbeat_at = ?, expires_at = ?
|
|
935
|
+
WHERE resource_id = ? AND owner_type = ? AND owner_id = ?
|
|
936
|
+
AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
937
|
+
)
|
|
938
|
+
.run(
|
|
939
|
+
now,
|
|
940
|
+
now + options.leaseMs,
|
|
941
|
+
row.resourceId,
|
|
942
|
+
lease.ownerType,
|
|
943
|
+
row.ownerId,
|
|
944
|
+
tokenHash(options.claimToken),
|
|
945
|
+
lease.generation,
|
|
946
|
+
now,
|
|
947
|
+
);
|
|
948
|
+
if (result.changes === 1) {
|
|
949
|
+
recordViewerDeltas(
|
|
950
|
+
this.state,
|
|
951
|
+
options.runId,
|
|
952
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
953
|
+
now,
|
|
954
|
+
);
|
|
955
|
+
return true;
|
|
956
|
+
}
|
|
957
|
+
return false;
|
|
958
|
+
});
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
verifyWorkflowRunClaim(options: { runId: string; claimToken: string; now?: string }): boolean {
|
|
962
|
+
const now = epoch(validTimestamp(options.now));
|
|
963
|
+
const row = this.workflowRunRow(options.runId);
|
|
964
|
+
if (row === undefined || row.ownerId === null || row.claimExpiresAt === null) return false;
|
|
965
|
+
const lease = this.requireLease(row.resourceId);
|
|
966
|
+
return (
|
|
967
|
+
lease.tokenHash !== null &&
|
|
968
|
+
lease.tokenHash.equals(tokenHash(options.claimToken)) &&
|
|
969
|
+
row.claimExpiresAt > now
|
|
970
|
+
);
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
workflowRunAuthority(
|
|
974
|
+
runId: string,
|
|
975
|
+
claimToken: string,
|
|
976
|
+
):
|
|
977
|
+
| {
|
|
978
|
+
actor: { type: "session" | "host"; id: string };
|
|
979
|
+
ownerType: "session" | "host";
|
|
980
|
+
ownerId: string;
|
|
981
|
+
token: string;
|
|
982
|
+
generation: number;
|
|
983
|
+
leaseMs: number;
|
|
984
|
+
}
|
|
985
|
+
| undefined {
|
|
986
|
+
const row = this.workflowRunRow(runId);
|
|
987
|
+
if (row === undefined || row.ownerId === null || row.claimExpiresAt === null) return undefined;
|
|
988
|
+
const lease = this.requireLease(row.resourceId);
|
|
989
|
+
if (lease.tokenHash === null || !lease.tokenHash.equals(tokenHash(claimToken)))
|
|
990
|
+
return undefined;
|
|
991
|
+
const ownerType = row.ownerId.startsWith("host-") ? "host" : "session";
|
|
992
|
+
return {
|
|
993
|
+
actor: { type: ownerType, id: row.ownerId },
|
|
994
|
+
ownerType,
|
|
995
|
+
ownerId: row.ownerId,
|
|
996
|
+
token: claimToken,
|
|
997
|
+
generation: row.leaseGeneration,
|
|
998
|
+
leaseMs: 30_000,
|
|
999
|
+
};
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
parkWorkflowRun(options: { runId: string; claimToken: string; now?: string }): boolean {
|
|
1003
|
+
return this.releaseRunClaim(options.runId, options.claimToken, "parked", options.now);
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
parkWorkflowRunForRunnerNoProgress(options: {
|
|
1007
|
+
runId: string;
|
|
1008
|
+
claimToken: string;
|
|
1009
|
+
detail: string;
|
|
1010
|
+
now?: string;
|
|
1011
|
+
}): boolean {
|
|
1012
|
+
const now = epoch(validTimestamp(options.now));
|
|
1013
|
+
return this.state.transaction(() => {
|
|
1014
|
+
if (
|
|
1015
|
+
!this.verifyWorkflowRunClaim({
|
|
1016
|
+
runId: options.runId,
|
|
1017
|
+
claimToken: options.claimToken,
|
|
1018
|
+
now: new Date(now).toISOString(),
|
|
1019
|
+
})
|
|
1020
|
+
) {
|
|
1021
|
+
return false;
|
|
1022
|
+
}
|
|
1023
|
+
const row = this.requireWorkflowRunRow(options.runId);
|
|
1024
|
+
const lease = this.requireLease(row.resourceId);
|
|
1025
|
+
const detail = options.detail.slice(0, 8_192);
|
|
1026
|
+
const errorHash = this.state.putText(detail, now);
|
|
1027
|
+
const run = this.state.connection
|
|
1028
|
+
.prepare(
|
|
1029
|
+
`UPDATE runs SET status_detail = ?, updated_at = ?, finished_at = NULL
|
|
1030
|
+
WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
|
|
1031
|
+
)
|
|
1032
|
+
.run(detail, now, options.runId);
|
|
1033
|
+
const queue = this.state.connection
|
|
1034
|
+
.prepare(
|
|
1035
|
+
`UPDATE run_queue
|
|
1036
|
+
SET status = 'parked', error_code = 'runnerNoProgress', error_hash = ?,
|
|
1037
|
+
available_at = ?, updated_at = ?, finished_at = NULL
|
|
1038
|
+
WHERE run_id = ? AND status IN ('queued', 'starting', 'running', 'parked')`,
|
|
1039
|
+
)
|
|
1040
|
+
.run(errorHash, now, now, options.runId);
|
|
1041
|
+
if (run.changes !== 1 || queue.changes !== 1) {
|
|
1042
|
+
throw new Error(`Workflow run ${options.runId} changed during runner recovery`);
|
|
1043
|
+
}
|
|
1044
|
+
const released = this.state.connection
|
|
1045
|
+
.prepare(
|
|
1046
|
+
`UPDATE leases
|
|
1047
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1048
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1049
|
+
WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1050
|
+
)
|
|
1051
|
+
.run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
|
|
1052
|
+
/* istanbul ignore if -- the exact live claim is stable in this transaction */
|
|
1053
|
+
if (released.changes !== 1) {
|
|
1054
|
+
throw new Error(`Workflow run ${options.runId} claim changed during runner recovery`);
|
|
1055
|
+
}
|
|
1056
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1057
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1058
|
+
this.insertEvent(
|
|
1059
|
+
row.resourceId,
|
|
1060
|
+
revision + 1,
|
|
1061
|
+
"run.worker_no_progress",
|
|
1062
|
+
lease.ownerType ?? "system",
|
|
1063
|
+
lease.ownerId,
|
|
1064
|
+
{ status: "parked", code: "runnerNoProgress" },
|
|
1065
|
+
now,
|
|
1066
|
+
lease.generation || undefined,
|
|
1067
|
+
);
|
|
1068
|
+
recordViewerDeltas(
|
|
1069
|
+
this.state,
|
|
1070
|
+
options.runId,
|
|
1071
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1072
|
+
now,
|
|
1073
|
+
);
|
|
1074
|
+
return true;
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
pauseParkedWorkflowRun(options: { runId: string; now?: string }): boolean {
|
|
1079
|
+
const now = epoch(validTimestamp(options.now));
|
|
1080
|
+
return this.state.transaction(() => {
|
|
1081
|
+
const row = this.workflowRunRow(options.runId);
|
|
1082
|
+
if (row === undefined || row.status !== "parked") return false;
|
|
1083
|
+
const lease = this.requireLease(row.resourceId);
|
|
1084
|
+
if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
|
|
1085
|
+
const pending = this.state.connection
|
|
1086
|
+
.prepare(
|
|
1087
|
+
`SELECT 1 FROM interactive_requests
|
|
1088
|
+
WHERE run_id = ? AND status = 'pending' LIMIT 1`,
|
|
1089
|
+
)
|
|
1090
|
+
.get(options.runId);
|
|
1091
|
+
if (pending === undefined) return false;
|
|
1092
|
+
if (row.paused === 1) return true;
|
|
1093
|
+
if (!["running", "waiting"].includes(row.runStatus)) return false;
|
|
1094
|
+
const changed = this.state.connection
|
|
1095
|
+
.prepare(
|
|
1096
|
+
`UPDATE runs SET paused = 1, status_detail = 'paused', updated_at = ?
|
|
1097
|
+
WHERE run_id = ? AND paused = 0 AND status IN ('running', 'waiting')`,
|
|
1098
|
+
)
|
|
1099
|
+
.run(now, options.runId);
|
|
1100
|
+
if (changed.changes !== 1) return false;
|
|
1101
|
+
closeRunTime(this.state, options.runId);
|
|
1102
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1103
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1104
|
+
this.insertEvent(
|
|
1105
|
+
row.resourceId,
|
|
1106
|
+
revision + 1,
|
|
1107
|
+
"run.paused",
|
|
1108
|
+
"control",
|
|
1109
|
+
null,
|
|
1110
|
+
{ status: "parked" },
|
|
1111
|
+
now,
|
|
1112
|
+
);
|
|
1113
|
+
recordViewerDeltas(
|
|
1114
|
+
this.state,
|
|
1115
|
+
options.runId,
|
|
1116
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1117
|
+
now,
|
|
1118
|
+
);
|
|
1119
|
+
return true;
|
|
1120
|
+
});
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
resumePausedInteraction(options: { runId: string; now?: string }): boolean {
|
|
1124
|
+
const now = epoch(validTimestamp(options.now));
|
|
1125
|
+
return this.state.transaction(() => {
|
|
1126
|
+
const row = this.workflowRunRow(options.runId);
|
|
1127
|
+
if (
|
|
1128
|
+
row === undefined ||
|
|
1129
|
+
row.status !== "parked" ||
|
|
1130
|
+
row.runStatus !== "waiting" ||
|
|
1131
|
+
row.paused !== 1
|
|
1132
|
+
) {
|
|
1133
|
+
return false;
|
|
1134
|
+
}
|
|
1135
|
+
const lease = this.requireLease(row.resourceId);
|
|
1136
|
+
if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
|
|
1137
|
+
const pending = this.state.connection
|
|
1138
|
+
.prepare(
|
|
1139
|
+
`SELECT 1 FROM interactive_requests
|
|
1140
|
+
WHERE run_id = ? AND status = 'pending' LIMIT 1`,
|
|
1141
|
+
)
|
|
1142
|
+
.get(options.runId);
|
|
1143
|
+
if (pending === undefined) return false;
|
|
1144
|
+
const changed = this.state.connection
|
|
1145
|
+
.prepare(
|
|
1146
|
+
`UPDATE runs
|
|
1147
|
+
SET paused = 0, status_detail = 'waiting for origin-session input', updated_at = ?
|
|
1148
|
+
WHERE run_id = ? AND status = 'waiting' AND paused = 1`,
|
|
1149
|
+
)
|
|
1150
|
+
.run(now, options.runId);
|
|
1151
|
+
if (changed.changes !== 1) return false;
|
|
1152
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1153
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1154
|
+
this.insertEvent(
|
|
1155
|
+
row.resourceId,
|
|
1156
|
+
revision + 1,
|
|
1157
|
+
"run.resumed",
|
|
1158
|
+
"control",
|
|
1159
|
+
null,
|
|
1160
|
+
{ status: "waiting" },
|
|
1161
|
+
now,
|
|
1162
|
+
);
|
|
1163
|
+
recordViewerDeltas(
|
|
1164
|
+
this.state,
|
|
1165
|
+
options.runId,
|
|
1166
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1167
|
+
now,
|
|
1168
|
+
);
|
|
1169
|
+
return true;
|
|
1170
|
+
});
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
requestWorkflowRunResume(options: { runId: string; now?: string }): boolean {
|
|
1174
|
+
const now = epoch(validTimestamp(options.now));
|
|
1175
|
+
return this.state.transaction(() => {
|
|
1176
|
+
const row = this.workflowRunRow(options.runId);
|
|
1177
|
+
if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return false;
|
|
1178
|
+
const lease = this.requireLease(row.resourceId);
|
|
1179
|
+
if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
|
|
1180
|
+
if (
|
|
1181
|
+
this.state.connection
|
|
1182
|
+
.prepare("SELECT 1 FROM interactive_requests WHERE run_id = ? AND status = 'pending'")
|
|
1183
|
+
.get(options.runId) !== undefined
|
|
1184
|
+
)
|
|
1185
|
+
return false;
|
|
1186
|
+
this.state.connection
|
|
1187
|
+
.prepare("UPDATE runs SET paused = 0, updated_at = ? WHERE run_id = ?")
|
|
1188
|
+
.run(now, options.runId);
|
|
1189
|
+
this.state.connection
|
|
1190
|
+
.prepare(
|
|
1191
|
+
"UPDATE run_queue SET status = 'queued', error_code = NULL, error_hash = NULL, available_at = ?, updated_at = ? WHERE run_id = ?",
|
|
1192
|
+
)
|
|
1193
|
+
.run(now, now, options.runId);
|
|
1194
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1195
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1196
|
+
this.insertEvent(
|
|
1197
|
+
row.resourceId,
|
|
1198
|
+
revision + 1,
|
|
1199
|
+
"run.resume_requested",
|
|
1200
|
+
"control",
|
|
1201
|
+
null,
|
|
1202
|
+
{ status: "queued" },
|
|
1203
|
+
now,
|
|
1204
|
+
);
|
|
1205
|
+
recordViewerDeltas(
|
|
1206
|
+
this.state,
|
|
1207
|
+
options.runId,
|
|
1208
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1209
|
+
now,
|
|
1210
|
+
);
|
|
1211
|
+
return true;
|
|
1212
|
+
});
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
isWorkflowRunPaused(runId: string): boolean {
|
|
1216
|
+
return this.workflowRunRow(runId)?.paused === 1;
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
parkWorkflowRunForSourceChange(options: {
|
|
1220
|
+
runId: string;
|
|
1221
|
+
claimToken: string;
|
|
1222
|
+
detail: string;
|
|
1223
|
+
now?: string;
|
|
1224
|
+
}): boolean {
|
|
1225
|
+
const now = epoch(validTimestamp(options.now));
|
|
1226
|
+
return this.state.transaction(() => {
|
|
1227
|
+
const row = this.workflowRunRow(options.runId);
|
|
1228
|
+
if (
|
|
1229
|
+
row === undefined ||
|
|
1230
|
+
!this.verifyWorkflowRunClaim({
|
|
1231
|
+
runId: options.runId,
|
|
1232
|
+
claimToken: options.claimToken,
|
|
1233
|
+
now: new Date(now).toISOString(),
|
|
1234
|
+
})
|
|
1235
|
+
) {
|
|
1236
|
+
return false;
|
|
1237
|
+
}
|
|
1238
|
+
const lease = this.requireLease(row.resourceId);
|
|
1239
|
+
const detail = options.detail.slice(0, 8_192);
|
|
1240
|
+
const errorHash = this.state.putText(detail, now);
|
|
1241
|
+
const run = this.state.connection
|
|
1242
|
+
.prepare(
|
|
1243
|
+
`UPDATE runs SET status_detail = ?, updated_at = ?, finished_at = NULL
|
|
1244
|
+
WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
|
|
1245
|
+
)
|
|
1246
|
+
.run(detail, now, options.runId);
|
|
1247
|
+
const queue = this.state.connection
|
|
1248
|
+
.prepare(
|
|
1249
|
+
`UPDATE run_queue
|
|
1250
|
+
SET status = 'parked', error_code = 'workflowSourceChanged', error_hash = ?,
|
|
1251
|
+
available_at = ?, updated_at = ?, finished_at = NULL
|
|
1252
|
+
WHERE run_id = ? AND status IN ('queued', 'starting', 'running', 'parked')`,
|
|
1253
|
+
)
|
|
1254
|
+
.run(errorHash, now, now, options.runId);
|
|
1255
|
+
/* istanbul ignore if -- exact live claim and nonterminal checks make both updates mandatory */
|
|
1256
|
+
if (run.changes !== 1 || queue.changes !== 1) {
|
|
1257
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent source-change state`);
|
|
1258
|
+
}
|
|
1259
|
+
const released = this.state.connection
|
|
1260
|
+
.prepare(
|
|
1261
|
+
`UPDATE leases
|
|
1262
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1263
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1264
|
+
WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1265
|
+
)
|
|
1266
|
+
.run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
|
|
1267
|
+
/* istanbul ignore if -- the exact live claim is stable in this transaction */
|
|
1268
|
+
if (released.changes !== 1) {
|
|
1269
|
+
throw new Error(`Workflow run ${options.runId} claim changed during source recovery`);
|
|
1270
|
+
}
|
|
1271
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1272
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1273
|
+
this.insertEvent(
|
|
1274
|
+
row.resourceId,
|
|
1275
|
+
revision + 1,
|
|
1276
|
+
"run.source_changed",
|
|
1277
|
+
lease.ownerType ?? "system",
|
|
1278
|
+
lease.ownerId,
|
|
1279
|
+
{ status: "parked", code: "workflowSourceChanged" },
|
|
1280
|
+
now,
|
|
1281
|
+
lease.generation || undefined,
|
|
1282
|
+
);
|
|
1283
|
+
recordViewerDeltas(
|
|
1284
|
+
this.state,
|
|
1285
|
+
options.runId,
|
|
1286
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1287
|
+
now,
|
|
1288
|
+
);
|
|
1289
|
+
return true;
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1293
|
+
parkWorkflowRunForAmbiguousEffect(options: {
|
|
1294
|
+
runId: string;
|
|
1295
|
+
claimToken: string;
|
|
1296
|
+
now?: string;
|
|
1297
|
+
}): boolean {
|
|
1298
|
+
const now = epoch(validTimestamp(options.now));
|
|
1299
|
+
return this.state.transaction(() => {
|
|
1300
|
+
const row = this.workflowRunRow(options.runId);
|
|
1301
|
+
if (
|
|
1302
|
+
row === undefined ||
|
|
1303
|
+
!this.verifyWorkflowRunClaim({
|
|
1304
|
+
runId: options.runId,
|
|
1305
|
+
claimToken: options.claimToken,
|
|
1306
|
+
now: new Date(now).toISOString(),
|
|
1307
|
+
})
|
|
1308
|
+
) {
|
|
1309
|
+
return false;
|
|
1310
|
+
}
|
|
1311
|
+
const lease = this.requireLease(row.resourceId);
|
|
1312
|
+
const detail = "effect outcome is ambiguous; explicit recovery is required";
|
|
1313
|
+
const errorHash = this.state.putText(detail, now);
|
|
1314
|
+
this.state.connection
|
|
1315
|
+
.prepare(
|
|
1316
|
+
`UPDATE node_attempts SET status = 'waiting', updated_at = ?
|
|
1317
|
+
WHERE run_id = ? AND status IN ('pending', 'running')`,
|
|
1318
|
+
)
|
|
1319
|
+
.run(now, options.runId);
|
|
1320
|
+
const run = this.state.connection
|
|
1321
|
+
.prepare(
|
|
1322
|
+
`UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
|
|
1323
|
+
WHERE run_id = ? AND status = 'running'`,
|
|
1324
|
+
)
|
|
1325
|
+
.run(detail, now, null, options.runId);
|
|
1326
|
+
const queue = this.state.connection
|
|
1327
|
+
.prepare(
|
|
1328
|
+
`UPDATE run_queue
|
|
1329
|
+
SET status = 'parked', error_code = 'effectAmbiguous', error_hash = ?, updated_at = ?
|
|
1330
|
+
WHERE run_id = ? AND status IN ('starting', 'running', 'parked')`,
|
|
1331
|
+
)
|
|
1332
|
+
.run(errorHash, now, options.runId);
|
|
1333
|
+
/* istanbul ignore if -- exact live claim and run checks make both updates mandatory */
|
|
1334
|
+
if (run.changes !== 1 || queue.changes !== 1) {
|
|
1335
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent ambiguous-effect state`);
|
|
1336
|
+
}
|
|
1337
|
+
const released = this.state.connection
|
|
1338
|
+
.prepare(
|
|
1339
|
+
`UPDATE leases
|
|
1340
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1341
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1342
|
+
WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1343
|
+
)
|
|
1344
|
+
.run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
|
|
1345
|
+
/* istanbul ignore if -- the exact live claim is stable in this transaction */
|
|
1346
|
+
if (released.changes !== 1) {
|
|
1347
|
+
throw new Error(`Workflow run ${options.runId} claim changed during effect recovery`);
|
|
1348
|
+
}
|
|
1349
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1350
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1351
|
+
this.insertEvent(
|
|
1352
|
+
row.resourceId,
|
|
1353
|
+
revision + 1,
|
|
1354
|
+
"run.effect_ambiguous",
|
|
1355
|
+
lease.ownerType ?? "system",
|
|
1356
|
+
lease.ownerId,
|
|
1357
|
+
{ status: "waiting", code: "effectAmbiguous" },
|
|
1358
|
+
now,
|
|
1359
|
+
lease.generation || undefined,
|
|
1360
|
+
);
|
|
1361
|
+
recordViewerDeltas(
|
|
1362
|
+
this.state,
|
|
1363
|
+
options.runId,
|
|
1364
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1365
|
+
now,
|
|
1366
|
+
);
|
|
1367
|
+
return true;
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
failWorkflowRun(options: {
|
|
1372
|
+
runId: string;
|
|
1373
|
+
claimToken?: string;
|
|
1374
|
+
errorCode: string;
|
|
1375
|
+
errorMessage: string;
|
|
1376
|
+
now?: string;
|
|
1377
|
+
}): boolean {
|
|
1378
|
+
return this.terminalRun(
|
|
1379
|
+
options.runId,
|
|
1380
|
+
options.claimToken,
|
|
1381
|
+
"failed",
|
|
1382
|
+
options.errorCode,
|
|
1383
|
+
options.errorMessage,
|
|
1384
|
+
options.now,
|
|
1385
|
+
);
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
cancelWorkflowRun(options: { runId: string; claimToken?: string; now?: string }): boolean {
|
|
1389
|
+
const cancelled = this.terminalRun(
|
|
1390
|
+
options.runId,
|
|
1391
|
+
options.claimToken,
|
|
1392
|
+
"cancelled",
|
|
1393
|
+
"cancelled",
|
|
1394
|
+
"Workflow run cancelled",
|
|
1395
|
+
options.now,
|
|
1396
|
+
);
|
|
1397
|
+
if (cancelled || options.claimToken !== undefined) return cancelled;
|
|
1398
|
+
return this.cancelStaleWorkflowRun({
|
|
1399
|
+
runId: options.runId,
|
|
1400
|
+
...(options.now === undefined ? {} : { now: options.now }),
|
|
1401
|
+
});
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
/** Cancel a nonterminal run only when its claim is absent or expired. */
|
|
1405
|
+
cancelStaleWorkflowRun(options: {
|
|
1406
|
+
runId: string;
|
|
1407
|
+
controlId?: string;
|
|
1408
|
+
claimToken?: string;
|
|
1409
|
+
now?: string;
|
|
1410
|
+
}): boolean {
|
|
1411
|
+
const now = epoch(validTimestamp(options.now));
|
|
1412
|
+
const controlId = options.controlId ?? "workflow-control";
|
|
1413
|
+
const claimToken = options.claimToken ?? randomUUID();
|
|
1414
|
+
return this.state.transaction(() => {
|
|
1415
|
+
const row = this.workflowRunRow(options.runId);
|
|
1416
|
+
if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return false;
|
|
1417
|
+
const lease = this.requireLease(row.resourceId);
|
|
1418
|
+
if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
|
|
1419
|
+
|
|
1420
|
+
const generation = lease.generation + 1;
|
|
1421
|
+
const claimed = this.state.connection
|
|
1422
|
+
.prepare(
|
|
1423
|
+
`UPDATE leases
|
|
1424
|
+
SET generation = ?, owner_type = 'system', owner_id = ?, token_hash = ?,
|
|
1425
|
+
acquired_at = ?, heartbeat_at = ?, expires_at = ?
|
|
1426
|
+
WHERE resource_id = ? AND generation = ?
|
|
1427
|
+
AND (owner_id IS NULL OR expires_at IS NULL OR expires_at <= ?)`,
|
|
1428
|
+
)
|
|
1429
|
+
.run(
|
|
1430
|
+
generation,
|
|
1431
|
+
controlId,
|
|
1432
|
+
tokenHash(claimToken),
|
|
1433
|
+
now,
|
|
1434
|
+
now,
|
|
1435
|
+
now + 30_000,
|
|
1436
|
+
row.resourceId,
|
|
1437
|
+
lease.generation,
|
|
1438
|
+
now,
|
|
1439
|
+
);
|
|
1440
|
+
if (claimed.changes !== 1) return false;
|
|
1441
|
+
|
|
1442
|
+
const errorHash = this.state.putText("Workflow run cancelled", now);
|
|
1443
|
+
const queue = this.state.connection
|
|
1444
|
+
.prepare(
|
|
1445
|
+
`UPDATE run_queue
|
|
1446
|
+
SET status = 'cancelled', error_code = 'cancelled', error_hash = ?,
|
|
1447
|
+
updated_at = ?, finished_at = ?
|
|
1448
|
+
WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
|
|
1449
|
+
)
|
|
1450
|
+
.run(errorHash, now, now, options.runId);
|
|
1451
|
+
/* istanbul ignore if -- the control claim selects one nonterminal queue row */
|
|
1452
|
+
if (queue.changes !== 1) {
|
|
1453
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent queue state`);
|
|
1454
|
+
}
|
|
1455
|
+
const run = this.state.connection
|
|
1456
|
+
.prepare(
|
|
1457
|
+
`UPDATE runs
|
|
1458
|
+
SET status = 'cancelled', paused = 0, status_detail = NULL, error_hash = ?,
|
|
1459
|
+
updated_at = ?, finished_at = ?
|
|
1460
|
+
WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
|
|
1461
|
+
)
|
|
1462
|
+
.run(errorHash, now, now, options.runId);
|
|
1463
|
+
/* istanbul ignore if -- the control claim selects one nonterminal durable run */
|
|
1464
|
+
if (run.changes !== 1) {
|
|
1465
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent durable state`);
|
|
1466
|
+
}
|
|
1467
|
+
this.settleWorkflowRunMessages(options.runId, now);
|
|
1468
|
+
this.cancelWorkflowRunDependents(options.runId, controlId, errorHash, now);
|
|
1469
|
+
|
|
1470
|
+
const released = this.state.connection
|
|
1471
|
+
.prepare(
|
|
1472
|
+
`UPDATE leases
|
|
1473
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1474
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1475
|
+
WHERE resource_id = ? AND owner_type = 'system' AND owner_id = ?
|
|
1476
|
+
AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1477
|
+
)
|
|
1478
|
+
.run(row.resourceId, controlId, tokenHash(claimToken), generation, now);
|
|
1479
|
+
/* istanbul ignore if -- the exact control claim was written in this transaction */
|
|
1480
|
+
if (released.changes !== 1) {
|
|
1481
|
+
throw new Error(`Workflow run ${options.runId} control claim changed during cancellation`);
|
|
1482
|
+
}
|
|
1483
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1484
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1485
|
+
this.insertEvent(
|
|
1486
|
+
row.resourceId,
|
|
1487
|
+
revision + 1,
|
|
1488
|
+
"run.queue_cancelled",
|
|
1489
|
+
"control",
|
|
1490
|
+
controlId,
|
|
1491
|
+
{ status: "cancelled", code: "cancelled", staleControl: true },
|
|
1492
|
+
now,
|
|
1493
|
+
generation,
|
|
1494
|
+
);
|
|
1495
|
+
recordViewerDeltas(
|
|
1496
|
+
this.state,
|
|
1497
|
+
options.runId,
|
|
1498
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1499
|
+
now,
|
|
1500
|
+
);
|
|
1501
|
+
return true;
|
|
1502
|
+
});
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
deleteWorkflowRun(options: { runId: string; claimToken: string }): boolean {
|
|
1506
|
+
const now = Date.now();
|
|
1507
|
+
return this.state.transaction(() => {
|
|
1508
|
+
const row = this.workflowRunRow(options.runId);
|
|
1509
|
+
if (row === undefined) return false;
|
|
1510
|
+
const lease = this.requireLease(row.resourceId);
|
|
1511
|
+
if (
|
|
1512
|
+
lease.ownerId === null ||
|
|
1513
|
+
lease.tokenHash === null ||
|
|
1514
|
+
lease.expiresAt === null ||
|
|
1515
|
+
lease.expiresAt <= now ||
|
|
1516
|
+
!lease.tokenHash.equals(tokenHash(options.claimToken))
|
|
1517
|
+
) {
|
|
1518
|
+
return false;
|
|
1519
|
+
}
|
|
1520
|
+
return (
|
|
1521
|
+
this.state.connection
|
|
1522
|
+
.prepare("DELETE FROM resources WHERE resource_id = ?")
|
|
1523
|
+
.run(row.resourceId).changes === 1
|
|
1524
|
+
);
|
|
1525
|
+
});
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
completeWorkflowRun(options: { runId: string; claimToken: string; now?: string }): boolean {
|
|
1529
|
+
return this.releaseRunClaim(options.runId, options.claimToken, "done", options.now);
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
recordRunEvent(options: {
|
|
1533
|
+
runId: string;
|
|
1534
|
+
workflowRef: string;
|
|
1535
|
+
type: string;
|
|
1536
|
+
payload?: JsonObject;
|
|
1537
|
+
runnerId?: string;
|
|
1538
|
+
now?: string;
|
|
1539
|
+
}): RunEventRecord {
|
|
1540
|
+
return this.state.transaction(() => {
|
|
1541
|
+
const row = this.requireWorkflowRunRow(options.runId);
|
|
1542
|
+
const now = epoch(validTimestamp(options.now));
|
|
1543
|
+
const revision = this.resourceRevision(row.resourceId) + 1;
|
|
1544
|
+
this.bumpResource(row.resourceId, revision - 1, now);
|
|
1545
|
+
const eventId = this.insertEvent(
|
|
1546
|
+
row.resourceId,
|
|
1547
|
+
revision,
|
|
1548
|
+
options.type,
|
|
1549
|
+
options.runnerId?.startsWith("host-") ? "host" : "session",
|
|
1550
|
+
options.runnerId ?? null,
|
|
1551
|
+
options.payload ?? {},
|
|
1552
|
+
now,
|
|
1553
|
+
row.leaseGeneration || undefined,
|
|
1554
|
+
);
|
|
1555
|
+
const seq = this.state.connection
|
|
1556
|
+
.prepare("SELECT event_seq AS seq FROM events WHERE event_id = ?")
|
|
1557
|
+
.get(eventId);
|
|
1558
|
+
/* istanbul ignore if -- exact schema and internal query shape */
|
|
1559
|
+
if (!isSequenceRow(seq)) throw new Error("Run event was not recorded");
|
|
1560
|
+
return {
|
|
1561
|
+
seq: seq.seq,
|
|
1562
|
+
recordedAt: new Date(now).toISOString(),
|
|
1563
|
+
runId: options.runId,
|
|
1564
|
+
workflowRef: options.workflowRef,
|
|
1565
|
+
type: options.type,
|
|
1566
|
+
runnerId: options.runnerId ?? null,
|
|
1567
|
+
payload: options.payload ?? {},
|
|
1568
|
+
};
|
|
1569
|
+
});
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
listRunEventsAfter(seq: number, options: { limit?: number } = {}): RunEventRecord[] {
|
|
1573
|
+
const rows = this.state.connection
|
|
1574
|
+
.prepare(
|
|
1575
|
+
`SELECT e.event_seq AS seq, e.recorded_at AS recordedAt, r.run_id AS runId,
|
|
1576
|
+
r.workflow_ref AS workflowRef, e.event_type AS eventType,
|
|
1577
|
+
e.actor_id AS runnerId, e.payload_hash AS payloadHash
|
|
1578
|
+
FROM events e JOIN runs r ON r.resource_id = e.resource_id
|
|
1579
|
+
WHERE e.event_seq > ? ORDER BY e.event_seq LIMIT ?`,
|
|
1580
|
+
)
|
|
1581
|
+
.all(seq, options.limit ?? 100);
|
|
1582
|
+
return rows.filter(isRunEventRow).map((row) => ({
|
|
1583
|
+
seq: row.seq,
|
|
1584
|
+
recordedAt: new Date(row.recordedAt).toISOString(),
|
|
1585
|
+
runId: row.runId,
|
|
1586
|
+
workflowRef: row.workflowRef,
|
|
1587
|
+
type: row.eventType,
|
|
1588
|
+
runnerId: row.runnerId,
|
|
1589
|
+
payload: row.payloadHash === null ? {} : (this.state.readJson(row.payloadHash) as JsonObject),
|
|
1590
|
+
}));
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
settleRunEffect(runId: string, effectType: "run.park_queue" | "run.settle_queue"): void {
|
|
1594
|
+
const rows = this.state.connection
|
|
1595
|
+
.prepare(
|
|
1596
|
+
`SELECT e.effect_id AS effectId, e.resource_id AS resourceId
|
|
1597
|
+
FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
|
|
1598
|
+
WHERE r.run_id = ? AND e.effect_type = ? AND e.status = 'pending'`,
|
|
1599
|
+
)
|
|
1600
|
+
.all(runId, effectType);
|
|
1601
|
+
for (const row of rows) {
|
|
1602
|
+
/* istanbul ignore if -- exact schema and internal query shape */
|
|
1603
|
+
if (!isEffectIdentityRow(row)) continue;
|
|
1604
|
+
this.state.transaction(() => {
|
|
1605
|
+
const now = Date.now();
|
|
1606
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1607
|
+
this.state.connection
|
|
1608
|
+
.prepare(
|
|
1609
|
+
`UPDATE effects SET status = 'applied', updated_at = ?, settled_at = ?
|
|
1610
|
+
WHERE effect_id = ? AND status = 'pending'`,
|
|
1611
|
+
)
|
|
1612
|
+
.run(now, now, row.effectId);
|
|
1613
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1614
|
+
this.insertEvent(
|
|
1615
|
+
row.resourceId,
|
|
1616
|
+
revision + 1,
|
|
1617
|
+
"effect.applied",
|
|
1618
|
+
"system",
|
|
1619
|
+
null,
|
|
1620
|
+
{ runId, effectType },
|
|
1621
|
+
now,
|
|
1622
|
+
);
|
|
1623
|
+
});
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
private workflowRunRow(runId: string): RunRow | undefined {
|
|
1628
|
+
const row = this.state.connection.prepare(workflowRunSelect("WHERE r.run_id = ?")).get(runId);
|
|
1629
|
+
return isRunRow(row) ? row : undefined;
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
private requireWorkflowRunRow(runId: string): RunRow {
|
|
1633
|
+
const row = this.workflowRunRow(runId);
|
|
1634
|
+
if (row === undefined) throw new Error(`Workflow run not found: ${runId}`);
|
|
1635
|
+
return row;
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
private requireWorkflowRun(runId: string): WorkflowRunQueueRecord {
|
|
1639
|
+
return this.mapWorkflowRun(this.requireWorkflowRunRow(runId));
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
private assertWorkflowRunPreparationCompatible(
|
|
1643
|
+
row: RunRow,
|
|
1644
|
+
options: WorkflowRunReservationOptions,
|
|
1645
|
+
definitionDigest: Buffer,
|
|
1646
|
+
): void {
|
|
1647
|
+
const expectedLineageKind =
|
|
1648
|
+
options.parentRunId === undefined ? null : (options.lineageKind ?? null);
|
|
1649
|
+
const parent =
|
|
1650
|
+
options.parentRunId === undefined
|
|
1651
|
+
? undefined
|
|
1652
|
+
: this.requireWorkflowRunRow(options.parentRunId);
|
|
1653
|
+
const expectedRootRunId = parent?.rootRunId ?? options.runId;
|
|
1654
|
+
const expectedRestartNumber =
|
|
1655
|
+
expectedLineageKind === "restart"
|
|
1656
|
+
? (options.restartNumber ?? (parent?.restartNumber ?? -1) + 1)
|
|
1657
|
+
: (parent?.restartNumber ?? 0);
|
|
1658
|
+
const expectedParentRunRevision = options.parentRunRevision ?? null;
|
|
1659
|
+
const compatible =
|
|
1660
|
+
row.workflowName === options.workflowName &&
|
|
1661
|
+
row.workflowRef === options.workflowSourceRef &&
|
|
1662
|
+
row.definitionDigest.equals(definitionDigest) &&
|
|
1663
|
+
canonicalJson(readQueuedRunSources(this.state, row)) ===
|
|
1664
|
+
canonicalJson(queuedWorkflowSource(options.workflowSource)) &&
|
|
1665
|
+
canonicalJson(this.state.readJson(row.inputHash)) === canonicalJson(options.input ?? null) &&
|
|
1666
|
+
canonicalJson(this.state.readJson(row.launchOptionsHash)) ===
|
|
1667
|
+
canonicalJson(options.launchOptions ?? {}) &&
|
|
1668
|
+
row.originSessionId === options.originSessionId &&
|
|
1669
|
+
row.executionMode === (options.executionMode ?? "interactive") &&
|
|
1670
|
+
row.parentRunId === (options.parentRunId ?? null) &&
|
|
1671
|
+
row.rootRunId === expectedRootRunId &&
|
|
1672
|
+
row.lineageKind === expectedLineageKind &&
|
|
1673
|
+
row.restartNumber === expectedRestartNumber &&
|
|
1674
|
+
row.parentRunRevision === expectedParentRunRevision;
|
|
1675
|
+
if (!compatible) {
|
|
1676
|
+
throw new Error(`Workflow run preparation conflicts: ${options.runId}`);
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
private workflowRunViewRows(clause: string, params: unknown[]): WorkflowRunViewRow[] {
|
|
1681
|
+
return this.state.connection
|
|
1682
|
+
.prepare(workflowRunViewSelect(clause))
|
|
1683
|
+
.all(...params)
|
|
1684
|
+
.filter(isWorkflowRunViewRow);
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
/* istanbul ignore next -- pure projection covered by integration tests */
|
|
1688
|
+
private mapWorkflowRun(row: RunRow): WorkflowRunQueueRecord {
|
|
1689
|
+
return {
|
|
1690
|
+
runId: row.runId,
|
|
1691
|
+
workflowName: row.workflowName,
|
|
1692
|
+
workflowSourceRef: row.workflowRef,
|
|
1693
|
+
workflowSource: readQueuedRunSources(this.state, row),
|
|
1694
|
+
initialized: row.runStatus !== "queued",
|
|
1695
|
+
definitionDigest: `sha256:${row.definitionDigest.toString("hex")}`,
|
|
1696
|
+
input: this.state.readJson(row.inputHash),
|
|
1697
|
+
launchOptions: this.state.readJson(row.launchOptionsHash),
|
|
1698
|
+
status: row.status,
|
|
1699
|
+
runnerId: row.ownerId,
|
|
1700
|
+
claimToken: null,
|
|
1701
|
+
claimGeneration: row.ownerId === null ? null : row.leaseGeneration,
|
|
1702
|
+
claimExpiresAt:
|
|
1703
|
+
row.claimExpiresAt === null ? null : new Date(row.claimExpiresAt).toISOString(),
|
|
1704
|
+
originSessionId: row.executionMode === "headless" ? null : row.originSessionId,
|
|
1705
|
+
executionMode: row.executionMode,
|
|
1706
|
+
parentRunId: row.parentRunId,
|
|
1707
|
+
rootRunId: row.rootRunId,
|
|
1708
|
+
lineageKind: row.lineageKind,
|
|
1709
|
+
restartNumber: row.restartNumber,
|
|
1710
|
+
parentRunRevision: row.parentRunRevision,
|
|
1711
|
+
errorCode: row.errorCode,
|
|
1712
|
+
errorMessage:
|
|
1713
|
+
row.errorHash === null
|
|
1714
|
+
? null
|
|
1715
|
+
: (this.state.readBlob(row.errorHash)?.content.toString("utf8") ?? null),
|
|
1716
|
+
createdAt: new Date(row.createdAt).toISOString(),
|
|
1717
|
+
updatedAt: new Date(row.updatedAt).toISOString(),
|
|
1718
|
+
startedAt: row.startedAt === null ? null : new Date(row.startedAt).toISOString(),
|
|
1719
|
+
finishedAt: row.finishedAt === null ? null : new Date(row.finishedAt).toISOString(),
|
|
1720
|
+
};
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
private claimRun(
|
|
1724
|
+
runId: string,
|
|
1725
|
+
runnerId: string,
|
|
1726
|
+
claimToken: string,
|
|
1727
|
+
leaseMs: number,
|
|
1728
|
+
now: number,
|
|
1729
|
+
options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
|
|
1730
|
+
): WorkflowRunQueueRecord | undefined {
|
|
1731
|
+
return this.state.transaction(() =>
|
|
1732
|
+
this.claimRunInTransaction(runId, runnerId, claimToken, leaseMs, now, options),
|
|
1733
|
+
);
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
private claimRunInTransaction(
|
|
1737
|
+
runId: string,
|
|
1738
|
+
runnerId: string,
|
|
1739
|
+
claimToken: string,
|
|
1740
|
+
leaseMs: number,
|
|
1741
|
+
now: number,
|
|
1742
|
+
options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
|
|
1743
|
+
): WorkflowRunQueueRecord | undefined {
|
|
1744
|
+
const row = this.workflowRunRow(runId);
|
|
1745
|
+
if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return undefined;
|
|
1746
|
+
if (
|
|
1747
|
+
options.allowPendingInteraction !== true &&
|
|
1748
|
+
this.state.connection
|
|
1749
|
+
.prepare(
|
|
1750
|
+
`SELECT 1 FROM interactive_requests
|
|
1751
|
+
WHERE run_id = ? AND status = 'pending' LIMIT 1`,
|
|
1752
|
+
)
|
|
1753
|
+
.get(runId) !== undefined
|
|
1754
|
+
) {
|
|
1755
|
+
return undefined;
|
|
1756
|
+
}
|
|
1757
|
+
const lease = this.requireLease(row.resourceId);
|
|
1758
|
+
if (
|
|
1759
|
+
lease.ownerId !== null &&
|
|
1760
|
+
lease.expiresAt !== null &&
|
|
1761
|
+
lease.expiresAt > now &&
|
|
1762
|
+
lease.ownerId !== runnerId
|
|
1763
|
+
)
|
|
1764
|
+
return undefined;
|
|
1765
|
+
const generation = lease.generation + 1;
|
|
1766
|
+
const expiresAt = now + leaseMs;
|
|
1767
|
+
const result = this.state.connection
|
|
1768
|
+
.prepare(
|
|
1769
|
+
`UPDATE leases SET generation = ?, owner_type = ?, owner_id = ?, token_hash = ?,
|
|
1770
|
+
acquired_at = ?, heartbeat_at = ?, expires_at = ?
|
|
1771
|
+
WHERE resource_id = ? AND generation = ?`,
|
|
1772
|
+
)
|
|
1773
|
+
.run(
|
|
1774
|
+
generation,
|
|
1775
|
+
runnerId.startsWith("host-") ? "host" : "session",
|
|
1776
|
+
runnerId,
|
|
1777
|
+
tokenHash(claimToken),
|
|
1778
|
+
now,
|
|
1779
|
+
now,
|
|
1780
|
+
expiresAt,
|
|
1781
|
+
row.resourceId,
|
|
1782
|
+
lease.generation,
|
|
1783
|
+
);
|
|
1784
|
+
/* istanbul ignore if -- impossible after exact schema and transaction checks */
|
|
1785
|
+
if (result.changes !== 1) return undefined;
|
|
1786
|
+
if (options.preserveQueueStatus !== true) {
|
|
1787
|
+
this.state.connection
|
|
1788
|
+
.prepare(
|
|
1789
|
+
`UPDATE run_queue
|
|
1790
|
+
SET status = 'starting', error_code = NULL, error_hash = NULL, updated_at = ?
|
|
1791
|
+
WHERE run_id = ?`,
|
|
1792
|
+
)
|
|
1793
|
+
.run(now, runId);
|
|
1794
|
+
}
|
|
1795
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1796
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1797
|
+
this.insertEvent(
|
|
1798
|
+
row.resourceId,
|
|
1799
|
+
revision + 1,
|
|
1800
|
+
"lease.claimed",
|
|
1801
|
+
runnerId.startsWith("host-") ? "host" : "session",
|
|
1802
|
+
runnerId,
|
|
1803
|
+
{ expiresAt },
|
|
1804
|
+
now,
|
|
1805
|
+
generation,
|
|
1806
|
+
);
|
|
1807
|
+
return { ...this.requireWorkflowRun(runId), claimToken };
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
private updateClaimedRunStatus(
|
|
1811
|
+
runId: string,
|
|
1812
|
+
claimToken: string,
|
|
1813
|
+
status: WorkflowRunLaunchStatus,
|
|
1814
|
+
nowValue?: string,
|
|
1815
|
+
): boolean {
|
|
1816
|
+
const now = epoch(validTimestamp(nowValue));
|
|
1817
|
+
return this.state.transaction(() => {
|
|
1818
|
+
if (
|
|
1819
|
+
!this.verifyWorkflowRunClaim({
|
|
1820
|
+
runId,
|
|
1821
|
+
claimToken,
|
|
1822
|
+
now: new Date(now).toISOString(),
|
|
1823
|
+
})
|
|
1824
|
+
) {
|
|
1825
|
+
return false;
|
|
1826
|
+
}
|
|
1827
|
+
const row = this.requireWorkflowRunRow(runId);
|
|
1828
|
+
const changed =
|
|
1829
|
+
this.state.connection
|
|
1830
|
+
.prepare(
|
|
1831
|
+
"UPDATE run_queue SET status = ?, started_at = COALESCE(started_at, ?), updated_at = ? WHERE run_id = ?",
|
|
1832
|
+
)
|
|
1833
|
+
.run(status, now, now, runId).changes === 1;
|
|
1834
|
+
/* istanbul ignore if -- impossible after exact schema and transaction checks */
|
|
1835
|
+
if (!changed) return false;
|
|
1836
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1837
|
+
const lease = this.requireLease(row.resourceId);
|
|
1838
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1839
|
+
this.insertEvent(
|
|
1840
|
+
row.resourceId,
|
|
1841
|
+
revision + 1,
|
|
1842
|
+
`run.queue_${status}`,
|
|
1843
|
+
lease.ownerType ?? "system",
|
|
1844
|
+
lease.ownerId,
|
|
1845
|
+
{ status },
|
|
1846
|
+
now,
|
|
1847
|
+
lease.generation || undefined,
|
|
1848
|
+
);
|
|
1849
|
+
return true;
|
|
1850
|
+
});
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
private releaseRunClaim(
|
|
1854
|
+
runId: string,
|
|
1855
|
+
claimToken: string,
|
|
1856
|
+
status: WorkflowRunLaunchStatus,
|
|
1857
|
+
nowValue?: string,
|
|
1858
|
+
): boolean {
|
|
1859
|
+
const now = epoch(validTimestamp(nowValue));
|
|
1860
|
+
return this.state.transaction(() => {
|
|
1861
|
+
const row = this.workflowRunRow(runId);
|
|
1862
|
+
if (
|
|
1863
|
+
row === undefined ||
|
|
1864
|
+
!this.verifyWorkflowRunClaim({ runId, claimToken, now: new Date(now).toISOString() })
|
|
1865
|
+
)
|
|
1866
|
+
return false;
|
|
1867
|
+
this.state.connection
|
|
1868
|
+
.prepare(
|
|
1869
|
+
"UPDATE run_queue SET status = ?, updated_at = ?, finished_at = ? WHERE run_id = ?",
|
|
1870
|
+
)
|
|
1871
|
+
.run(status, now, status === "done" ? now : null, runId);
|
|
1872
|
+
const lease = this.requireLease(row.resourceId);
|
|
1873
|
+
const update = this.state.connection
|
|
1874
|
+
.prepare(
|
|
1875
|
+
`UPDATE leases SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1876
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1877
|
+
WHERE resource_id = ? AND token_hash = ? AND generation = ?`,
|
|
1878
|
+
)
|
|
1879
|
+
.run(row.resourceId, tokenHash(claimToken), lease.generation);
|
|
1880
|
+
/* istanbul ignore if -- impossible after exact schema and transaction checks */
|
|
1881
|
+
if (update.changes !== 1) return false;
|
|
1882
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1883
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1884
|
+
this.insertEvent(
|
|
1885
|
+
row.resourceId,
|
|
1886
|
+
revision + 1,
|
|
1887
|
+
`run.queue_${status}`,
|
|
1888
|
+
lease.ownerType ?? "system",
|
|
1889
|
+
lease.ownerId,
|
|
1890
|
+
{ status },
|
|
1891
|
+
now,
|
|
1892
|
+
lease.generation || undefined,
|
|
1893
|
+
);
|
|
1894
|
+
this.settleRunEffect(runId, status === "parked" ? "run.park_queue" : "run.settle_queue");
|
|
1895
|
+
recordViewerDeltas(
|
|
1896
|
+
this.state,
|
|
1897
|
+
runId,
|
|
1898
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1899
|
+
now,
|
|
1900
|
+
);
|
|
1901
|
+
return true;
|
|
1902
|
+
});
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
private terminalRun(
|
|
1906
|
+
runId: string,
|
|
1907
|
+
claimToken: string | undefined,
|
|
1908
|
+
status: "failed" | "cancelled",
|
|
1909
|
+
code: string,
|
|
1910
|
+
error: string,
|
|
1911
|
+
nowValue?: string,
|
|
1912
|
+
): boolean {
|
|
1913
|
+
const now = epoch(validTimestamp(nowValue));
|
|
1914
|
+
return this.state.transaction(() => {
|
|
1915
|
+
const row = this.workflowRunRow(runId);
|
|
1916
|
+
if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return false;
|
|
1917
|
+
const lease = this.requireLease(row.resourceId);
|
|
1918
|
+
if (claimToken === undefined) {
|
|
1919
|
+
if (!(["queued", "starting"] as WorkflowRunLaunchStatus[]).includes(row.status)) {
|
|
1920
|
+
return false;
|
|
1921
|
+
}
|
|
1922
|
+
if (lease.ownerId !== null) return false;
|
|
1923
|
+
} else if (
|
|
1924
|
+
lease.ownerId === null ||
|
|
1925
|
+
lease.tokenHash === null ||
|
|
1926
|
+
lease.expiresAt === null ||
|
|
1927
|
+
lease.expiresAt <= now ||
|
|
1928
|
+
!lease.tokenHash.equals(tokenHash(claimToken))
|
|
1929
|
+
) {
|
|
1930
|
+
return false;
|
|
1931
|
+
}
|
|
1932
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1933
|
+
const errorHash = this.state.putText(error, now);
|
|
1934
|
+
const queueUpdate = this.state.connection
|
|
1935
|
+
.prepare(
|
|
1936
|
+
`UPDATE run_queue
|
|
1937
|
+
SET status = ?, error_code = ?, error_hash = ?, updated_at = ?, finished_at = ?
|
|
1938
|
+
WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
|
|
1939
|
+
)
|
|
1940
|
+
.run(status, code, errorHash, now, now, runId);
|
|
1941
|
+
if (queueUpdate.changes !== 1) return false;
|
|
1942
|
+
if (claimToken !== undefined) {
|
|
1943
|
+
const leaseUpdate = this.state.connection
|
|
1944
|
+
.prepare(
|
|
1945
|
+
`UPDATE leases
|
|
1946
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1947
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1948
|
+
WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1949
|
+
)
|
|
1950
|
+
.run(row.resourceId, tokenHash(claimToken), lease.generation, now);
|
|
1951
|
+
if (leaseUpdate.changes !== 1) {
|
|
1952
|
+
throw new Error(`Workflow run ${runId} claim changed during terminal transition`);
|
|
1953
|
+
}
|
|
1954
|
+
}
|
|
1955
|
+
this.state.connection
|
|
1956
|
+
.prepare(
|
|
1957
|
+
`UPDATE runs
|
|
1958
|
+
SET status = ?, paused = 0, status_detail = NULL, error_hash = ?,
|
|
1959
|
+
updated_at = ?, finished_at = ?
|
|
1960
|
+
WHERE run_id = ?`,
|
|
1961
|
+
)
|
|
1962
|
+
.run(status, errorHash, now, now, runId);
|
|
1963
|
+
this.settleWorkflowRunMessages(runId, now);
|
|
1964
|
+
if (status === "cancelled") {
|
|
1965
|
+
this.cancelWorkflowRunDependents(
|
|
1966
|
+
runId,
|
|
1967
|
+
lease.ownerId ?? "workflow-control",
|
|
1968
|
+
errorHash,
|
|
1969
|
+
now,
|
|
1970
|
+
);
|
|
1971
|
+
}
|
|
1972
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1973
|
+
this.insertEvent(
|
|
1974
|
+
row.resourceId,
|
|
1975
|
+
revision + 1,
|
|
1976
|
+
`run.queue_${status}`,
|
|
1977
|
+
lease.ownerType ?? "system",
|
|
1978
|
+
lease.ownerId,
|
|
1979
|
+
{ status, code, error },
|
|
1980
|
+
now,
|
|
1981
|
+
lease.generation || undefined,
|
|
1982
|
+
);
|
|
1983
|
+
recordViewerDeltas(
|
|
1984
|
+
this.state,
|
|
1985
|
+
runId,
|
|
1986
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1987
|
+
now,
|
|
1988
|
+
);
|
|
1989
|
+
return true;
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
private settleWorkflowRunMessages(runId: string, now: number): void {
|
|
1994
|
+
this.workflowMessages.settleOpenTurnsForRun(runId, "lost", now);
|
|
1995
|
+
this.workflowMessages.cancelPendingForRun(runId, now);
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
private cancelWorkflowRunDependents(
|
|
1999
|
+
runId: string,
|
|
2000
|
+
actorId: string,
|
|
2001
|
+
errorHash: Buffer,
|
|
2002
|
+
now: number,
|
|
2003
|
+
): void {
|
|
2004
|
+
closeRunTime(this.state, runId);
|
|
2005
|
+
this.state.connection
|
|
2006
|
+
.prepare(
|
|
2007
|
+
`UPDATE node_attempts
|
|
2008
|
+
SET status = 'cancelled', error_hash = COALESCE(error_hash, ?),
|
|
2009
|
+
updated_at = ?, finished_at = COALESCE(finished_at, ?)
|
|
2010
|
+
WHERE run_id = ? AND status IN ('pending', 'running', 'waiting', 'interrupted')`,
|
|
2011
|
+
)
|
|
2012
|
+
.run(errorHash, now, now, runId);
|
|
2013
|
+
|
|
2014
|
+
const effects = this.state.connection
|
|
2015
|
+
.prepare(
|
|
2016
|
+
`SELECT e.effect_id AS effectId, e.resource_id AS resourceId,
|
|
2017
|
+
e.status, e.attempt_count AS attemptCount
|
|
2018
|
+
FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
|
|
2019
|
+
WHERE r.run_id = ? AND e.owner_scope = 'run' AND e.status IN ('pending', 'applying')
|
|
2020
|
+
ORDER BY e.effect_id`,
|
|
2021
|
+
)
|
|
2022
|
+
.all(runId)
|
|
2023
|
+
.filter(isCancelledRunEffectRow);
|
|
2024
|
+
for (const effect of effects) {
|
|
2025
|
+
const status = effect.status === "applying" ? "ambiguous" : "cancelled";
|
|
2026
|
+
const revision = this.resourceRevision(effect.resourceId);
|
|
2027
|
+
const changed = this.state.connection
|
|
2028
|
+
.prepare(
|
|
2029
|
+
`UPDATE effects
|
|
2030
|
+
SET status = ?, next_attempt_at = NULL, error_hash = ?, updated_at = ?, settled_at = ?
|
|
2031
|
+
WHERE effect_id = ? AND status = ? AND attempt_count = ?`,
|
|
2032
|
+
)
|
|
2033
|
+
.run(status, errorHash, now, now, effect.effectId, effect.status, effect.attemptCount);
|
|
2034
|
+
/* istanbul ignore if -- cancellation serializes the selected effect transition */
|
|
2035
|
+
if (changed.changes !== 1) {
|
|
2036
|
+
throw new Error(`Workflow effect ${effect.effectId} changed during cancellation`);
|
|
2037
|
+
}
|
|
2038
|
+
if (effect.status === "applying") {
|
|
2039
|
+
this.state.connection
|
|
2040
|
+
.prepare(
|
|
2041
|
+
`UPDATE effect_attempts
|
|
2042
|
+
SET finished_at = ?, outcome = 'interrupted', error_hash = ?
|
|
2043
|
+
WHERE effect_id = ? AND attempt_number = ? AND finished_at IS NULL`,
|
|
2044
|
+
)
|
|
2045
|
+
.run(now, errorHash, effect.effectId, effect.attemptCount);
|
|
2046
|
+
}
|
|
2047
|
+
this.bumpResource(effect.resourceId, revision, now);
|
|
2048
|
+
this.insertEvent(
|
|
2049
|
+
effect.resourceId,
|
|
2050
|
+
revision + 1,
|
|
2051
|
+
`effect.${status}`,
|
|
2052
|
+
"control",
|
|
2053
|
+
actorId,
|
|
2054
|
+
{ runId, reason: "workflowCancelled" },
|
|
2055
|
+
now,
|
|
2056
|
+
);
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
this.state.connection
|
|
2060
|
+
.prepare(
|
|
2061
|
+
`INSERT INTO human_decision_resolutions(
|
|
2062
|
+
decision_id, outcome, provenance, response_hash, reason, channel,
|
|
2063
|
+
actor_id, request_digest, resolved_at
|
|
2064
|
+
)
|
|
2065
|
+
SELECT d.decision_id, 'cancelled', 'explicit_cancel', NULL,
|
|
2066
|
+
'Workflow run cancelled', NULL, ?, d.request_digest, ?
|
|
2067
|
+
FROM human_decisions d
|
|
2068
|
+
LEFT JOIN human_decision_resolutions r ON r.decision_id = d.decision_id
|
|
2069
|
+
WHERE d.run_id = ? AND r.decision_id IS NULL`,
|
|
2070
|
+
)
|
|
2071
|
+
.run(actorId, now, runId);
|
|
2072
|
+
const receiptHash = this.state.putJson(
|
|
2073
|
+
{ status: "rejected", error: "Workflow run cancelled" },
|
|
2074
|
+
now,
|
|
2075
|
+
);
|
|
2076
|
+
this.state.connection
|
|
2077
|
+
.prepare(
|
|
2078
|
+
`UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
|
|
2079
|
+
WHERE outcome = 'validating'
|
|
2080
|
+
AND request_id IN (SELECT request_id FROM interactive_requests WHERE run_id = ?)`,
|
|
2081
|
+
)
|
|
2082
|
+
.run(receiptHash, runId);
|
|
2083
|
+
this.state.connection
|
|
2084
|
+
.prepare(
|
|
2085
|
+
`UPDATE interactive_requests
|
|
2086
|
+
SET status = 'cancelled', revision = revision + 1, updated_at = ?
|
|
2087
|
+
WHERE run_id = ? AND status = 'pending'`,
|
|
2088
|
+
)
|
|
2089
|
+
.run(now, runId);
|
|
2090
|
+
}
|
|
2091
|
+
}
|