@osolmaz/pi-workflows 0.13.4 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +146 -160
- package/dist/builtins/autoimplement.workflow.js +3 -1
- package/dist/builtins/autoimplement.workflow.js.map +1 -1
- package/dist/builtins/change-verification.workflow.js +7 -2
- package/dist/builtins/change-verification.workflow.js.map +1 -1
- package/dist/builtins/metadata.d.ts +22 -0
- package/dist/builtins/metadata.js +10 -0
- package/dist/builtins/metadata.js.map +1 -0
- package/dist/builtins/monitor.workflow.js +4 -1
- package/dist/builtins/monitor.workflow.js.map +1 -1
- package/dist/builtins/sanity-check.workflow.js +4 -1
- package/dist/builtins/sanity-check.workflow.js.map +1 -1
- package/dist/builtins/workspace-preparation.workflow.js +3 -1
- package/dist/builtins/workspace-preparation.workflow.js.map +1 -1
- package/dist/controllers/index.d.ts +1 -2
- package/dist/controllers/index.js +0 -1
- package/dist/controllers/index.js.map +1 -1
- package/dist/controllers/sqlite.d.ts +85 -31
- package/dist/controllers/sqlite.js +541 -126
- package/dist/controllers/sqlite.js.map +1 -1
- package/dist/controllers/store.d.ts +1 -0
- package/dist/extension/controller-command.d.ts +22 -0
- package/dist/extension/controller-command.js +32 -0
- package/dist/extension/controller-command.js.map +1 -0
- package/dist/extension/index.d.ts +2 -9
- package/dist/extension/index.js +876 -2928
- package/dist/extension/index.js.map +1 -1
- package/dist/extension/message-card.d.ts +13 -0
- package/dist/extension/message-card.js +46 -0
- package/dist/extension/message-card.js.map +1 -0
- package/dist/extension/step-message.d.ts +5 -6
- package/dist/extension/step-message.js +41 -52
- package/dist/extension/step-message.js.map +1 -1
- package/dist/host/child-worker-supervisor.d.ts +50 -0
- package/dist/host/child-worker-supervisor.js +159 -0
- package/dist/host/child-worker-supervisor.js.map +1 -0
- package/dist/host/client.d.ts +48 -0
- package/dist/host/client.js +216 -0
- package/dist/host/client.js.map +1 -0
- package/dist/host/controller-worker-entry.d.ts +2 -0
- package/dist/host/controller-worker-entry.js +244 -0
- package/dist/host/controller-worker-entry.js.map +1 -0
- package/dist/host/controller-worker-protocol.d.ts +36 -0
- package/dist/host/controller-worker-protocol.js +49 -0
- package/dist/host/controller-worker-protocol.js.map +1 -0
- package/dist/host/controller-worker-supervisor.d.ts +21 -0
- package/dist/host/controller-worker-supervisor.js +54 -0
- package/dist/host/controller-worker-supervisor.js.map +1 -0
- package/dist/host/host-entry.d.ts +2 -0
- package/dist/host/host-entry.js +23 -0
- package/dist/host/host-entry.js.map +1 -0
- package/dist/host/processes.d.ts +17 -12
- package/dist/host/processes.js +154 -50
- package/dist/host/processes.js.map +1 -1
- package/dist/host/protocol.d.ts +38 -0
- package/dist/host/protocol.js +156 -0
- package/dist/host/protocol.js.map +1 -0
- package/dist/host/resolver-entry.d.ts +49 -0
- package/dist/host/resolver-entry.js +149 -0
- package/dist/host/resolver-entry.js.map +1 -0
- package/dist/host/rpc-executor.d.ts +14 -3
- package/dist/host/rpc-executor.js +63 -30
- package/dist/host/rpc-executor.js.map +1 -1
- package/dist/host/runner.d.ts +95 -29
- package/dist/host/runner.js +2311 -348
- package/dist/host/runner.js.map +1 -1
- package/dist/host/state.d.ts +174 -0
- package/dist/host/state.js +689 -0
- package/dist/host/state.js.map +1 -0
- package/dist/host/worker-entry.d.ts +10 -0
- package/dist/host/worker-entry.js +446 -0
- package/dist/host/worker-entry.js.map +1 -0
- package/dist/host/worker-protocol.d.ts +31 -0
- package/dist/host/worker-protocol.js +122 -0
- package/dist/host/worker-protocol.js.map +1 -0
- package/dist/host/worker-store.d.ts +80 -0
- package/dist/host/worker-store.js +91 -0
- package/dist/host/worker-store.js.map +1 -0
- package/dist/host/worker-supervisor.d.ts +22 -0
- package/dist/host/worker-supervisor.js +54 -0
- package/dist/host/worker-supervisor.js.map +1 -0
- package/dist/render/canvas.d.ts +2 -1
- package/dist/render/canvas.js +40 -15
- package/dist/render/canvas.js.map +1 -1
- package/dist/render/graph-render.d.ts +7 -3
- package/dist/render/graph-render.js +125 -74
- package/dist/render/graph-render.js.map +1 -1
- package/dist/state/database.js +3 -1
- package/dist/state/database.js.map +1 -1
- package/dist/state/index.d.ts +1 -0
- package/dist/state/index.js +1 -0
- package/dist/state/index.js.map +1 -1
- package/dist/state/prune.js +36 -10
- package/dist/state/prune.js.map +1 -1
- package/dist/state/schema.d.ts +1 -1
- package/dist/state/schema.js +171 -3
- package/dist/state/schema.js.map +1 -1
- package/dist/state/viewer.d.ts +46 -0
- package/dist/state/viewer.js +249 -0
- package/dist/state/viewer.js.map +1 -0
- package/dist/viewer/cli.d.ts +1 -1
- package/dist/viewer/cli.js +47 -18
- package/dist/viewer/cli.js.map +1 -1
- package/dist/workflows/composition.js +25 -2
- package/dist/workflows/composition.js.map +1 -1
- package/dist/workflows/definition.d.ts +3 -1
- package/dist/workflows/definition.js +25 -0
- package/dist/workflows/definition.js.map +1 -1
- package/dist/workflows/engine.d.ts +1 -0
- package/dist/workflows/engine.js +157 -42
- package/dist/workflows/engine.js.map +1 -1
- package/dist/workflows/errors.d.ts +3 -1
- package/dist/workflows/errors.js +4 -7
- package/dist/workflows/errors.js.map +1 -1
- package/dist/workflows/human-decision.d.ts +3 -0
- package/dist/workflows/human-decision.js +31 -0
- package/dist/workflows/human-decision.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +1 -1
- package/dist/workflows/index.js.map +1 -1
- package/dist/workflows/schema.js +17 -6
- package/dist/workflows/schema.js.map +1 -1
- package/dist/{viewer → workflows}/session-reducer.d.ts +3 -1
- package/dist/{viewer → workflows}/session-reducer.js +4 -0
- package/dist/workflows/session-reducer.js.map +1 -0
- package/dist/workflows/store.d.ts +87 -1
- package/dist/workflows/store.js +892 -93
- package/dist/workflows/store.js.map +1 -1
- package/dist/workflows/tool-input.d.ts +0 -22
- package/dist/workflows/tool-input.js +1 -44
- package/dist/workflows/tool-input.js.map +1 -1
- package/dist/workflows/types.d.ts +36 -4
- package/docs/2026-08-25-workflow-follow-ups.md +8 -6
- package/docs/2026-08-30-out-of-process-workflow-host-plan.md +353 -0
- package/docs/CONTROLLERS.md +11 -11
- package/docs/DEFERRED_TURNS.md +61 -27
- package/docs/HUMAN_DECISIONS.md +12 -4
- package/docs/SQLITE_STATE.md +42 -8
- package/docs/WORKFLOW_HOST.md +452 -0
- package/docs/development.md +46 -30
- package/docs/live-replay-protocol.md +129 -100
- package/docs/plans/2026-08-19-human-decision-gates-plan.md +34 -8
- package/docs/plans/2026-08-27-workflow-terminal-restart-plan.md +399 -0
- package/docs/plans/2026-08-28-piw-incremental-viewer-plan.md +293 -0
- package/docs/plans/piw-viewer-experience-implementation-plan.md +11 -1
- package/docs/plans/tui-viewer-implementation-plan.md +5 -0
- package/docs/tui-viewer.md +18 -3
- package/docs/workflows.md +164 -156
- package/examples/workflows/command-batch.workflow.ts +2 -0
- package/examples/workflows/shell.workflow.ts +2 -1
- package/herdr-plugin.toml +1 -1
- package/package.json +1 -1
- package/skills/autodoc/SKILL.md +1 -1
- package/skills/autoimplement/SKILL.md +1 -1
- package/skills/autoplan/SKILL.md +1 -1
- package/skills/pi-workflows/SKILL.md +2 -0
- package/src/builtins/autoimplement.workflow.ts +3 -0
- package/src/builtins/change-verification.workflow.ts +7 -2
- package/src/builtins/metadata.ts +9 -0
- package/src/builtins/monitor.workflow.ts +4 -0
- package/src/builtins/sanity-check.workflow.ts +4 -0
- package/src/builtins/workspace-preparation.workflow.ts +3 -1
- package/src/controllers/index.ts +3 -5
- package/src/controllers/sqlite.ts +929 -225
- package/src/controllers/store.ts +1 -0
- package/src/extension/controller-command.ts +45 -0
- package/src/extension/index.ts +948 -3506
- package/src/extension/message-card.ts +61 -0
- package/src/extension/step-message.ts +58 -63
- package/src/host/child-worker-supervisor.ts +183 -0
- package/src/host/client.ts +293 -0
- package/src/host/controller-worker-entry.ts +311 -0
- package/src/host/controller-worker-protocol.ts +104 -0
- package/src/host/controller-worker-supervisor.ts +79 -0
- package/src/host/host-entry.ts +23 -0
- package/src/host/processes.ts +171 -54
- package/src/host/protocol.ts +196 -0
- package/src/host/resolver-entry.ts +241 -0
- package/src/host/rpc-executor.ts +76 -34
- package/src/host/runner.ts +2813 -422
- package/src/host/state.ts +1160 -0
- package/src/host/worker-entry.ts +533 -0
- package/src/host/worker-protocol.ts +165 -0
- package/src/host/worker-store.ts +229 -0
- package/src/host/worker-supervisor.ts +74 -0
- package/src/render/canvas.ts +44 -10
- package/src/render/graph-render.ts +145 -90
- package/src/state/database.ts +2 -1
- package/src/state/index.ts +14 -0
- package/src/state/prune.ts +35 -9
- package/src/state/schema.ts +171 -3
- package/src/state/viewer.ts +356 -0
- package/src/viewer/cli.ts +49 -17
- package/src/workflows/composition.ts +36 -2
- package/src/workflows/definition.ts +32 -0
- package/src/workflows/engine.ts +157 -54
- package/src/workflows/errors.ts +11 -2
- package/src/workflows/human-decision.ts +49 -0
- package/src/workflows/index.ts +2 -0
- package/src/workflows/schema.ts +19 -6
- package/src/{viewer → workflows}/session-reducer.ts +11 -1
- package/src/workflows/store.ts +1316 -108
- package/src/workflows/tool-input.ts +2 -60
- package/src/workflows/types.ts +32 -4
- package/dist/controllers/workflow-engine-scheduler.d.ts +0 -29
- package/dist/controllers/workflow-engine-scheduler.js +0 -175
- package/dist/controllers/workflow-engine-scheduler.js.map +0 -1
- package/dist/extension/controller-host.d.ts +0 -48
- package/dist/extension/controller-host.js +0 -110
- package/dist/extension/controller-host.js.map +0 -1
- package/dist/extension/deferred-turn-coordinator.d.ts +0 -32
- package/dist/extension/deferred-turn-coordinator.js +0 -143
- package/dist/extension/deferred-turn-coordinator.js.map +0 -1
- package/dist/extension/deferred-turn.d.ts +0 -44
- package/dist/extension/deferred-turn.js +0 -110
- package/dist/extension/deferred-turn.js.map +0 -1
- package/dist/extension/executor.d.ts +0 -86
- package/dist/extension/executor.js +0 -311
- package/dist/extension/executor.js.map +0 -1
- package/dist/extension/follow-up-coordinator.d.ts +0 -27
- package/dist/extension/follow-up-coordinator.js +0 -131
- package/dist/extension/follow-up-coordinator.js.map +0 -1
- package/dist/extension/recorder.d.ts +0 -84
- package/dist/extension/recorder.js +0 -528
- package/dist/extension/recorder.js.map +0 -1
- package/dist/extension/session-events.d.ts +0 -133
- package/dist/extension/session-events.js +0 -61
- package/dist/extension/session-events.js.map +0 -1
- package/dist/viewer/session-reducer.js.map +0 -1
- package/src/controllers/workflow-engine-scheduler.ts +0 -246
- package/src/extension/controller-host.ts +0 -167
- package/src/extension/deferred-turn-coordinator.ts +0 -171
- package/src/extension/deferred-turn.ts +0 -166
- package/src/extension/executor.ts +0 -411
- package/src/extension/follow-up-coordinator.ts +0 -151
- package/src/extension/recorder.ts +0 -655
- package/src/extension/session-events.ts +0 -121
|
@@ -2,7 +2,9 @@ import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { StateDatabase, workflowStatePath } from "../state/database.js";
|
|
5
|
+
import { canonicalJson } from "../state/json.js";
|
|
5
6
|
import { resourceIdFor, tokenHash } from "../state/mutation.js";
|
|
7
|
+
import { initializeViewerRun, recordViewerDeltas } from "../state/viewer.js";
|
|
6
8
|
import {
|
|
7
9
|
EffectRequestConflictError,
|
|
8
10
|
ResourceConflictError,
|
|
@@ -39,6 +41,27 @@ export type WorkflowRunLaunchStatus =
|
|
|
39
41
|
| "failed"
|
|
40
42
|
| "cancelled";
|
|
41
43
|
|
|
44
|
+
export type WorkflowRunReservationOptions = {
|
|
45
|
+
runId: string;
|
|
46
|
+
workflowName: string;
|
|
47
|
+
workflowSourceRef: string;
|
|
48
|
+
workflowSource: unknown;
|
|
49
|
+
definitionDigest: string;
|
|
50
|
+
definitionSnapshot: unknown;
|
|
51
|
+
input: unknown;
|
|
52
|
+
launchOptions?: unknown;
|
|
53
|
+
runnerId: string;
|
|
54
|
+
originSessionId: string;
|
|
55
|
+
executionMode?: "interactive" | "headless";
|
|
56
|
+
parentRunId?: string;
|
|
57
|
+
now?: string;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export type WorkflowRunClaimOptions = WorkflowRunReservationOptions & {
|
|
61
|
+
claimToken: string;
|
|
62
|
+
leaseMs: number;
|
|
63
|
+
};
|
|
64
|
+
|
|
42
65
|
export type WorkflowRunQueueRecord = {
|
|
43
66
|
runId: string;
|
|
44
67
|
workflowName: string;
|
|
@@ -55,6 +78,7 @@ export type WorkflowRunQueueRecord = {
|
|
|
55
78
|
claimExpiresAt: string | null;
|
|
56
79
|
affinityRunnerId: string | null;
|
|
57
80
|
originSessionId: string | null;
|
|
81
|
+
executionMode: "interactive" | "headless";
|
|
58
82
|
parentRunId: string | null;
|
|
59
83
|
errorCode: string | null;
|
|
60
84
|
errorMessage: string | null;
|
|
@@ -64,6 +88,16 @@ export type WorkflowRunQueueRecord = {
|
|
|
64
88
|
finishedAt: string | null;
|
|
65
89
|
};
|
|
66
90
|
|
|
91
|
+
export type WorkflowRunPreparationResult =
|
|
92
|
+
| {
|
|
93
|
+
state: "claimed";
|
|
94
|
+
run: WorkflowRunQueueRecord & { claimToken: string };
|
|
95
|
+
}
|
|
96
|
+
| {
|
|
97
|
+
state: "adopted";
|
|
98
|
+
run: WorkflowRunQueueRecord;
|
|
99
|
+
};
|
|
100
|
+
|
|
67
101
|
export type WorkflowNotificationRecord = {
|
|
68
102
|
notificationId: string;
|
|
69
103
|
runId: string;
|
|
@@ -84,7 +118,9 @@ export type WorkflowTurnIntentCause =
|
|
|
84
118
|
| "failed"
|
|
85
119
|
| "launchFailed"
|
|
86
120
|
| "controllerInterrupted"
|
|
87
|
-
| "claimLost"
|
|
121
|
+
| "claimLost"
|
|
122
|
+
| "terminal"
|
|
123
|
+
| "cancelled";
|
|
88
124
|
|
|
89
125
|
export type WorkflowTurnIntentResolution = "workflowPrompt" | "presentation" | "fallback";
|
|
90
126
|
|
|
@@ -166,6 +202,7 @@ type RunRow = {
|
|
|
166
202
|
errorCode: string | null;
|
|
167
203
|
errorHash: Buffer | null;
|
|
168
204
|
originSessionId: string | null;
|
|
205
|
+
executionMode: "interactive" | "headless";
|
|
169
206
|
parentRunId: string | null;
|
|
170
207
|
createdAt: number;
|
|
171
208
|
updatedAt: number;
|
|
@@ -195,6 +232,18 @@ type EffectRow = {
|
|
|
195
232
|
errorHash: Buffer | null;
|
|
196
233
|
};
|
|
197
234
|
|
|
235
|
+
type CancelledRunEffectRow = {
|
|
236
|
+
effectId: string;
|
|
237
|
+
resourceId: string;
|
|
238
|
+
status: "pending" | "applying";
|
|
239
|
+
attemptCount: number;
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
type ExpiredInteractionRow = {
|
|
243
|
+
requestId: string;
|
|
244
|
+
attemptId: string;
|
|
245
|
+
};
|
|
246
|
+
|
|
198
247
|
type WorkflowRow = {
|
|
199
248
|
requestId: string;
|
|
200
249
|
resourceUid: string;
|
|
@@ -216,7 +265,13 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
216
265
|
|
|
217
266
|
constructor(
|
|
218
267
|
filePath: string = workflowStatePath(),
|
|
219
|
-
options: {
|
|
268
|
+
options: {
|
|
269
|
+
readOnly?: boolean;
|
|
270
|
+
projectPath?: string;
|
|
271
|
+
state?: StateDatabase;
|
|
272
|
+
/** Host-only global view. Project-scoped mutations still require projectPath. */
|
|
273
|
+
global?: boolean;
|
|
274
|
+
} = {},
|
|
220
275
|
) {
|
|
221
276
|
this.ownsState = options.state === undefined;
|
|
222
277
|
this.state =
|
|
@@ -227,9 +282,13 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
227
282
|
checkLegacyState: filePath === workflowStatePath(),
|
|
228
283
|
});
|
|
229
284
|
this.filePath = this.state.filePath;
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
285
|
+
if (options.global === true) {
|
|
286
|
+
this.projectId = null;
|
|
287
|
+
} else {
|
|
288
|
+
const projectPath = options.projectPath === undefined ? process.cwd() : options.projectPath;
|
|
289
|
+
this.projectId =
|
|
290
|
+
options.readOnly === true ? this.findProject(projectPath) : this.ensureProject(projectPath);
|
|
291
|
+
}
|
|
233
292
|
}
|
|
234
293
|
|
|
235
294
|
close(): void {
|
|
@@ -497,10 +556,16 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
497
556
|
ownerId: string;
|
|
498
557
|
leaseMs: number;
|
|
499
558
|
now?: string;
|
|
559
|
+
exclude?: ControllerResourceRef[];
|
|
500
560
|
}): ControllerQueueClaim | undefined {
|
|
501
561
|
if (options.controllers.length === 0) return undefined;
|
|
502
562
|
const now = epoch(validTimestamp(options.now));
|
|
503
563
|
const placeholders = options.controllers.map(() => "?").join(", ");
|
|
564
|
+
const exclusions = options.exclude ?? [];
|
|
565
|
+
const exclusionSql = exclusions
|
|
566
|
+
.map(() => "AND NOT (c.controller_name = ? AND c.resource_key = ?)")
|
|
567
|
+
.join("\n ");
|
|
568
|
+
const exclusionParams = exclusions.flatMap((ref) => [ref.controller, ref.key]);
|
|
504
569
|
return this.state.transaction(() => {
|
|
505
570
|
const row = this.state.connection
|
|
506
571
|
.prepare(
|
|
@@ -509,12 +574,13 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
509
574
|
JOIN leases l ON l.resource_id = c.resource_id
|
|
510
575
|
WHERE c.project_id = ?
|
|
511
576
|
AND c.controller_name IN (${placeholders})
|
|
577
|
+
${exclusionSql}
|
|
512
578
|
AND q.available_at <= ?
|
|
513
579
|
AND (l.owner_id IS NULL OR l.expires_at <= ?)
|
|
514
580
|
ORDER BY q.available_at, c.controller_name, c.resource_key
|
|
515
581
|
LIMIT 1`)}`,
|
|
516
582
|
)
|
|
517
|
-
.get(this.requireProjectId(), ...options.controllers, now, now);
|
|
583
|
+
.get(this.requireProjectId(), ...options.controllers, ...exclusionParams, now, now);
|
|
518
584
|
if (!isControllerRow(row)) return undefined;
|
|
519
585
|
const lease = this.requireLease(row.resourceId);
|
|
520
586
|
const token = randomBytes(32).toString("base64url");
|
|
@@ -996,139 +1062,148 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
996
1062
|
}));
|
|
997
1063
|
}
|
|
998
1064
|
|
|
999
|
-
reserveWorkflowRun(options: {
|
|
1000
|
-
runId: string;
|
|
1001
|
-
workflowName: string;
|
|
1002
|
-
workflowSourceRef: string;
|
|
1003
|
-
workflowSource: unknown;
|
|
1004
|
-
definitionDigest: string;
|
|
1005
|
-
definitionSnapshot: unknown;
|
|
1006
|
-
input: unknown;
|
|
1007
|
-
launchOptions?: unknown;
|
|
1008
|
-
runnerId: string;
|
|
1009
|
-
originSessionId: string;
|
|
1010
|
-
parentRunId?: string;
|
|
1011
|
-
now?: string;
|
|
1012
|
-
}): WorkflowRunQueueRecord {
|
|
1065
|
+
reserveWorkflowRun(options: WorkflowRunReservationOptions): WorkflowRunQueueRecord {
|
|
1013
1066
|
validateRunId(options.runId);
|
|
1014
1067
|
const now = epoch(validTimestamp(options.now));
|
|
1015
1068
|
const definitionDigest = digestBuffer(options.definitionDigest);
|
|
1016
|
-
return this.state.transaction(() =>
|
|
1017
|
-
|
|
1018
|
-
|
|
1069
|
+
return this.state.transaction(() =>
|
|
1070
|
+
this.reserveWorkflowRunInTransaction(options, now, definitionDigest),
|
|
1071
|
+
);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
private reserveWorkflowRunInTransaction(
|
|
1075
|
+
options: WorkflowRunReservationOptions,
|
|
1076
|
+
now: number,
|
|
1077
|
+
definitionDigest: Buffer,
|
|
1078
|
+
): WorkflowRunQueueRecord {
|
|
1079
|
+
if (this.getWorkflowRun(options.runId) !== undefined) {
|
|
1080
|
+
throw new Error(`Workflow run already reserved: ${options.runId}`);
|
|
1081
|
+
}
|
|
1082
|
+
if (options.parentRunId !== undefined) {
|
|
1083
|
+
const parent = this.requireWorkflowRunRow(options.parentRunId);
|
|
1084
|
+
if (parent.originSessionId !== options.originSessionId) {
|
|
1085
|
+
throw new Error("Continuation parent belongs to another Pi session");
|
|
1019
1086
|
}
|
|
1020
|
-
if (
|
|
1021
|
-
const
|
|
1022
|
-
if (
|
|
1023
|
-
throw new Error("Continuation parent
|
|
1087
|
+
if (parent.status === "parked") {
|
|
1088
|
+
const parentLease = this.requireLease(parent.resourceId);
|
|
1089
|
+
if (parentLease.ownerId !== null) {
|
|
1090
|
+
throw new Error("Continuation parent still has an active owner");
|
|
1024
1091
|
}
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
throw new Error("Continuation parent still has an active owner");
|
|
1029
|
-
}
|
|
1030
|
-
this.state.connection
|
|
1031
|
-
.prepare(
|
|
1032
|
-
`UPDATE run_queue
|
|
1092
|
+
this.state.connection
|
|
1093
|
+
.prepare(
|
|
1094
|
+
`UPDATE run_queue
|
|
1033
1095
|
SET status = 'done', updated_at = ?, finished_at = ?
|
|
1034
1096
|
WHERE run_id = ? AND status = 'parked'`,
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
}
|
|
1097
|
+
)
|
|
1098
|
+
.run(now, now, parent.runId);
|
|
1099
|
+
const parentRevision = this.resourceRevision(parent.resourceId);
|
|
1100
|
+
this.bumpResource(parent.resourceId, parentRevision, now);
|
|
1101
|
+
this.insertEvent(
|
|
1102
|
+
parent.resourceId,
|
|
1103
|
+
parentRevision + 1,
|
|
1104
|
+
"run.queue_done_for_continuation",
|
|
1105
|
+
"session",
|
|
1106
|
+
options.originSessionId,
|
|
1107
|
+
{ continuationRunId: options.runId },
|
|
1108
|
+
now,
|
|
1109
|
+
);
|
|
1110
|
+
} else if (parent.status === "done") {
|
|
1111
|
+
throw new Error("Continuation parent already has a reserved continuation");
|
|
1112
|
+
} else {
|
|
1113
|
+
throw new Error(`Continuation parent queue is ${parent.status}`);
|
|
1053
1114
|
}
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1115
|
+
}
|
|
1116
|
+
const resourceId = resourceIdFor("run", options.runId);
|
|
1117
|
+
const definitionHash = this.state.putJson(options.definitionSnapshot, now);
|
|
1118
|
+
const queuedSource = queuedWorkflowSource(options.workflowSource);
|
|
1119
|
+
const inputHash = this.state.putJson(options.input ?? null, now);
|
|
1120
|
+
const launchHash = this.state.putJson(options.launchOptions ?? {}, now);
|
|
1121
|
+
this.state.connection
|
|
1122
|
+
.prepare(
|
|
1123
|
+
`INSERT INTO workflow_definitions(
|
|
1062
1124
|
definition_digest, workflow_name, definition_hash, created_at
|
|
1063
1125
|
) VALUES (?, ?, ?, ?)
|
|
1064
1126
|
ON CONFLICT(definition_digest) DO NOTHING`,
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1127
|
+
)
|
|
1128
|
+
.run(definitionDigest, options.workflowName, definitionHash, now);
|
|
1129
|
+
this.state.connection
|
|
1130
|
+
.prepare(
|
|
1131
|
+
`INSERT INTO resources(resource_id, resource_type, aggregate_key, revision, created_at, updated_at)
|
|
1070
1132
|
VALUES (?, 'run', ?, 1, ?, ?)`,
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1133
|
+
)
|
|
1134
|
+
.run(resourceId, options.runId, now, now);
|
|
1135
|
+
this.state.connection
|
|
1136
|
+
.prepare("INSERT INTO leases(resource_id, generation) VALUES (?, 0)")
|
|
1137
|
+
.run(resourceId);
|
|
1138
|
+
this.state.connection
|
|
1139
|
+
.prepare(
|
|
1140
|
+
`INSERT INTO runs(
|
|
1079
1141
|
run_id, resource_id, project_id, parent_run_id, definition_digest,
|
|
1080
1142
|
workflow_ref, launch_options_hash, status, paused,
|
|
1081
1143
|
input_hash, created_at, updated_at
|
|
1082
1144
|
) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`,
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1145
|
+
)
|
|
1146
|
+
.run(
|
|
1147
|
+
options.runId,
|
|
1148
|
+
resourceId,
|
|
1149
|
+
this.requireProjectId(),
|
|
1150
|
+
options.parentRunId ?? null,
|
|
1151
|
+
definitionDigest,
|
|
1152
|
+
options.workflowSourceRef,
|
|
1153
|
+
launchHash,
|
|
1154
|
+
inputHash,
|
|
1155
|
+
now,
|
|
1156
|
+
now,
|
|
1157
|
+
);
|
|
1158
|
+
initializeViewerRun(this.state, options.runId, now);
|
|
1159
|
+
insertQueuedRunSources(this.state, options.runId, queuedSource);
|
|
1160
|
+
this.state.connection
|
|
1161
|
+
.prepare(
|
|
1162
|
+
`INSERT INTO run_bindings(run_id, origin_session_id, execution_mode, created_at)
|
|
1163
|
+
VALUES (?, ?, ?, ?)`,
|
|
1164
|
+
)
|
|
1165
|
+
.run(options.runId, options.originSessionId, options.executionMode ?? "interactive", now);
|
|
1166
|
+
this.state.connection
|
|
1167
|
+
.prepare(
|
|
1168
|
+
`INSERT INTO run_queue(
|
|
1106
1169
|
run_id, status, available_at, affinity_runner_id, origin_session_id,
|
|
1107
1170
|
consecutive_errors, created_at, updated_at
|
|
1108
1171
|
) VALUES (?, 'queued', ?, ?, ?, 0, ?, ?)`,
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1172
|
+
)
|
|
1173
|
+
.run(options.runId, now, options.runnerId, options.originSessionId, now, now);
|
|
1174
|
+
this.insertEvent(resourceId, 1, "run.queued", "session", options.originSessionId, {}, now);
|
|
1175
|
+
return this.requireWorkflowRun(options.runId);
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
prepareOrAdoptWorkflowRun(options: WorkflowRunClaimOptions): WorkflowRunPreparationResult {
|
|
1179
|
+
validateRunId(options.runId);
|
|
1180
|
+
const now = epoch(validTimestamp(options.now));
|
|
1181
|
+
const definitionDigest = digestBuffer(options.definitionDigest);
|
|
1182
|
+
return this.state.transaction(() => {
|
|
1183
|
+
const existing = this.workflowRunRow(options.runId);
|
|
1184
|
+
if (existing !== undefined) {
|
|
1185
|
+
this.assertWorkflowRunPreparationCompatible(existing, options, definitionDigest);
|
|
1186
|
+
return { state: "adopted", run: this.mapWorkflowRun(existing) };
|
|
1187
|
+
}
|
|
1188
|
+
this.reserveWorkflowRunInTransaction(options, now, definitionDigest);
|
|
1189
|
+
const claimed = this.claimRunInTransaction(
|
|
1190
|
+
options.runId,
|
|
1191
|
+
options.runnerId,
|
|
1192
|
+
options.claimToken,
|
|
1193
|
+
options.leaseMs,
|
|
1194
|
+
now,
|
|
1195
|
+
);
|
|
1196
|
+
if (claimed === undefined) {
|
|
1197
|
+
throw new Error(`Workflow run could not be claimed: ${options.runId}`);
|
|
1198
|
+
}
|
|
1199
|
+
return {
|
|
1200
|
+
state: "claimed",
|
|
1201
|
+
run: claimed as WorkflowRunQueueRecord & { claimToken: string },
|
|
1202
|
+
};
|
|
1113
1203
|
});
|
|
1114
1204
|
}
|
|
1115
1205
|
|
|
1116
|
-
enqueueWorkflowRun(options: {
|
|
1117
|
-
runId: string;
|
|
1118
|
-
workflowName: string;
|
|
1119
|
-
workflowSourceRef: string;
|
|
1120
|
-
workflowSource: unknown;
|
|
1121
|
-
definitionDigest: string;
|
|
1122
|
-
definitionSnapshot: unknown;
|
|
1123
|
-
input: unknown;
|
|
1124
|
-
launchOptions?: unknown;
|
|
1125
|
-
runnerId: string;
|
|
1126
|
-
claimToken: string;
|
|
1127
|
-
leaseMs: number;
|
|
1128
|
-
originSessionId: string;
|
|
1129
|
-
parentRunId?: string;
|
|
1130
|
-
now?: string;
|
|
1131
|
-
}): WorkflowRunQueueRecord {
|
|
1206
|
+
enqueueWorkflowRun(options: WorkflowRunClaimOptions): WorkflowRunQueueRecord {
|
|
1132
1207
|
if (this.getWorkflowRun(options.runId) === undefined) {
|
|
1133
1208
|
this.reserveWorkflowRun({
|
|
1134
1209
|
runId: options.runId,
|
|
@@ -1163,6 +1238,16 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1163
1238
|
return row === undefined ? undefined : this.mapWorkflowRun(row);
|
|
1164
1239
|
}
|
|
1165
1240
|
|
|
1241
|
+
workflowRunProjectPath(runId: string): string | undefined {
|
|
1242
|
+
const row = this.state.connection
|
|
1243
|
+
.prepare(
|
|
1244
|
+
`SELECT p.canonical_path AS canonicalPath
|
|
1245
|
+
FROM runs r JOIN projects p ON p.project_id = r.project_id WHERE r.run_id = ?`,
|
|
1246
|
+
)
|
|
1247
|
+
.get(runId);
|
|
1248
|
+
return isCanonicalPathRow(row) ? row.canonicalPath : undefined;
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1166
1251
|
listWorkflowRuns(
|
|
1167
1252
|
options: {
|
|
1168
1253
|
statuses?: WorkflowRunLaunchStatus[];
|
|
@@ -1170,8 +1255,12 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1170
1255
|
limit?: number;
|
|
1171
1256
|
} = {},
|
|
1172
1257
|
): WorkflowRunQueueRecord[] {
|
|
1173
|
-
const clauses
|
|
1174
|
-
const params: unknown[] = [
|
|
1258
|
+
const clauses: string[] = [];
|
|
1259
|
+
const params: unknown[] = [];
|
|
1260
|
+
if (this.projectId !== null) {
|
|
1261
|
+
clauses.push("r.project_id = ?");
|
|
1262
|
+
params.push(this.projectId);
|
|
1263
|
+
}
|
|
1175
1264
|
if (options.statuses !== undefined && options.statuses.length > 0) {
|
|
1176
1265
|
clauses.push(`q.status IN (${options.statuses.map(() => "?").join(", ")})`);
|
|
1177
1266
|
params.push(...options.statuses);
|
|
@@ -1183,7 +1272,9 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1183
1272
|
params.push(options.limit ?? 100);
|
|
1184
1273
|
const rows = this.state.connection
|
|
1185
1274
|
.prepare(
|
|
1186
|
-
workflowRunSelect(
|
|
1275
|
+
workflowRunSelect(
|
|
1276
|
+
`${clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`} ORDER BY q.created_at DESC LIMIT ?`,
|
|
1277
|
+
),
|
|
1187
1278
|
)
|
|
1188
1279
|
.all(...params);
|
|
1189
1280
|
return rows.filter(isRunRow).map((row) => this.mapWorkflowRun(row));
|
|
@@ -1211,6 +1302,132 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1211
1302
|
return this.claimRun(options.runId, options.runnerId, options.claimToken, options.leaseMs, now);
|
|
1212
1303
|
}
|
|
1213
1304
|
|
|
1305
|
+
/** Schedule a worker that validates one pending interactive submission. */
|
|
1306
|
+
claimWorkflowRunForInteractionValidation(options: {
|
|
1307
|
+
runId: string;
|
|
1308
|
+
runnerId: string;
|
|
1309
|
+
claimToken: string;
|
|
1310
|
+
leaseMs: number;
|
|
1311
|
+
now?: string;
|
|
1312
|
+
}): WorkflowRunQueueRecord | undefined {
|
|
1313
|
+
const now = epoch(validTimestamp(options.now));
|
|
1314
|
+
return this.claimRun(
|
|
1315
|
+
options.runId,
|
|
1316
|
+
options.runnerId,
|
|
1317
|
+
options.claimToken,
|
|
1318
|
+
options.leaseMs,
|
|
1319
|
+
now,
|
|
1320
|
+
{
|
|
1321
|
+
allowPendingInteraction: true,
|
|
1322
|
+
},
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/** Take a short host claim without scheduling a parked interactive run. */
|
|
1327
|
+
claimWorkflowRunForControl(options: {
|
|
1328
|
+
runId: string;
|
|
1329
|
+
runnerId: string;
|
|
1330
|
+
claimToken: string;
|
|
1331
|
+
leaseMs: number;
|
|
1332
|
+
now?: string;
|
|
1333
|
+
}): WorkflowRunQueueRecord | undefined {
|
|
1334
|
+
const now = epoch(validTimestamp(options.now));
|
|
1335
|
+
return this.claimRun(
|
|
1336
|
+
options.runId,
|
|
1337
|
+
options.runnerId,
|
|
1338
|
+
options.claimToken,
|
|
1339
|
+
options.leaseMs,
|
|
1340
|
+
now,
|
|
1341
|
+
{ allowPendingInteraction: true, preserveQueueStatus: true },
|
|
1342
|
+
);
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
beginWorkflowRunInteractionTimeout(options: {
|
|
1346
|
+
runId: string;
|
|
1347
|
+
claimToken: string;
|
|
1348
|
+
now?: string;
|
|
1349
|
+
}): boolean {
|
|
1350
|
+
const now = epoch(validTimestamp(options.now));
|
|
1351
|
+
return this.state.transaction(() => {
|
|
1352
|
+
if (
|
|
1353
|
+
!this.verifyWorkflowRunClaim({
|
|
1354
|
+
runId: options.runId,
|
|
1355
|
+
claimToken: options.claimToken,
|
|
1356
|
+
now: new Date(now).toISOString(),
|
|
1357
|
+
})
|
|
1358
|
+
) {
|
|
1359
|
+
return false;
|
|
1360
|
+
}
|
|
1361
|
+
const row = this.requireWorkflowRunRow(options.runId);
|
|
1362
|
+
const interaction = this.state.connection
|
|
1363
|
+
.prepare(
|
|
1364
|
+
`SELECT i.request_id AS requestId, i.attempt_id AS attemptId
|
|
1365
|
+
FROM interactive_requests i
|
|
1366
|
+
JOIN node_attempts a ON a.attempt_id = i.attempt_id
|
|
1367
|
+
WHERE i.run_id = ? AND i.status IN ('pending', 'presenting')
|
|
1368
|
+
AND a.deadline_at IS NOT NULL AND a.deadline_at <= ?
|
|
1369
|
+
ORDER BY a.deadline_at, i.request_id LIMIT 1`,
|
|
1370
|
+
)
|
|
1371
|
+
.get(options.runId, now);
|
|
1372
|
+
if (!isExpiredInteractionRow(interaction)) return false;
|
|
1373
|
+
const request = this.state.connection
|
|
1374
|
+
.prepare(
|
|
1375
|
+
`UPDATE interactive_requests
|
|
1376
|
+
SET status = 'cancelled', presenter_id = NULL,
|
|
1377
|
+
presentation_claim_expires_at = NULL, revision = revision + 1, updated_at = ?
|
|
1378
|
+
WHERE request_id = ? AND run_id = ? AND status IN ('pending', 'presenting')`,
|
|
1379
|
+
)
|
|
1380
|
+
.run(now, interaction.requestId, options.runId);
|
|
1381
|
+
const run = this.state.connection
|
|
1382
|
+
.prepare(
|
|
1383
|
+
`UPDATE runs
|
|
1384
|
+
SET status = 'running', status_detail = 'finishing expired interaction deadline',
|
|
1385
|
+
updated_at = ?, finished_at = NULL
|
|
1386
|
+
WHERE run_id = ? AND status = 'waiting'`,
|
|
1387
|
+
)
|
|
1388
|
+
.run(now, options.runId);
|
|
1389
|
+
const queue = this.state.connection
|
|
1390
|
+
.prepare(
|
|
1391
|
+
`UPDATE run_queue SET status = 'starting', error_code = NULL, error_hash = NULL,
|
|
1392
|
+
updated_at = ?, finished_at = NULL
|
|
1393
|
+
WHERE run_id = ? AND status = 'parked'`,
|
|
1394
|
+
)
|
|
1395
|
+
.run(now, options.runId);
|
|
1396
|
+
/* istanbul ignore if -- the expired parked interaction was selected above */
|
|
1397
|
+
if (request.changes !== 1 || run.changes !== 1 || queue.changes !== 1) {
|
|
1398
|
+
throw new Error(`Workflow run ${options.runId} changed during interaction timeout`);
|
|
1399
|
+
}
|
|
1400
|
+
const error = "Workflow node deadline expired before origin-session input arrived";
|
|
1401
|
+
const receiptHash = this.state.putJson({ status: "rejected", error }, now);
|
|
1402
|
+
this.state.connection
|
|
1403
|
+
.prepare(
|
|
1404
|
+
`UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
|
|
1405
|
+
WHERE request_id = ? AND outcome = 'validating'`,
|
|
1406
|
+
)
|
|
1407
|
+
.run(receiptHash, interaction.requestId);
|
|
1408
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1409
|
+
const lease = this.requireLease(row.resourceId);
|
|
1410
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1411
|
+
this.insertEvent(
|
|
1412
|
+
row.resourceId,
|
|
1413
|
+
revision + 1,
|
|
1414
|
+
"run.interaction_timeout_started",
|
|
1415
|
+
lease.ownerType ?? "system",
|
|
1416
|
+
lease.ownerId,
|
|
1417
|
+
{ requestId: interaction.requestId, attemptId: interaction.attemptId },
|
|
1418
|
+
now,
|
|
1419
|
+
lease.generation || undefined,
|
|
1420
|
+
);
|
|
1421
|
+
recordViewerDeltas(
|
|
1422
|
+
this.state,
|
|
1423
|
+
options.runId,
|
|
1424
|
+
[{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
|
|
1425
|
+
now,
|
|
1426
|
+
);
|
|
1427
|
+
return true;
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1214
1431
|
markWorkflowRunRunning(options: { runId: string; claimToken: string; now?: string }): boolean {
|
|
1215
1432
|
return this.updateClaimedRunStatus(options.runId, options.claimToken, "running", options.now);
|
|
1216
1433
|
}
|
|
@@ -1225,19 +1442,19 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1225
1442
|
}): WorkflowRunQueueRecord | undefined {
|
|
1226
1443
|
const now = epoch(validTimestamp(options.now));
|
|
1227
1444
|
const clauses = [
|
|
1228
|
-
"r.project_id = ?",
|
|
1229
1445
|
"q.status IN ('queued', 'parked', 'starting', 'running')",
|
|
1230
1446
|
"q.available_at <= ?",
|
|
1231
1447
|
"(l.owner_id IS NULL OR l.expires_at <= ? OR l.owner_id = ?)",
|
|
1232
1448
|
"(q.affinity_runner_id IS NULL OR q.affinity_runner_id = ? OR q.status IN ('parked', 'starting', 'running'))",
|
|
1449
|
+
"NOT (q.status = 'parked' AND (r.status = 'waiting' OR r.paused = 1))",
|
|
1450
|
+
"NOT EXISTS (SELECT 1 FROM interactive_requests i WHERE i.run_id = r.run_id AND i.status IN ('pending', 'presenting'))",
|
|
1451
|
+
"(q.error_code IS NULL OR q.error_code <> 'workflowSourceChanged')",
|
|
1233
1452
|
];
|
|
1234
|
-
const params: unknown[] = [
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
options.runnerId,
|
|
1240
|
-
];
|
|
1453
|
+
const params: unknown[] = [now, now, options.runnerId, options.runnerId];
|
|
1454
|
+
if (this.projectId !== null) {
|
|
1455
|
+
clauses.unshift("r.project_id = ?");
|
|
1456
|
+
params.unshift(this.projectId);
|
|
1457
|
+
}
|
|
1241
1458
|
if (options.sessionId !== undefined) {
|
|
1242
1459
|
clauses.push("b.origin_session_id = ?");
|
|
1243
1460
|
params.push(options.sessionId);
|
|
@@ -1266,22 +1483,37 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1266
1483
|
now?: string;
|
|
1267
1484
|
}): boolean {
|
|
1268
1485
|
const now = epoch(validTimestamp(options.now));
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
.
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1486
|
+
return this.state.transaction(() => {
|
|
1487
|
+
const row = this.workflowRunRow(options.runId);
|
|
1488
|
+
if (row === undefined || row.ownerId === null) return false;
|
|
1489
|
+
const lease = this.requireLease(row.resourceId);
|
|
1490
|
+
const result = this.state.connection
|
|
1491
|
+
.prepare(
|
|
1492
|
+
`UPDATE leases SET heartbeat_at = ?, expires_at = ?
|
|
1493
|
+
WHERE resource_id = ? AND owner_type = ? AND owner_id = ?
|
|
1494
|
+
AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1495
|
+
)
|
|
1496
|
+
.run(
|
|
1497
|
+
now,
|
|
1498
|
+
now + options.leaseMs,
|
|
1499
|
+
row.resourceId,
|
|
1500
|
+
lease.ownerType,
|
|
1501
|
+
row.ownerId,
|
|
1502
|
+
tokenHash(options.claimToken),
|
|
1503
|
+
lease.generation,
|
|
1504
|
+
now,
|
|
1505
|
+
);
|
|
1506
|
+
if (result.changes === 1) {
|
|
1507
|
+
recordViewerDeltas(
|
|
1508
|
+
this.state,
|
|
1509
|
+
options.runId,
|
|
1510
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1511
|
+
now,
|
|
1512
|
+
);
|
|
1513
|
+
return true;
|
|
1514
|
+
}
|
|
1515
|
+
return false;
|
|
1516
|
+
});
|
|
1285
1517
|
}
|
|
1286
1518
|
|
|
1287
1519
|
verifyWorkflowRunClaim(options: { runId: string; claimToken: string; now?: string }): boolean {
|
|
@@ -1306,16 +1538,11 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1306
1538
|
ownerId: string;
|
|
1307
1539
|
token: string;
|
|
1308
1540
|
generation: number;
|
|
1541
|
+
leaseMs: number;
|
|
1309
1542
|
}
|
|
1310
1543
|
| undefined {
|
|
1311
1544
|
const row = this.workflowRunRow(runId);
|
|
1312
|
-
if (
|
|
1313
|
-
row === undefined ||
|
|
1314
|
-
row.ownerId === null ||
|
|
1315
|
-
row.claimExpiresAt === null ||
|
|
1316
|
-
row.claimExpiresAt <= Date.now()
|
|
1317
|
-
)
|
|
1318
|
-
return undefined;
|
|
1545
|
+
if (row === undefined || row.ownerId === null || row.claimExpiresAt === null) return undefined;
|
|
1319
1546
|
const lease = this.requireLease(row.resourceId);
|
|
1320
1547
|
if (lease.tokenHash === null || !lease.tokenHash.equals(tokenHash(claimToken)))
|
|
1321
1548
|
return undefined;
|
|
@@ -1326,6 +1553,7 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1326
1553
|
ownerId: row.ownerId,
|
|
1327
1554
|
token: claimToken,
|
|
1328
1555
|
generation: row.leaseGeneration,
|
|
1556
|
+
leaseMs: 30_000,
|
|
1329
1557
|
};
|
|
1330
1558
|
}
|
|
1331
1559
|
|
|
@@ -1333,6 +1561,158 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1333
1561
|
return this.releaseRunClaim(options.runId, options.claimToken, "parked", options.now);
|
|
1334
1562
|
}
|
|
1335
1563
|
|
|
1564
|
+
parkWorkflowRunForSourceChange(options: {
|
|
1565
|
+
runId: string;
|
|
1566
|
+
claimToken: string;
|
|
1567
|
+
detail: string;
|
|
1568
|
+
now?: string;
|
|
1569
|
+
}): boolean {
|
|
1570
|
+
const now = epoch(validTimestamp(options.now));
|
|
1571
|
+
return this.state.transaction(() => {
|
|
1572
|
+
const row = this.workflowRunRow(options.runId);
|
|
1573
|
+
if (
|
|
1574
|
+
row === undefined ||
|
|
1575
|
+
!this.verifyWorkflowRunClaim({
|
|
1576
|
+
runId: options.runId,
|
|
1577
|
+
claimToken: options.claimToken,
|
|
1578
|
+
now: new Date(now).toISOString(),
|
|
1579
|
+
})
|
|
1580
|
+
) {
|
|
1581
|
+
return false;
|
|
1582
|
+
}
|
|
1583
|
+
const lease = this.requireLease(row.resourceId);
|
|
1584
|
+
const detail = options.detail.slice(0, 8_192);
|
|
1585
|
+
const errorHash = this.state.putText(detail, now);
|
|
1586
|
+
const run = this.state.connection
|
|
1587
|
+
.prepare(
|
|
1588
|
+
`UPDATE runs SET status_detail = ?, updated_at = ?, finished_at = NULL
|
|
1589
|
+
WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
|
|
1590
|
+
)
|
|
1591
|
+
.run(detail, now, options.runId);
|
|
1592
|
+
const queue = this.state.connection
|
|
1593
|
+
.prepare(
|
|
1594
|
+
`UPDATE run_queue
|
|
1595
|
+
SET status = 'parked', error_code = 'workflowSourceChanged', error_hash = ?,
|
|
1596
|
+
available_at = ?, updated_at = ?, finished_at = NULL
|
|
1597
|
+
WHERE run_id = ? AND status IN ('queued', 'starting', 'running', 'parked')`,
|
|
1598
|
+
)
|
|
1599
|
+
.run(errorHash, now, now, options.runId);
|
|
1600
|
+
/* istanbul ignore if -- exact live claim and nonterminal checks make both updates mandatory */
|
|
1601
|
+
if (run.changes !== 1 || queue.changes !== 1) {
|
|
1602
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent source-change state`);
|
|
1603
|
+
}
|
|
1604
|
+
const released = this.state.connection
|
|
1605
|
+
.prepare(
|
|
1606
|
+
`UPDATE leases
|
|
1607
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1608
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1609
|
+
WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1610
|
+
)
|
|
1611
|
+
.run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
|
|
1612
|
+
/* istanbul ignore if -- the exact live claim is stable in this transaction */
|
|
1613
|
+
if (released.changes !== 1) {
|
|
1614
|
+
throw new Error(`Workflow run ${options.runId} claim changed during source recovery`);
|
|
1615
|
+
}
|
|
1616
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1617
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1618
|
+
this.insertEvent(
|
|
1619
|
+
row.resourceId,
|
|
1620
|
+
revision + 1,
|
|
1621
|
+
"run.source_changed",
|
|
1622
|
+
lease.ownerType ?? "system",
|
|
1623
|
+
lease.ownerId,
|
|
1624
|
+
{ status: "parked", code: "workflowSourceChanged" },
|
|
1625
|
+
now,
|
|
1626
|
+
lease.generation || undefined,
|
|
1627
|
+
);
|
|
1628
|
+
recordViewerDeltas(
|
|
1629
|
+
this.state,
|
|
1630
|
+
options.runId,
|
|
1631
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1632
|
+
now,
|
|
1633
|
+
);
|
|
1634
|
+
return true;
|
|
1635
|
+
});
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
parkWorkflowRunForAmbiguousEffect(options: {
|
|
1639
|
+
runId: string;
|
|
1640
|
+
claimToken: string;
|
|
1641
|
+
now?: string;
|
|
1642
|
+
}): boolean {
|
|
1643
|
+
const now = epoch(validTimestamp(options.now));
|
|
1644
|
+
return this.state.transaction(() => {
|
|
1645
|
+
const row = this.workflowRunRow(options.runId);
|
|
1646
|
+
if (
|
|
1647
|
+
row === undefined ||
|
|
1648
|
+
!this.verifyWorkflowRunClaim({
|
|
1649
|
+
runId: options.runId,
|
|
1650
|
+
claimToken: options.claimToken,
|
|
1651
|
+
now: new Date(now).toISOString(),
|
|
1652
|
+
})
|
|
1653
|
+
) {
|
|
1654
|
+
return false;
|
|
1655
|
+
}
|
|
1656
|
+
const lease = this.requireLease(row.resourceId);
|
|
1657
|
+
const detail = "effect outcome is ambiguous; explicit recovery is required";
|
|
1658
|
+
const errorHash = this.state.putText(detail, now);
|
|
1659
|
+
this.state.connection
|
|
1660
|
+
.prepare(
|
|
1661
|
+
`UPDATE node_attempts SET status = 'waiting', updated_at = ?
|
|
1662
|
+
WHERE run_id = ? AND status IN ('pending', 'running')`,
|
|
1663
|
+
)
|
|
1664
|
+
.run(now, options.runId);
|
|
1665
|
+
const run = this.state.connection
|
|
1666
|
+
.prepare(
|
|
1667
|
+
`UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
|
|
1668
|
+
WHERE run_id = ? AND status = 'running'`,
|
|
1669
|
+
)
|
|
1670
|
+
.run(detail, now, now, options.runId);
|
|
1671
|
+
const queue = this.state.connection
|
|
1672
|
+
.prepare(
|
|
1673
|
+
`UPDATE run_queue
|
|
1674
|
+
SET status = 'parked', error_code = 'effectAmbiguous', error_hash = ?, updated_at = ?
|
|
1675
|
+
WHERE run_id = ? AND status IN ('starting', 'running', 'parked')`,
|
|
1676
|
+
)
|
|
1677
|
+
.run(errorHash, now, options.runId);
|
|
1678
|
+
/* istanbul ignore if -- exact live claim and run checks make both updates mandatory */
|
|
1679
|
+
if (run.changes !== 1 || queue.changes !== 1) {
|
|
1680
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent ambiguous-effect state`);
|
|
1681
|
+
}
|
|
1682
|
+
const released = this.state.connection
|
|
1683
|
+
.prepare(
|
|
1684
|
+
`UPDATE leases
|
|
1685
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1686
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1687
|
+
WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1688
|
+
)
|
|
1689
|
+
.run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
|
|
1690
|
+
/* istanbul ignore if -- the exact live claim is stable in this transaction */
|
|
1691
|
+
if (released.changes !== 1) {
|
|
1692
|
+
throw new Error(`Workflow run ${options.runId} claim changed during effect recovery`);
|
|
1693
|
+
}
|
|
1694
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1695
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1696
|
+
this.insertEvent(
|
|
1697
|
+
row.resourceId,
|
|
1698
|
+
revision + 1,
|
|
1699
|
+
"run.effect_ambiguous",
|
|
1700
|
+
lease.ownerType ?? "system",
|
|
1701
|
+
lease.ownerId,
|
|
1702
|
+
{ status: "waiting", code: "effectAmbiguous" },
|
|
1703
|
+
now,
|
|
1704
|
+
lease.generation || undefined,
|
|
1705
|
+
);
|
|
1706
|
+
recordViewerDeltas(
|
|
1707
|
+
this.state,
|
|
1708
|
+
options.runId,
|
|
1709
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1710
|
+
now,
|
|
1711
|
+
);
|
|
1712
|
+
return true;
|
|
1713
|
+
});
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1336
1716
|
failWorkflowRun(options: {
|
|
1337
1717
|
runId: string;
|
|
1338
1718
|
claimToken?: string;
|
|
@@ -1351,7 +1731,7 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1351
1731
|
}
|
|
1352
1732
|
|
|
1353
1733
|
cancelWorkflowRun(options: { runId: string; claimToken?: string; now?: string }): boolean {
|
|
1354
|
-
|
|
1734
|
+
const cancelled = this.terminalRun(
|
|
1355
1735
|
options.runId,
|
|
1356
1736
|
options.claimToken,
|
|
1357
1737
|
"cancelled",
|
|
@@ -1359,6 +1739,111 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1359
1739
|
"Workflow run cancelled",
|
|
1360
1740
|
options.now,
|
|
1361
1741
|
);
|
|
1742
|
+
if (cancelled || options.claimToken !== undefined) return cancelled;
|
|
1743
|
+
return this.cancelStaleWorkflowRun({
|
|
1744
|
+
runId: options.runId,
|
|
1745
|
+
...(options.now === undefined ? {} : { now: options.now }),
|
|
1746
|
+
});
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
/** Cancel a nonterminal run only when its claim is absent or expired. */
|
|
1750
|
+
cancelStaleWorkflowRun(options: {
|
|
1751
|
+
runId: string;
|
|
1752
|
+
controlId?: string;
|
|
1753
|
+
claimToken?: string;
|
|
1754
|
+
now?: string;
|
|
1755
|
+
}): boolean {
|
|
1756
|
+
const now = epoch(validTimestamp(options.now));
|
|
1757
|
+
const controlId = options.controlId ?? "workflow-control";
|
|
1758
|
+
const claimToken = options.claimToken ?? randomUUID();
|
|
1759
|
+
return this.state.transaction(() => {
|
|
1760
|
+
const row = this.workflowRunRow(options.runId);
|
|
1761
|
+
if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return false;
|
|
1762
|
+
const lease = this.requireLease(row.resourceId);
|
|
1763
|
+
if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
|
|
1764
|
+
|
|
1765
|
+
const generation = lease.generation + 1;
|
|
1766
|
+
const claimed = this.state.connection
|
|
1767
|
+
.prepare(
|
|
1768
|
+
`UPDATE leases
|
|
1769
|
+
SET generation = ?, owner_type = 'system', owner_id = ?, token_hash = ?,
|
|
1770
|
+
acquired_at = ?, heartbeat_at = ?, expires_at = ?
|
|
1771
|
+
WHERE resource_id = ? AND generation = ?
|
|
1772
|
+
AND (owner_id IS NULL OR expires_at IS NULL OR expires_at <= ?)`,
|
|
1773
|
+
)
|
|
1774
|
+
.run(
|
|
1775
|
+
generation,
|
|
1776
|
+
controlId,
|
|
1777
|
+
tokenHash(claimToken),
|
|
1778
|
+
now,
|
|
1779
|
+
now,
|
|
1780
|
+
now + 30_000,
|
|
1781
|
+
row.resourceId,
|
|
1782
|
+
lease.generation,
|
|
1783
|
+
now,
|
|
1784
|
+
);
|
|
1785
|
+
if (claimed.changes !== 1) return false;
|
|
1786
|
+
|
|
1787
|
+
const errorHash = this.state.putText("Workflow run cancelled", now);
|
|
1788
|
+
const queue = this.state.connection
|
|
1789
|
+
.prepare(
|
|
1790
|
+
`UPDATE run_queue
|
|
1791
|
+
SET status = 'cancelled', error_code = 'cancelled', error_hash = ?,
|
|
1792
|
+
updated_at = ?, finished_at = ?
|
|
1793
|
+
WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
|
|
1794
|
+
)
|
|
1795
|
+
.run(errorHash, now, now, options.runId);
|
|
1796
|
+
/* istanbul ignore if -- the control claim selects one nonterminal queue row */
|
|
1797
|
+
if (queue.changes !== 1) {
|
|
1798
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent queue state`);
|
|
1799
|
+
}
|
|
1800
|
+
const run = this.state.connection
|
|
1801
|
+
.prepare(
|
|
1802
|
+
`UPDATE runs
|
|
1803
|
+
SET status = 'cancelled', paused = 0, status_detail = NULL, error_hash = ?,
|
|
1804
|
+
updated_at = ?, finished_at = ?
|
|
1805
|
+
WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
|
|
1806
|
+
)
|
|
1807
|
+
.run(errorHash, now, now, options.runId);
|
|
1808
|
+
/* istanbul ignore if -- the control claim selects one nonterminal durable run */
|
|
1809
|
+
if (run.changes !== 1) {
|
|
1810
|
+
throw new Error(`Workflow run ${options.runId} has inconsistent durable state`);
|
|
1811
|
+
}
|
|
1812
|
+
this.cancelWorkflowRunDependents(options.runId, controlId, errorHash, now);
|
|
1813
|
+
|
|
1814
|
+
const released = this.state.connection
|
|
1815
|
+
.prepare(
|
|
1816
|
+
`UPDATE leases
|
|
1817
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
1818
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
1819
|
+
WHERE resource_id = ? AND owner_type = 'system' AND owner_id = ?
|
|
1820
|
+
AND token_hash = ? AND generation = ? AND expires_at > ?`,
|
|
1821
|
+
)
|
|
1822
|
+
.run(row.resourceId, controlId, tokenHash(claimToken), generation, now);
|
|
1823
|
+
/* istanbul ignore if -- the exact control claim was written in this transaction */
|
|
1824
|
+
if (released.changes !== 1) {
|
|
1825
|
+
throw new Error(`Workflow run ${options.runId} control claim changed during cancellation`);
|
|
1826
|
+
}
|
|
1827
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
1828
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
1829
|
+
this.insertEvent(
|
|
1830
|
+
row.resourceId,
|
|
1831
|
+
revision + 1,
|
|
1832
|
+
"run.queue_cancelled",
|
|
1833
|
+
"control",
|
|
1834
|
+
controlId,
|
|
1835
|
+
{ status: "cancelled", code: "cancelled", staleControl: true },
|
|
1836
|
+
now,
|
|
1837
|
+
generation,
|
|
1838
|
+
);
|
|
1839
|
+
recordViewerDeltas(
|
|
1840
|
+
this.state,
|
|
1841
|
+
options.runId,
|
|
1842
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
1843
|
+
now,
|
|
1844
|
+
);
|
|
1845
|
+
return true;
|
|
1846
|
+
});
|
|
1362
1847
|
}
|
|
1363
1848
|
|
|
1364
1849
|
deleteWorkflowRun(options: { runId: string; claimToken: string }): boolean {
|
|
@@ -1404,34 +1889,36 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1404
1889
|
runnerId?: string;
|
|
1405
1890
|
now?: string;
|
|
1406
1891
|
}): RunEventRecord {
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1892
|
+
return this.state.transaction(() => {
|
|
1893
|
+
const row = this.requireWorkflowRunRow(options.runId);
|
|
1894
|
+
const now = epoch(validTimestamp(options.now));
|
|
1895
|
+
const revision = this.resourceRevision(row.resourceId) + 1;
|
|
1896
|
+
this.bumpResource(row.resourceId, revision - 1, now);
|
|
1897
|
+
const eventId = this.insertEvent(
|
|
1898
|
+
row.resourceId,
|
|
1899
|
+
revision,
|
|
1900
|
+
options.type,
|
|
1901
|
+
options.runnerId?.startsWith("host-") ? "host" : "session",
|
|
1902
|
+
options.runnerId ?? null,
|
|
1903
|
+
options.payload ?? {},
|
|
1904
|
+
now,
|
|
1905
|
+
row.leaseGeneration || undefined,
|
|
1906
|
+
);
|
|
1907
|
+
const seq = this.state.connection
|
|
1908
|
+
.prepare("SELECT event_seq AS seq FROM events WHERE event_id = ?")
|
|
1909
|
+
.get(eventId);
|
|
1910
|
+
/* istanbul ignore if -- exact schema and internal query shape */
|
|
1911
|
+
if (!isSequenceRow(seq)) throw new Error("Run event was not recorded");
|
|
1912
|
+
return {
|
|
1913
|
+
seq: seq.seq,
|
|
1914
|
+
recordedAt: new Date(now).toISOString(),
|
|
1915
|
+
runId: options.runId,
|
|
1916
|
+
workflowRef: options.workflowRef,
|
|
1917
|
+
type: options.type,
|
|
1918
|
+
runnerId: options.runnerId ?? null,
|
|
1919
|
+
payload: options.payload ?? {},
|
|
1920
|
+
};
|
|
1921
|
+
});
|
|
1435
1922
|
}
|
|
1436
1923
|
|
|
1437
1924
|
listRunEventsAfter(seq: number, options: { limit?: number } = {}): RunEventRecord[] {
|
|
@@ -1753,6 +2240,20 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
1753
2240
|
return this.requireNotification(options.runId, options.attemptId, options.notificationIndex);
|
|
1754
2241
|
}
|
|
1755
2242
|
|
|
2243
|
+
listPendingWorkflowNotifications(options: {
|
|
2244
|
+
targetSessionId: string;
|
|
2245
|
+
limit?: number;
|
|
2246
|
+
}): WorkflowNotificationRecord[] {
|
|
2247
|
+
const now = Date.now();
|
|
2248
|
+
return this.notificationRows(options.targetSessionId, options.limit ?? 20)
|
|
2249
|
+
.map((row) => this.mapNotification(row))
|
|
2250
|
+
.filter(
|
|
2251
|
+
(notification) =>
|
|
2252
|
+
notification.deliveryClaimExpiresAt === null ||
|
|
2253
|
+
Date.parse(notification.deliveryClaimExpiresAt) <= now,
|
|
2254
|
+
);
|
|
2255
|
+
}
|
|
2256
|
+
|
|
1756
2257
|
claimPendingWorkflowNotifications(options: {
|
|
1757
2258
|
targetSessionId: string;
|
|
1758
2259
|
claimToken: string;
|
|
@@ -2100,6 +2601,26 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
2100
2601
|
)
|
|
2101
2602
|
.run(now, resourceId, expectedRevision);
|
|
2102
2603
|
if (result.changes !== 1) throw new Error("Resource revision conflict");
|
|
2604
|
+
const run = this.state.connection
|
|
2605
|
+
.prepare(
|
|
2606
|
+
`SELECT r.run_id AS runId
|
|
2607
|
+
FROM runs r JOIN viewer_runs v ON v.run_id = r.run_id
|
|
2608
|
+
WHERE r.resource_id = ?`,
|
|
2609
|
+
)
|
|
2610
|
+
.get(resourceId);
|
|
2611
|
+
if (isRecord(run) && typeof run.runId === "string") {
|
|
2612
|
+
recordViewerDeltas(
|
|
2613
|
+
this.state,
|
|
2614
|
+
run.runId,
|
|
2615
|
+
[
|
|
2616
|
+
{ targetType: "summary" },
|
|
2617
|
+
{ targetType: "graph" },
|
|
2618
|
+
{ targetType: "replay" },
|
|
2619
|
+
{ targetType: "inspector", targetKey: "run" },
|
|
2620
|
+
],
|
|
2621
|
+
now,
|
|
2622
|
+
);
|
|
2623
|
+
}
|
|
2103
2624
|
}
|
|
2104
2625
|
|
|
2105
2626
|
private insertEvent(
|
|
@@ -2228,6 +2749,28 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
2228
2749
|
return this.mapWorkflowRun(this.requireWorkflowRunRow(runId));
|
|
2229
2750
|
}
|
|
2230
2751
|
|
|
2752
|
+
private assertWorkflowRunPreparationCompatible(
|
|
2753
|
+
row: RunRow,
|
|
2754
|
+
options: WorkflowRunReservationOptions,
|
|
2755
|
+
definitionDigest: Buffer,
|
|
2756
|
+
): void {
|
|
2757
|
+
const compatible =
|
|
2758
|
+
row.workflowName === options.workflowName &&
|
|
2759
|
+
row.workflowRef === options.workflowSourceRef &&
|
|
2760
|
+
row.definitionDigest.equals(definitionDigest) &&
|
|
2761
|
+
canonicalJson(readQueuedRunSources(this.state, row)) ===
|
|
2762
|
+
canonicalJson(queuedWorkflowSource(options.workflowSource)) &&
|
|
2763
|
+
canonicalJson(this.state.readJson(row.inputHash)) === canonicalJson(options.input ?? null) &&
|
|
2764
|
+
canonicalJson(this.state.readJson(row.launchOptionsHash)) ===
|
|
2765
|
+
canonicalJson(options.launchOptions ?? {}) &&
|
|
2766
|
+
row.originSessionId === options.originSessionId &&
|
|
2767
|
+
row.executionMode === (options.executionMode ?? "interactive") &&
|
|
2768
|
+
row.parentRunId === (options.parentRunId ?? null);
|
|
2769
|
+
if (!compatible) {
|
|
2770
|
+
throw new Error(`Workflow run preparation conflicts: ${options.runId}`);
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
|
|
2231
2774
|
/* istanbul ignore next -- pure projection covered by integration tests */
|
|
2232
2775
|
private mapWorkflowRun(row: RunRow): WorkflowRunQueueRecord {
|
|
2233
2776
|
return {
|
|
@@ -2246,7 +2789,8 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
2246
2789
|
claimExpiresAt:
|
|
2247
2790
|
row.claimExpiresAt === null ? null : new Date(row.claimExpiresAt).toISOString(),
|
|
2248
2791
|
affinityRunnerId: row.affinityRunnerId,
|
|
2249
|
-
originSessionId: row.originSessionId,
|
|
2792
|
+
originSessionId: row.executionMode === "headless" ? null : row.originSessionId,
|
|
2793
|
+
executionMode: row.executionMode,
|
|
2250
2794
|
parentRunId: row.parentRunId,
|
|
2251
2795
|
errorCode: row.errorCode,
|
|
2252
2796
|
errorMessage:
|
|
@@ -2266,57 +2810,85 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
2266
2810
|
claimToken: string,
|
|
2267
2811
|
leaseMs: number,
|
|
2268
2812
|
now: number,
|
|
2813
|
+
options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
|
|
2269
2814
|
): WorkflowRunQueueRecord | undefined {
|
|
2270
|
-
return this.state.transaction(() =>
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2815
|
+
return this.state.transaction(() =>
|
|
2816
|
+
this.claimRunInTransaction(runId, runnerId, claimToken, leaseMs, now, options),
|
|
2817
|
+
);
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
private claimRunInTransaction(
|
|
2821
|
+
runId: string,
|
|
2822
|
+
runnerId: string,
|
|
2823
|
+
claimToken: string,
|
|
2824
|
+
leaseMs: number,
|
|
2825
|
+
now: number,
|
|
2826
|
+
options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
|
|
2827
|
+
): WorkflowRunQueueRecord | undefined {
|
|
2828
|
+
const row = this.workflowRunRow(runId);
|
|
2829
|
+
if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return undefined;
|
|
2830
|
+
if (
|
|
2831
|
+
options.allowPendingInteraction !== true &&
|
|
2832
|
+
this.state.connection
|
|
2285
2833
|
.prepare(
|
|
2286
|
-
`
|
|
2834
|
+
`SELECT 1 FROM interactive_requests
|
|
2835
|
+
WHERE run_id = ? AND status IN ('pending', 'presenting') LIMIT 1`,
|
|
2836
|
+
)
|
|
2837
|
+
.get(runId) !== undefined
|
|
2838
|
+
) {
|
|
2839
|
+
return undefined;
|
|
2840
|
+
}
|
|
2841
|
+
const lease = this.requireLease(row.resourceId);
|
|
2842
|
+
if (
|
|
2843
|
+
lease.ownerId !== null &&
|
|
2844
|
+
lease.expiresAt !== null &&
|
|
2845
|
+
lease.expiresAt > now &&
|
|
2846
|
+
lease.ownerId !== runnerId
|
|
2847
|
+
)
|
|
2848
|
+
return undefined;
|
|
2849
|
+
const generation = lease.generation + 1;
|
|
2850
|
+
const expiresAt = now + leaseMs;
|
|
2851
|
+
const result = this.state.connection
|
|
2852
|
+
.prepare(
|
|
2853
|
+
`UPDATE leases SET generation = ?, owner_type = ?, owner_id = ?, token_hash = ?,
|
|
2287
2854
|
acquired_at = ?, heartbeat_at = ?, expires_at = ?
|
|
2288
2855
|
WHERE resource_id = ? AND generation = ?`,
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
runnerId.startsWith("host-") ? "host" : "session",
|
|
2293
|
-
runnerId,
|
|
2294
|
-
tokenHash(claimToken),
|
|
2295
|
-
now,
|
|
2296
|
-
now,
|
|
2297
|
-
expiresAt,
|
|
2298
|
-
row.resourceId,
|
|
2299
|
-
lease.generation,
|
|
2300
|
-
);
|
|
2301
|
-
/* istanbul ignore if -- impossible after exact schema and transaction checks */
|
|
2302
|
-
if (result.changes !== 1) return undefined;
|
|
2303
|
-
this.state.connection
|
|
2304
|
-
.prepare("UPDATE run_queue SET status = 'starting', updated_at = ? WHERE run_id = ?")
|
|
2305
|
-
.run(now, runId);
|
|
2306
|
-
const revision = this.resourceRevision(row.resourceId);
|
|
2307
|
-
this.bumpResource(row.resourceId, revision, now);
|
|
2308
|
-
this.insertEvent(
|
|
2309
|
-
row.resourceId,
|
|
2310
|
-
revision + 1,
|
|
2311
|
-
"lease.claimed",
|
|
2856
|
+
)
|
|
2857
|
+
.run(
|
|
2858
|
+
generation,
|
|
2312
2859
|
runnerId.startsWith("host-") ? "host" : "session",
|
|
2313
2860
|
runnerId,
|
|
2314
|
-
|
|
2861
|
+
tokenHash(claimToken),
|
|
2315
2862
|
now,
|
|
2316
|
-
|
|
2863
|
+
now,
|
|
2864
|
+
expiresAt,
|
|
2865
|
+
row.resourceId,
|
|
2866
|
+
lease.generation,
|
|
2317
2867
|
);
|
|
2318
|
-
|
|
2319
|
-
|
|
2868
|
+
/* istanbul ignore if -- impossible after exact schema and transaction checks */
|
|
2869
|
+
if (result.changes !== 1) return undefined;
|
|
2870
|
+
if (options.preserveQueueStatus !== true) {
|
|
2871
|
+
this.state.connection
|
|
2872
|
+
.prepare(
|
|
2873
|
+
`UPDATE run_queue
|
|
2874
|
+
SET status = 'starting', error_code = NULL, error_hash = NULL, updated_at = ?
|
|
2875
|
+
WHERE run_id = ?`,
|
|
2876
|
+
)
|
|
2877
|
+
.run(now, runId);
|
|
2878
|
+
}
|
|
2879
|
+
const revision = this.resourceRevision(row.resourceId);
|
|
2880
|
+
this.bumpResource(row.resourceId, revision, now);
|
|
2881
|
+
this.insertEvent(
|
|
2882
|
+
row.resourceId,
|
|
2883
|
+
revision + 1,
|
|
2884
|
+
"lease.claimed",
|
|
2885
|
+
runnerId.startsWith("host-") ? "host" : "session",
|
|
2886
|
+
runnerId,
|
|
2887
|
+
{ expiresAt },
|
|
2888
|
+
now,
|
|
2889
|
+
generation,
|
|
2890
|
+
);
|
|
2891
|
+
return { ...this.requireWorkflowRun(runId), claimToken };
|
|
2320
2892
|
}
|
|
2321
2893
|
|
|
2322
2894
|
private updateClaimedRunStatus(
|
|
@@ -2403,6 +2975,12 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
2403
2975
|
now,
|
|
2404
2976
|
lease.generation || undefined,
|
|
2405
2977
|
);
|
|
2978
|
+
recordViewerDeltas(
|
|
2979
|
+
this.state,
|
|
2980
|
+
runId,
|
|
2981
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
2982
|
+
now,
|
|
2983
|
+
);
|
|
2406
2984
|
return true;
|
|
2407
2985
|
});
|
|
2408
2986
|
}
|
|
@@ -2465,6 +3043,14 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
2465
3043
|
WHERE run_id = ?`,
|
|
2466
3044
|
)
|
|
2467
3045
|
.run(status, errorHash, now, now, runId);
|
|
3046
|
+
if (status === "cancelled") {
|
|
3047
|
+
this.cancelWorkflowRunDependents(
|
|
3048
|
+
runId,
|
|
3049
|
+
lease.ownerId ?? "workflow-control",
|
|
3050
|
+
errorHash,
|
|
3051
|
+
now,
|
|
3052
|
+
);
|
|
3053
|
+
}
|
|
2468
3054
|
this.bumpResource(row.resourceId, revision, now);
|
|
2469
3055
|
this.insertEvent(
|
|
2470
3056
|
row.resourceId,
|
|
@@ -2476,10 +3062,110 @@ export class SqliteControllerStore implements ControllerStore {
|
|
|
2476
3062
|
now,
|
|
2477
3063
|
lease.generation || undefined,
|
|
2478
3064
|
);
|
|
3065
|
+
recordViewerDeltas(
|
|
3066
|
+
this.state,
|
|
3067
|
+
runId,
|
|
3068
|
+
[{ targetType: "summary" }, { targetType: "replay" }],
|
|
3069
|
+
now,
|
|
3070
|
+
);
|
|
2479
3071
|
return true;
|
|
2480
3072
|
});
|
|
2481
3073
|
}
|
|
2482
3074
|
|
|
3075
|
+
private cancelWorkflowRunDependents(
|
|
3076
|
+
runId: string,
|
|
3077
|
+
actorId: string,
|
|
3078
|
+
errorHash: Buffer,
|
|
3079
|
+
now: number,
|
|
3080
|
+
): void {
|
|
3081
|
+
this.state.connection
|
|
3082
|
+
.prepare(
|
|
3083
|
+
`UPDATE node_attempts
|
|
3084
|
+
SET status = 'cancelled', error_hash = COALESCE(error_hash, ?),
|
|
3085
|
+
updated_at = ?, finished_at = COALESCE(finished_at, ?)
|
|
3086
|
+
WHERE run_id = ? AND status IN ('pending', 'running', 'waiting', 'interrupted')`,
|
|
3087
|
+
)
|
|
3088
|
+
.run(errorHash, now, now, runId);
|
|
3089
|
+
|
|
3090
|
+
const effects = this.state.connection
|
|
3091
|
+
.prepare(
|
|
3092
|
+
`SELECT e.effect_id AS effectId, e.resource_id AS resourceId,
|
|
3093
|
+
e.status, e.attempt_count AS attemptCount
|
|
3094
|
+
FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
|
|
3095
|
+
WHERE r.run_id = ? AND e.owner_scope = 'run' AND e.status IN ('pending', 'applying')
|
|
3096
|
+
ORDER BY e.effect_id`,
|
|
3097
|
+
)
|
|
3098
|
+
.all(runId)
|
|
3099
|
+
.filter(isCancelledRunEffectRow);
|
|
3100
|
+
for (const effect of effects) {
|
|
3101
|
+
const status = effect.status === "applying" ? "ambiguous" : "cancelled";
|
|
3102
|
+
const revision = this.resourceRevision(effect.resourceId);
|
|
3103
|
+
const changed = this.state.connection
|
|
3104
|
+
.prepare(
|
|
3105
|
+
`UPDATE effects
|
|
3106
|
+
SET status = ?, next_attempt_at = NULL, error_hash = ?, updated_at = ?, settled_at = ?
|
|
3107
|
+
WHERE effect_id = ? AND status = ? AND attempt_count = ?`,
|
|
3108
|
+
)
|
|
3109
|
+
.run(status, errorHash, now, now, effect.effectId, effect.status, effect.attemptCount);
|
|
3110
|
+
/* istanbul ignore if -- cancellation serializes the selected effect transition */
|
|
3111
|
+
if (changed.changes !== 1) {
|
|
3112
|
+
throw new Error(`Workflow effect ${effect.effectId} changed during cancellation`);
|
|
3113
|
+
}
|
|
3114
|
+
if (effect.status === "applying") {
|
|
3115
|
+
this.state.connection
|
|
3116
|
+
.prepare(
|
|
3117
|
+
`UPDATE effect_attempts
|
|
3118
|
+
SET finished_at = ?, outcome = 'interrupted', error_hash = ?
|
|
3119
|
+
WHERE effect_id = ? AND attempt_number = ? AND finished_at IS NULL`,
|
|
3120
|
+
)
|
|
3121
|
+
.run(now, errorHash, effect.effectId, effect.attemptCount);
|
|
3122
|
+
}
|
|
3123
|
+
this.bumpResource(effect.resourceId, revision, now);
|
|
3124
|
+
this.insertEvent(
|
|
3125
|
+
effect.resourceId,
|
|
3126
|
+
revision + 1,
|
|
3127
|
+
`effect.${status}`,
|
|
3128
|
+
"control",
|
|
3129
|
+
actorId,
|
|
3130
|
+
{ runId, reason: "workflowCancelled" },
|
|
3131
|
+
now,
|
|
3132
|
+
);
|
|
3133
|
+
}
|
|
3134
|
+
|
|
3135
|
+
this.state.connection
|
|
3136
|
+
.prepare(
|
|
3137
|
+
`INSERT INTO human_decision_resolutions(
|
|
3138
|
+
decision_id, outcome, provenance, response_hash, reason, channel,
|
|
3139
|
+
actor_id, request_digest, resolved_at
|
|
3140
|
+
)
|
|
3141
|
+
SELECT d.decision_id, 'cancelled', 'explicit_cancel', NULL,
|
|
3142
|
+
'Workflow run cancelled', NULL, ?, d.request_digest, ?
|
|
3143
|
+
FROM human_decisions d
|
|
3144
|
+
LEFT JOIN human_decision_resolutions r ON r.decision_id = d.decision_id
|
|
3145
|
+
WHERE d.run_id = ? AND r.decision_id IS NULL`,
|
|
3146
|
+
)
|
|
3147
|
+
.run(actorId, now, runId);
|
|
3148
|
+
const receiptHash = this.state.putJson(
|
|
3149
|
+
{ status: "rejected", error: "Workflow run cancelled" },
|
|
3150
|
+
now,
|
|
3151
|
+
);
|
|
3152
|
+
this.state.connection
|
|
3153
|
+
.prepare(
|
|
3154
|
+
`UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
|
|
3155
|
+
WHERE outcome = 'validating'
|
|
3156
|
+
AND request_id IN (SELECT request_id FROM interactive_requests WHERE run_id = ?)`,
|
|
3157
|
+
)
|
|
3158
|
+
.run(receiptHash, runId);
|
|
3159
|
+
this.state.connection
|
|
3160
|
+
.prepare(
|
|
3161
|
+
`UPDATE interactive_requests
|
|
3162
|
+
SET status = 'cancelled', presenter_id = NULL,
|
|
3163
|
+
presentation_claim_expires_at = NULL, revision = revision + 1, updated_at = ?
|
|
3164
|
+
WHERE run_id = ? AND status IN ('pending', 'presenting')`,
|
|
3165
|
+
)
|
|
3166
|
+
.run(now, runId);
|
|
3167
|
+
}
|
|
3168
|
+
|
|
2483
3169
|
private turnIntent(intentId: string): WorkflowTurnIntentRecord | undefined {
|
|
2484
3170
|
const row = this.turnIntentRow(intentId);
|
|
2485
3171
|
return row === undefined ? undefined : this.mapTurnIntent(row);
|
|
@@ -2772,7 +3458,8 @@ function workflowRunSelect(clause: string): string {
|
|
|
2772
3458
|
q.status, q.available_at AS availableAt, q.affinity_runner_id AS affinityRunnerId,
|
|
2773
3459
|
q.consecutive_errors AS consecutiveErrors, q.error_code AS errorCode,
|
|
2774
3460
|
q.error_hash AS errorHash, b.origin_session_id AS originSessionId,
|
|
2775
|
-
|
|
3461
|
+
b.execution_mode AS executionMode, r.parent_run_id AS parentRunId,
|
|
3462
|
+
q.created_at AS createdAt, q.updated_at AS updatedAt,
|
|
2776
3463
|
q.started_at AS startedAt, q.finished_at AS finishedAt,
|
|
2777
3464
|
l.generation AS leaseGeneration, l.owner_id AS ownerId, l.expires_at AS claimExpiresAt
|
|
2778
3465
|
FROM runs r JOIN workflow_definitions d ON d.definition_digest = r.definition_digest
|
|
@@ -2881,6 +3568,20 @@ function isRunSourceIdentityRow(value: unknown): value is RunSourceIdentityRow {
|
|
|
2881
3568
|
function isEffectRow(value: unknown): value is EffectRow {
|
|
2882
3569
|
return isRecord(value);
|
|
2883
3570
|
}
|
|
3571
|
+
function isCancelledRunEffectRow(value: unknown): value is CancelledRunEffectRow {
|
|
3572
|
+
return (
|
|
3573
|
+
isRecord(value) &&
|
|
3574
|
+
typeof value.effectId === "string" &&
|
|
3575
|
+
typeof value.resourceId === "string" &&
|
|
3576
|
+
(value.status === "pending" || value.status === "applying") &&
|
|
3577
|
+
typeof value.attemptCount === "number"
|
|
3578
|
+
);
|
|
3579
|
+
}
|
|
3580
|
+
function isExpiredInteractionRow(value: unknown): value is ExpiredInteractionRow {
|
|
3581
|
+
return (
|
|
3582
|
+
isRecord(value) && typeof value.requestId === "string" && typeof value.attemptId === "string"
|
|
3583
|
+
);
|
|
3584
|
+
}
|
|
2884
3585
|
function isWorkflowRow(value: unknown): value is WorkflowRow {
|
|
2885
3586
|
return isRecord(value);
|
|
2886
3587
|
}
|
|
@@ -2902,6 +3603,9 @@ function isNotificationRow(value: unknown): value is NotificationRow {
|
|
|
2902
3603
|
function isProjectRow(value: unknown): value is { projectId: string } {
|
|
2903
3604
|
return isRecord(value);
|
|
2904
3605
|
}
|
|
3606
|
+
function isCanonicalPathRow(value: unknown): value is { canonicalPath: string } {
|
|
3607
|
+
return isRecord(value) && typeof value.canonicalPath === "string";
|
|
3608
|
+
}
|
|
2905
3609
|
function isFinalizerRow(value: unknown): value is { finalizer: string } {
|
|
2906
3610
|
return isRecord(value);
|
|
2907
3611
|
}
|