@osolmaz/pi-workflows 0.13.4 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +146 -160
- package/dist/builtins/autoimplement.workflow.js +3 -1
- package/dist/builtins/autoimplement.workflow.js.map +1 -1
- package/dist/builtins/change-verification.workflow.js +7 -2
- package/dist/builtins/change-verification.workflow.js.map +1 -1
- package/dist/builtins/metadata.d.ts +22 -0
- package/dist/builtins/metadata.js +10 -0
- package/dist/builtins/metadata.js.map +1 -0
- package/dist/builtins/monitor.workflow.js +4 -1
- package/dist/builtins/monitor.workflow.js.map +1 -1
- package/dist/builtins/sanity-check.workflow.js +4 -1
- package/dist/builtins/sanity-check.workflow.js.map +1 -1
- package/dist/builtins/workspace-preparation.workflow.js +3 -1
- package/dist/builtins/workspace-preparation.workflow.js.map +1 -1
- package/dist/controllers/index.d.ts +1 -2
- package/dist/controllers/index.js +0 -1
- package/dist/controllers/index.js.map +1 -1
- package/dist/controllers/sqlite.d.ts +85 -31
- package/dist/controllers/sqlite.js +541 -126
- package/dist/controllers/sqlite.js.map +1 -1
- package/dist/controllers/store.d.ts +1 -0
- package/dist/extension/controller-command.d.ts +22 -0
- package/dist/extension/controller-command.js +32 -0
- package/dist/extension/controller-command.js.map +1 -0
- package/dist/extension/index.d.ts +2 -9
- package/dist/extension/index.js +876 -2928
- package/dist/extension/index.js.map +1 -1
- package/dist/extension/message-card.d.ts +13 -0
- package/dist/extension/message-card.js +46 -0
- package/dist/extension/message-card.js.map +1 -0
- package/dist/extension/step-message.d.ts +5 -6
- package/dist/extension/step-message.js +41 -52
- package/dist/extension/step-message.js.map +1 -1
- package/dist/host/child-worker-supervisor.d.ts +50 -0
- package/dist/host/child-worker-supervisor.js +159 -0
- package/dist/host/child-worker-supervisor.js.map +1 -0
- package/dist/host/client.d.ts +48 -0
- package/dist/host/client.js +216 -0
- package/dist/host/client.js.map +1 -0
- package/dist/host/controller-worker-entry.d.ts +2 -0
- package/dist/host/controller-worker-entry.js +244 -0
- package/dist/host/controller-worker-entry.js.map +1 -0
- package/dist/host/controller-worker-protocol.d.ts +36 -0
- package/dist/host/controller-worker-protocol.js +49 -0
- package/dist/host/controller-worker-protocol.js.map +1 -0
- package/dist/host/controller-worker-supervisor.d.ts +21 -0
- package/dist/host/controller-worker-supervisor.js +54 -0
- package/dist/host/controller-worker-supervisor.js.map +1 -0
- package/dist/host/host-entry.d.ts +2 -0
- package/dist/host/host-entry.js +23 -0
- package/dist/host/host-entry.js.map +1 -0
- package/dist/host/processes.d.ts +17 -12
- package/dist/host/processes.js +154 -50
- package/dist/host/processes.js.map +1 -1
- package/dist/host/protocol.d.ts +38 -0
- package/dist/host/protocol.js +156 -0
- package/dist/host/protocol.js.map +1 -0
- package/dist/host/resolver-entry.d.ts +49 -0
- package/dist/host/resolver-entry.js +149 -0
- package/dist/host/resolver-entry.js.map +1 -0
- package/dist/host/rpc-executor.d.ts +14 -3
- package/dist/host/rpc-executor.js +63 -30
- package/dist/host/rpc-executor.js.map +1 -1
- package/dist/host/runner.d.ts +95 -29
- package/dist/host/runner.js +2311 -348
- package/dist/host/runner.js.map +1 -1
- package/dist/host/state.d.ts +174 -0
- package/dist/host/state.js +689 -0
- package/dist/host/state.js.map +1 -0
- package/dist/host/worker-entry.d.ts +10 -0
- package/dist/host/worker-entry.js +446 -0
- package/dist/host/worker-entry.js.map +1 -0
- package/dist/host/worker-protocol.d.ts +31 -0
- package/dist/host/worker-protocol.js +122 -0
- package/dist/host/worker-protocol.js.map +1 -0
- package/dist/host/worker-store.d.ts +80 -0
- package/dist/host/worker-store.js +91 -0
- package/dist/host/worker-store.js.map +1 -0
- package/dist/host/worker-supervisor.d.ts +22 -0
- package/dist/host/worker-supervisor.js +54 -0
- package/dist/host/worker-supervisor.js.map +1 -0
- package/dist/render/canvas.d.ts +2 -1
- package/dist/render/canvas.js +40 -15
- package/dist/render/canvas.js.map +1 -1
- package/dist/render/graph-render.d.ts +7 -3
- package/dist/render/graph-render.js +125 -74
- package/dist/render/graph-render.js.map +1 -1
- package/dist/state/database.js +3 -1
- package/dist/state/database.js.map +1 -1
- package/dist/state/index.d.ts +1 -0
- package/dist/state/index.js +1 -0
- package/dist/state/index.js.map +1 -1
- package/dist/state/prune.js +36 -10
- package/dist/state/prune.js.map +1 -1
- package/dist/state/schema.d.ts +1 -1
- package/dist/state/schema.js +171 -3
- package/dist/state/schema.js.map +1 -1
- package/dist/state/viewer.d.ts +46 -0
- package/dist/state/viewer.js +249 -0
- package/dist/state/viewer.js.map +1 -0
- package/dist/viewer/cli.d.ts +1 -1
- package/dist/viewer/cli.js +47 -18
- package/dist/viewer/cli.js.map +1 -1
- package/dist/workflows/composition.js +25 -2
- package/dist/workflows/composition.js.map +1 -1
- package/dist/workflows/definition.d.ts +3 -1
- package/dist/workflows/definition.js +25 -0
- package/dist/workflows/definition.js.map +1 -1
- package/dist/workflows/engine.d.ts +1 -0
- package/dist/workflows/engine.js +157 -42
- package/dist/workflows/engine.js.map +1 -1
- package/dist/workflows/errors.d.ts +3 -1
- package/dist/workflows/errors.js +4 -7
- package/dist/workflows/errors.js.map +1 -1
- package/dist/workflows/human-decision.d.ts +3 -0
- package/dist/workflows/human-decision.js +31 -0
- package/dist/workflows/human-decision.js.map +1 -1
- package/dist/workflows/index.d.ts +1 -1
- package/dist/workflows/index.js +1 -1
- package/dist/workflows/index.js.map +1 -1
- package/dist/workflows/schema.js +17 -6
- package/dist/workflows/schema.js.map +1 -1
- package/dist/{viewer → workflows}/session-reducer.d.ts +3 -1
- package/dist/{viewer → workflows}/session-reducer.js +4 -0
- package/dist/workflows/session-reducer.js.map +1 -0
- package/dist/workflows/store.d.ts +87 -1
- package/dist/workflows/store.js +892 -93
- package/dist/workflows/store.js.map +1 -1
- package/dist/workflows/tool-input.d.ts +0 -22
- package/dist/workflows/tool-input.js +1 -44
- package/dist/workflows/tool-input.js.map +1 -1
- package/dist/workflows/types.d.ts +36 -4
- package/docs/2026-08-25-workflow-follow-ups.md +8 -6
- package/docs/2026-08-30-out-of-process-workflow-host-plan.md +353 -0
- package/docs/CONTROLLERS.md +11 -11
- package/docs/DEFERRED_TURNS.md +61 -27
- package/docs/HUMAN_DECISIONS.md +12 -4
- package/docs/SQLITE_STATE.md +42 -8
- package/docs/WORKFLOW_HOST.md +452 -0
- package/docs/development.md +46 -30
- package/docs/live-replay-protocol.md +129 -100
- package/docs/plans/2026-08-19-human-decision-gates-plan.md +34 -8
- package/docs/plans/2026-08-27-workflow-terminal-restart-plan.md +399 -0
- package/docs/plans/2026-08-28-piw-incremental-viewer-plan.md +293 -0
- package/docs/plans/piw-viewer-experience-implementation-plan.md +11 -1
- package/docs/plans/tui-viewer-implementation-plan.md +5 -0
- package/docs/tui-viewer.md +18 -3
- package/docs/workflows.md +164 -156
- package/examples/workflows/command-batch.workflow.ts +2 -0
- package/examples/workflows/shell.workflow.ts +2 -1
- package/herdr-plugin.toml +1 -1
- package/package.json +1 -1
- package/skills/autodoc/SKILL.md +1 -1
- package/skills/autoimplement/SKILL.md +1 -1
- package/skills/autoplan/SKILL.md +1 -1
- package/skills/pi-workflows/SKILL.md +2 -0
- package/src/builtins/autoimplement.workflow.ts +3 -0
- package/src/builtins/change-verification.workflow.ts +7 -2
- package/src/builtins/metadata.ts +9 -0
- package/src/builtins/monitor.workflow.ts +4 -0
- package/src/builtins/sanity-check.workflow.ts +4 -0
- package/src/builtins/workspace-preparation.workflow.ts +3 -1
- package/src/controllers/index.ts +3 -5
- package/src/controllers/sqlite.ts +929 -225
- package/src/controllers/store.ts +1 -0
- package/src/extension/controller-command.ts +45 -0
- package/src/extension/index.ts +948 -3506
- package/src/extension/message-card.ts +61 -0
- package/src/extension/step-message.ts +58 -63
- package/src/host/child-worker-supervisor.ts +183 -0
- package/src/host/client.ts +293 -0
- package/src/host/controller-worker-entry.ts +311 -0
- package/src/host/controller-worker-protocol.ts +104 -0
- package/src/host/controller-worker-supervisor.ts +79 -0
- package/src/host/host-entry.ts +23 -0
- package/src/host/processes.ts +171 -54
- package/src/host/protocol.ts +196 -0
- package/src/host/resolver-entry.ts +241 -0
- package/src/host/rpc-executor.ts +76 -34
- package/src/host/runner.ts +2813 -422
- package/src/host/state.ts +1160 -0
- package/src/host/worker-entry.ts +533 -0
- package/src/host/worker-protocol.ts +165 -0
- package/src/host/worker-store.ts +229 -0
- package/src/host/worker-supervisor.ts +74 -0
- package/src/render/canvas.ts +44 -10
- package/src/render/graph-render.ts +145 -90
- package/src/state/database.ts +2 -1
- package/src/state/index.ts +14 -0
- package/src/state/prune.ts +35 -9
- package/src/state/schema.ts +171 -3
- package/src/state/viewer.ts +356 -0
- package/src/viewer/cli.ts +49 -17
- package/src/workflows/composition.ts +36 -2
- package/src/workflows/definition.ts +32 -0
- package/src/workflows/engine.ts +157 -54
- package/src/workflows/errors.ts +11 -2
- package/src/workflows/human-decision.ts +49 -0
- package/src/workflows/index.ts +2 -0
- package/src/workflows/schema.ts +19 -6
- package/src/{viewer → workflows}/session-reducer.ts +11 -1
- package/src/workflows/store.ts +1316 -108
- package/src/workflows/tool-input.ts +2 -60
- package/src/workflows/types.ts +32 -4
- package/dist/controllers/workflow-engine-scheduler.d.ts +0 -29
- package/dist/controllers/workflow-engine-scheduler.js +0 -175
- package/dist/controllers/workflow-engine-scheduler.js.map +0 -1
- package/dist/extension/controller-host.d.ts +0 -48
- package/dist/extension/controller-host.js +0 -110
- package/dist/extension/controller-host.js.map +0 -1
- package/dist/extension/deferred-turn-coordinator.d.ts +0 -32
- package/dist/extension/deferred-turn-coordinator.js +0 -143
- package/dist/extension/deferred-turn-coordinator.js.map +0 -1
- package/dist/extension/deferred-turn.d.ts +0 -44
- package/dist/extension/deferred-turn.js +0 -110
- package/dist/extension/deferred-turn.js.map +0 -1
- package/dist/extension/executor.d.ts +0 -86
- package/dist/extension/executor.js +0 -311
- package/dist/extension/executor.js.map +0 -1
- package/dist/extension/follow-up-coordinator.d.ts +0 -27
- package/dist/extension/follow-up-coordinator.js +0 -131
- package/dist/extension/follow-up-coordinator.js.map +0 -1
- package/dist/extension/recorder.d.ts +0 -84
- package/dist/extension/recorder.js +0 -528
- package/dist/extension/recorder.js.map +0 -1
- package/dist/extension/session-events.d.ts +0 -133
- package/dist/extension/session-events.js +0 -61
- package/dist/extension/session-events.js.map +0 -1
- package/dist/viewer/session-reducer.js.map +0 -1
- package/src/controllers/workflow-engine-scheduler.ts +0 -246
- package/src/extension/controller-host.ts +0 -167
- package/src/extension/deferred-turn-coordinator.ts +0 -171
- package/src/extension/deferred-turn.ts +0 -166
- package/src/extension/executor.ts +0 -411
- package/src/extension/follow-up-coordinator.ts +0 -151
- package/src/extension/recorder.ts +0 -655
- package/src/extension/session-events.ts +0 -121
package/src/host/runner.ts
CHANGED
|
@@ -1,519 +1,2910 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { once } from "node:events";
|
|
2
3
|
import fs from "node:fs";
|
|
4
|
+
import net, { type Socket } from "node:net";
|
|
3
5
|
import path from "node:path";
|
|
4
|
-
import {
|
|
6
|
+
import { applyStatusPatch } from "../controllers/conditions.js";
|
|
7
|
+
import { ResourceConflictError } from "../controllers/errors.js";
|
|
8
|
+
import {
|
|
9
|
+
controllerFileStem,
|
|
10
|
+
controllerSearchDirs,
|
|
11
|
+
discoverControllers,
|
|
12
|
+
} from "../controllers/loader.js";
|
|
5
13
|
import {
|
|
6
|
-
ControllerManager,
|
|
7
|
-
loadDiscoveredControllers,
|
|
8
14
|
SqliteControllerStore,
|
|
9
|
-
WorkflowEngineScheduler,
|
|
10
15
|
type WorkflowRunQueueRecord,
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
import {
|
|
14
|
-
|
|
16
|
+
type WorkflowTurnIntentFacts,
|
|
17
|
+
} from "../controllers/sqlite.js";
|
|
18
|
+
import type {
|
|
19
|
+
ControllerQueueClaim,
|
|
20
|
+
ControllerResource,
|
|
21
|
+
ControllerSettingsChangeRequest,
|
|
22
|
+
ControllerSettingsChangeResult,
|
|
23
|
+
ReconcileResult,
|
|
24
|
+
} from "../controllers/types.js";
|
|
25
|
+
import {
|
|
26
|
+
ControllerWorkflowCoordinator,
|
|
27
|
+
type ControllerWorkflowControlRequest,
|
|
28
|
+
type ControllerWorkflowScheduler,
|
|
29
|
+
type WorkflowSchedulerRequest,
|
|
30
|
+
type WorkflowSchedulerResult,
|
|
31
|
+
} from "../controllers/workflows.js";
|
|
32
|
+
import { StateDatabase, workflowStatePath } from "../state/database.js";
|
|
33
|
+
import { canonicalJson, type JsonValue } from "../state/json.js";
|
|
34
|
+
import { recordViewerDeltas } from "../state/viewer.js";
|
|
35
|
+
import { errorMessage } from "../workflows/errors.js";
|
|
36
|
+
import { HumanDecisionStore } from "../workflows/human-decision.js";
|
|
15
37
|
import {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
WorkflowSourceChangedError,
|
|
20
|
-
} from "../workflows/errors.js";
|
|
21
|
-
import { resolveWorkflowRef, resolveWorkflowSource } from "../workflows/loader.js";
|
|
38
|
+
type WorkflowSettingsDefinition,
|
|
39
|
+
type WorkflowSettingsPathRule,
|
|
40
|
+
} from "../workflows/settings.js";
|
|
22
41
|
import { WorkflowRunStore } from "../workflows/store.js";
|
|
23
|
-
import type {
|
|
24
|
-
|
|
25
|
-
|
|
42
|
+
import type {
|
|
43
|
+
HumanDecisionRequest,
|
|
44
|
+
HumanDecisionResponse,
|
|
45
|
+
ResolvedHumanDecision,
|
|
46
|
+
WorkflowDefinitionSnapshot,
|
|
47
|
+
WorkflowRunState,
|
|
48
|
+
WorkflowTraceEventDraft,
|
|
49
|
+
WorkflowUpdateInput,
|
|
50
|
+
} from "../workflows/types.js";
|
|
51
|
+
import { validateWorkflowUpdate } from "../workflows/updates.js";
|
|
52
|
+
import { WorkflowHostClient } from "./client.js";
|
|
53
|
+
import type {
|
|
54
|
+
ControllerWorkerLaunchEnvelope,
|
|
55
|
+
ControllerWorkerMessage,
|
|
56
|
+
ControllerWorkerResponse,
|
|
57
|
+
} from "./controller-worker-protocol.js";
|
|
58
|
+
import { ControllerWorkerSupervisor } from "./controller-worker-supervisor.js";
|
|
59
|
+
import {
|
|
60
|
+
HostProcessRegistry,
|
|
61
|
+
matchesProcessIdentity,
|
|
62
|
+
processParentPid,
|
|
63
|
+
processStartIdentity,
|
|
64
|
+
} from "./processes.js";
|
|
65
|
+
import {
|
|
66
|
+
encodeProtocolLine,
|
|
67
|
+
hostSocketPath,
|
|
68
|
+
NdjsonFrameDecoder,
|
|
69
|
+
parseHostRequest,
|
|
70
|
+
type HostRequest,
|
|
71
|
+
type HostResponse,
|
|
72
|
+
} from "./protocol.js";
|
|
73
|
+
import {
|
|
74
|
+
HostStateStore,
|
|
75
|
+
type HostClaim,
|
|
76
|
+
type InteractiveRequestRecord,
|
|
77
|
+
type InteractiveSubmissionRecord,
|
|
78
|
+
type WorkerLaunchEnvelope,
|
|
79
|
+
} from "./state.js";
|
|
80
|
+
import type { WorkerMessage, WorkerResponse } from "./worker-protocol.js";
|
|
81
|
+
import { WorkflowWorkerSupervisor } from "./worker-supervisor.js";
|
|
26
82
|
|
|
83
|
+
const HOST_LEASE_MS = 30_000;
|
|
84
|
+
const HOST_RENEW_MS = 10_000;
|
|
27
85
|
const CLAIM_POLL_MS = 2_000;
|
|
28
86
|
const RUN_CLAIM_LEASE_MS = 30_000;
|
|
29
|
-
const
|
|
87
|
+
const PRESENTATION_CLAIM_LEASE_MS = 10_000;
|
|
88
|
+
const DELIVERY_CLAIM_LEASE_MS = 10_000;
|
|
89
|
+
const CONTROLLER_CLAIM_LEASE_MS = 120_000;
|
|
90
|
+
const CONTROLLER_RENEW_MS = 30_000;
|
|
91
|
+
const MAX_CONTROLLER_WORKERS = 4;
|
|
92
|
+
const DEFAULT_CONTROLLER_TIMEOUT_MS = 60_000;
|
|
30
93
|
|
|
31
94
|
export type WorkflowHostOptions = {
|
|
32
|
-
|
|
95
|
+
/** Retained for callers; the host itself is global to the state database. */
|
|
96
|
+
cwd?: string;
|
|
33
97
|
runnerId?: string;
|
|
34
|
-
/** Explicit database path; defaults to the canonical workflow state database. */
|
|
35
98
|
databasePath?: string;
|
|
36
99
|
registry?: HostProcessRegistry;
|
|
37
100
|
piArgs?: string[];
|
|
38
|
-
/** Extra environment for headless children (for example a test provider). */
|
|
39
101
|
env?: Record<string, string>;
|
|
40
|
-
/** Poll interval for the claim loop; tests use a faster cadence. */
|
|
41
102
|
claimPollMs?: number;
|
|
103
|
+
hostLeaseMs?: number;
|
|
104
|
+
hostRenewMs?: number;
|
|
105
|
+
runClaimLeaseMs?: number;
|
|
106
|
+
workerEntryPath?: string;
|
|
107
|
+
workerStartupTimeoutMs?: number;
|
|
108
|
+
controllerWorkerEntryPath?: string;
|
|
42
109
|
onLog?: (message: string) => void;
|
|
43
110
|
};
|
|
44
111
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
112
|
+
type ActiveRun = {
|
|
113
|
+
record: WorkflowRunQueueRecord;
|
|
114
|
+
claimToken: string;
|
|
115
|
+
generation: number;
|
|
116
|
+
supervisor: WorkflowWorkerSupervisor;
|
|
117
|
+
workerPid?: number;
|
|
118
|
+
exiting: boolean;
|
|
119
|
+
workflowLoadFailure?: string;
|
|
120
|
+
control?: "cancel" | "pause" | "handoff";
|
|
121
|
+
claimLost?: boolean;
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
type DeliveryClaim = {
|
|
125
|
+
clientId: string;
|
|
126
|
+
token: string;
|
|
127
|
+
targetSessionId: string;
|
|
128
|
+
kind: "notification" | "turn";
|
|
129
|
+
resourceId: string;
|
|
130
|
+
expiresAt: number;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
type ActiveController = {
|
|
134
|
+
key: string;
|
|
135
|
+
projectPath: string;
|
|
136
|
+
store: SqliteControllerStore;
|
|
137
|
+
claim: ControllerQueueClaim;
|
|
138
|
+
resource: ControllerResource;
|
|
139
|
+
reconcileId: string;
|
|
140
|
+
startedAt: number;
|
|
141
|
+
supervisor: ControllerWorkerSupervisor;
|
|
142
|
+
renewTimer: ReturnType<typeof setInterval>;
|
|
143
|
+
settled: boolean;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
/** Global package-owned host. It writes state and supervises code-only workers. */
|
|
52
147
|
export class WorkflowHost {
|
|
53
148
|
private readonly options: WorkflowHostOptions;
|
|
54
|
-
private readonly
|
|
149
|
+
private readonly hostId: string;
|
|
150
|
+
private readonly databasePath: string;
|
|
151
|
+
private readonly stateDirectory: string;
|
|
152
|
+
private readonly socketPath: string;
|
|
153
|
+
private readonly lockPath: string;
|
|
154
|
+
private readonly state: StateDatabase;
|
|
155
|
+
private readonly hostState: HostStateStore;
|
|
156
|
+
private readonly queue: SqliteControllerStore;
|
|
157
|
+
private readonly decisions: HumanDecisionStore;
|
|
158
|
+
private readonly runStore: WorkflowRunStore;
|
|
55
159
|
private readonly registry: HostProcessRegistry;
|
|
56
|
-
private readonly
|
|
57
|
-
private readonly
|
|
58
|
-
private
|
|
160
|
+
private readonly activeRuns = new Map<string, ActiveRun>();
|
|
161
|
+
private readonly workerDescendants = new Map<string, Set<number>>();
|
|
162
|
+
private readonly activeControllers = new Map<string, ActiveController>();
|
|
163
|
+
private readonly controlClaims = new Map<string, string>();
|
|
164
|
+
private readonly activationTasks = new Map<string, Promise<void>>();
|
|
165
|
+
private readonly pendingStarts = new Set<string>();
|
|
166
|
+
private readonly pendingRunClaims = new Map<string, string>();
|
|
167
|
+
private readonly pendingResumes = new Set<string>();
|
|
168
|
+
private readonly blockedRuns = new Set<string>();
|
|
169
|
+
private readonly deliveryClaims = new Map<string, DeliveryClaim>();
|
|
170
|
+
private readonly sockets = new Set<Socket>();
|
|
171
|
+
private server: net.Server | null = null;
|
|
172
|
+
private claim: HostClaim | null = null;
|
|
173
|
+
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
|
59
174
|
private pollTimer: ReturnType<typeof setInterval> | null = null;
|
|
60
|
-
private readonly activeRuns = new Map<string, Promise<void>>();
|
|
61
|
-
private readonly schedulerExecutors = new Map<WorkflowEngine, RpcStepExecutor>();
|
|
62
|
-
/** Runs whose resume refused (edited source); skipped until a host restart. */
|
|
63
|
-
private readonly skippedRuns = new Set<string>();
|
|
64
175
|
private stopping = false;
|
|
176
|
+
private started = false;
|
|
177
|
+
private controllerPollActive = false;
|
|
178
|
+
private decisionTimeoutActive = false;
|
|
65
179
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
constructor(options: WorkflowHostOptions) {
|
|
180
|
+
constructor(options: WorkflowHostOptions = {}) {
|
|
69
181
|
this.options = options;
|
|
70
|
-
this.
|
|
71
|
-
|
|
72
|
-
this.
|
|
73
|
-
this.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
);
|
|
78
|
-
this.
|
|
79
|
-
|
|
182
|
+
this.hostId = options.runnerId ?? `host-${randomUUID()}`;
|
|
183
|
+
this.databasePath = path.resolve(options.databasePath ?? workflowStatePath());
|
|
184
|
+
this.stateDirectory = path.join(path.dirname(this.databasePath), "host");
|
|
185
|
+
this.socketPath = hostSocketPath(this.databasePath);
|
|
186
|
+
this.lockPath = path.join(this.stateDirectory, "host.lock.json");
|
|
187
|
+
this.state = new StateDatabase({ filePath: this.databasePath });
|
|
188
|
+
this.hostState = new HostStateStore(this.databasePath, { state: this.state });
|
|
189
|
+
this.queue = new SqliteControllerStore(this.databasePath, { state: this.state, global: true });
|
|
190
|
+
this.decisions = new HumanDecisionStore(this.databasePath, {
|
|
191
|
+
state: this.state,
|
|
192
|
+
authorityProvider: (runId) => {
|
|
193
|
+
const token = this.controlClaims.get(runId) ?? this.activeRuns.get(runId)?.claimToken;
|
|
194
|
+
return token === undefined ? undefined : this.queue.workflowRunAuthority(runId, token);
|
|
195
|
+
},
|
|
196
|
+
});
|
|
197
|
+
this.runStore = new WorkflowRunStore(this.databasePath, {
|
|
198
|
+
state: this.state,
|
|
199
|
+
authorityProvider: (runId) => {
|
|
200
|
+
const token = this.controlClaims.get(runId) ?? this.activeRuns.get(runId)?.claimToken;
|
|
201
|
+
return token === undefined ? undefined : this.queue.workflowRunAuthority(runId, token);
|
|
202
|
+
},
|
|
203
|
+
snapshotLifecycle: (context) => this.applyLifecycleProjection(context),
|
|
204
|
+
});
|
|
205
|
+
this.registry = options.registry ?? new HostProcessRegistry(this.stateDirectory);
|
|
80
206
|
}
|
|
81
207
|
|
|
82
|
-
|
|
83
|
-
this.
|
|
208
|
+
get endpoint(): string {
|
|
209
|
+
return this.socketPath;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private get runClaimLeaseMs(): number {
|
|
213
|
+
return this.options.runClaimLeaseMs ?? RUN_CLAIM_LEASE_MS;
|
|
84
214
|
}
|
|
85
215
|
|
|
86
|
-
/** Take the advisory lock, reap orphans, and start claiming. */
|
|
87
216
|
async start(): Promise<void> {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
builtinWorkflowCatalog,
|
|
106
|
-
);
|
|
107
|
-
return { workflow: resolved.definition, workflowSource: resolved.source };
|
|
108
|
-
},
|
|
109
|
-
createEngine: () => {
|
|
110
|
-
const executor = new RpcStepExecutor({
|
|
111
|
-
cwd: this.options.cwd,
|
|
112
|
-
registry: this.registry,
|
|
113
|
-
...(this.options.piArgs !== undefined ? { piArgs: this.options.piArgs } : {}),
|
|
114
|
-
...(this.options.env !== undefined ? { env: this.options.env } : {}),
|
|
115
|
-
});
|
|
116
|
-
const engine = new WorkflowEngine({ executor, store: this.childRunStore });
|
|
117
|
-
this.schedulerExecutors.set(engine, executor);
|
|
118
|
-
return engine;
|
|
119
|
-
},
|
|
120
|
-
disposeEngine: async (engine) => {
|
|
121
|
-
const executor = this.schedulerExecutors.get(engine);
|
|
122
|
-
if (executor !== undefined) {
|
|
123
|
-
this.schedulerExecutors.delete(engine);
|
|
124
|
-
await executor.close();
|
|
125
|
-
}
|
|
126
|
-
},
|
|
127
|
-
});
|
|
128
|
-
this.manager = new ControllerManager({
|
|
129
|
-
store: this.store,
|
|
130
|
-
controllers: definitions,
|
|
131
|
-
workflowScheduler: scheduler,
|
|
217
|
+
if (this.started) return;
|
|
218
|
+
fs.mkdirSync(this.stateDirectory, { recursive: true, mode: 0o700 });
|
|
219
|
+
fs.chmodSync(this.stateDirectory, 0o700);
|
|
220
|
+
const startIdentity = processStartIdentity(process.pid);
|
|
221
|
+
if (startIdentity === undefined) {
|
|
222
|
+
throw new Error("Cannot attest the workflow host process start identity");
|
|
223
|
+
}
|
|
224
|
+
acquireHostLock(this.lockPath, { pid: process.pid, startIdentity, hostId: this.hostId });
|
|
225
|
+
try {
|
|
226
|
+
const reaped = this.registry.reapOrphans();
|
|
227
|
+
if (reaped.length > 0) this.log(`reaped ${reaped.length} exact orphan process(es)`);
|
|
228
|
+
const previousHost = this.hostState.hostStatus();
|
|
229
|
+
this.claim = this.hostState.acquireHost({
|
|
230
|
+
hostId: this.hostId,
|
|
231
|
+
pid: process.pid,
|
|
232
|
+
processStartIdentity: startIdentity,
|
|
233
|
+
leaseMs: this.options.hostLeaseMs ?? HOST_LEASE_MS,
|
|
132
234
|
});
|
|
133
|
-
this.
|
|
134
|
-
this.
|
|
235
|
+
this.recoverPreviousHost(previousHost.hostId, this.claim.epoch);
|
|
236
|
+
await this.listen();
|
|
237
|
+
this.startTimers();
|
|
238
|
+
this.started = true;
|
|
239
|
+
this.log(`ready on ${this.socketPath} at epoch ${this.claim.epoch}`);
|
|
240
|
+
this.expireTimedOutInteraction();
|
|
241
|
+
void this.expireTimedOutDecision();
|
|
242
|
+
void this.claimOne();
|
|
243
|
+
void this.claimControllerOne();
|
|
244
|
+
} catch (error) {
|
|
245
|
+
const server = this.server;
|
|
246
|
+
this.server = null;
|
|
247
|
+
await this.closeServer(server);
|
|
248
|
+
if (this.claim !== null) {
|
|
249
|
+
try {
|
|
250
|
+
this.hostState.releaseHost(this.claim);
|
|
251
|
+
} catch {
|
|
252
|
+
// Preserve the startup error when cleanup cannot release an already-lost claim.
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
this.claim = null;
|
|
256
|
+
if (process.platform !== "win32") fs.rmSync(this.socketPath, { force: true });
|
|
257
|
+
this.releaseLock();
|
|
258
|
+
throw error;
|
|
135
259
|
}
|
|
136
|
-
|
|
137
|
-
this.pollTimer = setInterval(() => {
|
|
138
|
-
this.claimOnce();
|
|
139
|
-
}, this.options.claimPollMs ?? CLAIM_POLL_MS);
|
|
140
|
-
this.pollTimer.unref?.();
|
|
141
|
-
this.claimOnce();
|
|
142
|
-
this.log(`host ${this.runnerId} watching ${this.options.cwd}`);
|
|
143
260
|
}
|
|
144
261
|
|
|
145
|
-
/** Drain: stop claiming, park in-flight runs, stop controllers, kill children. */
|
|
146
262
|
async stop(): Promise<void> {
|
|
147
|
-
if (this.stopping)
|
|
148
|
-
return;
|
|
149
|
-
}
|
|
263
|
+
if (this.stopping) return;
|
|
150
264
|
this.stopping = true;
|
|
151
|
-
if (this.
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
await this.manager?.stop().catch(() => undefined);
|
|
161
|
-
for (const executor of this.schedulerExecutors.values()) {
|
|
162
|
-
await executor.close().catch(() => undefined);
|
|
265
|
+
if (this.heartbeatTimer !== null) clearInterval(this.heartbeatTimer);
|
|
266
|
+
if (this.pollTimer !== null) clearInterval(this.pollTimer);
|
|
267
|
+
this.heartbeatTimer = null;
|
|
268
|
+
this.pollTimer = null;
|
|
269
|
+
const server = this.server;
|
|
270
|
+
this.server = null;
|
|
271
|
+
await this.closeServer(server);
|
|
272
|
+
for (const [runId, claimToken] of this.pendingRunClaims) {
|
|
273
|
+
this.queue.parkWorkflowRun({ runId, claimToken });
|
|
163
274
|
}
|
|
164
|
-
this.
|
|
275
|
+
this.pendingRunClaims.clear();
|
|
276
|
+
this.pendingStarts.clear();
|
|
277
|
+
await Promise.allSettled(
|
|
278
|
+
[...this.activeRuns.values()].map(async (active) => {
|
|
279
|
+
active.control = "handoff";
|
|
280
|
+
await active.supervisor.stop("orphaned");
|
|
281
|
+
this.queue.parkWorkflowRun({ runId: active.record.runId, claimToken: active.claimToken });
|
|
282
|
+
}),
|
|
283
|
+
);
|
|
284
|
+
await Promise.allSettled(
|
|
285
|
+
[...this.activeControllers.values()].map(async (active) => {
|
|
286
|
+
clearInterval(active.renewTimer);
|
|
287
|
+
await active.supervisor.stop("orphaned");
|
|
288
|
+
if (!active.settled) {
|
|
289
|
+
active.store.requeueClaim(
|
|
290
|
+
active.claim,
|
|
291
|
+
{ availableAt: new Date().toISOString(), error: "Workflow host stopped" },
|
|
292
|
+
new Date().toISOString(),
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
}),
|
|
296
|
+
);
|
|
297
|
+
await Promise.allSettled(this.activationTasks.values());
|
|
165
298
|
this.registry.killAll();
|
|
166
|
-
|
|
167
|
-
this.
|
|
299
|
+
this.deliveryClaims.clear();
|
|
300
|
+
if (this.claim !== null) this.hostState.releaseHost(this.claim);
|
|
301
|
+
this.claim = null;
|
|
302
|
+
if (process.platform !== "win32") fs.rmSync(this.socketPath, { force: true });
|
|
303
|
+
this.releaseLock();
|
|
304
|
+
this.runStore.close();
|
|
305
|
+
this.queue.close();
|
|
306
|
+
this.hostState.close();
|
|
307
|
+
this.state.close();
|
|
308
|
+
this.started = false;
|
|
168
309
|
}
|
|
169
310
|
|
|
170
|
-
private
|
|
311
|
+
private async closeServer(server: net.Server | null): Promise<void> {
|
|
312
|
+
const closed = new Promise<void>((resolve) => {
|
|
313
|
+
if (server === null || !server.listening) {
|
|
314
|
+
resolve();
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
try {
|
|
318
|
+
server.close(() => resolve());
|
|
319
|
+
} catch {
|
|
320
|
+
resolve();
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
for (const socket of this.sockets) socket.destroy();
|
|
324
|
+
await closed;
|
|
325
|
+
this.sockets.clear();
|
|
326
|
+
}
|
|
171
327
|
|
|
172
|
-
private
|
|
173
|
-
if (
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
}
|
|
189
|
-
if (claimed === undefined) {
|
|
190
|
-
return;
|
|
191
|
-
}
|
|
192
|
-
const task = this.runClaimed(claimed).finally(() => {
|
|
193
|
-
this.activeRuns.delete(claimed.runId);
|
|
328
|
+
private recoverPreviousHost(previousHostId: string | null, hostEpoch: number): void {
|
|
329
|
+
if (previousHostId === null || previousHostId === this.hostId) return;
|
|
330
|
+
const now = Date.now();
|
|
331
|
+
this.state.transaction(() => {
|
|
332
|
+
this.state.connection
|
|
333
|
+
.prepare(
|
|
334
|
+
`UPDATE leases SET expires_at = ?
|
|
335
|
+
WHERE owner_id = ? AND owner_type IN ('host', 'controller') AND expires_at > ?`,
|
|
336
|
+
)
|
|
337
|
+
.run(now, previousHostId, now);
|
|
338
|
+
this.state.connection
|
|
339
|
+
.prepare(
|
|
340
|
+
`UPDATE run_workers SET status = 'orphaned', finished_at = ?
|
|
341
|
+
WHERE host_epoch < ? AND status IN ('starting', 'ready', 'running')`,
|
|
342
|
+
)
|
|
343
|
+
.run(now, hostEpoch);
|
|
194
344
|
});
|
|
195
|
-
this.activeRuns.set(claimed.runId, task);
|
|
196
345
|
}
|
|
197
346
|
|
|
198
|
-
private
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
347
|
+
private startTimers(): void {
|
|
348
|
+
this.heartbeatTimer = setInterval(() => {
|
|
349
|
+
try {
|
|
350
|
+
if (this.claim === null) return;
|
|
351
|
+
this.claim = this.hostState.renewHost(
|
|
352
|
+
this.claim,
|
|
353
|
+
this.options.hostLeaseMs ?? HOST_LEASE_MS,
|
|
354
|
+
);
|
|
355
|
+
for (const active of this.activeRuns.values()) {
|
|
356
|
+
if (active.control !== undefined || active.exiting) continue;
|
|
357
|
+
if (
|
|
358
|
+
!this.queue.renewWorkflowRunClaim({
|
|
359
|
+
runId: active.record.runId,
|
|
360
|
+
claimToken: active.claimToken,
|
|
361
|
+
leaseMs: this.runClaimLeaseMs,
|
|
362
|
+
})
|
|
363
|
+
) {
|
|
364
|
+
active.claimLost = true;
|
|
365
|
+
void active.supervisor.stop("claimLost");
|
|
366
|
+
}
|
|
211
367
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
throw new Error(`Workflow run ${runId} has no canonical workflow source`);
|
|
368
|
+
} catch (error) {
|
|
369
|
+
this.log(`host claim lost: ${errorMessage(error)}`);
|
|
370
|
+
void this.stop();
|
|
216
371
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
);
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
372
|
+
}, this.options.hostRenewMs ?? HOST_RENEW_MS);
|
|
373
|
+
this.heartbeatTimer.unref?.();
|
|
374
|
+
this.pollTimer = setInterval(() => {
|
|
375
|
+
this.expireTimedOutInteraction();
|
|
376
|
+
void this.expireTimedOutDecision();
|
|
377
|
+
void this.claimOne();
|
|
378
|
+
void this.claimControllerOne();
|
|
379
|
+
}, this.options.claimPollMs ?? CLAIM_POLL_MS);
|
|
380
|
+
this.pollTimer.unref?.();
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
private async listen(): Promise<void> {
|
|
384
|
+
if (process.platform !== "win32") fs.rmSync(this.socketPath, { force: true });
|
|
385
|
+
const server = net.createServer((socket) => this.handleConnection(socket));
|
|
386
|
+
this.server = server;
|
|
387
|
+
await new Promise<void>((resolve, reject) => {
|
|
388
|
+
const cleanup = () => {
|
|
389
|
+
server.off("listening", onListening);
|
|
390
|
+
server.off("error", onError);
|
|
391
|
+
};
|
|
392
|
+
const onListening = () => {
|
|
393
|
+
cleanup();
|
|
394
|
+
resolve();
|
|
395
|
+
};
|
|
396
|
+
const onError = (error: Error) => {
|
|
397
|
+
cleanup();
|
|
398
|
+
reject(error);
|
|
399
|
+
};
|
|
400
|
+
server.once("listening", onListening);
|
|
401
|
+
server.once("error", onError);
|
|
402
|
+
try {
|
|
403
|
+
server.listen(this.socketPath);
|
|
404
|
+
} catch (error) {
|
|
405
|
+
cleanup();
|
|
406
|
+
reject(error);
|
|
238
407
|
}
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
408
|
+
});
|
|
409
|
+
server.on("error", (error) => this.log(`socket error: ${errorMessage(error)}`));
|
|
410
|
+
if (process.platform !== "win32") fs.chmodSync(this.socketPath, 0o600);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
private handleConnection(socket: Socket): void {
|
|
414
|
+
this.sockets.add(socket);
|
|
415
|
+
const decoder = new NdjsonFrameDecoder();
|
|
416
|
+
let processing = Promise.resolve();
|
|
417
|
+
socket.on("data", (chunk: Buffer) => {
|
|
418
|
+
let frames: Buffer[];
|
|
244
419
|
try {
|
|
245
|
-
|
|
420
|
+
frames = decoder.push(chunk);
|
|
246
421
|
} catch {
|
|
247
|
-
|
|
422
|
+
socket.destroy();
|
|
423
|
+
return;
|
|
248
424
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
425
|
+
for (const frame of frames) {
|
|
426
|
+
processing = processing
|
|
427
|
+
.then(async () => {
|
|
428
|
+
const request = parseHostRequest(frame);
|
|
429
|
+
const response = this.handleRequest(request);
|
|
430
|
+
if (!socket.write(encodeProtocolLine(response))) await once(socket, "drain");
|
|
431
|
+
})
|
|
432
|
+
.catch(() => {
|
|
433
|
+
socket.destroy();
|
|
434
|
+
});
|
|
256
435
|
}
|
|
257
|
-
};
|
|
258
|
-
const fencedStore = new WorkflowRunStore(this.childRunStore.databasePath, {
|
|
259
|
-
authorityProvider: () => store.workflowRunAuthority(runId, claimToken),
|
|
260
436
|
});
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
...(record.originSessionId !== null ? { assistantMessageMode: "park" as const } : {}),
|
|
265
|
-
...(this.options.piArgs !== undefined ? { piArgs: this.options.piArgs } : {}),
|
|
266
|
-
...(this.options.env !== undefined ? { env: this.options.env } : {}),
|
|
267
|
-
});
|
|
268
|
-
const engine = new WorkflowEngine({
|
|
269
|
-
executor,
|
|
270
|
-
store: fencedStore,
|
|
271
|
-
notificationSink: {
|
|
272
|
-
notify: (request) => {
|
|
273
|
-
fence();
|
|
274
|
-
if (record.originSessionId === null) {
|
|
275
|
-
throw new Error(`Workflow run ${request.runId} has no origin session`);
|
|
276
|
-
}
|
|
277
|
-
const notification = store.enqueueWorkflowNotification({
|
|
278
|
-
...request,
|
|
279
|
-
targetSessionId: record.originSessionId,
|
|
280
|
-
});
|
|
281
|
-
return {
|
|
282
|
-
notificationId: notification.notificationId,
|
|
283
|
-
targetSessionId: notification.targetSessionId,
|
|
284
|
-
};
|
|
285
|
-
},
|
|
286
|
-
},
|
|
437
|
+
socket.on("error", (error) => {
|
|
438
|
+
this.log(`client socket error: ${errorMessage(error)}`);
|
|
439
|
+
socket.destroy();
|
|
287
440
|
});
|
|
288
|
-
|
|
289
|
-
|
|
441
|
+
socket.on("close", () => {
|
|
442
|
+
this.sockets.delete(socket);
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
private handleRequest(request: HostRequest): HostResponse {
|
|
447
|
+
if (this.claim === null || this.stopping) {
|
|
448
|
+
return {
|
|
449
|
+
schema: "pi-workflows.host-response.v1",
|
|
450
|
+
requestId: request.requestId,
|
|
451
|
+
outcome: "unavailable",
|
|
452
|
+
error: "Workflow host is stopping",
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
const afterCommit: Array<() => void> = [];
|
|
456
|
+
let response: HostResponse;
|
|
457
|
+
try {
|
|
458
|
+
response = this.hostState.executeCommand(request, this.claim.epoch, () =>
|
|
459
|
+
this.executeOperation(request, afterCommit),
|
|
460
|
+
);
|
|
461
|
+
} catch (error) {
|
|
462
|
+
response = {
|
|
463
|
+
schema: "pi-workflows.host-response.v1",
|
|
464
|
+
requestId: request.requestId,
|
|
465
|
+
outcome: "rejected",
|
|
466
|
+
error: errorMessage(error),
|
|
467
|
+
};
|
|
290
468
|
}
|
|
291
|
-
const
|
|
292
|
-
|
|
469
|
+
for (const effect of afterCommit) setImmediate(effect);
|
|
470
|
+
return response;
|
|
471
|
+
}
|
|
293
472
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
473
|
+
private executeOperation(
|
|
474
|
+
request: HostRequest,
|
|
475
|
+
afterCommit: Array<() => void>,
|
|
476
|
+
): Omit<HostResponse, "schema" | "requestId"> {
|
|
477
|
+
switch (request.operation) {
|
|
478
|
+
case "host.status":
|
|
479
|
+
return { outcome: "accepted", receipt: this.statusReceipt() };
|
|
480
|
+
case "host.stop":
|
|
481
|
+
afterCommit.push(() => void this.stop());
|
|
482
|
+
return { outcome: "accepted", receipt: { stopping: true } };
|
|
483
|
+
case "run.list":
|
|
484
|
+
return {
|
|
485
|
+
outcome: "accepted",
|
|
486
|
+
receipt: this.queue.listWorkflowRuns({
|
|
487
|
+
limit: payloadLimit(request.payload),
|
|
488
|
+
}) as JsonValue,
|
|
489
|
+
};
|
|
490
|
+
case "run.status": {
|
|
491
|
+
const runId = requireRunId(request);
|
|
492
|
+
const run = this.queue.getWorkflowRun(runId);
|
|
493
|
+
return run === undefined
|
|
494
|
+
? { outcome: "notFound", error: `Workflow run not found: ${runId}` }
|
|
495
|
+
: { outcome: "accepted", receipt: run as unknown as JsonValue };
|
|
301
496
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
status: result.state.status,
|
|
331
|
-
...(waitingForOrigin ? { reason: "waiting for origin Pi session" } : {}),
|
|
332
|
-
});
|
|
333
|
-
this.log(
|
|
334
|
-
`parked ${record.workflowName} run ${runId}${waitingForOrigin ? ": waiting for origin Pi session" : ""}`,
|
|
335
|
-
);
|
|
336
|
-
return;
|
|
497
|
+
case "run.cancel": {
|
|
498
|
+
const runId = requireRunId(request);
|
|
499
|
+
const active = this.activeRuns.get(runId);
|
|
500
|
+
if (active !== undefined) {
|
|
501
|
+
if (
|
|
502
|
+
!this.queue.cancelWorkflowRun({
|
|
503
|
+
runId: active.record.runId,
|
|
504
|
+
claimToken: active.claimToken,
|
|
505
|
+
})
|
|
506
|
+
) {
|
|
507
|
+
return { outcome: "claimLost", error: "Run claim was lost before cancellation" };
|
|
508
|
+
}
|
|
509
|
+
active.control = "cancel";
|
|
510
|
+
afterCommit.push(() => void active.supervisor.stop("cancelled"));
|
|
511
|
+
return { outcome: "accepted", receipt: { runId, status: "cancelled" } };
|
|
512
|
+
}
|
|
513
|
+
const pendingClaim = this.pendingRunClaims.get(runId);
|
|
514
|
+
if (pendingClaim !== undefined) {
|
|
515
|
+
if (!this.queue.cancelWorkflowRun({ runId, claimToken: pendingClaim })) {
|
|
516
|
+
return { outcome: "claimLost", error: "Run claim was lost before cancellation" };
|
|
517
|
+
}
|
|
518
|
+
this.clearPendingStart(runId, pendingClaim);
|
|
519
|
+
return { outcome: "accepted", receipt: { runId, status: "cancelled" } };
|
|
520
|
+
}
|
|
521
|
+
const cancelled = this.queue.cancelWorkflowRun({ runId });
|
|
522
|
+
return cancelled
|
|
523
|
+
? { outcome: "accepted", receipt: { runId, status: "cancelled" } }
|
|
524
|
+
: { outcome: "rejected", error: "Run has a live owner or is already terminal" };
|
|
337
525
|
}
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
clearInterval(renewTimer);
|
|
347
|
-
if (isClaimLostError(error)) {
|
|
348
|
-
this.log(`run ${runId} continues under another runner`);
|
|
349
|
-
return;
|
|
526
|
+
case "run.pause": {
|
|
527
|
+
const runId = requireRunId(request);
|
|
528
|
+
const active = this.activeRuns.get(runId);
|
|
529
|
+
if (active === undefined) return { outcome: "rejected", error: "Run is not active" };
|
|
530
|
+
this.commitActivePause(active);
|
|
531
|
+
active.control = "pause";
|
|
532
|
+
afterCommit.push(() => void active.supervisor.stop("cancelled"));
|
|
533
|
+
return { outcome: "accepted", receipt: { runId, status: "parked", paused: true } };
|
|
350
534
|
}
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
this.store.parkWorkflowRun({ runId, claimToken });
|
|
356
|
-
} catch {
|
|
357
|
-
// Best-effort.
|
|
535
|
+
case "run.resume": {
|
|
536
|
+
const runId = requireRunId(request);
|
|
537
|
+
if (this.activeRuns.has(runId) || this.pendingStarts.has(runId)) {
|
|
538
|
+
return { outcome: "adopted", receipt: { runId, active: true } };
|
|
358
539
|
}
|
|
359
|
-
|
|
360
|
-
this.
|
|
361
|
-
|
|
540
|
+
const token = randomUUID();
|
|
541
|
+
const claimed = this.queue.claimWorkflowRun({
|
|
542
|
+
runId,
|
|
543
|
+
runnerId: this.hostId,
|
|
544
|
+
claimToken: token,
|
|
545
|
+
leaseMs: this.runClaimLeaseMs,
|
|
362
546
|
});
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
);
|
|
366
|
-
return;
|
|
547
|
+
if (claimed === undefined) return { outcome: "rejected", error: "Run is not resumable" };
|
|
548
|
+
this.markPendingStart(runId, token);
|
|
549
|
+
afterCommit.push(() => void this.activateRun(claimed, token));
|
|
550
|
+
return { outcome: "accepted", receipt: { runId, generation: claimed.claimGeneration } };
|
|
367
551
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
552
|
+
case "run.start":
|
|
553
|
+
return this.startRun(request, afterCommit);
|
|
554
|
+
case "checkpoint.answer":
|
|
555
|
+
return this.answerCheckpoint(request, afterCommit);
|
|
556
|
+
case "notification.claim":
|
|
557
|
+
return this.claimNotification(request);
|
|
558
|
+
case "notification.deliver":
|
|
559
|
+
return this.deliverNotification(request);
|
|
560
|
+
case "turn.claim":
|
|
561
|
+
return this.claimTurn(request);
|
|
562
|
+
case "turn.resolve":
|
|
563
|
+
return this.resolveTurn(request);
|
|
564
|
+
case "controller.list":
|
|
565
|
+
case "controller.get":
|
|
566
|
+
case "controller.apply":
|
|
567
|
+
case "controller.reconcile":
|
|
568
|
+
case "controller.delete":
|
|
569
|
+
return this.executeControllerOperation(request);
|
|
570
|
+
case "interaction.update": {
|
|
571
|
+
const payload = requireRecord(request.payload, "interaction update payload");
|
|
572
|
+
if (payload.claimPresentation === true) {
|
|
573
|
+
const interaction = this.hostState.claimInteractionPresentation({
|
|
574
|
+
requestId: requireString(payload.requestId, "requestId"),
|
|
575
|
+
expectedRevision: requireNonNegativeInteger(
|
|
576
|
+
request.expectedRevision,
|
|
577
|
+
"expectedRevision",
|
|
578
|
+
),
|
|
579
|
+
presenterId: request.clientId,
|
|
580
|
+
leaseMs: PRESENTATION_CLAIM_LEASE_MS,
|
|
581
|
+
});
|
|
582
|
+
return {
|
|
583
|
+
outcome: "accepted",
|
|
584
|
+
revision: interaction.revision,
|
|
585
|
+
receipt: interaction as unknown as JsonValue,
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
if (typeof payload.sessionEntryId === "string") {
|
|
589
|
+
const interaction = this.hostState.markInteractionPresented({
|
|
590
|
+
requestId: requireString(payload.requestId, "requestId"),
|
|
591
|
+
expectedRevision: requireNonNegativeInteger(
|
|
592
|
+
request.expectedRevision,
|
|
593
|
+
"expectedRevision",
|
|
594
|
+
),
|
|
595
|
+
sessionEntryId: payload.sessionEntryId,
|
|
596
|
+
});
|
|
597
|
+
return {
|
|
598
|
+
outcome: "accepted",
|
|
599
|
+
revision: interaction.revision,
|
|
600
|
+
receipt: interaction as unknown as JsonValue,
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
return this.publishInteractionUpdate(request);
|
|
373
604
|
}
|
|
374
|
-
|
|
375
|
-
|
|
605
|
+
case "interaction.submit":
|
|
606
|
+
return this.submitInteraction(request);
|
|
607
|
+
case "decision.answer":
|
|
608
|
+
return this.answerDecision(request, afterCommit);
|
|
376
609
|
}
|
|
377
610
|
}
|
|
378
611
|
|
|
379
|
-
private
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
612
|
+
private executeControllerOperation(
|
|
613
|
+
request: HostRequest,
|
|
614
|
+
): Omit<HostResponse, "schema" | "requestId"> {
|
|
615
|
+
const payload = requireRecord(request.payload, "controller payload");
|
|
616
|
+
const projectPath = path.resolve(requireString(payload.projectPath, "projectPath"));
|
|
617
|
+
if (payload.projectPath !== projectPath) {
|
|
618
|
+
return {
|
|
619
|
+
outcome: "rejected",
|
|
620
|
+
error: "Controller projectPath must be absolute and normalized",
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
const store = new SqliteControllerStore(this.databasePath, {
|
|
624
|
+
state: this.state,
|
|
625
|
+
projectPath,
|
|
626
|
+
});
|
|
627
|
+
if (request.operation === "controller.list") {
|
|
628
|
+
return { outcome: "accepted", receipt: store.listResources() as unknown as JsonValue };
|
|
629
|
+
}
|
|
630
|
+
const controller = requireString(payload.controller, "controller");
|
|
631
|
+
const key = requireString(payload.key, "key");
|
|
632
|
+
const ref = { controller, key };
|
|
633
|
+
if (request.operation === "controller.get") {
|
|
634
|
+
const resource = store.getResource(ref);
|
|
635
|
+
return resource === undefined
|
|
636
|
+
? { outcome: "notFound", error: `Controller resource not found: ${controller}/${key}` }
|
|
637
|
+
: { outcome: "accepted", receipt: resource as unknown as JsonValue };
|
|
638
|
+
}
|
|
639
|
+
if (request.operation === "controller.apply") {
|
|
640
|
+
if (!Object.hasOwn(payload, "spec") || !Object.hasOwn(payload, "initialStatus")) {
|
|
641
|
+
return { outcome: "rejected", error: "Controller apply requires spec and initialStatus" };
|
|
397
642
|
}
|
|
398
|
-
|
|
399
|
-
|
|
643
|
+
const spec = payload.spec as JsonValue;
|
|
644
|
+
const initialStatus = payload.initialStatus as JsonValue;
|
|
645
|
+
const controllerPath = path.resolve(requireString(payload.controllerPath, "controllerPath"));
|
|
646
|
+
const sourceHash = requireString(payload.sourceHash, "sourceHash");
|
|
647
|
+
if (!controllerPathAllowed(projectPath, controller, controllerPath)) {
|
|
648
|
+
return { outcome: "rejected", error: "Controller source does not match discovery rules" };
|
|
649
|
+
}
|
|
650
|
+
const observedHash = createHash("sha256")
|
|
651
|
+
.update(fs.readFileSync(controllerPath))
|
|
652
|
+
.digest("hex");
|
|
653
|
+
if (observedHash !== sourceHash) {
|
|
654
|
+
return { outcome: "conflict", error: "Controller source changed before apply committed" };
|
|
655
|
+
}
|
|
656
|
+
const resource = store.putResource({ controller, key, spec, initialStatus });
|
|
657
|
+
return { outcome: "accepted", receipt: resource as unknown as JsonValue };
|
|
400
658
|
}
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
} catch {
|
|
405
|
-
// Best-effort.
|
|
659
|
+
const existing = store.getResource(ref);
|
|
660
|
+
if (existing === undefined) {
|
|
661
|
+
return { outcome: "notFound", error: `Controller resource not found: ${controller}/${key}` };
|
|
406
662
|
}
|
|
407
|
-
if (
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
return;
|
|
663
|
+
if (request.operation === "controller.reconcile") {
|
|
664
|
+
store.enqueue(ref);
|
|
665
|
+
return { outcome: "accepted", receipt: existing as unknown as JsonValue };
|
|
411
666
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
667
|
+
const deleting = store.requestDeletion(ref);
|
|
668
|
+
store.enqueue(ref);
|
|
669
|
+
return { outcome: "accepted", receipt: deleting as unknown as JsonValue };
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
private statusReceipt(): JsonValue {
|
|
673
|
+
const host = this.hostState.hostStatus();
|
|
674
|
+
const count = (sql: string, ...params: unknown[]): number => {
|
|
675
|
+
const row = this.state.connection.prepare(sql).get(...params) as
|
|
676
|
+
| { count?: unknown }
|
|
677
|
+
| undefined;
|
|
678
|
+
return typeof row?.count === "number" ? row.count : 0;
|
|
679
|
+
};
|
|
680
|
+
const now = Date.now();
|
|
681
|
+
return {
|
|
682
|
+
state: host.live ? "running" : "stale",
|
|
683
|
+
epoch: host.epoch,
|
|
684
|
+
socketAvailable: this.server?.listening === true,
|
|
685
|
+
startedAt: host.startedAt,
|
|
686
|
+
heartbeatAt: host.heartbeatAt,
|
|
687
|
+
expiresAt: host.expiresAt,
|
|
688
|
+
activeWorkers: this.activeRuns.size,
|
|
689
|
+
connectedClients: this.sockets.size,
|
|
690
|
+
queuedRuns: count("SELECT COUNT(*) AS count FROM run_queue WHERE status = 'queued'"),
|
|
691
|
+
runningRuns: count(
|
|
692
|
+
"SELECT COUNT(*) AS count FROM run_queue WHERE status IN ('starting', 'running')",
|
|
693
|
+
),
|
|
694
|
+
parkedRuns: count("SELECT COUNT(*) AS count FROM run_queue WHERE status = 'parked'"),
|
|
695
|
+
waitingRuns: count("SELECT COUNT(*) AS count FROM runs WHERE status = 'waiting'"),
|
|
696
|
+
expiredClaims: count(
|
|
697
|
+
`SELECT COUNT(*) AS count FROM run_queue q
|
|
698
|
+
JOIN runs r ON r.run_id = q.run_id
|
|
699
|
+
JOIN leases l ON l.resource_id = r.resource_id
|
|
700
|
+
WHERE q.status NOT IN ('done', 'failed', 'cancelled')
|
|
701
|
+
AND l.owner_id IS NOT NULL AND l.expires_at <= ?`,
|
|
702
|
+
now,
|
|
703
|
+
),
|
|
704
|
+
pendingInteractions: count(
|
|
705
|
+
"SELECT COUNT(*) AS count FROM interactive_requests WHERE status IN ('pending', 'presenting')",
|
|
706
|
+
),
|
|
707
|
+
ambiguousEffects: count("SELECT COUNT(*) AS count FROM effects WHERE status = 'ambiguous'"),
|
|
708
|
+
pendingControllers: count("SELECT COUNT(*) AS count FROM controller_queue"),
|
|
709
|
+
activeControllerWorkers: this.activeControllers.size,
|
|
710
|
+
lifecycleContradictions: count(
|
|
711
|
+
`SELECT COUNT(*) AS count
|
|
712
|
+
FROM runs r JOIN run_queue q ON q.run_id = r.run_id
|
|
713
|
+
WHERE NOT (
|
|
714
|
+
(r.status = 'queued' AND q.status = 'queued')
|
|
715
|
+
OR (r.status = 'running' AND q.status IN ('starting', 'running', 'parked'))
|
|
716
|
+
OR (r.status = 'waiting' AND q.status = 'parked')
|
|
717
|
+
OR (r.status = 'completed' AND q.status = 'done')
|
|
718
|
+
OR (r.status IN ('failed', 'timed_out') AND q.status = 'failed')
|
|
719
|
+
OR (r.status = 'cancelled' AND q.status = 'cancelled')
|
|
720
|
+
)`,
|
|
721
|
+
),
|
|
722
|
+
};
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
private rememberDeliveryClaim(options: Omit<DeliveryClaim, "expiresAt">): string {
|
|
726
|
+
const now = Date.now();
|
|
727
|
+
for (const [claimId, claim] of this.deliveryClaims) {
|
|
728
|
+
if (claim.expiresAt <= now) this.deliveryClaims.delete(claimId);
|
|
419
729
|
}
|
|
420
|
-
|
|
730
|
+
const claimId = randomUUID();
|
|
731
|
+
this.deliveryClaims.set(claimId, {
|
|
732
|
+
...options,
|
|
733
|
+
expiresAt: now + DELIVERY_CLAIM_LEASE_MS,
|
|
734
|
+
});
|
|
735
|
+
return claimId;
|
|
421
736
|
}
|
|
422
737
|
|
|
423
|
-
private
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
738
|
+
private deliveryClaim(
|
|
739
|
+
claimId: string,
|
|
740
|
+
clientId: string,
|
|
741
|
+
kind: DeliveryClaim["kind"],
|
|
742
|
+
resourceId: string,
|
|
743
|
+
targetSessionId: string,
|
|
744
|
+
): DeliveryClaim | undefined {
|
|
745
|
+
const claim = this.deliveryClaims.get(claimId);
|
|
746
|
+
if (
|
|
747
|
+
claim === undefined ||
|
|
748
|
+
claim.expiresAt <= Date.now() ||
|
|
749
|
+
claim.clientId !== clientId ||
|
|
750
|
+
claim.kind !== kind ||
|
|
751
|
+
claim.resourceId !== resourceId ||
|
|
752
|
+
claim.targetSessionId !== targetSessionId
|
|
753
|
+
) {
|
|
754
|
+
this.deliveryClaims.delete(claimId);
|
|
755
|
+
return undefined;
|
|
428
756
|
}
|
|
757
|
+
return claim;
|
|
429
758
|
}
|
|
430
|
-
}
|
|
431
759
|
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
760
|
+
private claimNotification(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
|
|
761
|
+
const payload = requireRecord(command.payload, "notification claim payload");
|
|
762
|
+
const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
|
|
763
|
+
const token = randomUUID();
|
|
764
|
+
const notification = this.queue.claimPendingWorkflowNotifications({
|
|
765
|
+
targetSessionId,
|
|
766
|
+
claimToken: token,
|
|
767
|
+
leaseMs: DELIVERY_CLAIM_LEASE_MS,
|
|
768
|
+
limit: 1,
|
|
769
|
+
})[0];
|
|
770
|
+
if (notification === undefined) {
|
|
771
|
+
return { outcome: "accepted", receipt: { notification: null } };
|
|
772
|
+
}
|
|
773
|
+
const claimId = this.rememberDeliveryClaim({
|
|
774
|
+
clientId: command.clientId,
|
|
775
|
+
token,
|
|
776
|
+
targetSessionId,
|
|
777
|
+
kind: "notification",
|
|
778
|
+
resourceId: notification.notificationId,
|
|
779
|
+
});
|
|
780
|
+
return {
|
|
781
|
+
outcome: "accepted",
|
|
782
|
+
receipt: { claimId, notification } as unknown as JsonValue,
|
|
783
|
+
};
|
|
435
784
|
}
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
785
|
+
|
|
786
|
+
private deliverNotification(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
|
|
787
|
+
const payload = requireRecord(command.payload, "notification delivery payload");
|
|
788
|
+
const notificationId = requireString(payload.notificationId, "notificationId");
|
|
789
|
+
const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
|
|
790
|
+
const claimId = requireString(payload.claimId, "claimId");
|
|
791
|
+
const claim = this.deliveryClaim(
|
|
792
|
+
claimId,
|
|
793
|
+
command.clientId,
|
|
794
|
+
"notification",
|
|
795
|
+
notificationId,
|
|
796
|
+
targetSessionId,
|
|
797
|
+
);
|
|
798
|
+
if (claim === undefined) {
|
|
799
|
+
return { outcome: "conflict", error: "Notification delivery claim is stale" };
|
|
800
|
+
}
|
|
801
|
+
const delivered = this.queue.markWorkflowNotificationDelivered({
|
|
802
|
+
notificationId,
|
|
803
|
+
targetSessionId,
|
|
804
|
+
claimToken: claim.token,
|
|
805
|
+
});
|
|
806
|
+
this.deliveryClaims.delete(claimId);
|
|
807
|
+
return delivered
|
|
808
|
+
? { outcome: "accepted", receipt: { notificationId, delivered: true } }
|
|
809
|
+
: { outcome: "conflict", error: "Notification delivery claim is stale" };
|
|
445
810
|
}
|
|
446
|
-
return decision as ResolvedHumanDecision;
|
|
447
|
-
}
|
|
448
811
|
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
812
|
+
private claimTurn(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
|
|
813
|
+
const payload = requireRecord(command.payload, "turn claim payload");
|
|
814
|
+
const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
|
|
815
|
+
const token = randomUUID();
|
|
816
|
+
const intent = this.queue.claimEligibleWorkflowTurnIntents({
|
|
817
|
+
targetSessionId,
|
|
818
|
+
claimToken: token,
|
|
819
|
+
leaseMs: DELIVERY_CLAIM_LEASE_MS,
|
|
820
|
+
limit: 1,
|
|
821
|
+
})[0];
|
|
822
|
+
if (intent === undefined) {
|
|
823
|
+
return { outcome: "accepted", receipt: { turn: null } };
|
|
824
|
+
}
|
|
825
|
+
const claimId = this.rememberDeliveryClaim({
|
|
826
|
+
clientId: command.clientId,
|
|
827
|
+
token,
|
|
828
|
+
targetSessionId,
|
|
829
|
+
kind: "turn",
|
|
830
|
+
resourceId: intent.intentId,
|
|
831
|
+
});
|
|
832
|
+
return {
|
|
833
|
+
outcome: "accepted",
|
|
834
|
+
receipt: { claimId, turn: intent } as unknown as JsonValue,
|
|
835
|
+
};
|
|
455
836
|
}
|
|
456
|
-
return createHash("sha256").update(canonical).digest("hex").slice(0, 24);
|
|
457
|
-
}
|
|
458
837
|
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
838
|
+
private resolveTurn(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
|
|
839
|
+
const payload = requireRecord(command.payload, "turn resolution payload");
|
|
840
|
+
const intentId = requireString(payload.intentId, "intentId");
|
|
841
|
+
const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
|
|
842
|
+
const claimId = requireString(payload.claimId, "claimId");
|
|
843
|
+
const claim = this.deliveryClaim(claimId, command.clientId, "turn", intentId, targetSessionId);
|
|
844
|
+
if (claim === undefined) {
|
|
845
|
+
return { outcome: "conflict", error: "Workflow turn delivery claim is stale" };
|
|
846
|
+
}
|
|
847
|
+
const resolved = this.queue.resolveWorkflowTurnIntent({
|
|
848
|
+
intentId,
|
|
849
|
+
targetSessionId,
|
|
850
|
+
claimToken: claim.token,
|
|
851
|
+
resolution: "presentation",
|
|
852
|
+
...(typeof payload.messageId === "string" ? { messageId: payload.messageId } : {}),
|
|
853
|
+
});
|
|
854
|
+
this.deliveryClaims.delete(claimId);
|
|
855
|
+
return resolved
|
|
856
|
+
? { outcome: "accepted", receipt: { intentId, resolved: true } }
|
|
857
|
+
: { outcome: "conflict", error: "Workflow turn delivery claim is stale" };
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
private answerCheckpoint(
|
|
861
|
+
command: HostRequest,
|
|
862
|
+
afterCommit: Array<() => void>,
|
|
863
|
+
): Omit<HostResponse, "schema" | "requestId"> {
|
|
864
|
+
const parentRunId = requireRunId(command);
|
|
865
|
+
const payload = requireRecord(command.payload, "checkpoint answer payload");
|
|
866
|
+
const continuationRunId = requireString(payload.continuationRunId, "continuationRunId");
|
|
867
|
+
const parent = this.queue.getWorkflowRun(parentRunId);
|
|
868
|
+
const bundle = this.runStore.readRun(parentRunId);
|
|
869
|
+
if (parent === undefined || bundle === null || bundle instanceof Promise) {
|
|
870
|
+
return { outcome: "notFound", error: `Checkpoint run not found: ${parentRunId}` };
|
|
871
|
+
}
|
|
872
|
+
const waitingOn = bundle.state.waitingOn;
|
|
873
|
+
if (bundle.state.status !== "waiting" || waitingOn === undefined) {
|
|
874
|
+
return { outcome: "rejected", error: "Workflow run is not waiting at a checkpoint" };
|
|
875
|
+
}
|
|
876
|
+
if (bundle.snapshot.nodes[waitingOn]?.humanDecision !== undefined) {
|
|
877
|
+
return { outcome: "rejected", error: "Protected human decisions require a human channel" };
|
|
878
|
+
}
|
|
879
|
+
const projectPath = this.queue.workflowRunProjectPath(parentRunId);
|
|
880
|
+
if (projectPath === undefined || parent.originSessionId === null) {
|
|
881
|
+
throw new Error("Checkpoint parent provenance is missing");
|
|
882
|
+
}
|
|
883
|
+
const prepared = this.prepareContinuation(
|
|
884
|
+
{
|
|
885
|
+
parent,
|
|
886
|
+
parentRunId,
|
|
887
|
+
continuationRunId,
|
|
888
|
+
projectPath,
|
|
889
|
+
definitionSnapshot: bundle.snapshot,
|
|
890
|
+
input: payload.input as JsonValue,
|
|
891
|
+
launchOptions: {},
|
|
892
|
+
originSessionId: parent.originSessionId,
|
|
893
|
+
},
|
|
894
|
+
afterCommit,
|
|
895
|
+
);
|
|
896
|
+
return {
|
|
897
|
+
outcome: prepared.state === "adopted" ? "adopted" : "accepted",
|
|
898
|
+
receipt: {
|
|
899
|
+
parentRunId,
|
|
900
|
+
runId: continuationRunId,
|
|
901
|
+
status: prepared.run.status,
|
|
902
|
+
} as JsonValue,
|
|
903
|
+
};
|
|
904
|
+
}
|
|
462
905
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
906
|
+
private answerDecision(
|
|
907
|
+
command: HostRequest,
|
|
908
|
+
afterCommit: Array<() => void>,
|
|
909
|
+
): Omit<HostResponse, "schema" | "requestId"> {
|
|
910
|
+
const payload = requireRecord(command.payload, "decision answer payload");
|
|
911
|
+
const requestId = requireString(payload.requestId, "requestId");
|
|
912
|
+
const interaction = this.hostState.getInteraction(requestId);
|
|
913
|
+
if (interaction === undefined || interaction.kind !== "decision") {
|
|
914
|
+
return { outcome: "notFound", error: `Decision request not found: ${requestId}` };
|
|
915
|
+
}
|
|
916
|
+
const request = interaction.contract as unknown as HumanDecisionRequest;
|
|
917
|
+
const response = payload.response as HumanDecisionResponse;
|
|
918
|
+
const accepted = this.decisions.acceptSync(request, {
|
|
919
|
+
...response,
|
|
920
|
+
decisionId: request.decisionId,
|
|
921
|
+
requestDigest: request.requestDigest,
|
|
922
|
+
source: {
|
|
923
|
+
channel: "pi",
|
|
924
|
+
actorId: interaction.targetSessionId,
|
|
925
|
+
eventId: command.idempotencyKey,
|
|
926
|
+
},
|
|
927
|
+
idempotencyKey: command.idempotencyKey,
|
|
928
|
+
});
|
|
929
|
+
if (accepted.status === "conflict") {
|
|
930
|
+
return { outcome: "conflict", error: "Another human decision answer already won" };
|
|
931
|
+
}
|
|
932
|
+
this.hostState.submitInteraction({
|
|
933
|
+
requestId,
|
|
934
|
+
submissionId: requireString(payload.submissionId, "submissionId"),
|
|
935
|
+
idempotencyKey: command.idempotencyKey,
|
|
936
|
+
expectedRevision: requireNonNegativeInteger(command.expectedRevision, "expectedRevision"),
|
|
937
|
+
payload: response as JsonValue,
|
|
938
|
+
accepted: true,
|
|
939
|
+
receipt: accepted.decision as unknown as JsonValue,
|
|
940
|
+
});
|
|
941
|
+
|
|
942
|
+
const continuationRunId = this.prepareDecisionContinuation(
|
|
943
|
+
interaction,
|
|
944
|
+
request,
|
|
945
|
+
accepted.decision,
|
|
946
|
+
afterCommit,
|
|
474
947
|
);
|
|
948
|
+
return {
|
|
949
|
+
outcome: accepted.status === "adopted" ? "adopted" : "accepted",
|
|
950
|
+
receipt: {
|
|
951
|
+
requestId,
|
|
952
|
+
parentRunId: interaction.runId,
|
|
953
|
+
runId: continuationRunId,
|
|
954
|
+
decision: accepted.decision,
|
|
955
|
+
} as JsonValue,
|
|
956
|
+
};
|
|
475
957
|
}
|
|
476
|
-
fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 });
|
|
477
|
-
fs.writeFileSync(
|
|
478
|
-
lockPath,
|
|
479
|
-
`${JSON.stringify({ pid: process.pid, runnerId, startedAt: new Date().toISOString() })}\n`,
|
|
480
|
-
{ encoding: "utf8", mode: 0o600 },
|
|
481
|
-
);
|
|
482
|
-
}
|
|
483
958
|
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
959
|
+
private prepareDecisionContinuation(
|
|
960
|
+
interaction: InteractiveRequestRecord,
|
|
961
|
+
request: HumanDecisionRequest,
|
|
962
|
+
decision: ResolvedHumanDecision,
|
|
963
|
+
afterCommit: Array<() => void>,
|
|
964
|
+
): string {
|
|
965
|
+
const parent = this.queue.getWorkflowRun(interaction.runId);
|
|
966
|
+
const bundle = this.runStore.readRun(interaction.runId);
|
|
967
|
+
if (parent === undefined || bundle === null || bundle instanceof Promise) {
|
|
968
|
+
throw new Error(`Decision parent run is unreadable: ${interaction.runId}`);
|
|
969
|
+
}
|
|
970
|
+
const continuationRunId = `continuation-${request.decisionId.replace(/^decision-/, "")}`;
|
|
971
|
+
const projectPath = this.queue.workflowRunProjectPath(interaction.runId);
|
|
972
|
+
if (projectPath === undefined) throw new Error("Decision parent project is missing");
|
|
973
|
+
this.prepareContinuation(
|
|
974
|
+
{
|
|
975
|
+
parent,
|
|
976
|
+
parentRunId: interaction.runId,
|
|
977
|
+
continuationRunId,
|
|
978
|
+
projectPath,
|
|
979
|
+
definitionSnapshot: bundle.snapshot,
|
|
980
|
+
input: {},
|
|
981
|
+
launchOptions: { humanDecision: decision },
|
|
982
|
+
originSessionId: interaction.targetSessionId,
|
|
983
|
+
},
|
|
984
|
+
afterCommit,
|
|
985
|
+
);
|
|
986
|
+
return continuationRunId;
|
|
489
987
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
988
|
+
|
|
989
|
+
private prepareContinuation(
|
|
990
|
+
options: {
|
|
991
|
+
parent: WorkflowRunQueueRecord;
|
|
992
|
+
parentRunId: string;
|
|
993
|
+
continuationRunId: string;
|
|
994
|
+
projectPath: string;
|
|
995
|
+
definitionSnapshot: WorkflowDefinitionSnapshot;
|
|
996
|
+
input: unknown;
|
|
997
|
+
launchOptions: JsonValue;
|
|
998
|
+
originSessionId: string;
|
|
999
|
+
},
|
|
1000
|
+
afterCommit: Array<() => void>,
|
|
1001
|
+
) {
|
|
1002
|
+
const token = randomUUID();
|
|
1003
|
+
const scoped = new SqliteControllerStore(this.databasePath, {
|
|
1004
|
+
state: this.state,
|
|
1005
|
+
projectPath: options.projectPath,
|
|
1006
|
+
});
|
|
1007
|
+
const prepared = scoped.prepareOrAdoptWorkflowRun({
|
|
1008
|
+
runId: options.continuationRunId,
|
|
1009
|
+
workflowName: options.parent.workflowName,
|
|
1010
|
+
workflowSourceRef: options.parent.workflowSourceRef,
|
|
1011
|
+
workflowSource: options.parent.workflowSource,
|
|
1012
|
+
definitionDigest: options.parent.definitionDigest,
|
|
1013
|
+
definitionSnapshot: options.definitionSnapshot,
|
|
1014
|
+
input: options.input,
|
|
1015
|
+
launchOptions: options.launchOptions,
|
|
1016
|
+
runnerId: this.hostId,
|
|
1017
|
+
claimToken: token,
|
|
1018
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1019
|
+
originSessionId: options.originSessionId,
|
|
1020
|
+
executionMode: options.parent.executionMode,
|
|
1021
|
+
parentRunId: options.parentRunId,
|
|
1022
|
+
});
|
|
1023
|
+
if (prepared.state === "claimed") {
|
|
1024
|
+
this.markPendingStart(options.continuationRunId, token);
|
|
1025
|
+
afterCommit.push(() => void this.activateRun(prepared.run, token));
|
|
1026
|
+
}
|
|
1027
|
+
return prepared;
|
|
494
1028
|
}
|
|
495
|
-
}
|
|
496
1029
|
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
1030
|
+
private startRun(
|
|
1031
|
+
request: HostRequest,
|
|
1032
|
+
afterCommit: Array<() => void>,
|
|
1033
|
+
): Omit<HostResponse, "schema" | "requestId"> {
|
|
1034
|
+
const runId = requireRunId(request);
|
|
1035
|
+
const payload = requireRecord(request.payload, "run.start payload");
|
|
1036
|
+
const projectPath = requireAbsolutePath(payload.projectPath, "projectPath");
|
|
1037
|
+
const workflowName = requireString(payload.workflowName, "workflowName");
|
|
1038
|
+
const workflowSourceRef = requireString(payload.workflowSourceRef, "workflowSourceRef");
|
|
1039
|
+
const definitionDigest = requireString(payload.definitionDigest, "definitionDigest");
|
|
1040
|
+
const originSessionId = requireString(payload.originSessionId, "originSessionId");
|
|
1041
|
+
const executionMode = payload.executionMode === "headless" ? "headless" : "interactive";
|
|
1042
|
+
const token = randomUUID();
|
|
1043
|
+
const scoped = new SqliteControllerStore(this.databasePath, {
|
|
1044
|
+
state: this.state,
|
|
1045
|
+
projectPath,
|
|
1046
|
+
});
|
|
1047
|
+
const prepared = scoped.prepareOrAdoptWorkflowRun({
|
|
1048
|
+
runId,
|
|
1049
|
+
workflowName,
|
|
1050
|
+
workflowSourceRef,
|
|
1051
|
+
workflowSource: payload.workflowSource,
|
|
1052
|
+
definitionDigest,
|
|
1053
|
+
definitionSnapshot: payload.definitionSnapshot,
|
|
1054
|
+
input: payload.input,
|
|
1055
|
+
launchOptions: payload.launchOptions ?? {},
|
|
1056
|
+
runnerId: this.hostId,
|
|
1057
|
+
claimToken: token,
|
|
1058
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1059
|
+
originSessionId,
|
|
1060
|
+
executionMode,
|
|
1061
|
+
...(typeof payload.parentRunId === "string" ? { parentRunId: payload.parentRunId } : {}),
|
|
1062
|
+
});
|
|
1063
|
+
if (prepared.state === "adopted") {
|
|
1064
|
+
return {
|
|
1065
|
+
outcome: "adopted",
|
|
1066
|
+
receipt: { runId, status: prepared.run.status } as JsonValue,
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
this.markPendingStart(runId, token);
|
|
1070
|
+
afterCommit.push(() => void this.activateRun(prepared.run, token));
|
|
1071
|
+
return {
|
|
1072
|
+
outcome: "accepted",
|
|
1073
|
+
revision: runRevision(this.state, runId),
|
|
1074
|
+
receipt: { runId, generation: prepared.run.claimGeneration } as JsonValue,
|
|
502
1075
|
};
|
|
503
|
-
|
|
504
|
-
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
private publishInteractionUpdate(
|
|
1079
|
+
request: HostRequest,
|
|
1080
|
+
): Omit<HostResponse, "schema" | "requestId"> {
|
|
1081
|
+
const runId = requireRunId(request);
|
|
1082
|
+
const payload = requireRecord(request.payload, "interaction update payload");
|
|
1083
|
+
const requestId = requireString(payload.requestId, "requestId");
|
|
1084
|
+
const interaction = this.hostState.getInteraction(requestId);
|
|
1085
|
+
if (interaction === undefined || interaction.runId !== runId) {
|
|
1086
|
+
return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
|
|
1087
|
+
}
|
|
1088
|
+
const attemptId = requireString(payload.attempt, "attempt");
|
|
1089
|
+
const nodeId = requireString(payload.step, "step");
|
|
1090
|
+
const storedContract = requireRecord(interaction.contract, "interactive contract");
|
|
1091
|
+
const stepContract = requireRecord(storedContract.contract, "workflow step contract");
|
|
1092
|
+
if (
|
|
1093
|
+
interaction.attemptId !== attemptId ||
|
|
1094
|
+
requireString(stepContract.nodeId, "contract.nodeId") !== nodeId ||
|
|
1095
|
+
!["pending", "presenting"].includes(interaction.status)
|
|
1096
|
+
) {
|
|
1097
|
+
return { outcome: "conflict", error: "Interactive request attempt is stale" };
|
|
1098
|
+
}
|
|
1099
|
+
if (
|
|
1100
|
+
interaction.revision !==
|
|
1101
|
+
requireNonNegativeInteger(request.expectedRevision, "expectedRevision")
|
|
1102
|
+
) {
|
|
1103
|
+
return { outcome: "conflict", error: "Interactive request revision is stale" };
|
|
1104
|
+
}
|
|
1105
|
+
const value = requireRecord(payload.value, "interaction update value");
|
|
1106
|
+
const update = validateWorkflowUpdate(value.update);
|
|
1107
|
+
const token = randomUUID();
|
|
1108
|
+
const claimed = this.queue.claimWorkflowRunForControl({
|
|
1109
|
+
runId,
|
|
1110
|
+
runnerId: this.hostId,
|
|
1111
|
+
claimToken: token,
|
|
1112
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1113
|
+
});
|
|
1114
|
+
if (claimed === undefined) {
|
|
1115
|
+
return { outcome: "claimLost", error: "Interactive run is not available for update" };
|
|
1116
|
+
}
|
|
1117
|
+
this.controlClaims.set(runId, token);
|
|
1118
|
+
try {
|
|
1119
|
+
this.runStore.synchronizeRevision(runId);
|
|
1120
|
+
const loaded = this.runStore.readRun(runId);
|
|
1121
|
+
if (loaded === null || loaded.state.status !== "waiting") {
|
|
1122
|
+
throw new Error("Interactive update does not match a waiting workflow run");
|
|
1123
|
+
}
|
|
1124
|
+
const result = this.runStore.publishUpdateSynchronous(
|
|
1125
|
+
runId,
|
|
1126
|
+
loaded.state,
|
|
1127
|
+
nodeId,
|
|
1128
|
+
attemptId,
|
|
1129
|
+
update,
|
|
1130
|
+
);
|
|
1131
|
+
if (!this.queue.parkWorkflowRun({ runId, claimToken: token })) {
|
|
1132
|
+
throw new Error("Interactive update could not release its control claim");
|
|
1133
|
+
}
|
|
1134
|
+
return {
|
|
1135
|
+
outcome: "accepted",
|
|
1136
|
+
revision: result.event.seq,
|
|
1137
|
+
receipt: result.record as unknown as JsonValue,
|
|
1138
|
+
};
|
|
1139
|
+
} finally {
|
|
1140
|
+
this.controlClaims.delete(runId);
|
|
505
1141
|
}
|
|
506
|
-
return { pid: parsed.pid, runnerId: parsed.runnerId };
|
|
507
|
-
} catch {
|
|
508
|
-
return null;
|
|
509
1142
|
}
|
|
510
|
-
}
|
|
511
1143
|
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
1144
|
+
private submitInteraction(request: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
|
|
1145
|
+
const payload = requireRecord(request.payload, "interaction payload");
|
|
1146
|
+
const requestId = requireString(payload.requestId, "requestId");
|
|
1147
|
+
const current = this.hostState.getInteraction(requestId);
|
|
1148
|
+
if (current === undefined || current.runId !== requireRunId(request)) {
|
|
1149
|
+
return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
|
|
1150
|
+
}
|
|
1151
|
+
const attemptId = requireString(payload.attempt, "attempt");
|
|
1152
|
+
const nodeId = requireString(payload.step, "step");
|
|
1153
|
+
const storedContract = requireRecord(current.contract, "interactive contract");
|
|
1154
|
+
const stepContract = requireRecord(storedContract.contract, "workflow step contract");
|
|
1155
|
+
if (
|
|
1156
|
+
current.attemptId !== attemptId ||
|
|
1157
|
+
requireString(stepContract.nodeId, "contract.nodeId") !== nodeId
|
|
1158
|
+
) {
|
|
1159
|
+
return { outcome: "conflict", error: "Interactive request attempt is stale" };
|
|
1160
|
+
}
|
|
1161
|
+
const submissionId = requireString(payload.submissionId, "submissionId");
|
|
1162
|
+
const submission = this.hostState.beginInteractionValidation({
|
|
1163
|
+
requestId,
|
|
1164
|
+
submissionId,
|
|
1165
|
+
idempotencyKey: request.idempotencyKey,
|
|
1166
|
+
expectedRevision: requireNonNegativeInteger(request.expectedRevision, "expectedRevision"),
|
|
1167
|
+
payload: (payload.value ?? null) as JsonValue,
|
|
1168
|
+
receipt: {
|
|
1169
|
+
operation: request.operation,
|
|
1170
|
+
requestId,
|
|
1171
|
+
submissionId,
|
|
1172
|
+
status: "validating",
|
|
1173
|
+
},
|
|
1174
|
+
});
|
|
1175
|
+
const interaction = submission.interaction;
|
|
1176
|
+
if (this.activeRuns.has(interaction.runId) || this.activationTasks.has(interaction.runId)) {
|
|
1177
|
+
this.pendingResumes.add(interaction.runId);
|
|
1178
|
+
} else {
|
|
1179
|
+
const token = randomUUID();
|
|
1180
|
+
const claimed = this.queue.claimWorkflowRunForInteractionValidation({
|
|
1181
|
+
runId: interaction.runId,
|
|
1182
|
+
runnerId: this.hostId,
|
|
1183
|
+
claimToken: token,
|
|
1184
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1185
|
+
});
|
|
1186
|
+
if (claimed !== undefined) {
|
|
1187
|
+
this.markPendingStart(interaction.runId, token);
|
|
1188
|
+
setImmediate(() => void this.activateRun(claimed, token));
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
return {
|
|
1192
|
+
outcome: submission.outcome,
|
|
1193
|
+
revision: interaction.revision,
|
|
1194
|
+
receipt: submission.receipt,
|
|
1195
|
+
};
|
|
518
1196
|
}
|
|
1197
|
+
|
|
1198
|
+
private async claimControllerOne(): Promise<void> {
|
|
1199
|
+
if (
|
|
1200
|
+
this.stopping ||
|
|
1201
|
+
this.claim === null ||
|
|
1202
|
+
this.controllerPollActive ||
|
|
1203
|
+
this.activeControllers.size >= MAX_CONTROLLER_WORKERS
|
|
1204
|
+
) {
|
|
1205
|
+
return;
|
|
1206
|
+
}
|
|
1207
|
+
this.controllerPollActive = true;
|
|
1208
|
+
try {
|
|
1209
|
+
const rows = this.state.connection
|
|
1210
|
+
.prepare(
|
|
1211
|
+
`SELECT DISTINCT p.canonical_path AS projectPath
|
|
1212
|
+
FROM controller_queue q
|
|
1213
|
+
JOIN controller_resources c ON c.controller_resource_id = q.controller_resource_id
|
|
1214
|
+
JOIN projects p ON p.project_id = c.project_id
|
|
1215
|
+
JOIN leases l ON l.resource_id = c.resource_id
|
|
1216
|
+
WHERE q.available_at <= ? AND (l.owner_id IS NULL OR l.expires_at <= ?)
|
|
1217
|
+
ORDER BY q.available_at, p.canonical_path`,
|
|
1218
|
+
)
|
|
1219
|
+
.all(Date.now(), Date.now()) as Array<{ projectPath?: unknown }>;
|
|
1220
|
+
for (const row of rows) {
|
|
1221
|
+
if (typeof row.projectPath !== "string") continue;
|
|
1222
|
+
const discovered = await discoverControllers({ cwd: row.projectPath });
|
|
1223
|
+
if (discovered.length === 0) continue;
|
|
1224
|
+
const byName = new Map(discovered.map((item) => [item.name, item.path]));
|
|
1225
|
+
const store = new SqliteControllerStore(this.databasePath, {
|
|
1226
|
+
state: this.state,
|
|
1227
|
+
projectPath: row.projectPath,
|
|
1228
|
+
});
|
|
1229
|
+
const claim = store.claimNext({
|
|
1230
|
+
controllers: [...byName.keys()],
|
|
1231
|
+
ownerId: this.hostId,
|
|
1232
|
+
leaseMs: CONTROLLER_CLAIM_LEASE_MS,
|
|
1233
|
+
exclude: [...this.activeControllers.values()]
|
|
1234
|
+
.filter((active) => active.projectPath === row.projectPath)
|
|
1235
|
+
.map((active) => ({
|
|
1236
|
+
controller: active.claim.controller,
|
|
1237
|
+
key: active.claim.key,
|
|
1238
|
+
})),
|
|
1239
|
+
});
|
|
1240
|
+
if (claim === undefined) continue;
|
|
1241
|
+
const definitionPath = byName.get(claim.controller);
|
|
1242
|
+
const resource = store.getResource({ controller: claim.controller, key: claim.key });
|
|
1243
|
+
if (definitionPath === undefined || resource === undefined) {
|
|
1244
|
+
store.requeueClaim(claim, {
|
|
1245
|
+
availableAt: new Date().toISOString(),
|
|
1246
|
+
error: "Controller source is unavailable",
|
|
1247
|
+
});
|
|
1248
|
+
continue;
|
|
1249
|
+
}
|
|
1250
|
+
void this.activateController(row.projectPath, definitionPath, store, claim, resource);
|
|
1251
|
+
break;
|
|
1252
|
+
}
|
|
1253
|
+
} catch (error) {
|
|
1254
|
+
this.log(`controller scheduling failed: ${errorMessage(error)}`);
|
|
1255
|
+
} finally {
|
|
1256
|
+
this.controllerPollActive = false;
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
private async activateController(
|
|
1261
|
+
projectPath: string,
|
|
1262
|
+
definitionPath: string,
|
|
1263
|
+
store: SqliteControllerStore,
|
|
1264
|
+
claim: ControllerQueueClaim,
|
|
1265
|
+
resource: ControllerResource,
|
|
1266
|
+
): Promise<void> {
|
|
1267
|
+
if (this.claim === null || this.stopping) {
|
|
1268
|
+
store.requeueClaim(claim, { availableAt: new Date().toISOString() });
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
const workerEpoch = randomUUID();
|
|
1272
|
+
const key = `${projectPath}\u0000${claim.controller}\u0000${claim.key}`;
|
|
1273
|
+
const reconcileId = randomUUID();
|
|
1274
|
+
const envelope: ControllerWorkerLaunchEnvelope = {
|
|
1275
|
+
schema: "pi-workflows.controller-worker-launch.v1",
|
|
1276
|
+
workerEpoch,
|
|
1277
|
+
hostEpoch: this.claim.epoch,
|
|
1278
|
+
generation: claim.generation,
|
|
1279
|
+
projectPath,
|
|
1280
|
+
definitionPath,
|
|
1281
|
+
controllerName: claim.controller,
|
|
1282
|
+
resource,
|
|
1283
|
+
timeoutMs: DEFAULT_CONTROLLER_TIMEOUT_MS,
|
|
1284
|
+
};
|
|
1285
|
+
let active: ActiveController;
|
|
1286
|
+
const supervisor = new ControllerWorkerSupervisor(envelope, {
|
|
1287
|
+
registry: this.registry,
|
|
1288
|
+
onMessage: async (message) => await this.handleControllerWorkerMessage(active, message),
|
|
1289
|
+
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
1290
|
+
...(this.options.controllerWorkerEntryPath === undefined
|
|
1291
|
+
? {}
|
|
1292
|
+
: { workerEntryPath: this.options.controllerWorkerEntryPath }),
|
|
1293
|
+
...(this.options.workerStartupTimeoutMs === undefined
|
|
1294
|
+
? {}
|
|
1295
|
+
: { startupTimeoutMs: this.options.workerStartupTimeoutMs }),
|
|
1296
|
+
onDiagnostic: (message) => this.log(`controller worker ${workerEpoch}: ${message}`),
|
|
1297
|
+
});
|
|
1298
|
+
const renewTimer = setInterval(() => {
|
|
1299
|
+
if (!store.renewClaim(claim, CONTROLLER_CLAIM_LEASE_MS)) {
|
|
1300
|
+
void supervisor.stop("claimLost");
|
|
1301
|
+
}
|
|
1302
|
+
}, CONTROLLER_RENEW_MS);
|
|
1303
|
+
renewTimer.unref?.();
|
|
1304
|
+
active = {
|
|
1305
|
+
key,
|
|
1306
|
+
projectPath,
|
|
1307
|
+
store,
|
|
1308
|
+
claim,
|
|
1309
|
+
resource,
|
|
1310
|
+
reconcileId,
|
|
1311
|
+
startedAt: Date.now(),
|
|
1312
|
+
supervisor,
|
|
1313
|
+
renewTimer,
|
|
1314
|
+
settled: false,
|
|
1315
|
+
};
|
|
1316
|
+
this.activeControllers.set(key, active);
|
|
1317
|
+
store.recordEvent({
|
|
1318
|
+
controller: claim.controller,
|
|
1319
|
+
key: claim.key,
|
|
1320
|
+
claim,
|
|
1321
|
+
type: "reconcile_started",
|
|
1322
|
+
payload: { reconcileId, generation: resource.metadata.generation },
|
|
1323
|
+
});
|
|
1324
|
+
try {
|
|
1325
|
+
await supervisor.start();
|
|
1326
|
+
const result = await supervisor.wait();
|
|
1327
|
+
if (result.outcome !== "exited" && !active.settled) {
|
|
1328
|
+
this.finishControllerFailure(
|
|
1329
|
+
active,
|
|
1330
|
+
result.diagnostic ?? `Controller worker ${result.outcome}`,
|
|
1331
|
+
);
|
|
1332
|
+
}
|
|
1333
|
+
} catch (error) {
|
|
1334
|
+
await supervisor.stop("crashed");
|
|
1335
|
+
if (!active.settled) this.finishControllerFailure(active, errorMessage(error));
|
|
1336
|
+
} finally {
|
|
1337
|
+
clearInterval(renewTimer);
|
|
1338
|
+
this.activeControllers.delete(key);
|
|
1339
|
+
if (!this.stopping) setImmediate(() => void this.claimControllerOne());
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
private async handleControllerWorkerMessage(
|
|
1344
|
+
active: ActiveController,
|
|
1345
|
+
message: ControllerWorkerMessage,
|
|
1346
|
+
): Promise<ControllerWorkerResponse> {
|
|
1347
|
+
const current = this.activeControllers.get(active.key);
|
|
1348
|
+
if (
|
|
1349
|
+
current !== active ||
|
|
1350
|
+
message.workerEpoch !== active.supervisor.envelope.workerEpoch ||
|
|
1351
|
+
message.generation !== active.claim.generation
|
|
1352
|
+
) {
|
|
1353
|
+
return controllerWorkerResponse(
|
|
1354
|
+
message,
|
|
1355
|
+
"claimLost",
|
|
1356
|
+
undefined,
|
|
1357
|
+
"Controller worker is stale",
|
|
1358
|
+
);
|
|
1359
|
+
}
|
|
1360
|
+
if (!active.store.renewClaim(active.claim, CONTROLLER_CLAIM_LEASE_MS)) {
|
|
1361
|
+
return controllerWorkerResponse(message, "claimLost", undefined, "Controller claim expired");
|
|
1362
|
+
}
|
|
1363
|
+
try {
|
|
1364
|
+
const payload = requireRecord(message.payload, "controller worker payload");
|
|
1365
|
+
switch (message.operation) {
|
|
1366
|
+
case "worker.ready":
|
|
1367
|
+
return controllerWorkerResponse(message, "accepted", {});
|
|
1368
|
+
case "effect.reserve": {
|
|
1369
|
+
const reservation = active.store.reserveEffect({
|
|
1370
|
+
key: requireString(payload.key, "key"),
|
|
1371
|
+
resourceUid: active.resource.metadata.uid,
|
|
1372
|
+
claim: active.claim,
|
|
1373
|
+
generation: active.resource.metadata.generation,
|
|
1374
|
+
kind: requireString(payload.kind, "kind"),
|
|
1375
|
+
requestFingerprint: requireString(payload.requestFingerprint, "requestFingerprint"),
|
|
1376
|
+
});
|
|
1377
|
+
return controllerWorkerResponse(message, "accepted", reservation as unknown as JsonValue);
|
|
1378
|
+
}
|
|
1379
|
+
case "effect.settle": {
|
|
1380
|
+
const state = requireString(payload.state, "state");
|
|
1381
|
+
if (!["applied", "rejected", "indeterminate"].includes(state)) {
|
|
1382
|
+
throw new Error("Controller effect state is invalid");
|
|
1383
|
+
}
|
|
1384
|
+
const record = active.store.updateEffect({
|
|
1385
|
+
resourceUid: active.resource.metadata.uid,
|
|
1386
|
+
key: requireString(payload.key, "key"),
|
|
1387
|
+
claim: active.claim,
|
|
1388
|
+
state: state as "applied" | "rejected" | "indeterminate",
|
|
1389
|
+
...(typeof payload.externalRef === "string"
|
|
1390
|
+
? { externalRef: payload.externalRef }
|
|
1391
|
+
: {}),
|
|
1392
|
+
...(typeof payload.error === "string" ? { error: payload.error } : {}),
|
|
1393
|
+
});
|
|
1394
|
+
active.store.recordEvent({
|
|
1395
|
+
controller: active.claim.controller,
|
|
1396
|
+
key: active.claim.key,
|
|
1397
|
+
claim: active.claim,
|
|
1398
|
+
type: `effect_${state}`,
|
|
1399
|
+
payload: { effectKey: record.key },
|
|
1400
|
+
});
|
|
1401
|
+
return controllerWorkerResponse(message, "accepted", record as unknown as JsonValue);
|
|
1402
|
+
}
|
|
1403
|
+
case "workflow.ensure":
|
|
1404
|
+
case "workflow.changeSettings":
|
|
1405
|
+
case "workflow.queueFollowUp":
|
|
1406
|
+
case "workflow.removeFollowUp": {
|
|
1407
|
+
const workflows = new ControllerWorkflowCoordinator(
|
|
1408
|
+
active.store,
|
|
1409
|
+
this.controllerWorkflowScheduler(active),
|
|
1410
|
+
).forResource(active.resource, active.claim, new AbortController().signal);
|
|
1411
|
+
const result =
|
|
1412
|
+
message.operation === "workflow.ensure"
|
|
1413
|
+
? await workflows.ensure(payload as never)
|
|
1414
|
+
: message.operation === "workflow.changeSettings"
|
|
1415
|
+
? await workflows.changeSettings(payload as never)
|
|
1416
|
+
: message.operation === "workflow.queueFollowUp"
|
|
1417
|
+
? await workflows.queueFollowUp(payload as never)
|
|
1418
|
+
: await workflows.removeFollowUp(payload as never);
|
|
1419
|
+
return controllerWorkerResponse(message, "accepted", result as unknown as JsonValue);
|
|
1420
|
+
}
|
|
1421
|
+
case "worker.finished":
|
|
1422
|
+
this.finishControllerSuccess(
|
|
1423
|
+
active,
|
|
1424
|
+
message.payload as unknown as ReconcileResult<unknown>,
|
|
1425
|
+
);
|
|
1426
|
+
return controllerWorkerResponse(message, "accepted", {});
|
|
1427
|
+
case "worker.failed":
|
|
1428
|
+
this.finishControllerFailure(active, requireString(payload.error, "error"));
|
|
1429
|
+
return controllerWorkerResponse(message, "accepted", {});
|
|
1430
|
+
}
|
|
1431
|
+
} catch (error) {
|
|
1432
|
+
return controllerWorkerResponse(message, "rejected", undefined, errorMessage(error));
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
private controllerWorkflowScheduler(active: ActiveController): ControllerWorkflowScheduler {
|
|
1437
|
+
return {
|
|
1438
|
+
ensure: async (request, _signal, _onComplete) =>
|
|
1439
|
+
await this.ensureControllerWorkflow(active, request),
|
|
1440
|
+
changeSettings: async (request) =>
|
|
1441
|
+
await this.changeControllerWorkflowSettings(active, request),
|
|
1442
|
+
queueFollowUp: async (request) => {
|
|
1443
|
+
const targetSessionId = this.runStore.originSessionId(request.runId);
|
|
1444
|
+
if (targetSessionId === undefined) {
|
|
1445
|
+
throw new Error("Controller follow-up requires an origin Pi session");
|
|
1446
|
+
}
|
|
1447
|
+
const result = this.runStore.queueFollowUp({
|
|
1448
|
+
runId: request.runId,
|
|
1449
|
+
requestId: request.actorRequestKey,
|
|
1450
|
+
targetSessionId,
|
|
1451
|
+
actor: { type: "controller", id: request.controllerResourceUid },
|
|
1452
|
+
source: "controller-request",
|
|
1453
|
+
prompt: request.prompt,
|
|
1454
|
+
});
|
|
1455
|
+
return {
|
|
1456
|
+
runId: request.runId,
|
|
1457
|
+
followUpId: result.followUp.followUpId,
|
|
1458
|
+
order: result.followUp.order,
|
|
1459
|
+
state: result.followUp.state,
|
|
1460
|
+
adopted: result.adopted,
|
|
1461
|
+
};
|
|
1462
|
+
},
|
|
1463
|
+
removeFollowUp: async (request) => {
|
|
1464
|
+
const result = this.runStore.removeFollowUp({
|
|
1465
|
+
runId: request.runId,
|
|
1466
|
+
followUpId: request.followUpId,
|
|
1467
|
+
actor: { type: "controller", id: request.controllerResourceUid },
|
|
1468
|
+
source: "controller-request",
|
|
1469
|
+
});
|
|
1470
|
+
return {
|
|
1471
|
+
runId: request.runId,
|
|
1472
|
+
followUpId: result.followUpId,
|
|
1473
|
+
order: result.order,
|
|
1474
|
+
state: result.state,
|
|
1475
|
+
adopted: false,
|
|
1476
|
+
};
|
|
1477
|
+
},
|
|
1478
|
+
};
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
private async changeControllerWorkflowSettings(
|
|
1482
|
+
_active: ActiveController,
|
|
1483
|
+
request: ControllerWorkflowControlRequest<ControllerSettingsChangeRequest>,
|
|
1484
|
+
): Promise<ControllerSettingsChangeResult> {
|
|
1485
|
+
const run = this.queue.getWorkflowRun(request.runId);
|
|
1486
|
+
if (run === undefined) throw new Error(`Workflow run not found: ${request.runId}`);
|
|
1487
|
+
const scopes = this.runStore.listSettingsScopes(request.runId);
|
|
1488
|
+
const scopeId = request.scopeId ?? (scopes.length === 1 ? scopes[0]?.scopeId : undefined);
|
|
1489
|
+
if (scopeId === undefined) {
|
|
1490
|
+
throw new Error("Controller settings changes require scopeId when several scopes exist");
|
|
1491
|
+
}
|
|
1492
|
+
const scope = this.runStore.getSettingsScope(scopeId);
|
|
1493
|
+
if (scope === undefined || scope.activeRunId !== request.runId) {
|
|
1494
|
+
throw new Error(`Workflow settings scope not found: ${scopeId}`);
|
|
1495
|
+
}
|
|
1496
|
+
const prior = this.state.connection
|
|
1497
|
+
.prepare(
|
|
1498
|
+
`SELECT change_number AS changeNumber
|
|
1499
|
+
FROM workflow_setting_changes WHERE scope_id = ? AND request_id = ?`,
|
|
1500
|
+
)
|
|
1501
|
+
.get(scopeId, request.actorRequestKey) as { changeNumber?: unknown } | undefined;
|
|
1502
|
+
if (typeof prior?.changeNumber === "number") {
|
|
1503
|
+
return { runId: request.runId, scopeId, changeNumber: prior.changeNumber, adopted: true };
|
|
1504
|
+
}
|
|
1505
|
+
const projectPath = this.queue.workflowRunProjectPath(request.runId);
|
|
1506
|
+
if (projectPath === undefined) throw new Error("Workflow run project is missing");
|
|
1507
|
+
const resolver = new WorkflowHostClient({
|
|
1508
|
+
databasePath: this.databasePath,
|
|
1509
|
+
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
1510
|
+
});
|
|
1511
|
+
const proposal = await resolver.resolveSettingsChange({
|
|
1512
|
+
cwd: projectPath,
|
|
1513
|
+
workflowRef: run.workflowSourceRef,
|
|
1514
|
+
definitionDigest: run.definitionDigest,
|
|
1515
|
+
mountPath: scope.mountPath,
|
|
1516
|
+
current: scope.settings,
|
|
1517
|
+
patch: request.patch as unknown as JsonValue,
|
|
1518
|
+
actorId: request.controllerResourceUid,
|
|
1519
|
+
});
|
|
1520
|
+
if (proposal.definitionDigest !== run.definitionDigest || !Array.isArray(proposal.paths)) {
|
|
1521
|
+
throw new Error("Workflow settings proposal does not match the durable source");
|
|
1522
|
+
}
|
|
1523
|
+
const proposedSettings = canonicalJson(proposal.settings);
|
|
1524
|
+
const definition: WorkflowSettingsDefinition<JsonValue> = {
|
|
1525
|
+
initial: proposal.settings,
|
|
1526
|
+
paths: proposal.paths as unknown as WorkflowSettingsPathRule[],
|
|
1527
|
+
parse: (value) => {
|
|
1528
|
+
if (canonicalJson(value) !== proposedSettings) {
|
|
1529
|
+
throw new Error("Workflow settings proposal changed before commit");
|
|
1530
|
+
}
|
|
1531
|
+
return proposal.settings;
|
|
1532
|
+
},
|
|
1533
|
+
};
|
|
1534
|
+
const result = await this.runStore.changeSettings(definition, {
|
|
1535
|
+
runId: request.runId,
|
|
1536
|
+
scopeId,
|
|
1537
|
+
requestId: request.actorRequestKey,
|
|
1538
|
+
expectedChangeNumber: request.expectedChangeNumber ?? scope.changeNumber,
|
|
1539
|
+
actor: { type: "controller", id: request.controllerResourceUid },
|
|
1540
|
+
source: "controller-request",
|
|
1541
|
+
patch: proposal.patch,
|
|
1542
|
+
});
|
|
1543
|
+
return {
|
|
1544
|
+
runId: request.runId,
|
|
1545
|
+
scopeId,
|
|
1546
|
+
changeNumber: result.scope.changeNumber,
|
|
1547
|
+
adopted: result.adopted,
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
private async ensureControllerWorkflow(
|
|
1552
|
+
active: ActiveController,
|
|
1553
|
+
request: WorkflowSchedulerRequest,
|
|
1554
|
+
): Promise<WorkflowSchedulerResult> {
|
|
1555
|
+
const existing = this.queue.getWorkflowRun(request.runId);
|
|
1556
|
+
if (existing !== undefined) return controllerWorkflowResult(existing);
|
|
1557
|
+
const resolver = new WorkflowHostClient({
|
|
1558
|
+
databasePath: this.databasePath,
|
|
1559
|
+
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
1560
|
+
});
|
|
1561
|
+
const resolved = await resolver.resolveWorkflow({
|
|
1562
|
+
cwd: active.projectPath,
|
|
1563
|
+
workflowRef: request.workflow,
|
|
1564
|
+
});
|
|
1565
|
+
const token = randomUUID();
|
|
1566
|
+
const scoped = new SqliteControllerStore(this.databasePath, {
|
|
1567
|
+
state: this.state,
|
|
1568
|
+
projectPath: active.projectPath,
|
|
1569
|
+
});
|
|
1570
|
+
const prepared = scoped.prepareOrAdoptWorkflowRun({
|
|
1571
|
+
runId: request.runId,
|
|
1572
|
+
workflowName: resolved.workflowName,
|
|
1573
|
+
workflowSourceRef: resolved.workflowSourceRef,
|
|
1574
|
+
workflowSource: resolved.workflowSource,
|
|
1575
|
+
definitionDigest: resolved.definitionDigest,
|
|
1576
|
+
definitionSnapshot: resolved.definitionSnapshot,
|
|
1577
|
+
input: request.input,
|
|
1578
|
+
launchOptions: {},
|
|
1579
|
+
runnerId: this.hostId,
|
|
1580
|
+
claimToken: token,
|
|
1581
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1582
|
+
originSessionId: `controller-${active.resource.metadata.uid}`,
|
|
1583
|
+
executionMode: "headless",
|
|
1584
|
+
});
|
|
1585
|
+
if (prepared.state === "claimed") {
|
|
1586
|
+
this.markPendingStart(request.runId, token);
|
|
1587
|
+
setImmediate(() => void this.activateRun(prepared.run, token));
|
|
1588
|
+
}
|
|
1589
|
+
return controllerWorkflowResult(prepared.run);
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
private finishControllerSuccess(
|
|
1593
|
+
active: ActiveController,
|
|
1594
|
+
result: ReconcileResult<unknown>,
|
|
1595
|
+
): void {
|
|
1596
|
+
if (active.settled) return;
|
|
1597
|
+
const now = new Date();
|
|
1598
|
+
const ref = { controller: active.claim.controller, key: active.claim.key };
|
|
1599
|
+
const status = applyStatusPatch(
|
|
1600
|
+
active.resource.status,
|
|
1601
|
+
result.status,
|
|
1602
|
+
active.resource.metadata.generation,
|
|
1603
|
+
now.toISOString(),
|
|
1604
|
+
);
|
|
1605
|
+
try {
|
|
1606
|
+
const updated = active.store.updateStatus({
|
|
1607
|
+
ref,
|
|
1608
|
+
expectedResourceVersion: active.claim.resourceVersion,
|
|
1609
|
+
claim: active.claim,
|
|
1610
|
+
status,
|
|
1611
|
+
...(result.status?.finalizers === undefined
|
|
1612
|
+
? {}
|
|
1613
|
+
: { finalizers: result.status.finalizers }),
|
|
1614
|
+
now: now.toISOString(),
|
|
1615
|
+
});
|
|
1616
|
+
active.store.recordEvent({
|
|
1617
|
+
...ref,
|
|
1618
|
+
claim: active.claim,
|
|
1619
|
+
type: "reconcile_finished",
|
|
1620
|
+
payload: {
|
|
1621
|
+
reconcileId: active.reconcileId,
|
|
1622
|
+
result: result.kind,
|
|
1623
|
+
generation: active.resource.metadata.generation,
|
|
1624
|
+
durationMs: Math.max(0, now.getTime() - active.startedAt),
|
|
1625
|
+
...(result.kind === "requeue" && result.afterMs !== undefined
|
|
1626
|
+
? { requeueAfterMs: result.afterMs }
|
|
1627
|
+
: {}),
|
|
1628
|
+
},
|
|
1629
|
+
});
|
|
1630
|
+
if (
|
|
1631
|
+
result.kind === "settled" &&
|
|
1632
|
+
updated.metadata.deletionTimestamp !== undefined &&
|
|
1633
|
+
updated.metadata.finalizers.length === 0
|
|
1634
|
+
) {
|
|
1635
|
+
active.store.recordEvent({
|
|
1636
|
+
...ref,
|
|
1637
|
+
claim: active.claim,
|
|
1638
|
+
type: "resource_deleted",
|
|
1639
|
+
payload: { reconcileId: active.reconcileId },
|
|
1640
|
+
});
|
|
1641
|
+
active.store.deleteResource(ref, active.claim.resourceVersion, active.claim);
|
|
1642
|
+
} else if (result.kind === "settled") {
|
|
1643
|
+
active.store.settleClaim(active.claim, now.toISOString());
|
|
1644
|
+
} else {
|
|
1645
|
+
active.store.requeueClaim(
|
|
1646
|
+
active.claim,
|
|
1647
|
+
{ availableAt: new Date(now.getTime() + (result.afterMs ?? 0)).toISOString() },
|
|
1648
|
+
now.toISOString(),
|
|
1649
|
+
);
|
|
1650
|
+
}
|
|
1651
|
+
active.settled = true;
|
|
1652
|
+
} catch (error) {
|
|
1653
|
+
if (error instanceof ResourceConflictError) {
|
|
1654
|
+
active.store.recordEvent({
|
|
1655
|
+
...ref,
|
|
1656
|
+
claim: active.claim,
|
|
1657
|
+
type: "reconcile_conflict",
|
|
1658
|
+
payload: { reconcileId: active.reconcileId },
|
|
1659
|
+
});
|
|
1660
|
+
active.store.requeueClaim(
|
|
1661
|
+
active.claim,
|
|
1662
|
+
{ availableAt: now.toISOString() },
|
|
1663
|
+
now.toISOString(),
|
|
1664
|
+
);
|
|
1665
|
+
active.settled = true;
|
|
1666
|
+
return;
|
|
1667
|
+
}
|
|
1668
|
+
throw error;
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
private finishControllerFailure(active: ActiveController, failure: string): void {
|
|
1673
|
+
if (active.settled) return;
|
|
1674
|
+
const now = new Date();
|
|
1675
|
+
const delay = Math.min(60_000, 1_000 * 2 ** Math.min(active.claim.consecutiveErrors, 6));
|
|
1676
|
+
const error = failure.slice(0, 8_192);
|
|
1677
|
+
active.store.recordEvent({
|
|
1678
|
+
controller: active.claim.controller,
|
|
1679
|
+
key: active.claim.key,
|
|
1680
|
+
claim: active.claim,
|
|
1681
|
+
type: "reconcile_failed",
|
|
1682
|
+
payload: {
|
|
1683
|
+
reconcileId: active.reconcileId,
|
|
1684
|
+
error,
|
|
1685
|
+
durationMs: Math.max(0, now.getTime() - active.startedAt),
|
|
1686
|
+
requeueAfterMs: delay,
|
|
1687
|
+
},
|
|
1688
|
+
});
|
|
1689
|
+
active.store.requeueClaim(
|
|
1690
|
+
active.claim,
|
|
1691
|
+
{ availableAt: new Date(now.getTime() + delay).toISOString(), error },
|
|
1692
|
+
now.toISOString(),
|
|
1693
|
+
);
|
|
1694
|
+
active.settled = true;
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
private async expireTimedOutDecision(): Promise<void> {
|
|
1698
|
+
if (this.stopping || this.claim === null || this.decisionTimeoutActive) return;
|
|
1699
|
+
this.decisionTimeoutActive = true;
|
|
1700
|
+
try {
|
|
1701
|
+
const now = new Date();
|
|
1702
|
+
const requests = await this.decisions.listExpiredDefaultRequests(now);
|
|
1703
|
+
const candidate = requests
|
|
1704
|
+
.map((request) => ({
|
|
1705
|
+
request,
|
|
1706
|
+
interaction: this.hostState.getInteraction(request.decisionId),
|
|
1707
|
+
}))
|
|
1708
|
+
.find(
|
|
1709
|
+
({ request, interaction }) =>
|
|
1710
|
+
interaction?.kind === "decision" &&
|
|
1711
|
+
interaction.runId === request.runId &&
|
|
1712
|
+
["pending", "presenting"].includes(interaction.status) &&
|
|
1713
|
+
!this.activeRuns.has(request.runId) &&
|
|
1714
|
+
!this.pendingStarts.has(request.runId),
|
|
1715
|
+
);
|
|
1716
|
+
if (
|
|
1717
|
+
candidate === undefined ||
|
|
1718
|
+
candidate.interaction === undefined ||
|
|
1719
|
+
candidate.request.defaultResponse === undefined
|
|
1720
|
+
) {
|
|
1721
|
+
return;
|
|
1722
|
+
}
|
|
1723
|
+
const { request, interaction } = candidate;
|
|
1724
|
+
const claimToken = randomUUID();
|
|
1725
|
+
const claimed = this.queue.claimWorkflowRunForControl({
|
|
1726
|
+
runId: request.runId,
|
|
1727
|
+
runnerId: this.hostId,
|
|
1728
|
+
claimToken,
|
|
1729
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1730
|
+
});
|
|
1731
|
+
if (claimed === undefined) return;
|
|
1732
|
+
this.controlClaims.set(request.runId, claimToken);
|
|
1733
|
+
let committed = false;
|
|
1734
|
+
const afterCommit: Array<() => void> = [];
|
|
1735
|
+
try {
|
|
1736
|
+
this.state.transaction(() => {
|
|
1737
|
+
const accepted = this.decisions.resolveTimeoutSync(request, new Date());
|
|
1738
|
+
const submissionId = `timeout-${request.decisionId}`;
|
|
1739
|
+
this.hostState.submitInteraction({
|
|
1740
|
+
requestId: interaction.requestId,
|
|
1741
|
+
submissionId,
|
|
1742
|
+
idempotencyKey: submissionId,
|
|
1743
|
+
expectedRevision: interaction.revision,
|
|
1744
|
+
payload: request.defaultResponse as JsonValue,
|
|
1745
|
+
accepted: true,
|
|
1746
|
+
receipt: accepted.decision as unknown as JsonValue,
|
|
1747
|
+
});
|
|
1748
|
+
if (!this.queue.parkWorkflowRun({ runId: request.runId, claimToken })) {
|
|
1749
|
+
throw new Error("Decision timeout could not release the parent run claim");
|
|
1750
|
+
}
|
|
1751
|
+
this.prepareDecisionContinuation(interaction, request, accepted.decision, afterCommit);
|
|
1752
|
+
});
|
|
1753
|
+
committed = true;
|
|
1754
|
+
for (const effect of afterCommit) setImmediate(effect);
|
|
1755
|
+
this.log(`human decision ${request.decisionId} resolved by its timeout policy`);
|
|
1756
|
+
} catch (error) {
|
|
1757
|
+
this.log(`human decision timeout failed for ${request.decisionId}: ${errorMessage(error)}`);
|
|
1758
|
+
} finally {
|
|
1759
|
+
this.controlClaims.delete(request.runId);
|
|
1760
|
+
if (!committed) this.queue.parkWorkflowRun({ runId: request.runId, claimToken });
|
|
1761
|
+
}
|
|
1762
|
+
} catch (error) {
|
|
1763
|
+
this.log(`human decision timeout scan failed: ${errorMessage(error)}`);
|
|
1764
|
+
} finally {
|
|
1765
|
+
this.decisionTimeoutActive = false;
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
private expireTimedOutInteraction(): void {
|
|
1770
|
+
if (this.stopping || this.claim === null) return;
|
|
1771
|
+
const runId = this.hostState
|
|
1772
|
+
.expiredInteractionRunIds()
|
|
1773
|
+
.find((candidate) => !this.activeRuns.has(candidate) && !this.pendingStarts.has(candidate));
|
|
1774
|
+
if (runId === undefined) return;
|
|
1775
|
+
const claimToken = randomUUID();
|
|
1776
|
+
const claimed = this.queue.claimWorkflowRunForControl({
|
|
1777
|
+
runId,
|
|
1778
|
+
runnerId: this.hostId,
|
|
1779
|
+
claimToken,
|
|
1780
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1781
|
+
});
|
|
1782
|
+
if (claimed === undefined) return;
|
|
1783
|
+
let activated = false;
|
|
1784
|
+
try {
|
|
1785
|
+
activated = this.queue.beginWorkflowRunInteractionTimeout({ runId, claimToken });
|
|
1786
|
+
if (activated) {
|
|
1787
|
+
this.markPendingStart(runId, claimToken);
|
|
1788
|
+
setImmediate(() => void this.activateRun(claimed, claimToken));
|
|
1789
|
+
this.log(`run ${runId} is finishing its expired origin-session deadline`);
|
|
1790
|
+
}
|
|
1791
|
+
} catch (error) {
|
|
1792
|
+
this.log(`interaction timeout failed for run ${runId}: ${errorMessage(error)}`);
|
|
1793
|
+
} finally {
|
|
1794
|
+
if (!activated) this.queue.parkWorkflowRun({ runId, claimToken });
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
private markPendingStart(runId: string, claimToken: string): void {
|
|
1799
|
+
this.pendingStarts.add(runId);
|
|
1800
|
+
this.pendingRunClaims.set(runId, claimToken);
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
private clearPendingStart(runId: string, claimToken: string): void {
|
|
1804
|
+
if (this.pendingRunClaims.get(runId) !== claimToken) return;
|
|
1805
|
+
this.pendingRunClaims.delete(runId);
|
|
1806
|
+
this.pendingStarts.delete(runId);
|
|
1807
|
+
}
|
|
1808
|
+
|
|
1809
|
+
private async claimOne(): Promise<void> {
|
|
1810
|
+
if (this.stopping || this.claim === null) return;
|
|
1811
|
+
const excluded = new Set([
|
|
1812
|
+
...this.activeRuns.keys(),
|
|
1813
|
+
...this.pendingStarts,
|
|
1814
|
+
...this.blockedRuns,
|
|
1815
|
+
]);
|
|
1816
|
+
const validatingRunId = this.hostState
|
|
1817
|
+
.validatingInteractionRunIds()
|
|
1818
|
+
.find((runId) => !excluded.has(runId));
|
|
1819
|
+
if (validatingRunId !== undefined) {
|
|
1820
|
+
const validationToken = randomUUID();
|
|
1821
|
+
const validationRun = this.queue.claimWorkflowRunForInteractionValidation({
|
|
1822
|
+
runId: validatingRunId,
|
|
1823
|
+
runnerId: this.hostId,
|
|
1824
|
+
claimToken: validationToken,
|
|
1825
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1826
|
+
});
|
|
1827
|
+
if (validationRun !== undefined) {
|
|
1828
|
+
this.markPendingStart(validatingRunId, validationToken);
|
|
1829
|
+
await this.activateRun(validationRun, validationToken);
|
|
1830
|
+
return;
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
const token = randomUUID();
|
|
1834
|
+
const claimed = this.queue.claimNextWorkflowRun({
|
|
1835
|
+
runnerId: this.hostId,
|
|
1836
|
+
claimToken: token,
|
|
1837
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1838
|
+
excludeRunIds: [...excluded],
|
|
1839
|
+
});
|
|
1840
|
+
if (claimed === undefined) return;
|
|
1841
|
+
this.markPendingStart(claimed.runId, token);
|
|
1842
|
+
await this.activateRun(claimed, token);
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
private activateRun(record: WorkflowRunQueueRecord, claimToken: string): Promise<void> {
|
|
1846
|
+
const existing = this.activationTasks.get(record.runId);
|
|
1847
|
+
if (existing !== undefined) return existing;
|
|
1848
|
+
const task = this.activateRunNow(record, claimToken).finally(() => {
|
|
1849
|
+
if (this.activationTasks.get(record.runId) === task) {
|
|
1850
|
+
this.activationTasks.delete(record.runId);
|
|
1851
|
+
}
|
|
1852
|
+
if (this.stopping) return;
|
|
1853
|
+
if (this.pendingResumes.delete(record.runId)) {
|
|
1854
|
+
setImmediate(() => void this.resumePendingRun(record.runId));
|
|
1855
|
+
} else {
|
|
1856
|
+
setImmediate(() => void this.claimOne());
|
|
1857
|
+
}
|
|
1858
|
+
});
|
|
1859
|
+
this.activationTasks.set(record.runId, task);
|
|
1860
|
+
return task;
|
|
1861
|
+
}
|
|
1862
|
+
|
|
1863
|
+
private async activateRunNow(record: WorkflowRunQueueRecord, claimToken: string): Promise<void> {
|
|
1864
|
+
const runId = record.runId;
|
|
1865
|
+
if (this.pendingRunClaims.get(runId) !== claimToken) return;
|
|
1866
|
+
if (this.stopping) {
|
|
1867
|
+
this.clearPendingStart(runId, claimToken);
|
|
1868
|
+
this.queue.parkWorkflowRun({ runId, claimToken });
|
|
1869
|
+
return;
|
|
1870
|
+
}
|
|
1871
|
+
const existingActive = this.activeRuns.get(runId);
|
|
1872
|
+
if (existingActive !== undefined) {
|
|
1873
|
+
this.clearPendingStart(runId, claimToken);
|
|
1874
|
+
if (existingActive.claimToken !== claimToken) {
|
|
1875
|
+
this.queue.parkWorkflowRun({ runId, claimToken });
|
|
1876
|
+
}
|
|
1877
|
+
return;
|
|
1878
|
+
}
|
|
1879
|
+
const generation = record.claimGeneration;
|
|
1880
|
+
const projectPath = this.queue.workflowRunProjectPath(runId);
|
|
1881
|
+
if (generation === null || projectPath === undefined || this.claim === null) {
|
|
1882
|
+
this.clearPendingStart(runId, claimToken);
|
|
1883
|
+
this.queue.parkWorkflowRun({ runId, claimToken });
|
|
1884
|
+
return;
|
|
1885
|
+
}
|
|
1886
|
+
const envelope: WorkerLaunchEnvelope = {
|
|
1887
|
+
schema: "pi-workflows.worker-launch.v1",
|
|
1888
|
+
runId,
|
|
1889
|
+
generation,
|
|
1890
|
+
workerEpoch: randomUUID(),
|
|
1891
|
+
projectPath,
|
|
1892
|
+
workflowSource: record.workflowSource as JsonValue,
|
|
1893
|
+
definitionDigest: record.definitionDigest,
|
|
1894
|
+
inputHash: `sha256:${createHash("sha256").update(canonicalJson(record.input)).digest("hex")}`,
|
|
1895
|
+
protocolVersion: 1,
|
|
1896
|
+
};
|
|
1897
|
+
this.hostState.recordWorkerStart(envelope, this.claim.epoch);
|
|
1898
|
+
let active: ActiveRun;
|
|
1899
|
+
const supervisor = new WorkflowWorkerSupervisor(envelope, {
|
|
1900
|
+
registry: this.registry,
|
|
1901
|
+
onMessage: async (message) => await this.handleWorkerMessage(message),
|
|
1902
|
+
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
1903
|
+
...(this.options.workerEntryPath === undefined
|
|
1904
|
+
? {}
|
|
1905
|
+
: { workerEntryPath: this.options.workerEntryPath }),
|
|
1906
|
+
...(this.options.workerStartupTimeoutMs === undefined
|
|
1907
|
+
? {}
|
|
1908
|
+
: { startupTimeoutMs: this.options.workerStartupTimeoutMs }),
|
|
1909
|
+
onSpawn: (identity) => {
|
|
1910
|
+
active.workerPid = identity.pid;
|
|
1911
|
+
this.hostState.attachWorkerProcess(
|
|
1912
|
+
envelope.workerEpoch,
|
|
1913
|
+
identity.pid,
|
|
1914
|
+
identity.startIdentity,
|
|
1915
|
+
);
|
|
1916
|
+
},
|
|
1917
|
+
onDiagnostic: (message) => this.log(`worker ${envelope.workerEpoch}: ${message}`),
|
|
1918
|
+
});
|
|
1919
|
+
active = { record, claimToken, generation, supervisor, exiting: false };
|
|
1920
|
+
this.activeRuns.set(runId, active);
|
|
1921
|
+
this.clearPendingStart(runId, claimToken);
|
|
1922
|
+
try {
|
|
1923
|
+
this.runStore.synchronizeRevision(runId);
|
|
1924
|
+
const effectRecovery = await this.runStore.recoverApplyingEffects(runId);
|
|
1925
|
+
if (effectRecovery === "ambiguous" || hasUncertainEffect(this.state, runId)) {
|
|
1926
|
+
if (!this.queue.parkWorkflowRunForAmbiguousEffect({ runId, claimToken })) {
|
|
1927
|
+
throw new Error("Run claim was lost during ambiguous-effect recovery");
|
|
1928
|
+
}
|
|
1929
|
+
this.hostState.finishWorker({
|
|
1930
|
+
workerEpoch: envelope.workerEpoch,
|
|
1931
|
+
outcome: "orphaned",
|
|
1932
|
+
diagnostic: "effect outcome is ambiguous",
|
|
1933
|
+
});
|
|
1934
|
+
return;
|
|
1935
|
+
}
|
|
1936
|
+
await supervisor.start();
|
|
1937
|
+
const result = await supervisor.wait();
|
|
1938
|
+
this.hostState.finishWorker({
|
|
1939
|
+
workerEpoch: envelope.workerEpoch,
|
|
1940
|
+
outcome: result.outcome,
|
|
1941
|
+
exitCode: result.exitCode,
|
|
1942
|
+
signal: result.signal,
|
|
1943
|
+
...(result.diagnostic === undefined ? {} : { diagnostic: result.diagnostic }),
|
|
1944
|
+
});
|
|
1945
|
+
if (
|
|
1946
|
+
active.control === undefined &&
|
|
1947
|
+
(result.outcome !== "exited" ||
|
|
1948
|
+
this.hostState.validatingInteraction(active.record.runId) !== undefined)
|
|
1949
|
+
) {
|
|
1950
|
+
await this.recoverWorkerExit(active, result.outcome);
|
|
1951
|
+
}
|
|
1952
|
+
} catch (error) {
|
|
1953
|
+
this.log(`worker ${envelope.workerEpoch} failed: ${errorMessage(error)}`);
|
|
1954
|
+
await supervisor.stop("crashed");
|
|
1955
|
+
const result = await supervisor.wait();
|
|
1956
|
+
try {
|
|
1957
|
+
this.hostState.finishWorker({
|
|
1958
|
+
workerEpoch: envelope.workerEpoch,
|
|
1959
|
+
outcome: "crashed",
|
|
1960
|
+
exitCode: result.exitCode,
|
|
1961
|
+
signal: result.signal,
|
|
1962
|
+
diagnostic: result.diagnostic ?? errorMessage(error),
|
|
1963
|
+
});
|
|
1964
|
+
} catch {
|
|
1965
|
+
// A completed worker record wins.
|
|
1966
|
+
}
|
|
1967
|
+
if (active.control === undefined) {
|
|
1968
|
+
await this.recoverWorkerExit(active, "crashed");
|
|
1969
|
+
}
|
|
1970
|
+
} finally {
|
|
1971
|
+
this.reapWorkerDescendants(envelope.workerEpoch);
|
|
1972
|
+
this.activeRuns.delete(runId);
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
private async resumePendingRun(runId: string): Promise<void> {
|
|
1977
|
+
if (this.stopping || this.claim === null) return;
|
|
1978
|
+
const token = randomUUID();
|
|
1979
|
+
const claimed = this.queue.claimWorkflowRunForInteractionValidation({
|
|
1980
|
+
runId,
|
|
1981
|
+
runnerId: this.hostId,
|
|
1982
|
+
claimToken: token,
|
|
1983
|
+
leaseMs: this.runClaimLeaseMs,
|
|
1984
|
+
});
|
|
1985
|
+
if (claimed === undefined) {
|
|
1986
|
+
await this.claimOne();
|
|
1987
|
+
return;
|
|
1988
|
+
}
|
|
1989
|
+
this.markPendingStart(runId, token);
|
|
1990
|
+
await this.activateRun(claimed, token);
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
private async handleWorkerMessage(message: WorkerMessage): Promise<WorkerResponse> {
|
|
1994
|
+
const active = this.activeRuns.get(message.runId);
|
|
1995
|
+
if (
|
|
1996
|
+
active === undefined ||
|
|
1997
|
+
active.generation !== message.generation ||
|
|
1998
|
+
active.supervisor.envelope.workerEpoch !== message.workerEpoch
|
|
1999
|
+
) {
|
|
2000
|
+
return workerResponse(message, "claimLost", undefined, "Worker epoch is stale");
|
|
2001
|
+
}
|
|
2002
|
+
const stored = this.hostState.readWorkerMessage(message);
|
|
2003
|
+
if (stored !== undefined) return stored;
|
|
2004
|
+
const response = await this.handleFreshWorkerMessage(active, message);
|
|
2005
|
+
return this.hostState.recordWorkerMessage(message, response);
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
private async handleFreshWorkerMessage(
|
|
2009
|
+
active: ActiveRun,
|
|
2010
|
+
message: WorkerMessage,
|
|
2011
|
+
): Promise<WorkerResponse> {
|
|
2012
|
+
if (message.operation === "worker.ready") {
|
|
2013
|
+
if (
|
|
2014
|
+
!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
|
|
2015
|
+
) {
|
|
2016
|
+
active.claimLost = true;
|
|
2017
|
+
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
2018
|
+
return workerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
2019
|
+
}
|
|
2020
|
+
this.hostState.markWorkerReady(message.workerEpoch);
|
|
2021
|
+
this.queue.markWorkflowRunRunning({ runId: message.runId, claimToken: active.claimToken });
|
|
2022
|
+
const revision = this.runStore.synchronizeRevision(message.runId);
|
|
2023
|
+
const current = this.queue.getWorkflowRun(message.runId);
|
|
2024
|
+
const candidateInteraction =
|
|
2025
|
+
this.hostState.acceptedInteraction(message.runId) ??
|
|
2026
|
+
this.hostState.validatingInteraction(message.runId);
|
|
2027
|
+
const timedOutInteraction = this.hostState.timedOutInteraction(message.runId);
|
|
2028
|
+
const resumeInteractionAttemptId =
|
|
2029
|
+
candidateInteraction?.attemptId ?? timedOutInteraction?.attemptId;
|
|
2030
|
+
return workerResponse(
|
|
2031
|
+
message,
|
|
2032
|
+
"accepted",
|
|
2033
|
+
{
|
|
2034
|
+
initialized: current?.initialized ?? active.record.initialized,
|
|
2035
|
+
input: active.record.input,
|
|
2036
|
+
launchOptions: active.record.launchOptions,
|
|
2037
|
+
parentRunId: active.record.parentRunId,
|
|
2038
|
+
originSessionId: active.record.originSessionId,
|
|
2039
|
+
stateDirectory: this.stateDirectory,
|
|
2040
|
+
...(resumeInteractionAttemptId === undefined ? {} : { resumeInteractionAttemptId }),
|
|
2041
|
+
...(candidateInteraction === undefined ? {} : { candidateInteraction }),
|
|
2042
|
+
...(this.options.piArgs === undefined ? {} : { piArgs: this.options.piArgs }),
|
|
2043
|
+
} as JsonValue,
|
|
2044
|
+
undefined,
|
|
2045
|
+
revision,
|
|
2046
|
+
);
|
|
2047
|
+
}
|
|
2048
|
+
if (message.operation === "worker.exiting") {
|
|
2049
|
+
const payload = requireRecord(message.payload, "worker exit payload");
|
|
2050
|
+
if (payload.status === "workflowLoadFailed") {
|
|
2051
|
+
active.workflowLoadFailure = requireString(payload.error, "workflow load error").slice(
|
|
2052
|
+
0,
|
|
2053
|
+
8_192,
|
|
2054
|
+
);
|
|
2055
|
+
}
|
|
2056
|
+
active.exiting = true;
|
|
2057
|
+
return workerResponse(message, "accepted", {});
|
|
2058
|
+
}
|
|
2059
|
+
if (message.operation === "process.register" || message.operation === "process.unregister") {
|
|
2060
|
+
return this.handleWorkerProcessOperation(active, message);
|
|
2061
|
+
}
|
|
2062
|
+
const currentRevision = runRevision(this.state, message.runId);
|
|
2063
|
+
if (message.expectedRevision !== currentRevision) {
|
|
2064
|
+
return workerResponse(
|
|
2065
|
+
message,
|
|
2066
|
+
"rejected",
|
|
2067
|
+
undefined,
|
|
2068
|
+
`Workflow run revision conflict: expected ${message.expectedRevision}, got ${currentRevision}`,
|
|
2069
|
+
currentRevision,
|
|
2070
|
+
);
|
|
2071
|
+
}
|
|
2072
|
+
if (
|
|
2073
|
+
!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
|
|
2074
|
+
) {
|
|
2075
|
+
active.claimLost = true;
|
|
2076
|
+
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
2077
|
+
return workerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
2078
|
+
}
|
|
2079
|
+
try {
|
|
2080
|
+
const payload = requireRecord(message.payload, "worker payload");
|
|
2081
|
+
let result: unknown;
|
|
2082
|
+
switch (message.operation) {
|
|
2083
|
+
case "store.initializeRun":
|
|
2084
|
+
result = await this.runStore.initializeRunFromSnapshot(
|
|
2085
|
+
payload.snapshot as WorkflowDefinitionSnapshot,
|
|
2086
|
+
requireString(payload.workflowName, "workflowName"),
|
|
2087
|
+
payload.state as WorkflowRunState,
|
|
2088
|
+
payload.options as never,
|
|
2089
|
+
);
|
|
2090
|
+
break;
|
|
2091
|
+
case "store.prepareRunResume":
|
|
2092
|
+
await this.prepareInteractionResume(message.runId);
|
|
2093
|
+
result = await this.runStore.prepareRunResume(message.runId);
|
|
2094
|
+
break;
|
|
2095
|
+
case "store.readRun":
|
|
2096
|
+
result = this.runStore.readRun(
|
|
2097
|
+
requireString(payload.runId, "runId"),
|
|
2098
|
+
payload.options as never,
|
|
2099
|
+
);
|
|
2100
|
+
break;
|
|
2101
|
+
case "store.writeSnapshot":
|
|
2102
|
+
result = await this.runStore.writeSnapshot(
|
|
2103
|
+
message.runId,
|
|
2104
|
+
payload.state as WorkflowRunState,
|
|
2105
|
+
payload.event as WorkflowTraceEventDraft,
|
|
2106
|
+
);
|
|
2107
|
+
break;
|
|
2108
|
+
case "store.publishUpdate":
|
|
2109
|
+
result = await this.runStore.publishUpdate(
|
|
2110
|
+
message.runId,
|
|
2111
|
+
payload.state as WorkflowRunState,
|
|
2112
|
+
requireString(payload.nodeId, "nodeId"),
|
|
2113
|
+
requireString(payload.attemptId, "attemptId"),
|
|
2114
|
+
payload.update as WorkflowUpdateInput,
|
|
2115
|
+
);
|
|
2116
|
+
break;
|
|
2117
|
+
case "store.findSettingsScope":
|
|
2118
|
+
result = this.runStore.findSettingsScope(
|
|
2119
|
+
message.runId,
|
|
2120
|
+
requireString(payload.mountPath, "mountPath", true),
|
|
2121
|
+
requireNonNegativeInteger(payload.invocation, "invocation"),
|
|
2122
|
+
);
|
|
2123
|
+
break;
|
|
2124
|
+
case "store.ensureSettingsScope":
|
|
2125
|
+
result = this.runStore.ensureSettingsScope(payload.options as never);
|
|
2126
|
+
break;
|
|
2127
|
+
case "store.getSettingsScopeAtChange":
|
|
2128
|
+
result = this.runStore.getSettingsScopeAtChange(
|
|
2129
|
+
requireString(payload.scopeId, "scopeId"),
|
|
2130
|
+
requireNonNegativeInteger(payload.changeNumber, "changeNumber"),
|
|
2131
|
+
);
|
|
2132
|
+
break;
|
|
2133
|
+
case "store.createHumanDecisionRequest": {
|
|
2134
|
+
const decision = payload.request as HumanDecisionRequest;
|
|
2135
|
+
result = await this.runStore.createHumanDecisionRequest(decision);
|
|
2136
|
+
if (active.record.originSessionId !== null) {
|
|
2137
|
+
this.hostState.createInteractiveRequest({
|
|
2138
|
+
requestId: decision.decisionId,
|
|
2139
|
+
runId: decision.runId,
|
|
2140
|
+
attemptId: decision.attemptId,
|
|
2141
|
+
targetSessionId: active.record.originSessionId,
|
|
2142
|
+
kind: "decision",
|
|
2143
|
+
contract: decision as unknown as JsonValue,
|
|
2144
|
+
});
|
|
2145
|
+
}
|
|
2146
|
+
break;
|
|
2147
|
+
}
|
|
2148
|
+
case "store.readResolvedHumanDecision":
|
|
2149
|
+
result = await this.runStore.readResolvedHumanDecision(
|
|
2150
|
+
requireString(payload.decisionId, "decisionId"),
|
|
2151
|
+
);
|
|
2152
|
+
break;
|
|
2153
|
+
case "store.reserveEffect":
|
|
2154
|
+
result = await this.runStore.reserveEffect(payload.options as never);
|
|
2155
|
+
if (
|
|
2156
|
+
typeof result === "object" &&
|
|
2157
|
+
result !== null &&
|
|
2158
|
+
(result as { disposition?: unknown }).disposition === "ambiguous" &&
|
|
2159
|
+
!this.queue.parkWorkflowRunForAmbiguousEffect({
|
|
2160
|
+
runId: message.runId,
|
|
2161
|
+
claimToken: active.claimToken,
|
|
2162
|
+
})
|
|
2163
|
+
) {
|
|
2164
|
+
throw new Error("Run claim was lost during ambiguous-effect recovery");
|
|
2165
|
+
}
|
|
2166
|
+
break;
|
|
2167
|
+
case "store.settleEffect": {
|
|
2168
|
+
const options = payload.options as {
|
|
2169
|
+
outcome?: unknown;
|
|
2170
|
+
};
|
|
2171
|
+
await this.runStore.settleEffect(payload.options as never);
|
|
2172
|
+
if (
|
|
2173
|
+
options.outcome === "ambiguous" &&
|
|
2174
|
+
!this.queue.parkWorkflowRunForAmbiguousEffect({
|
|
2175
|
+
runId: message.runId,
|
|
2176
|
+
claimToken: active.claimToken,
|
|
2177
|
+
})
|
|
2178
|
+
) {
|
|
2179
|
+
throw new Error("Run claim was lost during ambiguous-effect recovery");
|
|
2180
|
+
}
|
|
2181
|
+
result = null;
|
|
2182
|
+
break;
|
|
2183
|
+
}
|
|
2184
|
+
case "interaction.accept":
|
|
2185
|
+
result = this.acceptWorkerInteraction(message, payload);
|
|
2186
|
+
break;
|
|
2187
|
+
case "interaction.reject":
|
|
2188
|
+
result = this.rejectWorkerInteraction(active, message, payload);
|
|
2189
|
+
break;
|
|
2190
|
+
case "notification.request":
|
|
2191
|
+
result = this.enqueueWorkerNotification(active, message, payload);
|
|
2192
|
+
break;
|
|
2193
|
+
case "presentation.request":
|
|
2194
|
+
result = this.requestWorkerPresentation(active, message, payload);
|
|
2195
|
+
break;
|
|
2196
|
+
case "interaction.request": {
|
|
2197
|
+
result = this.parkForInteraction(active, message, payload);
|
|
2198
|
+
const exitDeadline = setTimeout(() => {
|
|
2199
|
+
if (this.activeRuns.get(message.runId) === active && !active.exiting) {
|
|
2200
|
+
active.control = "handoff";
|
|
2201
|
+
void active.supervisor.stop("orphaned");
|
|
2202
|
+
}
|
|
2203
|
+
}, 1_000);
|
|
2204
|
+
exitDeadline.unref?.();
|
|
2205
|
+
break;
|
|
2206
|
+
}
|
|
2207
|
+
default:
|
|
2208
|
+
return workerResponse(message, "rejected", undefined, "Unknown worker operation");
|
|
2209
|
+
}
|
|
2210
|
+
return workerResponse(
|
|
2211
|
+
message,
|
|
2212
|
+
"accepted",
|
|
2213
|
+
result === undefined ? null : (result as JsonValue),
|
|
2214
|
+
undefined,
|
|
2215
|
+
runRevision(this.state, message.runId),
|
|
2216
|
+
);
|
|
2217
|
+
} catch (error) {
|
|
2218
|
+
return workerResponse(message, "rejected", undefined, errorMessage(error));
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
private handleWorkerProcessOperation(active: ActiveRun, message: WorkerMessage): WorkerResponse {
|
|
2223
|
+
if (
|
|
2224
|
+
message.operation === "process.register" &&
|
|
2225
|
+
!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
|
|
2226
|
+
) {
|
|
2227
|
+
active.claimLost = true;
|
|
2228
|
+
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
2229
|
+
return workerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
2230
|
+
}
|
|
2231
|
+
try {
|
|
2232
|
+
const payload = requireRecord(message.payload, "worker process payload");
|
|
2233
|
+
const pid = requireNonNegativeInteger(payload.pid, "pid");
|
|
2234
|
+
if (pid === 0) throw new Error("Worker process PID must be positive");
|
|
2235
|
+
const owned = this.workerDescendants.get(message.workerEpoch);
|
|
2236
|
+
if (message.operation === "process.unregister") {
|
|
2237
|
+
if (owned?.has(pid) !== true) {
|
|
2238
|
+
throw new Error("Worker process is not registered to this worker epoch");
|
|
2239
|
+
}
|
|
2240
|
+
this.registry.unregister(pid);
|
|
2241
|
+
owned.delete(pid);
|
|
2242
|
+
if (owned.size === 0) this.workerDescendants.delete(message.workerEpoch);
|
|
2243
|
+
return workerResponse(message, "accepted", { pid });
|
|
2244
|
+
}
|
|
2245
|
+
if (active.workerPid === undefined || processParentPid(pid) !== active.workerPid) {
|
|
2246
|
+
throw new Error("Worker process is not a direct child of the active worker");
|
|
2247
|
+
}
|
|
2248
|
+
for (const [workerEpoch, pids] of this.workerDescendants) {
|
|
2249
|
+
if (workerEpoch !== message.workerEpoch && pids.has(pid)) {
|
|
2250
|
+
throw new Error("Worker process is already registered to another worker epoch");
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
this.registry.register(pid);
|
|
2254
|
+
const descendants = owned ?? new Set<number>();
|
|
2255
|
+
descendants.add(pid);
|
|
2256
|
+
this.workerDescendants.set(message.workerEpoch, descendants);
|
|
2257
|
+
return workerResponse(message, "accepted", { pid });
|
|
2258
|
+
} catch (error) {
|
|
2259
|
+
return workerResponse(message, "rejected", undefined, errorMessage(error));
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
private reapWorkerDescendants(workerEpoch: string): void {
|
|
2264
|
+
const descendants = this.workerDescendants.get(workerEpoch);
|
|
2265
|
+
if (descendants === undefined) return;
|
|
2266
|
+
this.workerDescendants.delete(workerEpoch);
|
|
2267
|
+
for (const pid of descendants) this.registry.kill(pid);
|
|
2268
|
+
}
|
|
2269
|
+
|
|
2270
|
+
private acceptWorkerInteraction(
|
|
2271
|
+
message: WorkerMessage,
|
|
2272
|
+
payload: Record<string, unknown>,
|
|
2273
|
+
): InteractiveSubmissionRecord {
|
|
2274
|
+
const candidate = this.requireWorkerInteraction(message, payload, true);
|
|
2275
|
+
return this.hostState.finishInteractionValidation({
|
|
2276
|
+
requestId: candidate.requestId,
|
|
2277
|
+
submissionId: candidate.submissionId,
|
|
2278
|
+
accepted: true,
|
|
2279
|
+
receipt: { status: "accepted" },
|
|
2280
|
+
});
|
|
2281
|
+
}
|
|
2282
|
+
|
|
2283
|
+
private rejectWorkerInteraction(
|
|
2284
|
+
active: ActiveRun,
|
|
2285
|
+
message: WorkerMessage,
|
|
2286
|
+
payload: Record<string, unknown>,
|
|
2287
|
+
): InteractiveSubmissionRecord {
|
|
2288
|
+
const candidate = this.requireWorkerInteraction(message, payload, false);
|
|
2289
|
+
return this.settleRejectedInteraction(
|
|
2290
|
+
active,
|
|
2291
|
+
candidate,
|
|
2292
|
+
requireString(payload.error, "interaction validation error"),
|
|
2293
|
+
);
|
|
2294
|
+
}
|
|
2295
|
+
|
|
2296
|
+
private requireWorkerInteraction(
|
|
2297
|
+
message: WorkerMessage,
|
|
2298
|
+
payload: Record<string, unknown>,
|
|
2299
|
+
acceptSettled: boolean,
|
|
2300
|
+
) {
|
|
2301
|
+
const candidate =
|
|
2302
|
+
this.hostState.validatingInteraction(message.runId) ??
|
|
2303
|
+
(acceptSettled ? this.hostState.acceptedInteraction(message.runId) : undefined);
|
|
2304
|
+
if (
|
|
2305
|
+
candidate === undefined ||
|
|
2306
|
+
candidate.requestId !== requireString(payload.requestId, "interaction requestId") ||
|
|
2307
|
+
candidate.submissionId !== requireString(payload.submissionId, "interaction submissionId") ||
|
|
2308
|
+
candidate.attemptId !== message.attemptId ||
|
|
2309
|
+
candidate.attemptId !== requireString(payload.attemptId, "interaction attemptId")
|
|
2310
|
+
) {
|
|
2311
|
+
throw new Error("Interactive submission does not match the worker candidate");
|
|
2312
|
+
}
|
|
2313
|
+
return candidate;
|
|
2314
|
+
}
|
|
2315
|
+
|
|
2316
|
+
private settleRejectedInteraction(
|
|
2317
|
+
active: ActiveRun,
|
|
2318
|
+
candidate: {
|
|
2319
|
+
requestId: string;
|
|
2320
|
+
submissionId: string;
|
|
2321
|
+
attemptId: string;
|
|
2322
|
+
},
|
|
2323
|
+
error: string,
|
|
2324
|
+
): InteractiveSubmissionRecord {
|
|
2325
|
+
const result = this.state.transaction(() => {
|
|
2326
|
+
const submission = this.hostState.finishInteractionValidation({
|
|
2327
|
+
requestId: candidate.requestId,
|
|
2328
|
+
submissionId: candidate.submissionId,
|
|
2329
|
+
accepted: false,
|
|
2330
|
+
receipt: { status: "rejected", error },
|
|
2331
|
+
});
|
|
2332
|
+
const now = Date.now();
|
|
2333
|
+
this.state.connection
|
|
2334
|
+
.prepare(
|
|
2335
|
+
`UPDATE node_attempts SET status = 'waiting', updated_at = ?
|
|
2336
|
+
WHERE attempt_id = ? AND run_id = ? AND status IN ('running', 'interrupted')`,
|
|
2337
|
+
)
|
|
2338
|
+
.run(now, candidate.attemptId, active.record.runId);
|
|
2339
|
+
this.state.connection
|
|
2340
|
+
.prepare(
|
|
2341
|
+
`UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
|
|
2342
|
+
WHERE run_id = ? AND status = 'running'`,
|
|
2343
|
+
)
|
|
2344
|
+
.run(error, now, now, active.record.runId);
|
|
2345
|
+
if (
|
|
2346
|
+
!this.queue.parkWorkflowRun({
|
|
2347
|
+
runId: active.record.runId,
|
|
2348
|
+
claimToken: active.claimToken,
|
|
2349
|
+
})
|
|
2350
|
+
) {
|
|
2351
|
+
throw new Error("Rejected interaction could not release the run claim");
|
|
2352
|
+
}
|
|
2353
|
+
recordViewerDeltas(
|
|
2354
|
+
this.state,
|
|
2355
|
+
active.record.runId,
|
|
2356
|
+
[{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
|
|
2357
|
+
now,
|
|
2358
|
+
);
|
|
2359
|
+
return submission;
|
|
2360
|
+
});
|
|
2361
|
+
active.control = "handoff";
|
|
2362
|
+
return result;
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
private enqueueWorkerNotification(
|
|
2366
|
+
active: ActiveRun,
|
|
2367
|
+
message: WorkerMessage,
|
|
2368
|
+
payload: Record<string, unknown>,
|
|
2369
|
+
): JsonValue {
|
|
2370
|
+
if (active.record.originSessionId === null) {
|
|
2371
|
+
throw new Error("Workflow notification has no origin Pi session");
|
|
2372
|
+
}
|
|
2373
|
+
const request = requireRecord(payload.request, "notification request");
|
|
2374
|
+
if (
|
|
2375
|
+
requireString(request.runId, "notification.runId") !== message.runId ||
|
|
2376
|
+
requireString(request.workflowName, "notification.workflowName") !==
|
|
2377
|
+
active.record.workflowName
|
|
2378
|
+
) {
|
|
2379
|
+
throw new Error("Workflow notification provenance does not match the active run");
|
|
2380
|
+
}
|
|
2381
|
+
const notificationIndex = requireNonNegativeInteger(
|
|
2382
|
+
request.notificationIndex,
|
|
2383
|
+
"notificationIndex",
|
|
2384
|
+
);
|
|
2385
|
+
if (notificationIndex === 0) throw new Error("notificationIndex must be positive");
|
|
2386
|
+
const kind = request.kind;
|
|
2387
|
+
if (kind !== "progress" && kind !== "final") {
|
|
2388
|
+
throw new Error("Workflow notification kind is invalid");
|
|
2389
|
+
}
|
|
2390
|
+
const record = this.queue.enqueueWorkflowNotification({
|
|
2391
|
+
runId: message.runId,
|
|
2392
|
+
nodeId: requireString(request.nodeId, "notification.nodeId"),
|
|
2393
|
+
attemptId: requireString(request.attemptId, "notification.attemptId"),
|
|
2394
|
+
notificationIndex,
|
|
2395
|
+
targetSessionId: active.record.originSessionId,
|
|
2396
|
+
kind,
|
|
2397
|
+
content: requireString(request.content, "notification.content"),
|
|
2398
|
+
});
|
|
2399
|
+
return {
|
|
2400
|
+
notificationId: record.notificationId,
|
|
2401
|
+
targetSessionId: record.targetSessionId,
|
|
2402
|
+
};
|
|
2403
|
+
}
|
|
2404
|
+
|
|
2405
|
+
private requestWorkerPresentation(
|
|
2406
|
+
active: ActiveRun,
|
|
2407
|
+
message: WorkerMessage,
|
|
2408
|
+
payload: Record<string, unknown>,
|
|
2409
|
+
): JsonValue {
|
|
2410
|
+
if (active.record.originSessionId === null) {
|
|
2411
|
+
throw new Error("Workflow presentation has no origin Pi session");
|
|
2412
|
+
}
|
|
2413
|
+
const instructions = requireString(payload.instructions, "presentation instructions").trim();
|
|
2414
|
+
if (instructions.length === 0) throw new Error("Presentation instructions must not be empty");
|
|
2415
|
+
const facts: WorkflowTurnIntentFacts & { presentationPrompt: string } = {
|
|
2416
|
+
schema: "pi-workflows.deferred-turn-facts.v1",
|
|
2417
|
+
workflowName: active.record.workflowName,
|
|
2418
|
+
runId: message.runId,
|
|
2419
|
+
observedState: "completed",
|
|
2420
|
+
cause: "terminal",
|
|
2421
|
+
nodeId: null,
|
|
2422
|
+
attemptId: null,
|
|
2423
|
+
reason: null,
|
|
2424
|
+
handoff: false,
|
|
2425
|
+
presentationPrompt: instructions,
|
|
2426
|
+
};
|
|
2427
|
+
const intent = this.queue.ensureWorkflowTurnIntent({
|
|
2428
|
+
intentId: `presentation-${message.runId}`,
|
|
2429
|
+
sourceEventId: `presentation-request-${message.runId}`,
|
|
2430
|
+
runId: message.runId,
|
|
2431
|
+
workflowRef: active.record.workflowSourceRef,
|
|
2432
|
+
targetSessionId: active.record.originSessionId,
|
|
2433
|
+
cause: "terminal",
|
|
2434
|
+
fallbackFacts: facts,
|
|
2435
|
+
eligible: false,
|
|
2436
|
+
});
|
|
2437
|
+
return intent as unknown as JsonValue;
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
private parkForInteraction(
|
|
2441
|
+
active: ActiveRun,
|
|
2442
|
+
message: WorkerMessage,
|
|
2443
|
+
payload: Record<string, unknown>,
|
|
2444
|
+
): JsonValue {
|
|
2445
|
+
if (active.record.originSessionId === null) {
|
|
2446
|
+
throw new Error("Interactive request has no origin Pi session");
|
|
2447
|
+
}
|
|
2448
|
+
const attemptId = requireString(payload.attemptId, "attemptId");
|
|
2449
|
+
const requestId = `interaction-${message.runId}-${attemptId}`;
|
|
2450
|
+
return this.state.transaction(() => {
|
|
2451
|
+
const request = this.hostState.createInteractiveRequest({
|
|
2452
|
+
requestId,
|
|
2453
|
+
runId: message.runId,
|
|
2454
|
+
attemptId,
|
|
2455
|
+
targetSessionId: active.record.originSessionId as string,
|
|
2456
|
+
kind: payload.kind === "assistant" ? "assistant" : "agent",
|
|
2457
|
+
contract: payload.contract as JsonValue,
|
|
2458
|
+
});
|
|
2459
|
+
const now = Date.now();
|
|
2460
|
+
this.state.connection
|
|
2461
|
+
.prepare(
|
|
2462
|
+
`UPDATE node_attempts SET status = 'waiting', updated_at = ?
|
|
2463
|
+
WHERE attempt_id = ? AND run_id = ? AND status = 'running'`,
|
|
2464
|
+
)
|
|
2465
|
+
.run(now, attemptId, message.runId);
|
|
2466
|
+
this.state.connection
|
|
2467
|
+
.prepare(
|
|
2468
|
+
`UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
|
|
2469
|
+
WHERE run_id = ? AND status = 'running'`,
|
|
2470
|
+
)
|
|
2471
|
+
.run("waiting for origin Pi session", now, now, message.runId);
|
|
2472
|
+
if (!this.queue.parkWorkflowRun({ runId: message.runId, claimToken: active.claimToken })) {
|
|
2473
|
+
throw new Error("Interactive request could not release the run claim");
|
|
2474
|
+
}
|
|
2475
|
+
active.control = "handoff";
|
|
2476
|
+
recordViewerDeltas(
|
|
2477
|
+
this.state,
|
|
2478
|
+
message.runId,
|
|
2479
|
+
[{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
|
|
2480
|
+
now,
|
|
2481
|
+
);
|
|
2482
|
+
return request as unknown as JsonValue;
|
|
2483
|
+
});
|
|
2484
|
+
}
|
|
2485
|
+
|
|
2486
|
+
private async prepareInteractionResume(runId: string): Promise<void> {
|
|
2487
|
+
const accepted = this.hostState.acceptedInteraction(runId);
|
|
2488
|
+
const candidate = accepted ?? this.hostState.validatingInteraction(runId);
|
|
2489
|
+
const timedOut = this.hostState.timedOutInteraction(runId);
|
|
2490
|
+
if (candidate === undefined && timedOut === undefined) return;
|
|
2491
|
+
const loaded = this.runStore.readRun(runId);
|
|
2492
|
+
if (loaded === null) return;
|
|
2493
|
+
if (timedOut !== undefined) {
|
|
2494
|
+
if (loaded.state.status !== "running") {
|
|
2495
|
+
throw new Error(`Timed-out interaction run ${runId} is not running`);
|
|
2496
|
+
}
|
|
2497
|
+
return;
|
|
2498
|
+
}
|
|
2499
|
+
if (candidate === undefined || loaded.state.status !== "waiting") return;
|
|
2500
|
+
loaded.state.status = "running";
|
|
2501
|
+
delete loaded.state.statusDetail;
|
|
2502
|
+
delete loaded.state.finishedAt;
|
|
2503
|
+
await this.runStore.writeSnapshot(runId, loaded.state, {
|
|
2504
|
+
scope: "node",
|
|
2505
|
+
type: accepted === undefined ? "interaction_validation_started" : "interaction_accepted",
|
|
2506
|
+
nodeId: candidate.nodeId,
|
|
2507
|
+
attemptId: candidate.attemptId,
|
|
2508
|
+
payload: {
|
|
2509
|
+
requestId: candidate.requestId,
|
|
2510
|
+
submissionId: candidate.submissionId,
|
|
2511
|
+
},
|
|
2512
|
+
});
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
private applyLifecycleProjection(context: {
|
|
2516
|
+
runId: string;
|
|
2517
|
+
state: WorkflowRunState;
|
|
2518
|
+
event: { type: string; attemptId?: string; payload?: Record<string, unknown> };
|
|
2519
|
+
database: StateDatabase;
|
|
2520
|
+
now: number;
|
|
2521
|
+
}): void {
|
|
2522
|
+
const active = this.activeRuns.get(context.runId);
|
|
2523
|
+
if (active === undefined) return;
|
|
2524
|
+
if (context.event.type === "node_finished" && context.event.attemptId !== undefined) {
|
|
2525
|
+
this.hostState.consumeAcceptedInteraction(
|
|
2526
|
+
context.runId,
|
|
2527
|
+
context.event.attemptId,
|
|
2528
|
+
context.now,
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
if (context.state.status === "running") {
|
|
2532
|
+
if (context.event.type === "run_started" || context.event.type === "run_resumed") {
|
|
2533
|
+
context.database.connection
|
|
2534
|
+
.prepare(
|
|
2535
|
+
`UPDATE run_queue SET status = 'running', started_at = COALESCE(started_at, ?), updated_at = ?
|
|
2536
|
+
WHERE run_id = ? AND status IN ('starting', 'parked')`,
|
|
2537
|
+
)
|
|
2538
|
+
.run(context.now, context.now, context.runId);
|
|
2539
|
+
}
|
|
2540
|
+
return;
|
|
2541
|
+
}
|
|
2542
|
+
this.completeControllerWorkflow(context.runId, context.state);
|
|
2543
|
+
const queueStatus =
|
|
2544
|
+
context.state.status === "completed"
|
|
2545
|
+
? "done"
|
|
2546
|
+
: context.state.status === "waiting"
|
|
2547
|
+
? "parked"
|
|
2548
|
+
: context.state.status === "cancelled"
|
|
2549
|
+
? "cancelled"
|
|
2550
|
+
: "failed";
|
|
2551
|
+
const run = context.database.connection
|
|
2552
|
+
.prepare("SELECT error_hash AS errorHash FROM runs WHERE run_id = ?")
|
|
2553
|
+
.get(context.runId) as { errorHash?: Buffer | null } | undefined;
|
|
2554
|
+
context.database.connection
|
|
2555
|
+
.prepare(
|
|
2556
|
+
`UPDATE run_queue
|
|
2557
|
+
SET status = ?, error_code = ?, error_hash = ?, updated_at = ?, finished_at = ?
|
|
2558
|
+
WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
|
|
2559
|
+
)
|
|
2560
|
+
.run(
|
|
2561
|
+
queueStatus,
|
|
2562
|
+
queueStatus === "failed" ? context.state.status : null,
|
|
2563
|
+
run?.errorHash ?? null,
|
|
2564
|
+
context.now,
|
|
2565
|
+
["done", "failed", "cancelled"].includes(queueStatus) ? context.now : null,
|
|
2566
|
+
context.runId,
|
|
2567
|
+
);
|
|
2568
|
+
if (context.state.status === "completed") {
|
|
2569
|
+
if (
|
|
2570
|
+
context.event.payload?.presentationRequired === true &&
|
|
2571
|
+
active.record.originSessionId !== null &&
|
|
2572
|
+
this.queue.findPendingWorkflowTurnIntent({
|
|
2573
|
+
runId: context.runId,
|
|
2574
|
+
targetSessionId: active.record.originSessionId,
|
|
2575
|
+
}) === undefined
|
|
2576
|
+
) {
|
|
2577
|
+
const facts: WorkflowTurnIntentFacts & { presentationPrompt: string } = {
|
|
2578
|
+
schema: "pi-workflows.deferred-turn-facts.v1",
|
|
2579
|
+
workflowName: active.record.workflowName,
|
|
2580
|
+
runId: context.runId,
|
|
2581
|
+
observedState: "completed",
|
|
2582
|
+
cause: "terminal",
|
|
2583
|
+
nodeId: null,
|
|
2584
|
+
attemptId: null,
|
|
2585
|
+
reason: "workflow presentation instructions were unavailable",
|
|
2586
|
+
handoff: false,
|
|
2587
|
+
presentationPrompt:
|
|
2588
|
+
"Summarize the completed workflow result for the user in a normal response.",
|
|
2589
|
+
};
|
|
2590
|
+
this.queue.ensureWorkflowTurnIntent({
|
|
2591
|
+
intentId: `presentation-${context.runId}`,
|
|
2592
|
+
sourceEventId: `presentation-request-${context.runId}`,
|
|
2593
|
+
runId: context.runId,
|
|
2594
|
+
workflowRef: active.record.workflowSourceRef,
|
|
2595
|
+
targetSessionId: active.record.originSessionId,
|
|
2596
|
+
cause: "terminal",
|
|
2597
|
+
fallbackFacts: facts,
|
|
2598
|
+
eligible: false,
|
|
2599
|
+
});
|
|
2600
|
+
}
|
|
2601
|
+
context.database.connection
|
|
2602
|
+
.prepare(
|
|
2603
|
+
`UPDATE turn_intents SET eligible_at = ?
|
|
2604
|
+
WHERE run_id = ? AND resolved_at IS NULL AND eligible_at IS NULL`,
|
|
2605
|
+
)
|
|
2606
|
+
.run(context.now, context.runId);
|
|
2607
|
+
}
|
|
2608
|
+
context.database.connection
|
|
2609
|
+
.prepare(
|
|
2610
|
+
`UPDATE leases
|
|
2611
|
+
SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
|
|
2612
|
+
acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
|
|
2613
|
+
WHERE resource_id = (SELECT resource_id FROM runs WHERE run_id = ?)
|
|
2614
|
+
AND generation = ?`,
|
|
2615
|
+
)
|
|
2616
|
+
.run(context.runId, active.generation);
|
|
2617
|
+
}
|
|
2618
|
+
|
|
2619
|
+
private completeControllerWorkflow(runId: string, state: WorkflowRunState): void {
|
|
2620
|
+
const row = this.state.connection
|
|
2621
|
+
.prepare(
|
|
2622
|
+
`SELECT w.request_id AS requestId, c.controller_name AS controller,
|
|
2623
|
+
c.resource_key AS resourceKey, p.canonical_path AS projectPath
|
|
2624
|
+
FROM controller_workflows w
|
|
2625
|
+
JOIN controller_resources c ON c.controller_resource_id = w.controller_resource_id
|
|
2626
|
+
JOIN projects p ON p.project_id = c.project_id
|
|
2627
|
+
WHERE w.run_id = ? LIMIT 1`,
|
|
2628
|
+
)
|
|
2629
|
+
.get(runId) as
|
|
2630
|
+
| {
|
|
2631
|
+
requestId?: unknown;
|
|
2632
|
+
controller?: unknown;
|
|
2633
|
+
resourceKey?: unknown;
|
|
2634
|
+
projectPath?: unknown;
|
|
2635
|
+
}
|
|
2636
|
+
| undefined;
|
|
2637
|
+
if (
|
|
2638
|
+
typeof row?.requestId !== "string" ||
|
|
2639
|
+
typeof row.controller !== "string" ||
|
|
2640
|
+
typeof row.resourceKey !== "string" ||
|
|
2641
|
+
typeof row.projectPath !== "string"
|
|
2642
|
+
) {
|
|
2643
|
+
return;
|
|
2644
|
+
}
|
|
2645
|
+
const store = new SqliteControllerStore(this.databasePath, {
|
|
2646
|
+
state: this.state,
|
|
2647
|
+
projectPath: row.projectPath,
|
|
2648
|
+
});
|
|
2649
|
+
const childState =
|
|
2650
|
+
state.status === "completed"
|
|
2651
|
+
? "succeeded"
|
|
2652
|
+
: state.status === "waiting"
|
|
2653
|
+
? "waiting"
|
|
2654
|
+
: "failed";
|
|
2655
|
+
store.completeWorkflow(row.requestId, {
|
|
2656
|
+
state: childState,
|
|
2657
|
+
runId,
|
|
2658
|
+
...(state.error === undefined ? {} : { error: state.error }),
|
|
2659
|
+
});
|
|
2660
|
+
store.enqueue({ controller: row.controller, key: row.resourceKey });
|
|
2661
|
+
}
|
|
2662
|
+
|
|
2663
|
+
private commitActivePause(active: ActiveRun): void {
|
|
2664
|
+
const now = Date.now();
|
|
2665
|
+
this.state.transaction(() => {
|
|
2666
|
+
this.state.connection
|
|
2667
|
+
.prepare("UPDATE runs SET paused = 1, status_detail = ?, updated_at = ? WHERE run_id = ?")
|
|
2668
|
+
.run("paused", now, active.record.runId);
|
|
2669
|
+
if (
|
|
2670
|
+
!this.queue.parkWorkflowRun({
|
|
2671
|
+
runId: active.record.runId,
|
|
2672
|
+
claimToken: active.claimToken,
|
|
2673
|
+
})
|
|
2674
|
+
) {
|
|
2675
|
+
throw new Error("Run claim was lost before pause committed");
|
|
2676
|
+
}
|
|
2677
|
+
});
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2680
|
+
private async recoverWorkerExit(active: ActiveRun, outcome: string): Promise<void> {
|
|
2681
|
+
if (
|
|
2682
|
+
!this.queue.verifyWorkflowRunClaim({
|
|
2683
|
+
runId: active.record.runId,
|
|
2684
|
+
claimToken: active.claimToken,
|
|
2685
|
+
})
|
|
2686
|
+
) {
|
|
2687
|
+
return;
|
|
2688
|
+
}
|
|
2689
|
+
const recovery = await this.runStore.recoverApplyingEffects(active.record.runId);
|
|
2690
|
+
if (recovery === "ambiguous" || hasUncertainEffect(this.state, active.record.runId)) {
|
|
2691
|
+
this.blockedRuns.add(active.record.runId);
|
|
2692
|
+
if (
|
|
2693
|
+
!this.queue.parkWorkflowRunForAmbiguousEffect({
|
|
2694
|
+
runId: active.record.runId,
|
|
2695
|
+
claimToken: active.claimToken,
|
|
2696
|
+
})
|
|
2697
|
+
) {
|
|
2698
|
+
return;
|
|
2699
|
+
}
|
|
2700
|
+
this.log(`run ${active.record.runId} parked for ambiguous effect recovery`);
|
|
2701
|
+
return;
|
|
2702
|
+
}
|
|
2703
|
+
const validating = this.hostState.validatingInteraction(active.record.runId);
|
|
2704
|
+
if (validating !== undefined) {
|
|
2705
|
+
this.settleRejectedInteraction(
|
|
2706
|
+
active,
|
|
2707
|
+
validating,
|
|
2708
|
+
`Workflow worker ${outcome} before it completed interaction validation`,
|
|
2709
|
+
);
|
|
2710
|
+
this.log(`run ${active.record.runId} rejected an interrupted interaction validation`);
|
|
2711
|
+
return;
|
|
2712
|
+
}
|
|
2713
|
+
if (active.workflowLoadFailure !== undefined) {
|
|
2714
|
+
if (
|
|
2715
|
+
this.queue.parkWorkflowRunForSourceChange({
|
|
2716
|
+
runId: active.record.runId,
|
|
2717
|
+
claimToken: active.claimToken,
|
|
2718
|
+
detail: active.workflowLoadFailure,
|
|
2719
|
+
})
|
|
2720
|
+
) {
|
|
2721
|
+
this.log(`run ${active.record.runId} parked because its workflow source could not load`);
|
|
2722
|
+
}
|
|
2723
|
+
return;
|
|
2724
|
+
}
|
|
2725
|
+
this.queue.parkWorkflowRun({ runId: active.record.runId, claimToken: active.claimToken });
|
|
2726
|
+
this.log(`run ${active.record.runId} parked after worker ${outcome}`);
|
|
2727
|
+
}
|
|
2728
|
+
|
|
2729
|
+
private releaseLock(): void {
|
|
2730
|
+
try {
|
|
2731
|
+
const current = JSON.parse(fs.readFileSync(this.lockPath, "utf8")) as unknown;
|
|
2732
|
+
if (isLockRecord(current) && current.hostId === this.hostId) fs.rmSync(this.lockPath);
|
|
2733
|
+
} catch {
|
|
2734
|
+
// The lock is already gone.
|
|
2735
|
+
}
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
private log(message: string): void {
|
|
2739
|
+
this.options.onLog?.(message);
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
|
|
2743
|
+
function acquireHostLock(
|
|
2744
|
+
lockPath: string,
|
|
2745
|
+
record: { pid: number; startIdentity: string; hostId: string },
|
|
2746
|
+
): void {
|
|
2747
|
+
try {
|
|
2748
|
+
const existing = JSON.parse(fs.readFileSync(lockPath, "utf8")) as unknown;
|
|
2749
|
+
if (isLockRecord(existing) && matchesProcessIdentity(existing)) {
|
|
2750
|
+
throw new Error(`A workflow host is already running with PID ${existing.pid}`);
|
|
2751
|
+
}
|
|
2752
|
+
fs.rmSync(lockPath, { force: true });
|
|
2753
|
+
} catch (error) {
|
|
2754
|
+
if (error instanceof Error && error.message.startsWith("A workflow host is already"))
|
|
2755
|
+
throw error;
|
|
2756
|
+
}
|
|
2757
|
+
fs.writeFileSync(
|
|
2758
|
+
lockPath,
|
|
2759
|
+
`${JSON.stringify({ schema: "pi-workflows.host-lock.v1", ...record })}\n`,
|
|
2760
|
+
{ encoding: "utf8", mode: 0o600, flag: "wx" },
|
|
2761
|
+
);
|
|
2762
|
+
}
|
|
2763
|
+
|
|
2764
|
+
function workerResponse(
|
|
2765
|
+
message: WorkerMessage,
|
|
2766
|
+
outcome: WorkerResponse["outcome"],
|
|
2767
|
+
result?: JsonValue,
|
|
2768
|
+
error?: string,
|
|
2769
|
+
revision?: number,
|
|
2770
|
+
): WorkerResponse {
|
|
2771
|
+
return {
|
|
2772
|
+
schema: "pi-workflows.worker-response.v1",
|
|
2773
|
+
messageId: message.messageId,
|
|
2774
|
+
outcome,
|
|
2775
|
+
...(revision === undefined ? {} : { revision }),
|
|
2776
|
+
...(result === undefined ? {} : { result }),
|
|
2777
|
+
...(error === undefined ? {} : { error }),
|
|
2778
|
+
};
|
|
2779
|
+
}
|
|
2780
|
+
|
|
2781
|
+
function controllerWorkerResponse(
|
|
2782
|
+
message: ControllerWorkerMessage,
|
|
2783
|
+
outcome: ControllerWorkerResponse["outcome"],
|
|
2784
|
+
result?: JsonValue,
|
|
2785
|
+
error?: string,
|
|
2786
|
+
): ControllerWorkerResponse {
|
|
2787
|
+
return {
|
|
2788
|
+
schema: "pi-workflows.controller-worker-response.v1",
|
|
2789
|
+
messageId: message.messageId,
|
|
2790
|
+
outcome,
|
|
2791
|
+
...(result === undefined ? {} : { result }),
|
|
2792
|
+
...(error === undefined ? {} : { error }),
|
|
2793
|
+
};
|
|
2794
|
+
}
|
|
2795
|
+
|
|
2796
|
+
function controllerWorkflowResult(record: WorkflowRunQueueRecord): WorkflowSchedulerResult {
|
|
2797
|
+
switch (record.status) {
|
|
2798
|
+
case "queued":
|
|
2799
|
+
case "starting":
|
|
2800
|
+
case "running":
|
|
2801
|
+
return { state: "running", runId: record.runId };
|
|
2802
|
+
case "parked":
|
|
2803
|
+
return { state: "waiting", runId: record.runId };
|
|
2804
|
+
case "done":
|
|
2805
|
+
return { state: "succeeded", runId: record.runId };
|
|
2806
|
+
case "failed":
|
|
2807
|
+
case "cancelled":
|
|
2808
|
+
return {
|
|
2809
|
+
state: "failed",
|
|
2810
|
+
runId: record.runId,
|
|
2811
|
+
...(record.errorMessage === null ? {} : { error: record.errorMessage }),
|
|
2812
|
+
};
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2815
|
+
|
|
2816
|
+
function runRevision(state: StateDatabase, runId: string): number {
|
|
2817
|
+
const row = state.connection
|
|
2818
|
+
.prepare(
|
|
2819
|
+
`SELECT resources.revision FROM resources
|
|
2820
|
+
JOIN runs ON runs.resource_id = resources.resource_id WHERE runs.run_id = ?`,
|
|
2821
|
+
)
|
|
2822
|
+
.get(runId);
|
|
2823
|
+
if (!isRevisionRow(row)) throw new Error(`Workflow run not found: ${runId}`);
|
|
2824
|
+
return row.revision;
|
|
2825
|
+
}
|
|
2826
|
+
|
|
2827
|
+
function hasUncertainEffect(state: StateDatabase, runId: string): boolean {
|
|
2828
|
+
const row = state.connection
|
|
2829
|
+
.prepare(
|
|
2830
|
+
`SELECT 1 AS present FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
|
|
2831
|
+
WHERE r.run_id = ? AND e.status IN ('applying', 'ambiguous') LIMIT 1`,
|
|
2832
|
+
)
|
|
2833
|
+
.get(runId);
|
|
2834
|
+
return row !== undefined;
|
|
2835
|
+
}
|
|
2836
|
+
|
|
2837
|
+
function controllerPathAllowed(
|
|
2838
|
+
projectPath: string,
|
|
2839
|
+
controllerName: string,
|
|
2840
|
+
controllerPath: string,
|
|
2841
|
+
): boolean {
|
|
2842
|
+
if (
|
|
2843
|
+
controllerFileStem(controllerPath) !== controllerName ||
|
|
2844
|
+
!/\.controller\.(?:ts|js|mts|mjs)$/u.test(controllerPath)
|
|
2845
|
+
) {
|
|
2846
|
+
return false;
|
|
2847
|
+
}
|
|
2848
|
+
return controllerSearchDirs({ cwd: projectPath }).some(
|
|
2849
|
+
({ dir }) => path.dirname(controllerPath) === path.resolve(dir),
|
|
2850
|
+
);
|
|
2851
|
+
}
|
|
2852
|
+
|
|
2853
|
+
function payloadLimit(payload: JsonValue): number {
|
|
2854
|
+
if (typeof payload !== "object" || payload === null || Array.isArray(payload)) return 100;
|
|
2855
|
+
const limit = (payload as Record<string, unknown>).limit;
|
|
2856
|
+
return typeof limit === "number" && Number.isSafeInteger(limit) && limit > 0 ? limit : 100;
|
|
2857
|
+
}
|
|
2858
|
+
|
|
2859
|
+
function requireRunId(request: HostRequest): string {
|
|
2860
|
+
if (request.runId === undefined) throw new Error(`${request.operation} requires runId`);
|
|
2861
|
+
return request.runId;
|
|
2862
|
+
}
|
|
2863
|
+
|
|
2864
|
+
function requireRecord(value: unknown, name: string): Record<string, unknown> {
|
|
2865
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
2866
|
+
throw new Error(`${name} must be an object`);
|
|
2867
|
+
}
|
|
2868
|
+
return value as Record<string, unknown>;
|
|
2869
|
+
}
|
|
2870
|
+
|
|
2871
|
+
function requireString(value: unknown, name: string, allowEmpty = false): string {
|
|
2872
|
+
if (typeof value !== "string" || (!allowEmpty && value.length === 0)) {
|
|
2873
|
+
throw new Error(`${name} must be text`);
|
|
2874
|
+
}
|
|
2875
|
+
return value;
|
|
2876
|
+
}
|
|
2877
|
+
|
|
2878
|
+
function requireAbsolutePath(value: unknown, name: string): string {
|
|
2879
|
+
const parsed = requireString(value, name);
|
|
2880
|
+
if (!path.isAbsolute(parsed)) throw new Error(`${name} must be an absolute path`);
|
|
2881
|
+
return parsed;
|
|
2882
|
+
}
|
|
2883
|
+
|
|
2884
|
+
function requireNonNegativeInteger(value: unknown, name: string): number {
|
|
2885
|
+
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
|
2886
|
+
throw new Error(`${name} must be a non-negative integer`);
|
|
2887
|
+
}
|
|
2888
|
+
return value as number;
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2891
|
+
function isRevisionRow(value: unknown): value is { revision: number } {
|
|
2892
|
+
return (
|
|
2893
|
+
typeof value === "object" &&
|
|
2894
|
+
value !== null &&
|
|
2895
|
+
typeof (value as { revision?: unknown }).revision === "number"
|
|
2896
|
+
);
|
|
2897
|
+
}
|
|
2898
|
+
|
|
2899
|
+
function isLockRecord(
|
|
2900
|
+
value: unknown,
|
|
2901
|
+
): value is { schema: string; pid: number; startIdentity: string; hostId: string } {
|
|
2902
|
+
return (
|
|
2903
|
+
typeof value === "object" &&
|
|
2904
|
+
value !== null &&
|
|
2905
|
+
(value as { schema?: unknown }).schema === "pi-workflows.host-lock.v1" &&
|
|
2906
|
+
typeof (value as { pid?: unknown }).pid === "number" &&
|
|
2907
|
+
typeof (value as { startIdentity?: unknown }).startIdentity === "string" &&
|
|
2908
|
+
typeof (value as { hostId?: unknown }).hostId === "string"
|
|
2909
|
+
);
|
|
519
2910
|
}
|