@osolmaz/pi-workflows 0.13.4 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/README.md +146 -160
  2. package/dist/builtins/autoimplement.workflow.js +3 -1
  3. package/dist/builtins/autoimplement.workflow.js.map +1 -1
  4. package/dist/builtins/change-verification.workflow.js +7 -2
  5. package/dist/builtins/change-verification.workflow.js.map +1 -1
  6. package/dist/builtins/metadata.d.ts +22 -0
  7. package/dist/builtins/metadata.js +10 -0
  8. package/dist/builtins/metadata.js.map +1 -0
  9. package/dist/builtins/monitor.workflow.js +4 -1
  10. package/dist/builtins/monitor.workflow.js.map +1 -1
  11. package/dist/builtins/sanity-check.workflow.js +4 -1
  12. package/dist/builtins/sanity-check.workflow.js.map +1 -1
  13. package/dist/builtins/workspace-preparation.workflow.js +3 -1
  14. package/dist/builtins/workspace-preparation.workflow.js.map +1 -1
  15. package/dist/controllers/index.d.ts +1 -2
  16. package/dist/controllers/index.js +0 -1
  17. package/dist/controllers/index.js.map +1 -1
  18. package/dist/controllers/sqlite.d.ts +85 -31
  19. package/dist/controllers/sqlite.js +541 -126
  20. package/dist/controllers/sqlite.js.map +1 -1
  21. package/dist/controllers/store.d.ts +1 -0
  22. package/dist/extension/controller-command.d.ts +22 -0
  23. package/dist/extension/controller-command.js +32 -0
  24. package/dist/extension/controller-command.js.map +1 -0
  25. package/dist/extension/index.d.ts +2 -9
  26. package/dist/extension/index.js +876 -2928
  27. package/dist/extension/index.js.map +1 -1
  28. package/dist/extension/message-card.d.ts +13 -0
  29. package/dist/extension/message-card.js +46 -0
  30. package/dist/extension/message-card.js.map +1 -0
  31. package/dist/extension/step-message.d.ts +5 -6
  32. package/dist/extension/step-message.js +41 -52
  33. package/dist/extension/step-message.js.map +1 -1
  34. package/dist/host/child-worker-supervisor.d.ts +50 -0
  35. package/dist/host/child-worker-supervisor.js +159 -0
  36. package/dist/host/child-worker-supervisor.js.map +1 -0
  37. package/dist/host/client.d.ts +48 -0
  38. package/dist/host/client.js +216 -0
  39. package/dist/host/client.js.map +1 -0
  40. package/dist/host/controller-worker-entry.d.ts +2 -0
  41. package/dist/host/controller-worker-entry.js +244 -0
  42. package/dist/host/controller-worker-entry.js.map +1 -0
  43. package/dist/host/controller-worker-protocol.d.ts +36 -0
  44. package/dist/host/controller-worker-protocol.js +49 -0
  45. package/dist/host/controller-worker-protocol.js.map +1 -0
  46. package/dist/host/controller-worker-supervisor.d.ts +21 -0
  47. package/dist/host/controller-worker-supervisor.js +54 -0
  48. package/dist/host/controller-worker-supervisor.js.map +1 -0
  49. package/dist/host/host-entry.d.ts +2 -0
  50. package/dist/host/host-entry.js +23 -0
  51. package/dist/host/host-entry.js.map +1 -0
  52. package/dist/host/processes.d.ts +17 -12
  53. package/dist/host/processes.js +154 -50
  54. package/dist/host/processes.js.map +1 -1
  55. package/dist/host/protocol.d.ts +38 -0
  56. package/dist/host/protocol.js +156 -0
  57. package/dist/host/protocol.js.map +1 -0
  58. package/dist/host/resolver-entry.d.ts +49 -0
  59. package/dist/host/resolver-entry.js +149 -0
  60. package/dist/host/resolver-entry.js.map +1 -0
  61. package/dist/host/rpc-executor.d.ts +14 -3
  62. package/dist/host/rpc-executor.js +63 -30
  63. package/dist/host/rpc-executor.js.map +1 -1
  64. package/dist/host/runner.d.ts +95 -29
  65. package/dist/host/runner.js +2311 -348
  66. package/dist/host/runner.js.map +1 -1
  67. package/dist/host/state.d.ts +174 -0
  68. package/dist/host/state.js +689 -0
  69. package/dist/host/state.js.map +1 -0
  70. package/dist/host/worker-entry.d.ts +10 -0
  71. package/dist/host/worker-entry.js +446 -0
  72. package/dist/host/worker-entry.js.map +1 -0
  73. package/dist/host/worker-protocol.d.ts +31 -0
  74. package/dist/host/worker-protocol.js +122 -0
  75. package/dist/host/worker-protocol.js.map +1 -0
  76. package/dist/host/worker-store.d.ts +80 -0
  77. package/dist/host/worker-store.js +91 -0
  78. package/dist/host/worker-store.js.map +1 -0
  79. package/dist/host/worker-supervisor.d.ts +22 -0
  80. package/dist/host/worker-supervisor.js +54 -0
  81. package/dist/host/worker-supervisor.js.map +1 -0
  82. package/dist/render/canvas.d.ts +2 -1
  83. package/dist/render/canvas.js +40 -15
  84. package/dist/render/canvas.js.map +1 -1
  85. package/dist/render/graph-render.d.ts +7 -3
  86. package/dist/render/graph-render.js +125 -74
  87. package/dist/render/graph-render.js.map +1 -1
  88. package/dist/state/database.js +3 -1
  89. package/dist/state/database.js.map +1 -1
  90. package/dist/state/index.d.ts +1 -0
  91. package/dist/state/index.js +1 -0
  92. package/dist/state/index.js.map +1 -1
  93. package/dist/state/prune.js +36 -10
  94. package/dist/state/prune.js.map +1 -1
  95. package/dist/state/schema.d.ts +1 -1
  96. package/dist/state/schema.js +171 -3
  97. package/dist/state/schema.js.map +1 -1
  98. package/dist/state/viewer.d.ts +46 -0
  99. package/dist/state/viewer.js +249 -0
  100. package/dist/state/viewer.js.map +1 -0
  101. package/dist/viewer/cli.d.ts +1 -1
  102. package/dist/viewer/cli.js +47 -18
  103. package/dist/viewer/cli.js.map +1 -1
  104. package/dist/workflows/composition.js +25 -2
  105. package/dist/workflows/composition.js.map +1 -1
  106. package/dist/workflows/definition.d.ts +3 -1
  107. package/dist/workflows/definition.js +25 -0
  108. package/dist/workflows/definition.js.map +1 -1
  109. package/dist/workflows/engine.d.ts +1 -0
  110. package/dist/workflows/engine.js +157 -42
  111. package/dist/workflows/engine.js.map +1 -1
  112. package/dist/workflows/errors.d.ts +3 -1
  113. package/dist/workflows/errors.js +4 -7
  114. package/dist/workflows/errors.js.map +1 -1
  115. package/dist/workflows/human-decision.d.ts +3 -0
  116. package/dist/workflows/human-decision.js +31 -0
  117. package/dist/workflows/human-decision.js.map +1 -1
  118. package/dist/workflows/index.d.ts +1 -1
  119. package/dist/workflows/index.js +1 -1
  120. package/dist/workflows/index.js.map +1 -1
  121. package/dist/workflows/schema.js +17 -6
  122. package/dist/workflows/schema.js.map +1 -1
  123. package/dist/{viewer → workflows}/session-reducer.d.ts +3 -1
  124. package/dist/{viewer → workflows}/session-reducer.js +4 -0
  125. package/dist/workflows/session-reducer.js.map +1 -0
  126. package/dist/workflows/store.d.ts +87 -1
  127. package/dist/workflows/store.js +892 -93
  128. package/dist/workflows/store.js.map +1 -1
  129. package/dist/workflows/tool-input.d.ts +0 -22
  130. package/dist/workflows/tool-input.js +1 -44
  131. package/dist/workflows/tool-input.js.map +1 -1
  132. package/dist/workflows/types.d.ts +36 -4
  133. package/docs/2026-08-25-workflow-follow-ups.md +8 -6
  134. package/docs/2026-08-30-out-of-process-workflow-host-plan.md +353 -0
  135. package/docs/CONTROLLERS.md +11 -11
  136. package/docs/DEFERRED_TURNS.md +61 -27
  137. package/docs/HUMAN_DECISIONS.md +12 -4
  138. package/docs/SQLITE_STATE.md +42 -8
  139. package/docs/WORKFLOW_HOST.md +452 -0
  140. package/docs/development.md +46 -30
  141. package/docs/live-replay-protocol.md +129 -100
  142. package/docs/plans/2026-08-19-human-decision-gates-plan.md +34 -8
  143. package/docs/plans/2026-08-27-workflow-terminal-restart-plan.md +399 -0
  144. package/docs/plans/2026-08-28-piw-incremental-viewer-plan.md +293 -0
  145. package/docs/plans/piw-viewer-experience-implementation-plan.md +11 -1
  146. package/docs/plans/tui-viewer-implementation-plan.md +5 -0
  147. package/docs/tui-viewer.md +18 -3
  148. package/docs/workflows.md +164 -156
  149. package/examples/workflows/command-batch.workflow.ts +2 -0
  150. package/examples/workflows/shell.workflow.ts +2 -1
  151. package/herdr-plugin.toml +1 -1
  152. package/package.json +1 -1
  153. package/skills/autodoc/SKILL.md +1 -1
  154. package/skills/autoimplement/SKILL.md +1 -1
  155. package/skills/autoplan/SKILL.md +1 -1
  156. package/skills/pi-workflows/SKILL.md +2 -0
  157. package/src/builtins/autoimplement.workflow.ts +3 -0
  158. package/src/builtins/change-verification.workflow.ts +7 -2
  159. package/src/builtins/metadata.ts +9 -0
  160. package/src/builtins/monitor.workflow.ts +4 -0
  161. package/src/builtins/sanity-check.workflow.ts +4 -0
  162. package/src/builtins/workspace-preparation.workflow.ts +3 -1
  163. package/src/controllers/index.ts +3 -5
  164. package/src/controllers/sqlite.ts +929 -225
  165. package/src/controllers/store.ts +1 -0
  166. package/src/extension/controller-command.ts +45 -0
  167. package/src/extension/index.ts +948 -3506
  168. package/src/extension/message-card.ts +61 -0
  169. package/src/extension/step-message.ts +58 -63
  170. package/src/host/child-worker-supervisor.ts +183 -0
  171. package/src/host/client.ts +293 -0
  172. package/src/host/controller-worker-entry.ts +311 -0
  173. package/src/host/controller-worker-protocol.ts +104 -0
  174. package/src/host/controller-worker-supervisor.ts +79 -0
  175. package/src/host/host-entry.ts +23 -0
  176. package/src/host/processes.ts +171 -54
  177. package/src/host/protocol.ts +196 -0
  178. package/src/host/resolver-entry.ts +241 -0
  179. package/src/host/rpc-executor.ts +76 -34
  180. package/src/host/runner.ts +2813 -422
  181. package/src/host/state.ts +1160 -0
  182. package/src/host/worker-entry.ts +533 -0
  183. package/src/host/worker-protocol.ts +165 -0
  184. package/src/host/worker-store.ts +229 -0
  185. package/src/host/worker-supervisor.ts +74 -0
  186. package/src/render/canvas.ts +44 -10
  187. package/src/render/graph-render.ts +145 -90
  188. package/src/state/database.ts +2 -1
  189. package/src/state/index.ts +14 -0
  190. package/src/state/prune.ts +35 -9
  191. package/src/state/schema.ts +171 -3
  192. package/src/state/viewer.ts +356 -0
  193. package/src/viewer/cli.ts +49 -17
  194. package/src/workflows/composition.ts +36 -2
  195. package/src/workflows/definition.ts +32 -0
  196. package/src/workflows/engine.ts +157 -54
  197. package/src/workflows/errors.ts +11 -2
  198. package/src/workflows/human-decision.ts +49 -0
  199. package/src/workflows/index.ts +2 -0
  200. package/src/workflows/schema.ts +19 -6
  201. package/src/{viewer → workflows}/session-reducer.ts +11 -1
  202. package/src/workflows/store.ts +1316 -108
  203. package/src/workflows/tool-input.ts +2 -60
  204. package/src/workflows/types.ts +32 -4
  205. package/dist/controllers/workflow-engine-scheduler.d.ts +0 -29
  206. package/dist/controllers/workflow-engine-scheduler.js +0 -175
  207. package/dist/controllers/workflow-engine-scheduler.js.map +0 -1
  208. package/dist/extension/controller-host.d.ts +0 -48
  209. package/dist/extension/controller-host.js +0 -110
  210. package/dist/extension/controller-host.js.map +0 -1
  211. package/dist/extension/deferred-turn-coordinator.d.ts +0 -32
  212. package/dist/extension/deferred-turn-coordinator.js +0 -143
  213. package/dist/extension/deferred-turn-coordinator.js.map +0 -1
  214. package/dist/extension/deferred-turn.d.ts +0 -44
  215. package/dist/extension/deferred-turn.js +0 -110
  216. package/dist/extension/deferred-turn.js.map +0 -1
  217. package/dist/extension/executor.d.ts +0 -86
  218. package/dist/extension/executor.js +0 -311
  219. package/dist/extension/executor.js.map +0 -1
  220. package/dist/extension/follow-up-coordinator.d.ts +0 -27
  221. package/dist/extension/follow-up-coordinator.js +0 -131
  222. package/dist/extension/follow-up-coordinator.js.map +0 -1
  223. package/dist/extension/recorder.d.ts +0 -84
  224. package/dist/extension/recorder.js +0 -528
  225. package/dist/extension/recorder.js.map +0 -1
  226. package/dist/extension/session-events.d.ts +0 -133
  227. package/dist/extension/session-events.js +0 -61
  228. package/dist/extension/session-events.js.map +0 -1
  229. package/dist/viewer/session-reducer.js.map +0 -1
  230. package/src/controllers/workflow-engine-scheduler.ts +0 -246
  231. package/src/extension/controller-host.ts +0 -167
  232. package/src/extension/deferred-turn-coordinator.ts +0 -171
  233. package/src/extension/deferred-turn.ts +0 -166
  234. package/src/extension/executor.ts +0 -411
  235. package/src/extension/follow-up-coordinator.ts +0 -151
  236. package/src/extension/recorder.ts +0 -655
  237. package/src/extension/session-events.ts +0 -121
@@ -1,519 +1,2910 @@
1
1
  import { createHash, randomUUID } from "node:crypto";
2
+ import { once } from "node:events";
2
3
  import fs from "node:fs";
4
+ import net, { type Socket } from "node:net";
3
5
  import path from "node:path";
4
- import { builtinWorkflowCatalog } from "../builtins/catalog.js";
6
+ import { applyStatusPatch } from "../controllers/conditions.js";
7
+ import { ResourceConflictError } from "../controllers/errors.js";
8
+ import {
9
+ controllerFileStem,
10
+ controllerSearchDirs,
11
+ discoverControllers,
12
+ } from "../controllers/loader.js";
5
13
  import {
6
- ControllerManager,
7
- loadDiscoveredControllers,
8
14
  SqliteControllerStore,
9
- WorkflowEngineScheduler,
10
15
  type WorkflowRunQueueRecord,
11
- } from "../controllers/index.js";
12
- import type { JsonObject } from "../controllers/types.js";
13
- import { workflowStatePath } from "../state/database.js";
14
- import { WorkflowEngine } from "../workflows/engine.js";
16
+ type WorkflowTurnIntentFacts,
17
+ } from "../controllers/sqlite.js";
18
+ import type {
19
+ ControllerQueueClaim,
20
+ ControllerResource,
21
+ ControllerSettingsChangeRequest,
22
+ ControllerSettingsChangeResult,
23
+ ReconcileResult,
24
+ } from "../controllers/types.js";
25
+ import {
26
+ ControllerWorkflowCoordinator,
27
+ type ControllerWorkflowControlRequest,
28
+ type ControllerWorkflowScheduler,
29
+ type WorkflowSchedulerRequest,
30
+ type WorkflowSchedulerResult,
31
+ } from "../controllers/workflows.js";
32
+ import { StateDatabase, workflowStatePath } from "../state/database.js";
33
+ import { canonicalJson, type JsonValue } from "../state/json.js";
34
+ import { recordViewerDeltas } from "../state/viewer.js";
35
+ import { errorMessage } from "../workflows/errors.js";
36
+ import { HumanDecisionStore } from "../workflows/human-decision.js";
15
37
  import {
16
- ClaimLostError,
17
- errorMessage,
18
- isClaimLostError,
19
- WorkflowSourceChangedError,
20
- } from "../workflows/errors.js";
21
- import { resolveWorkflowRef, resolveWorkflowSource } from "../workflows/loader.js";
38
+ type WorkflowSettingsDefinition,
39
+ type WorkflowSettingsPathRule,
40
+ } from "../workflows/settings.js";
22
41
  import { WorkflowRunStore } from "../workflows/store.js";
23
- import type { ResolvedHumanDecision, WorkflowDefinition } from "../workflows/types.js";
24
- import { HostProcessRegistry } from "./processes.js";
25
- import { RpcStepExecutor } from "./rpc-executor.js";
42
+ import type {
43
+ HumanDecisionRequest,
44
+ HumanDecisionResponse,
45
+ ResolvedHumanDecision,
46
+ WorkflowDefinitionSnapshot,
47
+ WorkflowRunState,
48
+ WorkflowTraceEventDraft,
49
+ WorkflowUpdateInput,
50
+ } from "../workflows/types.js";
51
+ import { validateWorkflowUpdate } from "../workflows/updates.js";
52
+ import { WorkflowHostClient } from "./client.js";
53
+ import type {
54
+ ControllerWorkerLaunchEnvelope,
55
+ ControllerWorkerMessage,
56
+ ControllerWorkerResponse,
57
+ } from "./controller-worker-protocol.js";
58
+ import { ControllerWorkerSupervisor } from "./controller-worker-supervisor.js";
59
+ import {
60
+ HostProcessRegistry,
61
+ matchesProcessIdentity,
62
+ processParentPid,
63
+ processStartIdentity,
64
+ } from "./processes.js";
65
+ import {
66
+ encodeProtocolLine,
67
+ hostSocketPath,
68
+ NdjsonFrameDecoder,
69
+ parseHostRequest,
70
+ type HostRequest,
71
+ type HostResponse,
72
+ } from "./protocol.js";
73
+ import {
74
+ HostStateStore,
75
+ type HostClaim,
76
+ type InteractiveRequestRecord,
77
+ type InteractiveSubmissionRecord,
78
+ type WorkerLaunchEnvelope,
79
+ } from "./state.js";
80
+ import type { WorkerMessage, WorkerResponse } from "./worker-protocol.js";
81
+ import { WorkflowWorkerSupervisor } from "./worker-supervisor.js";
26
82
 
83
+ const HOST_LEASE_MS = 30_000;
84
+ const HOST_RENEW_MS = 10_000;
27
85
  const CLAIM_POLL_MS = 2_000;
28
86
  const RUN_CLAIM_LEASE_MS = 30_000;
29
- const RUN_CLAIM_RENEW_MS = 10_000;
87
+ const PRESENTATION_CLAIM_LEASE_MS = 10_000;
88
+ const DELIVERY_CLAIM_LEASE_MS = 10_000;
89
+ const CONTROLLER_CLAIM_LEASE_MS = 120_000;
90
+ const CONTROLLER_RENEW_MS = 30_000;
91
+ const MAX_CONTROLLER_WORKERS = 4;
92
+ const DEFAULT_CONTROLLER_TIMEOUT_MS = 60_000;
30
93
 
31
94
  export type WorkflowHostOptions = {
32
- cwd: string;
95
+ /** Retained for callers; the host itself is global to the state database. */
96
+ cwd?: string;
33
97
  runnerId?: string;
34
- /** Explicit database path; defaults to the canonical workflow state database. */
35
98
  databasePath?: string;
36
99
  registry?: HostProcessRegistry;
37
100
  piArgs?: string[];
38
- /** Extra environment for headless children (for example a test provider). */
39
101
  env?: Record<string, string>;
40
- /** Poll interval for the claim loop; tests use a faster cadence. */
41
102
  claimPollMs?: number;
103
+ hostLeaseMs?: number;
104
+ hostRenewMs?: number;
105
+ runClaimLeaseMs?: number;
106
+ workerEntryPath?: string;
107
+ workerStartupTimeoutMs?: number;
108
+ controllerWorkerEntryPath?: string;
42
109
  onLog?: (message: string) => void;
43
110
  };
44
111
 
45
- /**
46
- * The always-on runner: claims parked workflow runs and reconciles durable
47
- * controllers without a Pi session. Conversation nodes execute in headless
48
- * `pi --mode rpc` children. Everything the host does is recoverable: claims
49
- * expire, SQLite leases fence stale writers, and child processes are reaped by
50
- * the next host.
51
- */
112
+ type ActiveRun = {
113
+ record: WorkflowRunQueueRecord;
114
+ claimToken: string;
115
+ generation: number;
116
+ supervisor: WorkflowWorkerSupervisor;
117
+ workerPid?: number;
118
+ exiting: boolean;
119
+ workflowLoadFailure?: string;
120
+ control?: "cancel" | "pause" | "handoff";
121
+ claimLost?: boolean;
122
+ };
123
+
124
+ type DeliveryClaim = {
125
+ clientId: string;
126
+ token: string;
127
+ targetSessionId: string;
128
+ kind: "notification" | "turn";
129
+ resourceId: string;
130
+ expiresAt: number;
131
+ };
132
+
133
+ type ActiveController = {
134
+ key: string;
135
+ projectPath: string;
136
+ store: SqliteControllerStore;
137
+ claim: ControllerQueueClaim;
138
+ resource: ControllerResource;
139
+ reconcileId: string;
140
+ startedAt: number;
141
+ supervisor: ControllerWorkerSupervisor;
142
+ renewTimer: ReturnType<typeof setInterval>;
143
+ settled: boolean;
144
+ };
145
+
146
+ /** Global package-owned host. It writes state and supervises code-only workers. */
52
147
  export class WorkflowHost {
53
148
  private readonly options: WorkflowHostOptions;
54
- private readonly runnerId: string;
149
+ private readonly hostId: string;
150
+ private readonly databasePath: string;
151
+ private readonly stateDirectory: string;
152
+ private readonly socketPath: string;
153
+ private readonly lockPath: string;
154
+ private readonly state: StateDatabase;
155
+ private readonly hostState: HostStateStore;
156
+ private readonly queue: SqliteControllerStore;
157
+ private readonly decisions: HumanDecisionStore;
158
+ private readonly runStore: WorkflowRunStore;
55
159
  private readonly registry: HostProcessRegistry;
56
- private readonly store: SqliteControllerStore;
57
- private readonly childRunStore: WorkflowRunStore;
58
- private manager: ControllerManager | null = null;
160
+ private readonly activeRuns = new Map<string, ActiveRun>();
161
+ private readonly workerDescendants = new Map<string, Set<number>>();
162
+ private readonly activeControllers = new Map<string, ActiveController>();
163
+ private readonly controlClaims = new Map<string, string>();
164
+ private readonly activationTasks = new Map<string, Promise<void>>();
165
+ private readonly pendingStarts = new Set<string>();
166
+ private readonly pendingRunClaims = new Map<string, string>();
167
+ private readonly pendingResumes = new Set<string>();
168
+ private readonly blockedRuns = new Set<string>();
169
+ private readonly deliveryClaims = new Map<string, DeliveryClaim>();
170
+ private readonly sockets = new Set<Socket>();
171
+ private server: net.Server | null = null;
172
+ private claim: HostClaim | null = null;
173
+ private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
59
174
  private pollTimer: ReturnType<typeof setInterval> | null = null;
60
- private readonly activeRuns = new Map<string, Promise<void>>();
61
- private readonly schedulerExecutors = new Map<WorkflowEngine, RpcStepExecutor>();
62
- /** Runs whose resume refused (edited source); skipped until a host restart. */
63
- private readonly skippedRuns = new Set<string>();
64
175
  private stopping = false;
176
+ private started = false;
177
+ private controllerPollActive = false;
178
+ private decisionTimeoutActive = false;
65
179
 
66
- private readonly stateDir: string;
67
-
68
- constructor(options: WorkflowHostOptions) {
180
+ constructor(options: WorkflowHostOptions = {}) {
69
181
  this.options = options;
70
- this.runnerId = options.runnerId ?? `host-${randomUUID().slice(0, 8)}`;
71
- const databasePath = options.databasePath ?? workflowStatePath();
72
- this.store = new SqliteControllerStore(databasePath, { projectPath: options.cwd });
73
- this.stateDir = path.join(
74
- path.dirname(this.store.filePath),
75
- "hosts",
76
- hostProjectScope(options.cwd),
77
- );
78
- this.registry = options.registry ?? new HostProcessRegistry(this.stateDir);
79
- this.childRunStore = new WorkflowRunStore(databasePath);
182
+ this.hostId = options.runnerId ?? `host-${randomUUID()}`;
183
+ this.databasePath = path.resolve(options.databasePath ?? workflowStatePath());
184
+ this.stateDirectory = path.join(path.dirname(this.databasePath), "host");
185
+ this.socketPath = hostSocketPath(this.databasePath);
186
+ this.lockPath = path.join(this.stateDirectory, "host.lock.json");
187
+ this.state = new StateDatabase({ filePath: this.databasePath });
188
+ this.hostState = new HostStateStore(this.databasePath, { state: this.state });
189
+ this.queue = new SqliteControllerStore(this.databasePath, { state: this.state, global: true });
190
+ this.decisions = new HumanDecisionStore(this.databasePath, {
191
+ state: this.state,
192
+ authorityProvider: (runId) => {
193
+ const token = this.controlClaims.get(runId) ?? this.activeRuns.get(runId)?.claimToken;
194
+ return token === undefined ? undefined : this.queue.workflowRunAuthority(runId, token);
195
+ },
196
+ });
197
+ this.runStore = new WorkflowRunStore(this.databasePath, {
198
+ state: this.state,
199
+ authorityProvider: (runId) => {
200
+ const token = this.controlClaims.get(runId) ?? this.activeRuns.get(runId)?.claimToken;
201
+ return token === undefined ? undefined : this.queue.workflowRunAuthority(runId, token);
202
+ },
203
+ snapshotLifecycle: (context) => this.applyLifecycleProjection(context),
204
+ });
205
+ this.registry = options.registry ?? new HostProcessRegistry(this.stateDirectory);
80
206
  }
81
207
 
82
- private log(message: string): void {
83
- this.options.onLog?.(message);
208
+ get endpoint(): string {
209
+ return this.socketPath;
210
+ }
211
+
212
+ private get runClaimLeaseMs(): number {
213
+ return this.options.runClaimLeaseMs ?? RUN_CLAIM_LEASE_MS;
84
214
  }
85
215
 
86
- /** Take the advisory lock, reap orphans, and start claiming. */
87
216
  async start(): Promise<void> {
88
- // The lock comes first: a second host must refuse before touching the
89
- // children registry, or it would kill the live host's child processes
90
- // as supposed orphans.
91
- acquireHostLock(this.stateDir, this.runnerId, this.options.cwd);
92
- const reaped = this.registry.reapOrphans();
93
- if (reaped.length > 0) {
94
- this.log(`reaped ${reaped.length} orphaned headless session(s): ${reaped.join(", ")}`);
95
- }
96
-
97
- const definitions = await loadDiscoveredControllers({ cwd: this.options.cwd });
98
- if (definitions.length > 0) {
99
- const scheduler = new WorkflowEngineScheduler({
100
- store: this.childRunStore,
101
- resolveWorkflow: async (name) => {
102
- const resolved = await resolveWorkflowRef(
103
- name,
104
- { cwd: this.options.cwd },
105
- builtinWorkflowCatalog,
106
- );
107
- return { workflow: resolved.definition, workflowSource: resolved.source };
108
- },
109
- createEngine: () => {
110
- const executor = new RpcStepExecutor({
111
- cwd: this.options.cwd,
112
- registry: this.registry,
113
- ...(this.options.piArgs !== undefined ? { piArgs: this.options.piArgs } : {}),
114
- ...(this.options.env !== undefined ? { env: this.options.env } : {}),
115
- });
116
- const engine = new WorkflowEngine({ executor, store: this.childRunStore });
117
- this.schedulerExecutors.set(engine, executor);
118
- return engine;
119
- },
120
- disposeEngine: async (engine) => {
121
- const executor = this.schedulerExecutors.get(engine);
122
- if (executor !== undefined) {
123
- this.schedulerExecutors.delete(engine);
124
- await executor.close();
125
- }
126
- },
127
- });
128
- this.manager = new ControllerManager({
129
- store: this.store,
130
- controllers: definitions,
131
- workflowScheduler: scheduler,
217
+ if (this.started) return;
218
+ fs.mkdirSync(this.stateDirectory, { recursive: true, mode: 0o700 });
219
+ fs.chmodSync(this.stateDirectory, 0o700);
220
+ const startIdentity = processStartIdentity(process.pid);
221
+ if (startIdentity === undefined) {
222
+ throw new Error("Cannot attest the workflow host process start identity");
223
+ }
224
+ acquireHostLock(this.lockPath, { pid: process.pid, startIdentity, hostId: this.hostId });
225
+ try {
226
+ const reaped = this.registry.reapOrphans();
227
+ if (reaped.length > 0) this.log(`reaped ${reaped.length} exact orphan process(es)`);
228
+ const previousHost = this.hostState.hostStatus();
229
+ this.claim = this.hostState.acquireHost({
230
+ hostId: this.hostId,
231
+ pid: process.pid,
232
+ processStartIdentity: startIdentity,
233
+ leaseMs: this.options.hostLeaseMs ?? HOST_LEASE_MS,
132
234
  });
133
- this.manager.start();
134
- this.log(`controller workers started for ${definitions.length} controller(s)`);
235
+ this.recoverPreviousHost(previousHost.hostId, this.claim.epoch);
236
+ await this.listen();
237
+ this.startTimers();
238
+ this.started = true;
239
+ this.log(`ready on ${this.socketPath} at epoch ${this.claim.epoch}`);
240
+ this.expireTimedOutInteraction();
241
+ void this.expireTimedOutDecision();
242
+ void this.claimOne();
243
+ void this.claimControllerOne();
244
+ } catch (error) {
245
+ const server = this.server;
246
+ this.server = null;
247
+ await this.closeServer(server);
248
+ if (this.claim !== null) {
249
+ try {
250
+ this.hostState.releaseHost(this.claim);
251
+ } catch {
252
+ // Preserve the startup error when cleanup cannot release an already-lost claim.
253
+ }
254
+ }
255
+ this.claim = null;
256
+ if (process.platform !== "win32") fs.rmSync(this.socketPath, { force: true });
257
+ this.releaseLock();
258
+ throw error;
135
259
  }
136
-
137
- this.pollTimer = setInterval(() => {
138
- this.claimOnce();
139
- }, this.options.claimPollMs ?? CLAIM_POLL_MS);
140
- this.pollTimer.unref?.();
141
- this.claimOnce();
142
- this.log(`host ${this.runnerId} watching ${this.options.cwd}`);
143
260
  }
144
261
 
145
- /** Drain: stop claiming, park in-flight runs, stop controllers, kill children. */
146
262
  async stop(): Promise<void> {
147
- if (this.stopping) {
148
- return;
149
- }
263
+ if (this.stopping) return;
150
264
  this.stopping = true;
151
- if (this.pollTimer !== null) {
152
- clearInterval(this.pollTimer);
153
- this.pollTimer = null;
154
- }
155
- const pending = [...this.activeRuns.values()];
156
- for (const run of this.parkedEngines.splice(0)) {
157
- run();
158
- }
159
- await Promise.allSettled(pending);
160
- await this.manager?.stop().catch(() => undefined);
161
- for (const executor of this.schedulerExecutors.values()) {
162
- await executor.close().catch(() => undefined);
265
+ if (this.heartbeatTimer !== null) clearInterval(this.heartbeatTimer);
266
+ if (this.pollTimer !== null) clearInterval(this.pollTimer);
267
+ this.heartbeatTimer = null;
268
+ this.pollTimer = null;
269
+ const server = this.server;
270
+ this.server = null;
271
+ await this.closeServer(server);
272
+ for (const [runId, claimToken] of this.pendingRunClaims) {
273
+ this.queue.parkWorkflowRun({ runId, claimToken });
163
274
  }
164
- this.schedulerExecutors.clear();
275
+ this.pendingRunClaims.clear();
276
+ this.pendingStarts.clear();
277
+ await Promise.allSettled(
278
+ [...this.activeRuns.values()].map(async (active) => {
279
+ active.control = "handoff";
280
+ await active.supervisor.stop("orphaned");
281
+ this.queue.parkWorkflowRun({ runId: active.record.runId, claimToken: active.claimToken });
282
+ }),
283
+ );
284
+ await Promise.allSettled(
285
+ [...this.activeControllers.values()].map(async (active) => {
286
+ clearInterval(active.renewTimer);
287
+ await active.supervisor.stop("orphaned");
288
+ if (!active.settled) {
289
+ active.store.requeueClaim(
290
+ active.claim,
291
+ { availableAt: new Date().toISOString(), error: "Workflow host stopped" },
292
+ new Date().toISOString(),
293
+ );
294
+ }
295
+ }),
296
+ );
297
+ await Promise.allSettled(this.activationTasks.values());
165
298
  this.registry.killAll();
166
- releaseHostLock(this.stateDir, this.runnerId);
167
- this.store.close();
299
+ this.deliveryClaims.clear();
300
+ if (this.claim !== null) this.hostState.releaseHost(this.claim);
301
+ this.claim = null;
302
+ if (process.platform !== "win32") fs.rmSync(this.socketPath, { force: true });
303
+ this.releaseLock();
304
+ this.runStore.close();
305
+ this.queue.close();
306
+ this.hostState.close();
307
+ this.state.close();
308
+ this.started = false;
168
309
  }
169
310
 
170
- private readonly parkedEngines: Array<() => void> = [];
311
+ private async closeServer(server: net.Server | null): Promise<void> {
312
+ const closed = new Promise<void>((resolve) => {
313
+ if (server === null || !server.listening) {
314
+ resolve();
315
+ return;
316
+ }
317
+ try {
318
+ server.close(() => resolve());
319
+ } catch {
320
+ resolve();
321
+ }
322
+ });
323
+ for (const socket of this.sockets) socket.destroy();
324
+ await closed;
325
+ this.sockets.clear();
326
+ }
171
327
 
172
- private claimOnce(): void {
173
- if (this.stopping || this.activeRuns.size > 0) {
174
- return;
175
- }
176
- let claimed: WorkflowRunQueueRecord | undefined;
177
- try {
178
- claimed = this.store.claimNextWorkflowRun({
179
- runnerId: this.runnerId,
180
- claimToken: randomUUID(),
181
- leaseMs: RUN_CLAIM_LEASE_MS,
182
- excludeRunIds: [...this.skippedRuns],
183
- });
184
- } catch (error) {
185
- // Store contention or corruption must not kill the host's loop.
186
- this.log(`claim failed, retrying shortly: ${errorMessage(error)}`);
187
- return;
188
- }
189
- if (claimed === undefined) {
190
- return;
191
- }
192
- const task = this.runClaimed(claimed).finally(() => {
193
- this.activeRuns.delete(claimed.runId);
328
+ private recoverPreviousHost(previousHostId: string | null, hostEpoch: number): void {
329
+ if (previousHostId === null || previousHostId === this.hostId) return;
330
+ const now = Date.now();
331
+ this.state.transaction(() => {
332
+ this.state.connection
333
+ .prepare(
334
+ `UPDATE leases SET expires_at = ?
335
+ WHERE owner_id = ? AND owner_type IN ('host', 'controller') AND expires_at > ?`,
336
+ )
337
+ .run(now, previousHostId, now);
338
+ this.state.connection
339
+ .prepare(
340
+ `UPDATE run_workers SET status = 'orphaned', finished_at = ?
341
+ WHERE host_epoch < ? AND status IN ('starting', 'ready', 'running')`,
342
+ )
343
+ .run(now, hostEpoch);
194
344
  });
195
- this.activeRuns.set(claimed.runId, task);
196
345
  }
197
346
 
198
- private async runClaimed(record: WorkflowRunQueueRecord): Promise<void> {
199
- const claimToken = record.claimToken as string;
200
- const runId = record.runId;
201
- this.log(`resuming ${record.workflowName} run ${runId}`);
202
- let workflow: WorkflowDefinition;
203
- let workflowSource: import("../workflows/types.js").WorkflowSource;
204
- try {
205
- const bundle = this.childRunStore.readRun(runId);
206
- if (bundle?.state.status === "waiting") {
207
- if (this.store.parkWorkflowRun({ runId, claimToken })) {
208
- this.store.settleRunEffect(runId, "run.park_queue");
209
- this.skippedRuns.add(runId);
210
- this.log(`left ${record.workflowName} run ${runId} waiting for its decision owner`);
347
+ private startTimers(): void {
348
+ this.heartbeatTimer = setInterval(() => {
349
+ try {
350
+ if (this.claim === null) return;
351
+ this.claim = this.hostState.renewHost(
352
+ this.claim,
353
+ this.options.hostLeaseMs ?? HOST_LEASE_MS,
354
+ );
355
+ for (const active of this.activeRuns.values()) {
356
+ if (active.control !== undefined || active.exiting) continue;
357
+ if (
358
+ !this.queue.renewWorkflowRunClaim({
359
+ runId: active.record.runId,
360
+ claimToken: active.claimToken,
361
+ leaseMs: this.runClaimLeaseMs,
362
+ })
363
+ ) {
364
+ active.claimLost = true;
365
+ void active.supervisor.stop("claimLost");
366
+ }
211
367
  }
212
- return;
213
- }
214
- if (bundle?.state.workflowSource === undefined) {
215
- throw new Error(`Workflow run ${runId} has no canonical workflow source`);
368
+ } catch (error) {
369
+ this.log(`host claim lost: ${errorMessage(error)}`);
370
+ void this.stop();
216
371
  }
217
- workflow = await resolveWorkflowSource(
218
- bundle.state.workflowSource,
219
- builtinWorkflowCatalog,
220
- runId,
221
- );
222
- workflowSource = bundle.state.workflowSource;
223
- } catch (error) {
224
- if (error instanceof WorkflowSourceChangedError) {
225
- try {
226
- this.store.parkWorkflowRun({ runId, claimToken });
227
- } catch {
228
- // Best-effort.
229
- }
230
- this.skippedRuns.add(runId);
231
- this.recordEvent(runId, record.workflowName, "parked", {
232
- reason: "workflow source changed",
233
- });
234
- this.log(
235
- `run ${runId} skipped: workflow source changed; install the matching package revision, then restart the host`,
236
- );
237
- return;
372
+ }, this.options.hostRenewMs ?? HOST_RENEW_MS);
373
+ this.heartbeatTimer.unref?.();
374
+ this.pollTimer = setInterval(() => {
375
+ this.expireTimedOutInteraction();
376
+ void this.expireTimedOutDecision();
377
+ void this.claimOne();
378
+ void this.claimControllerOne();
379
+ }, this.options.claimPollMs ?? CLAIM_POLL_MS);
380
+ this.pollTimer.unref?.();
381
+ }
382
+
383
+ private async listen(): Promise<void> {
384
+ if (process.platform !== "win32") fs.rmSync(this.socketPath, { force: true });
385
+ const server = net.createServer((socket) => this.handleConnection(socket));
386
+ this.server = server;
387
+ await new Promise<void>((resolve, reject) => {
388
+ const cleanup = () => {
389
+ server.off("listening", onListening);
390
+ server.off("error", onError);
391
+ };
392
+ const onListening = () => {
393
+ cleanup();
394
+ resolve();
395
+ };
396
+ const onError = (error: Error) => {
397
+ cleanup();
398
+ reject(error);
399
+ };
400
+ server.once("listening", onListening);
401
+ server.once("error", onError);
402
+ try {
403
+ server.listen(this.socketPath);
404
+ } catch (error) {
405
+ cleanup();
406
+ reject(error);
238
407
  }
239
- await this.failUnresumable(record, claimToken, errorMessage(error));
240
- return;
241
- }
242
- if (this.stopping) {
243
- // The drain started during setup: park before anything executes.
408
+ });
409
+ server.on("error", (error) => this.log(`socket error: ${errorMessage(error)}`));
410
+ if (process.platform !== "win32") fs.chmodSync(this.socketPath, 0o600);
411
+ }
412
+
413
+ private handleConnection(socket: Socket): void {
414
+ this.sockets.add(socket);
415
+ const decoder = new NdjsonFrameDecoder();
416
+ let processing = Promise.resolve();
417
+ socket.on("data", (chunk: Buffer) => {
418
+ let frames: Buffer[];
244
419
  try {
245
- this.store.parkWorkflowRun({ runId, claimToken });
420
+ frames = decoder.push(chunk);
246
421
  } catch {
247
- // Best-effort.
422
+ socket.destroy();
423
+ return;
248
424
  }
249
- return;
250
- }
251
-
252
- const store = this.store;
253
- const fence = () => {
254
- if (!store.verifyWorkflowRunClaim({ runId, claimToken })) {
255
- throw new ClaimLostError(runId);
425
+ for (const frame of frames) {
426
+ processing = processing
427
+ .then(async () => {
428
+ const request = parseHostRequest(frame);
429
+ const response = this.handleRequest(request);
430
+ if (!socket.write(encodeProtocolLine(response))) await once(socket, "drain");
431
+ })
432
+ .catch(() => {
433
+ socket.destroy();
434
+ });
256
435
  }
257
- };
258
- const fencedStore = new WorkflowRunStore(this.childRunStore.databasePath, {
259
- authorityProvider: () => store.workflowRunAuthority(runId, claimToken),
260
436
  });
261
- const executor = new RpcStepExecutor({
262
- cwd: this.options.cwd,
263
- registry: this.registry,
264
- ...(record.originSessionId !== null ? { assistantMessageMode: "park" as const } : {}),
265
- ...(this.options.piArgs !== undefined ? { piArgs: this.options.piArgs } : {}),
266
- ...(this.options.env !== undefined ? { env: this.options.env } : {}),
267
- });
268
- const engine = new WorkflowEngine({
269
- executor,
270
- store: fencedStore,
271
- notificationSink: {
272
- notify: (request) => {
273
- fence();
274
- if (record.originSessionId === null) {
275
- throw new Error(`Workflow run ${request.runId} has no origin session`);
276
- }
277
- const notification = store.enqueueWorkflowNotification({
278
- ...request,
279
- targetSessionId: record.originSessionId,
280
- });
281
- return {
282
- notificationId: notification.notificationId,
283
- targetSessionId: notification.targetSessionId,
284
- };
285
- },
286
- },
437
+ socket.on("error", (error) => {
438
+ this.log(`client socket error: ${errorMessage(error)}`);
439
+ socket.destroy();
287
440
  });
288
- if (!store.markWorkflowRunRunning({ runId, claimToken })) {
289
- throw new ClaimLostError(runId);
441
+ socket.on("close", () => {
442
+ this.sockets.delete(socket);
443
+ });
444
+ }
445
+
446
+ private handleRequest(request: HostRequest): HostResponse {
447
+ if (this.claim === null || this.stopping) {
448
+ return {
449
+ schema: "pi-workflows.host-response.v1",
450
+ requestId: request.requestId,
451
+ outcome: "unavailable",
452
+ error: "Workflow host is stopping",
453
+ };
454
+ }
455
+ const afterCommit: Array<() => void> = [];
456
+ let response: HostResponse;
457
+ try {
458
+ response = this.hostState.executeCommand(request, this.claim.epoch, () =>
459
+ this.executeOperation(request, afterCommit),
460
+ );
461
+ } catch (error) {
462
+ response = {
463
+ schema: "pi-workflows.host-response.v1",
464
+ requestId: request.requestId,
465
+ outcome: "rejected",
466
+ error: errorMessage(error),
467
+ };
290
468
  }
291
- const parkEngine = () => engine.park();
292
- this.parkedEngines.push(parkEngine);
469
+ for (const effect of afterCommit) setImmediate(effect);
470
+ return response;
471
+ }
293
472
 
294
- const renewTimer = setInterval(() => {
295
- try {
296
- if (!store.renewWorkflowRunClaim({ runId, claimToken, leaseMs: RUN_CLAIM_LEASE_MS })) {
297
- engine.cancel();
298
- }
299
- } catch {
300
- // Transient store errors leave fencing to decide ownership.
473
+ private executeOperation(
474
+ request: HostRequest,
475
+ afterCommit: Array<() => void>,
476
+ ): Omit<HostResponse, "schema" | "requestId"> {
477
+ switch (request.operation) {
478
+ case "host.status":
479
+ return { outcome: "accepted", receipt: this.statusReceipt() };
480
+ case "host.stop":
481
+ afterCommit.push(() => void this.stop());
482
+ return { outcome: "accepted", receipt: { stopping: true } };
483
+ case "run.list":
484
+ return {
485
+ outcome: "accepted",
486
+ receipt: this.queue.listWorkflowRuns({
487
+ limit: payloadLimit(request.payload),
488
+ }) as JsonValue,
489
+ };
490
+ case "run.status": {
491
+ const runId = requireRunId(request);
492
+ const run = this.queue.getWorkflowRun(runId);
493
+ return run === undefined
494
+ ? { outcome: "notFound", error: `Workflow run not found: ${runId}` }
495
+ : { outcome: "accepted", receipt: run as unknown as JsonValue };
301
496
  }
302
- }, RUN_CLAIM_RENEW_MS);
303
- renewTimer.unref?.();
304
-
305
- this.recordEvent(runId, record.workflowName, "resumed", { runnerId: this.runnerId });
306
- try {
307
- const humanDecision =
308
- record.parentRunId === null ? undefined : hostHumanDecision(record.launchOptions);
309
- const result = record.initialized
310
- ? await engine.resumeRun(workflow, runId, { workflowSource })
311
- : record.parentRunId === null
312
- ? await engine.run(workflow, record.input, { workflowSource, runId })
313
- : await engine.continueRun(workflow, record.parentRunId, record.input, {
314
- workflowSource,
315
- runId,
316
- ...(humanDecision === undefined ? {} : { humanDecision }),
317
- });
318
- clearInterval(renewTimer);
319
- if (result.state.status === "running" || result.state.status === "waiting") {
320
- // A drained run, visible-assistant handoff, or new checkpoint stays
321
- // available to its next owner. Skip waits in this host process so it
322
- // cannot reclaim the same run in a tight loop.
323
- this.store.parkWorkflowRun({ runId, claimToken });
324
- this.store.settleRunEffect(runId, "run.park_queue");
325
- const waitingForOrigin =
326
- result.state.status === "running" &&
327
- result.state.statusDetail === "waiting for origin Pi session";
328
- if (result.state.status === "waiting" || waitingForOrigin) this.skippedRuns.add(runId);
329
- this.recordEvent(runId, record.workflowName, "parked", {
330
- status: result.state.status,
331
- ...(waitingForOrigin ? { reason: "waiting for origin Pi session" } : {}),
332
- });
333
- this.log(
334
- `parked ${record.workflowName} run ${runId}${waitingForOrigin ? ": waiting for origin Pi session" : ""}`,
335
- );
336
- return;
497
+ case "run.cancel": {
498
+ const runId = requireRunId(request);
499
+ const active = this.activeRuns.get(runId);
500
+ if (active !== undefined) {
501
+ if (
502
+ !this.queue.cancelWorkflowRun({
503
+ runId: active.record.runId,
504
+ claimToken: active.claimToken,
505
+ })
506
+ ) {
507
+ return { outcome: "claimLost", error: "Run claim was lost before cancellation" };
508
+ }
509
+ active.control = "cancel";
510
+ afterCommit.push(() => void active.supervisor.stop("cancelled"));
511
+ return { outcome: "accepted", receipt: { runId, status: "cancelled" } };
512
+ }
513
+ const pendingClaim = this.pendingRunClaims.get(runId);
514
+ if (pendingClaim !== undefined) {
515
+ if (!this.queue.cancelWorkflowRun({ runId, claimToken: pendingClaim })) {
516
+ return { outcome: "claimLost", error: "Run claim was lost before cancellation" };
517
+ }
518
+ this.clearPendingStart(runId, pendingClaim);
519
+ return { outcome: "accepted", receipt: { runId, status: "cancelled" } };
520
+ }
521
+ const cancelled = this.queue.cancelWorkflowRun({ runId });
522
+ return cancelled
523
+ ? { outcome: "accepted", receipt: { runId, status: "cancelled" } }
524
+ : { outcome: "rejected", error: "Run has a live owner or is already terminal" };
337
525
  }
338
- this.store.completeWorkflowRun({ runId, claimToken });
339
- this.store.settleRunEffect(runId, "run.settle_queue");
340
- this.recordEvent(runId, record.workflowName, result.state.status, {
341
- ...(result.state.error !== undefined ? { error: result.state.error } : {}),
342
- ...(result.state.waitingOn !== undefined ? { waitingOn: result.state.waitingOn } : {}),
343
- });
344
- this.log(`${record.workflowName} run ${runId} ${result.state.status}`);
345
- } catch (error) {
346
- clearInterval(renewTimer);
347
- if (isClaimLostError(error)) {
348
- this.log(`run ${runId} continues under another runner`);
349
- return;
526
+ case "run.pause": {
527
+ const runId = requireRunId(request);
528
+ const active = this.activeRuns.get(runId);
529
+ if (active === undefined) return { outcome: "rejected", error: "Run is not active" };
530
+ this.commitActivePause(active);
531
+ active.control = "pause";
532
+ afterCommit.push(() => void active.supervisor.stop("cancelled"));
533
+ return { outcome: "accepted", receipt: { runId, status: "parked", paused: true } };
350
534
  }
351
- if (error instanceof WorkflowSourceChangedError) {
352
- // Edited source is a refusal, not a failure: keep the run claimable
353
- // for a later fix, and stop spinning on it for this host's lifetime.
354
- try {
355
- this.store.parkWorkflowRun({ runId, claimToken });
356
- } catch {
357
- // Best-effort.
535
+ case "run.resume": {
536
+ const runId = requireRunId(request);
537
+ if (this.activeRuns.has(runId) || this.pendingStarts.has(runId)) {
538
+ return { outcome: "adopted", receipt: { runId, active: true } };
358
539
  }
359
- this.skippedRuns.add(runId);
360
- this.recordEvent(runId, record.workflowName, "parked", {
361
- reason: "workflow source changed",
540
+ const token = randomUUID();
541
+ const claimed = this.queue.claimWorkflowRun({
542
+ runId,
543
+ runnerId: this.hostId,
544
+ claimToken: token,
545
+ leaseMs: this.runClaimLeaseMs,
362
546
  });
363
- this.log(
364
- `run ${runId} skipped: workflow source changed; revert or force-resume it, then restart the host`,
365
- );
366
- return;
547
+ if (claimed === undefined) return { outcome: "rejected", error: "Run is not resumable" };
548
+ this.markPendingStart(runId, token);
549
+ afterCommit.push(() => void this.activateRun(claimed, token));
550
+ return { outcome: "accepted", receipt: { runId, generation: claimed.claimGeneration } };
367
551
  }
368
- await this.failUnresumable(record, claimToken, errorMessage(error));
369
- } finally {
370
- const index = this.parkedEngines.indexOf(parkEngine);
371
- if (index !== -1) {
372
- this.parkedEngines.splice(index, 1);
552
+ case "run.start":
553
+ return this.startRun(request, afterCommit);
554
+ case "checkpoint.answer":
555
+ return this.answerCheckpoint(request, afterCommit);
556
+ case "notification.claim":
557
+ return this.claimNotification(request);
558
+ case "notification.deliver":
559
+ return this.deliverNotification(request);
560
+ case "turn.claim":
561
+ return this.claimTurn(request);
562
+ case "turn.resolve":
563
+ return this.resolveTurn(request);
564
+ case "controller.list":
565
+ case "controller.get":
566
+ case "controller.apply":
567
+ case "controller.reconcile":
568
+ case "controller.delete":
569
+ return this.executeControllerOperation(request);
570
+ case "interaction.update": {
571
+ const payload = requireRecord(request.payload, "interaction update payload");
572
+ if (payload.claimPresentation === true) {
573
+ const interaction = this.hostState.claimInteractionPresentation({
574
+ requestId: requireString(payload.requestId, "requestId"),
575
+ expectedRevision: requireNonNegativeInteger(
576
+ request.expectedRevision,
577
+ "expectedRevision",
578
+ ),
579
+ presenterId: request.clientId,
580
+ leaseMs: PRESENTATION_CLAIM_LEASE_MS,
581
+ });
582
+ return {
583
+ outcome: "accepted",
584
+ revision: interaction.revision,
585
+ receipt: interaction as unknown as JsonValue,
586
+ };
587
+ }
588
+ if (typeof payload.sessionEntryId === "string") {
589
+ const interaction = this.hostState.markInteractionPresented({
590
+ requestId: requireString(payload.requestId, "requestId"),
591
+ expectedRevision: requireNonNegativeInteger(
592
+ request.expectedRevision,
593
+ "expectedRevision",
594
+ ),
595
+ sessionEntryId: payload.sessionEntryId,
596
+ });
597
+ return {
598
+ outcome: "accepted",
599
+ revision: interaction.revision,
600
+ receipt: interaction as unknown as JsonValue,
601
+ };
602
+ }
603
+ return this.publishInteractionUpdate(request);
373
604
  }
374
- fencedStore.close();
375
- await executor.close().catch(() => undefined);
605
+ case "interaction.submit":
606
+ return this.submitInteraction(request);
607
+ case "decision.answer":
608
+ return this.answerDecision(request, afterCommit);
376
609
  }
377
610
  }
378
611
 
379
- private async failUnresumable(
380
- record: WorkflowRunQueueRecord,
381
- claimToken: string,
382
- message: string,
383
- ): Promise<void> {
384
- let actualStatus: string | undefined;
385
- let heldClaim = false;
386
- try {
387
- // The interruption write obeys the same fencing rule as every other
388
- // state write: without a live claim, the current owner decides.
389
- heldClaim = this.store.verifyWorkflowRunClaim({ runId: record.runId, claimToken });
390
- if (heldClaim) {
391
- const claimedStore = new WorkflowRunStore(this.store.filePath, {
392
- state: this.store.state,
393
- authorityProvider: () => this.store.workflowRunAuthority(record.runId, claimToken),
394
- });
395
- const bundle = await claimedStore.markRunInterrupted(record.runId, message);
396
- actualStatus = bundle?.state.status;
612
+ private executeControllerOperation(
613
+ request: HostRequest,
614
+ ): Omit<HostResponse, "schema" | "requestId"> {
615
+ const payload = requireRecord(request.payload, "controller payload");
616
+ const projectPath = path.resolve(requireString(payload.projectPath, "projectPath"));
617
+ if (payload.projectPath !== projectPath) {
618
+ return {
619
+ outcome: "rejected",
620
+ error: "Controller projectPath must be absolute and normalized",
621
+ };
622
+ }
623
+ const store = new SqliteControllerStore(this.databasePath, {
624
+ state: this.state,
625
+ projectPath,
626
+ });
627
+ if (request.operation === "controller.list") {
628
+ return { outcome: "accepted", receipt: store.listResources() as unknown as JsonValue };
629
+ }
630
+ const controller = requireString(payload.controller, "controller");
631
+ const key = requireString(payload.key, "key");
632
+ const ref = { controller, key };
633
+ if (request.operation === "controller.get") {
634
+ const resource = store.getResource(ref);
635
+ return resource === undefined
636
+ ? { outcome: "notFound", error: `Controller resource not found: ${controller}/${key}` }
637
+ : { outcome: "accepted", receipt: resource as unknown as JsonValue };
638
+ }
639
+ if (request.operation === "controller.apply") {
640
+ if (!Object.hasOwn(payload, "spec") || !Object.hasOwn(payload, "initialStatus")) {
641
+ return { outcome: "rejected", error: "Controller apply requires spec and initialStatus" };
397
642
  }
398
- } catch {
399
- // The run state may be unreadable; the queue row still needs closure.
643
+ const spec = payload.spec as JsonValue;
644
+ const initialStatus = payload.initialStatus as JsonValue;
645
+ const controllerPath = path.resolve(requireString(payload.controllerPath, "controllerPath"));
646
+ const sourceHash = requireString(payload.sourceHash, "sourceHash");
647
+ if (!controllerPathAllowed(projectPath, controller, controllerPath)) {
648
+ return { outcome: "rejected", error: "Controller source does not match discovery rules" };
649
+ }
650
+ const observedHash = createHash("sha256")
651
+ .update(fs.readFileSync(controllerPath))
652
+ .digest("hex");
653
+ if (observedHash !== sourceHash) {
654
+ return { outcome: "conflict", error: "Controller source changed before apply committed" };
655
+ }
656
+ const resource = store.putResource({ controller, key, spec, initialStatus });
657
+ return { outcome: "accepted", receipt: resource as unknown as JsonValue };
400
658
  }
401
- try {
402
- this.store.completeWorkflowRun({ runId: record.runId, claimToken });
403
- this.store.settleRunEffect(record.runId, "run.settle_queue");
404
- } catch {
405
- // Best-effort.
659
+ const existing = store.getResource(ref);
660
+ if (existing === undefined) {
661
+ return { outcome: "notFound", error: `Controller resource not found: ${controller}/${key}` };
406
662
  }
407
- if (!heldClaim) {
408
- // Another runner owns the run; its owner reports from here.
409
- this.log(`run ${record.runId} continues under another runner`);
410
- return;
663
+ if (request.operation === "controller.reconcile") {
664
+ store.enqueue(ref);
665
+ return { outcome: "accepted", receipt: existing as unknown as JsonValue };
411
666
  }
412
- // Report the run's real terminal state when the interruption was a
413
- // no-op (the run was already waiting or completed), so the feed
414
- // stays truthful for sessions syncing from it.
415
- if (actualStatus !== undefined && actualStatus !== "failed") {
416
- this.recordEvent(record.runId, record.workflowName, actualStatus, {});
417
- } else {
418
- this.recordEvent(record.runId, record.workflowName, "failed", { error: message });
667
+ const deleting = store.requestDeletion(ref);
668
+ store.enqueue(ref);
669
+ return { outcome: "accepted", receipt: deleting as unknown as JsonValue };
670
+ }
671
+
672
+ private statusReceipt(): JsonValue {
673
+ const host = this.hostState.hostStatus();
674
+ const count = (sql: string, ...params: unknown[]): number => {
675
+ const row = this.state.connection.prepare(sql).get(...params) as
676
+ | { count?: unknown }
677
+ | undefined;
678
+ return typeof row?.count === "number" ? row.count : 0;
679
+ };
680
+ const now = Date.now();
681
+ return {
682
+ state: host.live ? "running" : "stale",
683
+ epoch: host.epoch,
684
+ socketAvailable: this.server?.listening === true,
685
+ startedAt: host.startedAt,
686
+ heartbeatAt: host.heartbeatAt,
687
+ expiresAt: host.expiresAt,
688
+ activeWorkers: this.activeRuns.size,
689
+ connectedClients: this.sockets.size,
690
+ queuedRuns: count("SELECT COUNT(*) AS count FROM run_queue WHERE status = 'queued'"),
691
+ runningRuns: count(
692
+ "SELECT COUNT(*) AS count FROM run_queue WHERE status IN ('starting', 'running')",
693
+ ),
694
+ parkedRuns: count("SELECT COUNT(*) AS count FROM run_queue WHERE status = 'parked'"),
695
+ waitingRuns: count("SELECT COUNT(*) AS count FROM runs WHERE status = 'waiting'"),
696
+ expiredClaims: count(
697
+ `SELECT COUNT(*) AS count FROM run_queue q
698
+ JOIN runs r ON r.run_id = q.run_id
699
+ JOIN leases l ON l.resource_id = r.resource_id
700
+ WHERE q.status NOT IN ('done', 'failed', 'cancelled')
701
+ AND l.owner_id IS NOT NULL AND l.expires_at <= ?`,
702
+ now,
703
+ ),
704
+ pendingInteractions: count(
705
+ "SELECT COUNT(*) AS count FROM interactive_requests WHERE status IN ('pending', 'presenting')",
706
+ ),
707
+ ambiguousEffects: count("SELECT COUNT(*) AS count FROM effects WHERE status = 'ambiguous'"),
708
+ pendingControllers: count("SELECT COUNT(*) AS count FROM controller_queue"),
709
+ activeControllerWorkers: this.activeControllers.size,
710
+ lifecycleContradictions: count(
711
+ `SELECT COUNT(*) AS count
712
+ FROM runs r JOIN run_queue q ON q.run_id = r.run_id
713
+ WHERE NOT (
714
+ (r.status = 'queued' AND q.status = 'queued')
715
+ OR (r.status = 'running' AND q.status IN ('starting', 'running', 'parked'))
716
+ OR (r.status = 'waiting' AND q.status = 'parked')
717
+ OR (r.status = 'completed' AND q.status = 'done')
718
+ OR (r.status IN ('failed', 'timed_out') AND q.status = 'failed')
719
+ OR (r.status = 'cancelled' AND q.status = 'cancelled')
720
+ )`,
721
+ ),
722
+ };
723
+ }
724
+
725
+ private rememberDeliveryClaim(options: Omit<DeliveryClaim, "expiresAt">): string {
726
+ const now = Date.now();
727
+ for (const [claimId, claim] of this.deliveryClaims) {
728
+ if (claim.expiresAt <= now) this.deliveryClaims.delete(claimId);
419
729
  }
420
- this.log(`run ${record.runId} cannot resume: ${message}`);
730
+ const claimId = randomUUID();
731
+ this.deliveryClaims.set(claimId, {
732
+ ...options,
733
+ expiresAt: now + DELIVERY_CLAIM_LEASE_MS,
734
+ });
735
+ return claimId;
421
736
  }
422
737
 
423
- private recordEvent(runId: string, workflowRef: string, type: string, payload: JsonObject): void {
424
- try {
425
- this.store.recordRunEvent({ runId, workflowRef, type, payload, runnerId: this.runnerId });
426
- } catch {
427
- // The event feed is best-effort.
738
+ private deliveryClaim(
739
+ claimId: string,
740
+ clientId: string,
741
+ kind: DeliveryClaim["kind"],
742
+ resourceId: string,
743
+ targetSessionId: string,
744
+ ): DeliveryClaim | undefined {
745
+ const claim = this.deliveryClaims.get(claimId);
746
+ if (
747
+ claim === undefined ||
748
+ claim.expiresAt <= Date.now() ||
749
+ claim.clientId !== clientId ||
750
+ claim.kind !== kind ||
751
+ claim.resourceId !== resourceId ||
752
+ claim.targetSessionId !== targetSessionId
753
+ ) {
754
+ this.deliveryClaims.delete(claimId);
755
+ return undefined;
428
756
  }
757
+ return claim;
429
758
  }
430
- }
431
759
 
432
- function hostHumanDecision(value: unknown): ResolvedHumanDecision | undefined {
433
- if (typeof value !== "object" || value === null || Array.isArray(value)) {
434
- throw new Error("Stored workflow launch options are invalid");
760
+ private claimNotification(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
761
+ const payload = requireRecord(command.payload, "notification claim payload");
762
+ const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
763
+ const token = randomUUID();
764
+ const notification = this.queue.claimPendingWorkflowNotifications({
765
+ targetSessionId,
766
+ claimToken: token,
767
+ leaseMs: DELIVERY_CLAIM_LEASE_MS,
768
+ limit: 1,
769
+ })[0];
770
+ if (notification === undefined) {
771
+ return { outcome: "accepted", receipt: { notification: null } };
772
+ }
773
+ const claimId = this.rememberDeliveryClaim({
774
+ clientId: command.clientId,
775
+ token,
776
+ targetSessionId,
777
+ kind: "notification",
778
+ resourceId: notification.notificationId,
779
+ });
780
+ return {
781
+ outcome: "accepted",
782
+ receipt: { claimId, notification } as unknown as JsonValue,
783
+ };
435
784
  }
436
- const decision = Reflect.get(value, "humanDecision");
437
- if (decision === undefined) return undefined;
438
- if (
439
- typeof decision !== "object" ||
440
- decision === null ||
441
- Array.isArray(decision) ||
442
- Reflect.get(decision, "schema") !== "pi-workflows.human-decision-accepted.v1"
443
- ) {
444
- throw new Error("Stored human decision launch option is invalid");
785
+
786
+ private deliverNotification(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
787
+ const payload = requireRecord(command.payload, "notification delivery payload");
788
+ const notificationId = requireString(payload.notificationId, "notificationId");
789
+ const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
790
+ const claimId = requireString(payload.claimId, "claimId");
791
+ const claim = this.deliveryClaim(
792
+ claimId,
793
+ command.clientId,
794
+ "notification",
795
+ notificationId,
796
+ targetSessionId,
797
+ );
798
+ if (claim === undefined) {
799
+ return { outcome: "conflict", error: "Notification delivery claim is stale" };
800
+ }
801
+ const delivered = this.queue.markWorkflowNotificationDelivered({
802
+ notificationId,
803
+ targetSessionId,
804
+ claimToken: claim.token,
805
+ });
806
+ this.deliveryClaims.delete(claimId);
807
+ return delivered
808
+ ? { outcome: "accepted", receipt: { notificationId, delivered: true } }
809
+ : { outcome: "conflict", error: "Notification delivery claim is stale" };
445
810
  }
446
- return decision as ResolvedHumanDecision;
447
- }
448
811
 
449
- function hostProjectScope(cwd: string): string {
450
- let canonical: string;
451
- try {
452
- canonical = fs.realpathSync.native(cwd);
453
- } catch {
454
- canonical = path.resolve(cwd);
812
+ private claimTurn(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
813
+ const payload = requireRecord(command.payload, "turn claim payload");
814
+ const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
815
+ const token = randomUUID();
816
+ const intent = this.queue.claimEligibleWorkflowTurnIntents({
817
+ targetSessionId,
818
+ claimToken: token,
819
+ leaseMs: DELIVERY_CLAIM_LEASE_MS,
820
+ limit: 1,
821
+ })[0];
822
+ if (intent === undefined) {
823
+ return { outcome: "accepted", receipt: { turn: null } };
824
+ }
825
+ const claimId = this.rememberDeliveryClaim({
826
+ clientId: command.clientId,
827
+ token,
828
+ targetSessionId,
829
+ kind: "turn",
830
+ resourceId: intent.intentId,
831
+ });
832
+ return {
833
+ outcome: "accepted",
834
+ receipt: { claimId, turn: intent } as unknown as JsonValue,
835
+ };
455
836
  }
456
- return createHash("sha256").update(canonical).digest("hex").slice(0, 24);
457
- }
458
837
 
459
- function hostLockPath(stateDir: string): string {
460
- return path.join(stateDir, "host.lock");
461
- }
838
+ private resolveTurn(command: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
839
+ const payload = requireRecord(command.payload, "turn resolution payload");
840
+ const intentId = requireString(payload.intentId, "intentId");
841
+ const targetSessionId = requireString(payload.targetSessionId, "targetSessionId");
842
+ const claimId = requireString(payload.claimId, "claimId");
843
+ const claim = this.deliveryClaim(claimId, command.clientId, "turn", intentId, targetSessionId);
844
+ if (claim === undefined) {
845
+ return { outcome: "conflict", error: "Workflow turn delivery claim is stale" };
846
+ }
847
+ const resolved = this.queue.resolveWorkflowTurnIntent({
848
+ intentId,
849
+ targetSessionId,
850
+ claimToken: claim.token,
851
+ resolution: "presentation",
852
+ ...(typeof payload.messageId === "string" ? { messageId: payload.messageId } : {}),
853
+ });
854
+ this.deliveryClaims.delete(claimId);
855
+ return resolved
856
+ ? { outcome: "accepted", receipt: { intentId, resolved: true } }
857
+ : { outcome: "conflict", error: "Workflow turn delivery claim is stale" };
858
+ }
859
+
860
+ private answerCheckpoint(
861
+ command: HostRequest,
862
+ afterCommit: Array<() => void>,
863
+ ): Omit<HostResponse, "schema" | "requestId"> {
864
+ const parentRunId = requireRunId(command);
865
+ const payload = requireRecord(command.payload, "checkpoint answer payload");
866
+ const continuationRunId = requireString(payload.continuationRunId, "continuationRunId");
867
+ const parent = this.queue.getWorkflowRun(parentRunId);
868
+ const bundle = this.runStore.readRun(parentRunId);
869
+ if (parent === undefined || bundle === null || bundle instanceof Promise) {
870
+ return { outcome: "notFound", error: `Checkpoint run not found: ${parentRunId}` };
871
+ }
872
+ const waitingOn = bundle.state.waitingOn;
873
+ if (bundle.state.status !== "waiting" || waitingOn === undefined) {
874
+ return { outcome: "rejected", error: "Workflow run is not waiting at a checkpoint" };
875
+ }
876
+ if (bundle.snapshot.nodes[waitingOn]?.humanDecision !== undefined) {
877
+ return { outcome: "rejected", error: "Protected human decisions require a human channel" };
878
+ }
879
+ const projectPath = this.queue.workflowRunProjectPath(parentRunId);
880
+ if (projectPath === undefined || parent.originSessionId === null) {
881
+ throw new Error("Checkpoint parent provenance is missing");
882
+ }
883
+ const prepared = this.prepareContinuation(
884
+ {
885
+ parent,
886
+ parentRunId,
887
+ continuationRunId,
888
+ projectPath,
889
+ definitionSnapshot: bundle.snapshot,
890
+ input: payload.input as JsonValue,
891
+ launchOptions: {},
892
+ originSessionId: parent.originSessionId,
893
+ },
894
+ afterCommit,
895
+ );
896
+ return {
897
+ outcome: prepared.state === "adopted" ? "adopted" : "accepted",
898
+ receipt: {
899
+ parentRunId,
900
+ runId: continuationRunId,
901
+ status: prepared.run.status,
902
+ } as JsonValue,
903
+ };
904
+ }
462
905
 
463
- /**
464
- * The advisory lock guards host-versus-host only: the embedded runner in a
465
- * Pi session does not take it. A second host refuses to start while the
466
- * recorded PID is alive.
467
- */
468
- function acquireHostLock(stateDir: string, runnerId: string, cwd: string): void {
469
- const lockPath = hostLockPath(stateDir);
470
- const existing = readLock(lockPath);
471
- if (existing !== null && existing.runnerId !== runnerId && isAlive(existing.pid)) {
472
- throw new Error(
473
- `Another workflow host (pid ${existing.pid}, ${existing.runnerId}) is already running for ${cwd}`,
906
+ private answerDecision(
907
+ command: HostRequest,
908
+ afterCommit: Array<() => void>,
909
+ ): Omit<HostResponse, "schema" | "requestId"> {
910
+ const payload = requireRecord(command.payload, "decision answer payload");
911
+ const requestId = requireString(payload.requestId, "requestId");
912
+ const interaction = this.hostState.getInteraction(requestId);
913
+ if (interaction === undefined || interaction.kind !== "decision") {
914
+ return { outcome: "notFound", error: `Decision request not found: ${requestId}` };
915
+ }
916
+ const request = interaction.contract as unknown as HumanDecisionRequest;
917
+ const response = payload.response as HumanDecisionResponse;
918
+ const accepted = this.decisions.acceptSync(request, {
919
+ ...response,
920
+ decisionId: request.decisionId,
921
+ requestDigest: request.requestDigest,
922
+ source: {
923
+ channel: "pi",
924
+ actorId: interaction.targetSessionId,
925
+ eventId: command.idempotencyKey,
926
+ },
927
+ idempotencyKey: command.idempotencyKey,
928
+ });
929
+ if (accepted.status === "conflict") {
930
+ return { outcome: "conflict", error: "Another human decision answer already won" };
931
+ }
932
+ this.hostState.submitInteraction({
933
+ requestId,
934
+ submissionId: requireString(payload.submissionId, "submissionId"),
935
+ idempotencyKey: command.idempotencyKey,
936
+ expectedRevision: requireNonNegativeInteger(command.expectedRevision, "expectedRevision"),
937
+ payload: response as JsonValue,
938
+ accepted: true,
939
+ receipt: accepted.decision as unknown as JsonValue,
940
+ });
941
+
942
+ const continuationRunId = this.prepareDecisionContinuation(
943
+ interaction,
944
+ request,
945
+ accepted.decision,
946
+ afterCommit,
474
947
  );
948
+ return {
949
+ outcome: accepted.status === "adopted" ? "adopted" : "accepted",
950
+ receipt: {
951
+ requestId,
952
+ parentRunId: interaction.runId,
953
+ runId: continuationRunId,
954
+ decision: accepted.decision,
955
+ } as JsonValue,
956
+ };
475
957
  }
476
- fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 });
477
- fs.writeFileSync(
478
- lockPath,
479
- `${JSON.stringify({ pid: process.pid, runnerId, startedAt: new Date().toISOString() })}\n`,
480
- { encoding: "utf8", mode: 0o600 },
481
- );
482
- }
483
958
 
484
- function releaseHostLock(stateDir: string, runnerId: string): void {
485
- const lockPath = hostLockPath(stateDir);
486
- const existing = readLock(lockPath);
487
- if (existing?.runnerId !== runnerId) {
488
- return;
959
+ private prepareDecisionContinuation(
960
+ interaction: InteractiveRequestRecord,
961
+ request: HumanDecisionRequest,
962
+ decision: ResolvedHumanDecision,
963
+ afterCommit: Array<() => void>,
964
+ ): string {
965
+ const parent = this.queue.getWorkflowRun(interaction.runId);
966
+ const bundle = this.runStore.readRun(interaction.runId);
967
+ if (parent === undefined || bundle === null || bundle instanceof Promise) {
968
+ throw new Error(`Decision parent run is unreadable: ${interaction.runId}`);
969
+ }
970
+ const continuationRunId = `continuation-${request.decisionId.replace(/^decision-/, "")}`;
971
+ const projectPath = this.queue.workflowRunProjectPath(interaction.runId);
972
+ if (projectPath === undefined) throw new Error("Decision parent project is missing");
973
+ this.prepareContinuation(
974
+ {
975
+ parent,
976
+ parentRunId: interaction.runId,
977
+ continuationRunId,
978
+ projectPath,
979
+ definitionSnapshot: bundle.snapshot,
980
+ input: {},
981
+ launchOptions: { humanDecision: decision },
982
+ originSessionId: interaction.targetSessionId,
983
+ },
984
+ afterCommit,
985
+ );
986
+ return continuationRunId;
489
987
  }
490
- try {
491
- fs.rmSync(lockPath);
492
- } catch {
493
- // Already gone.
988
+
989
+ private prepareContinuation(
990
+ options: {
991
+ parent: WorkflowRunQueueRecord;
992
+ parentRunId: string;
993
+ continuationRunId: string;
994
+ projectPath: string;
995
+ definitionSnapshot: WorkflowDefinitionSnapshot;
996
+ input: unknown;
997
+ launchOptions: JsonValue;
998
+ originSessionId: string;
999
+ },
1000
+ afterCommit: Array<() => void>,
1001
+ ) {
1002
+ const token = randomUUID();
1003
+ const scoped = new SqliteControllerStore(this.databasePath, {
1004
+ state: this.state,
1005
+ projectPath: options.projectPath,
1006
+ });
1007
+ const prepared = scoped.prepareOrAdoptWorkflowRun({
1008
+ runId: options.continuationRunId,
1009
+ workflowName: options.parent.workflowName,
1010
+ workflowSourceRef: options.parent.workflowSourceRef,
1011
+ workflowSource: options.parent.workflowSource,
1012
+ definitionDigest: options.parent.definitionDigest,
1013
+ definitionSnapshot: options.definitionSnapshot,
1014
+ input: options.input,
1015
+ launchOptions: options.launchOptions,
1016
+ runnerId: this.hostId,
1017
+ claimToken: token,
1018
+ leaseMs: this.runClaimLeaseMs,
1019
+ originSessionId: options.originSessionId,
1020
+ executionMode: options.parent.executionMode,
1021
+ parentRunId: options.parentRunId,
1022
+ });
1023
+ if (prepared.state === "claimed") {
1024
+ this.markPendingStart(options.continuationRunId, token);
1025
+ afterCommit.push(() => void this.activateRun(prepared.run, token));
1026
+ }
1027
+ return prepared;
494
1028
  }
495
- }
496
1029
 
497
- function readLock(lockPath: string): { pid: number; runnerId: string } | null {
498
- try {
499
- const parsed = JSON.parse(fs.readFileSync(lockPath, "utf8")) as {
500
- pid?: unknown;
501
- runnerId?: unknown;
1030
+ private startRun(
1031
+ request: HostRequest,
1032
+ afterCommit: Array<() => void>,
1033
+ ): Omit<HostResponse, "schema" | "requestId"> {
1034
+ const runId = requireRunId(request);
1035
+ const payload = requireRecord(request.payload, "run.start payload");
1036
+ const projectPath = requireAbsolutePath(payload.projectPath, "projectPath");
1037
+ const workflowName = requireString(payload.workflowName, "workflowName");
1038
+ const workflowSourceRef = requireString(payload.workflowSourceRef, "workflowSourceRef");
1039
+ const definitionDigest = requireString(payload.definitionDigest, "definitionDigest");
1040
+ const originSessionId = requireString(payload.originSessionId, "originSessionId");
1041
+ const executionMode = payload.executionMode === "headless" ? "headless" : "interactive";
1042
+ const token = randomUUID();
1043
+ const scoped = new SqliteControllerStore(this.databasePath, {
1044
+ state: this.state,
1045
+ projectPath,
1046
+ });
1047
+ const prepared = scoped.prepareOrAdoptWorkflowRun({
1048
+ runId,
1049
+ workflowName,
1050
+ workflowSourceRef,
1051
+ workflowSource: payload.workflowSource,
1052
+ definitionDigest,
1053
+ definitionSnapshot: payload.definitionSnapshot,
1054
+ input: payload.input,
1055
+ launchOptions: payload.launchOptions ?? {},
1056
+ runnerId: this.hostId,
1057
+ claimToken: token,
1058
+ leaseMs: this.runClaimLeaseMs,
1059
+ originSessionId,
1060
+ executionMode,
1061
+ ...(typeof payload.parentRunId === "string" ? { parentRunId: payload.parentRunId } : {}),
1062
+ });
1063
+ if (prepared.state === "adopted") {
1064
+ return {
1065
+ outcome: "adopted",
1066
+ receipt: { runId, status: prepared.run.status } as JsonValue,
1067
+ };
1068
+ }
1069
+ this.markPendingStart(runId, token);
1070
+ afterCommit.push(() => void this.activateRun(prepared.run, token));
1071
+ return {
1072
+ outcome: "accepted",
1073
+ revision: runRevision(this.state, runId),
1074
+ receipt: { runId, generation: prepared.run.claimGeneration } as JsonValue,
502
1075
  };
503
- if (typeof parsed.pid !== "number" || typeof parsed.runnerId !== "string") {
504
- return null;
1076
+ }
1077
+
1078
+ private publishInteractionUpdate(
1079
+ request: HostRequest,
1080
+ ): Omit<HostResponse, "schema" | "requestId"> {
1081
+ const runId = requireRunId(request);
1082
+ const payload = requireRecord(request.payload, "interaction update payload");
1083
+ const requestId = requireString(payload.requestId, "requestId");
1084
+ const interaction = this.hostState.getInteraction(requestId);
1085
+ if (interaction === undefined || interaction.runId !== runId) {
1086
+ return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
1087
+ }
1088
+ const attemptId = requireString(payload.attempt, "attempt");
1089
+ const nodeId = requireString(payload.step, "step");
1090
+ const storedContract = requireRecord(interaction.contract, "interactive contract");
1091
+ const stepContract = requireRecord(storedContract.contract, "workflow step contract");
1092
+ if (
1093
+ interaction.attemptId !== attemptId ||
1094
+ requireString(stepContract.nodeId, "contract.nodeId") !== nodeId ||
1095
+ !["pending", "presenting"].includes(interaction.status)
1096
+ ) {
1097
+ return { outcome: "conflict", error: "Interactive request attempt is stale" };
1098
+ }
1099
+ if (
1100
+ interaction.revision !==
1101
+ requireNonNegativeInteger(request.expectedRevision, "expectedRevision")
1102
+ ) {
1103
+ return { outcome: "conflict", error: "Interactive request revision is stale" };
1104
+ }
1105
+ const value = requireRecord(payload.value, "interaction update value");
1106
+ const update = validateWorkflowUpdate(value.update);
1107
+ const token = randomUUID();
1108
+ const claimed = this.queue.claimWorkflowRunForControl({
1109
+ runId,
1110
+ runnerId: this.hostId,
1111
+ claimToken: token,
1112
+ leaseMs: this.runClaimLeaseMs,
1113
+ });
1114
+ if (claimed === undefined) {
1115
+ return { outcome: "claimLost", error: "Interactive run is not available for update" };
1116
+ }
1117
+ this.controlClaims.set(runId, token);
1118
+ try {
1119
+ this.runStore.synchronizeRevision(runId);
1120
+ const loaded = this.runStore.readRun(runId);
1121
+ if (loaded === null || loaded.state.status !== "waiting") {
1122
+ throw new Error("Interactive update does not match a waiting workflow run");
1123
+ }
1124
+ const result = this.runStore.publishUpdateSynchronous(
1125
+ runId,
1126
+ loaded.state,
1127
+ nodeId,
1128
+ attemptId,
1129
+ update,
1130
+ );
1131
+ if (!this.queue.parkWorkflowRun({ runId, claimToken: token })) {
1132
+ throw new Error("Interactive update could not release its control claim");
1133
+ }
1134
+ return {
1135
+ outcome: "accepted",
1136
+ revision: result.event.seq,
1137
+ receipt: result.record as unknown as JsonValue,
1138
+ };
1139
+ } finally {
1140
+ this.controlClaims.delete(runId);
505
1141
  }
506
- return { pid: parsed.pid, runnerId: parsed.runnerId };
507
- } catch {
508
- return null;
509
1142
  }
510
- }
511
1143
 
512
- function isAlive(pid: number): boolean {
513
- try {
514
- process.kill(pid, 0);
515
- return true;
516
- } catch {
517
- return false;
1144
+ private submitInteraction(request: HostRequest): Omit<HostResponse, "schema" | "requestId"> {
1145
+ const payload = requireRecord(request.payload, "interaction payload");
1146
+ const requestId = requireString(payload.requestId, "requestId");
1147
+ const current = this.hostState.getInteraction(requestId);
1148
+ if (current === undefined || current.runId !== requireRunId(request)) {
1149
+ return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
1150
+ }
1151
+ const attemptId = requireString(payload.attempt, "attempt");
1152
+ const nodeId = requireString(payload.step, "step");
1153
+ const storedContract = requireRecord(current.contract, "interactive contract");
1154
+ const stepContract = requireRecord(storedContract.contract, "workflow step contract");
1155
+ if (
1156
+ current.attemptId !== attemptId ||
1157
+ requireString(stepContract.nodeId, "contract.nodeId") !== nodeId
1158
+ ) {
1159
+ return { outcome: "conflict", error: "Interactive request attempt is stale" };
1160
+ }
1161
+ const submissionId = requireString(payload.submissionId, "submissionId");
1162
+ const submission = this.hostState.beginInteractionValidation({
1163
+ requestId,
1164
+ submissionId,
1165
+ idempotencyKey: request.idempotencyKey,
1166
+ expectedRevision: requireNonNegativeInteger(request.expectedRevision, "expectedRevision"),
1167
+ payload: (payload.value ?? null) as JsonValue,
1168
+ receipt: {
1169
+ operation: request.operation,
1170
+ requestId,
1171
+ submissionId,
1172
+ status: "validating",
1173
+ },
1174
+ });
1175
+ const interaction = submission.interaction;
1176
+ if (this.activeRuns.has(interaction.runId) || this.activationTasks.has(interaction.runId)) {
1177
+ this.pendingResumes.add(interaction.runId);
1178
+ } else {
1179
+ const token = randomUUID();
1180
+ const claimed = this.queue.claimWorkflowRunForInteractionValidation({
1181
+ runId: interaction.runId,
1182
+ runnerId: this.hostId,
1183
+ claimToken: token,
1184
+ leaseMs: this.runClaimLeaseMs,
1185
+ });
1186
+ if (claimed !== undefined) {
1187
+ this.markPendingStart(interaction.runId, token);
1188
+ setImmediate(() => void this.activateRun(claimed, token));
1189
+ }
1190
+ }
1191
+ return {
1192
+ outcome: submission.outcome,
1193
+ revision: interaction.revision,
1194
+ receipt: submission.receipt,
1195
+ };
518
1196
  }
1197
+
1198
+ private async claimControllerOne(): Promise<void> {
1199
+ if (
1200
+ this.stopping ||
1201
+ this.claim === null ||
1202
+ this.controllerPollActive ||
1203
+ this.activeControllers.size >= MAX_CONTROLLER_WORKERS
1204
+ ) {
1205
+ return;
1206
+ }
1207
+ this.controllerPollActive = true;
1208
+ try {
1209
+ const rows = this.state.connection
1210
+ .prepare(
1211
+ `SELECT DISTINCT p.canonical_path AS projectPath
1212
+ FROM controller_queue q
1213
+ JOIN controller_resources c ON c.controller_resource_id = q.controller_resource_id
1214
+ JOIN projects p ON p.project_id = c.project_id
1215
+ JOIN leases l ON l.resource_id = c.resource_id
1216
+ WHERE q.available_at <= ? AND (l.owner_id IS NULL OR l.expires_at <= ?)
1217
+ ORDER BY q.available_at, p.canonical_path`,
1218
+ )
1219
+ .all(Date.now(), Date.now()) as Array<{ projectPath?: unknown }>;
1220
+ for (const row of rows) {
1221
+ if (typeof row.projectPath !== "string") continue;
1222
+ const discovered = await discoverControllers({ cwd: row.projectPath });
1223
+ if (discovered.length === 0) continue;
1224
+ const byName = new Map(discovered.map((item) => [item.name, item.path]));
1225
+ const store = new SqliteControllerStore(this.databasePath, {
1226
+ state: this.state,
1227
+ projectPath: row.projectPath,
1228
+ });
1229
+ const claim = store.claimNext({
1230
+ controllers: [...byName.keys()],
1231
+ ownerId: this.hostId,
1232
+ leaseMs: CONTROLLER_CLAIM_LEASE_MS,
1233
+ exclude: [...this.activeControllers.values()]
1234
+ .filter((active) => active.projectPath === row.projectPath)
1235
+ .map((active) => ({
1236
+ controller: active.claim.controller,
1237
+ key: active.claim.key,
1238
+ })),
1239
+ });
1240
+ if (claim === undefined) continue;
1241
+ const definitionPath = byName.get(claim.controller);
1242
+ const resource = store.getResource({ controller: claim.controller, key: claim.key });
1243
+ if (definitionPath === undefined || resource === undefined) {
1244
+ store.requeueClaim(claim, {
1245
+ availableAt: new Date().toISOString(),
1246
+ error: "Controller source is unavailable",
1247
+ });
1248
+ continue;
1249
+ }
1250
+ void this.activateController(row.projectPath, definitionPath, store, claim, resource);
1251
+ break;
1252
+ }
1253
+ } catch (error) {
1254
+ this.log(`controller scheduling failed: ${errorMessage(error)}`);
1255
+ } finally {
1256
+ this.controllerPollActive = false;
1257
+ }
1258
+ }
1259
+
1260
+ private async activateController(
1261
+ projectPath: string,
1262
+ definitionPath: string,
1263
+ store: SqliteControllerStore,
1264
+ claim: ControllerQueueClaim,
1265
+ resource: ControllerResource,
1266
+ ): Promise<void> {
1267
+ if (this.claim === null || this.stopping) {
1268
+ store.requeueClaim(claim, { availableAt: new Date().toISOString() });
1269
+ return;
1270
+ }
1271
+ const workerEpoch = randomUUID();
1272
+ const key = `${projectPath}\u0000${claim.controller}\u0000${claim.key}`;
1273
+ const reconcileId = randomUUID();
1274
+ const envelope: ControllerWorkerLaunchEnvelope = {
1275
+ schema: "pi-workflows.controller-worker-launch.v1",
1276
+ workerEpoch,
1277
+ hostEpoch: this.claim.epoch,
1278
+ generation: claim.generation,
1279
+ projectPath,
1280
+ definitionPath,
1281
+ controllerName: claim.controller,
1282
+ resource,
1283
+ timeoutMs: DEFAULT_CONTROLLER_TIMEOUT_MS,
1284
+ };
1285
+ let active: ActiveController;
1286
+ const supervisor = new ControllerWorkerSupervisor(envelope, {
1287
+ registry: this.registry,
1288
+ onMessage: async (message) => await this.handleControllerWorkerMessage(active, message),
1289
+ ...(this.options.env === undefined ? {} : { env: this.options.env }),
1290
+ ...(this.options.controllerWorkerEntryPath === undefined
1291
+ ? {}
1292
+ : { workerEntryPath: this.options.controllerWorkerEntryPath }),
1293
+ ...(this.options.workerStartupTimeoutMs === undefined
1294
+ ? {}
1295
+ : { startupTimeoutMs: this.options.workerStartupTimeoutMs }),
1296
+ onDiagnostic: (message) => this.log(`controller worker ${workerEpoch}: ${message}`),
1297
+ });
1298
+ const renewTimer = setInterval(() => {
1299
+ if (!store.renewClaim(claim, CONTROLLER_CLAIM_LEASE_MS)) {
1300
+ void supervisor.stop("claimLost");
1301
+ }
1302
+ }, CONTROLLER_RENEW_MS);
1303
+ renewTimer.unref?.();
1304
+ active = {
1305
+ key,
1306
+ projectPath,
1307
+ store,
1308
+ claim,
1309
+ resource,
1310
+ reconcileId,
1311
+ startedAt: Date.now(),
1312
+ supervisor,
1313
+ renewTimer,
1314
+ settled: false,
1315
+ };
1316
+ this.activeControllers.set(key, active);
1317
+ store.recordEvent({
1318
+ controller: claim.controller,
1319
+ key: claim.key,
1320
+ claim,
1321
+ type: "reconcile_started",
1322
+ payload: { reconcileId, generation: resource.metadata.generation },
1323
+ });
1324
+ try {
1325
+ await supervisor.start();
1326
+ const result = await supervisor.wait();
1327
+ if (result.outcome !== "exited" && !active.settled) {
1328
+ this.finishControllerFailure(
1329
+ active,
1330
+ result.diagnostic ?? `Controller worker ${result.outcome}`,
1331
+ );
1332
+ }
1333
+ } catch (error) {
1334
+ await supervisor.stop("crashed");
1335
+ if (!active.settled) this.finishControllerFailure(active, errorMessage(error));
1336
+ } finally {
1337
+ clearInterval(renewTimer);
1338
+ this.activeControllers.delete(key);
1339
+ if (!this.stopping) setImmediate(() => void this.claimControllerOne());
1340
+ }
1341
+ }
1342
+
1343
+ private async handleControllerWorkerMessage(
1344
+ active: ActiveController,
1345
+ message: ControllerWorkerMessage,
1346
+ ): Promise<ControllerWorkerResponse> {
1347
+ const current = this.activeControllers.get(active.key);
1348
+ if (
1349
+ current !== active ||
1350
+ message.workerEpoch !== active.supervisor.envelope.workerEpoch ||
1351
+ message.generation !== active.claim.generation
1352
+ ) {
1353
+ return controllerWorkerResponse(
1354
+ message,
1355
+ "claimLost",
1356
+ undefined,
1357
+ "Controller worker is stale",
1358
+ );
1359
+ }
1360
+ if (!active.store.renewClaim(active.claim, CONTROLLER_CLAIM_LEASE_MS)) {
1361
+ return controllerWorkerResponse(message, "claimLost", undefined, "Controller claim expired");
1362
+ }
1363
+ try {
1364
+ const payload = requireRecord(message.payload, "controller worker payload");
1365
+ switch (message.operation) {
1366
+ case "worker.ready":
1367
+ return controllerWorkerResponse(message, "accepted", {});
1368
+ case "effect.reserve": {
1369
+ const reservation = active.store.reserveEffect({
1370
+ key: requireString(payload.key, "key"),
1371
+ resourceUid: active.resource.metadata.uid,
1372
+ claim: active.claim,
1373
+ generation: active.resource.metadata.generation,
1374
+ kind: requireString(payload.kind, "kind"),
1375
+ requestFingerprint: requireString(payload.requestFingerprint, "requestFingerprint"),
1376
+ });
1377
+ return controllerWorkerResponse(message, "accepted", reservation as unknown as JsonValue);
1378
+ }
1379
+ case "effect.settle": {
1380
+ const state = requireString(payload.state, "state");
1381
+ if (!["applied", "rejected", "indeterminate"].includes(state)) {
1382
+ throw new Error("Controller effect state is invalid");
1383
+ }
1384
+ const record = active.store.updateEffect({
1385
+ resourceUid: active.resource.metadata.uid,
1386
+ key: requireString(payload.key, "key"),
1387
+ claim: active.claim,
1388
+ state: state as "applied" | "rejected" | "indeterminate",
1389
+ ...(typeof payload.externalRef === "string"
1390
+ ? { externalRef: payload.externalRef }
1391
+ : {}),
1392
+ ...(typeof payload.error === "string" ? { error: payload.error } : {}),
1393
+ });
1394
+ active.store.recordEvent({
1395
+ controller: active.claim.controller,
1396
+ key: active.claim.key,
1397
+ claim: active.claim,
1398
+ type: `effect_${state}`,
1399
+ payload: { effectKey: record.key },
1400
+ });
1401
+ return controllerWorkerResponse(message, "accepted", record as unknown as JsonValue);
1402
+ }
1403
+ case "workflow.ensure":
1404
+ case "workflow.changeSettings":
1405
+ case "workflow.queueFollowUp":
1406
+ case "workflow.removeFollowUp": {
1407
+ const workflows = new ControllerWorkflowCoordinator(
1408
+ active.store,
1409
+ this.controllerWorkflowScheduler(active),
1410
+ ).forResource(active.resource, active.claim, new AbortController().signal);
1411
+ const result =
1412
+ message.operation === "workflow.ensure"
1413
+ ? await workflows.ensure(payload as never)
1414
+ : message.operation === "workflow.changeSettings"
1415
+ ? await workflows.changeSettings(payload as never)
1416
+ : message.operation === "workflow.queueFollowUp"
1417
+ ? await workflows.queueFollowUp(payload as never)
1418
+ : await workflows.removeFollowUp(payload as never);
1419
+ return controllerWorkerResponse(message, "accepted", result as unknown as JsonValue);
1420
+ }
1421
+ case "worker.finished":
1422
+ this.finishControllerSuccess(
1423
+ active,
1424
+ message.payload as unknown as ReconcileResult<unknown>,
1425
+ );
1426
+ return controllerWorkerResponse(message, "accepted", {});
1427
+ case "worker.failed":
1428
+ this.finishControllerFailure(active, requireString(payload.error, "error"));
1429
+ return controllerWorkerResponse(message, "accepted", {});
1430
+ }
1431
+ } catch (error) {
1432
+ return controllerWorkerResponse(message, "rejected", undefined, errorMessage(error));
1433
+ }
1434
+ }
1435
+
1436
+ private controllerWorkflowScheduler(active: ActiveController): ControllerWorkflowScheduler {
1437
+ return {
1438
+ ensure: async (request, _signal, _onComplete) =>
1439
+ await this.ensureControllerWorkflow(active, request),
1440
+ changeSettings: async (request) =>
1441
+ await this.changeControllerWorkflowSettings(active, request),
1442
+ queueFollowUp: async (request) => {
1443
+ const targetSessionId = this.runStore.originSessionId(request.runId);
1444
+ if (targetSessionId === undefined) {
1445
+ throw new Error("Controller follow-up requires an origin Pi session");
1446
+ }
1447
+ const result = this.runStore.queueFollowUp({
1448
+ runId: request.runId,
1449
+ requestId: request.actorRequestKey,
1450
+ targetSessionId,
1451
+ actor: { type: "controller", id: request.controllerResourceUid },
1452
+ source: "controller-request",
1453
+ prompt: request.prompt,
1454
+ });
1455
+ return {
1456
+ runId: request.runId,
1457
+ followUpId: result.followUp.followUpId,
1458
+ order: result.followUp.order,
1459
+ state: result.followUp.state,
1460
+ adopted: result.adopted,
1461
+ };
1462
+ },
1463
+ removeFollowUp: async (request) => {
1464
+ const result = this.runStore.removeFollowUp({
1465
+ runId: request.runId,
1466
+ followUpId: request.followUpId,
1467
+ actor: { type: "controller", id: request.controllerResourceUid },
1468
+ source: "controller-request",
1469
+ });
1470
+ return {
1471
+ runId: request.runId,
1472
+ followUpId: result.followUpId,
1473
+ order: result.order,
1474
+ state: result.state,
1475
+ adopted: false,
1476
+ };
1477
+ },
1478
+ };
1479
+ }
1480
+
1481
+ private async changeControllerWorkflowSettings(
1482
+ _active: ActiveController,
1483
+ request: ControllerWorkflowControlRequest<ControllerSettingsChangeRequest>,
1484
+ ): Promise<ControllerSettingsChangeResult> {
1485
+ const run = this.queue.getWorkflowRun(request.runId);
1486
+ if (run === undefined) throw new Error(`Workflow run not found: ${request.runId}`);
1487
+ const scopes = this.runStore.listSettingsScopes(request.runId);
1488
+ const scopeId = request.scopeId ?? (scopes.length === 1 ? scopes[0]?.scopeId : undefined);
1489
+ if (scopeId === undefined) {
1490
+ throw new Error("Controller settings changes require scopeId when several scopes exist");
1491
+ }
1492
+ const scope = this.runStore.getSettingsScope(scopeId);
1493
+ if (scope === undefined || scope.activeRunId !== request.runId) {
1494
+ throw new Error(`Workflow settings scope not found: ${scopeId}`);
1495
+ }
1496
+ const prior = this.state.connection
1497
+ .prepare(
1498
+ `SELECT change_number AS changeNumber
1499
+ FROM workflow_setting_changes WHERE scope_id = ? AND request_id = ?`,
1500
+ )
1501
+ .get(scopeId, request.actorRequestKey) as { changeNumber?: unknown } | undefined;
1502
+ if (typeof prior?.changeNumber === "number") {
1503
+ return { runId: request.runId, scopeId, changeNumber: prior.changeNumber, adopted: true };
1504
+ }
1505
+ const projectPath = this.queue.workflowRunProjectPath(request.runId);
1506
+ if (projectPath === undefined) throw new Error("Workflow run project is missing");
1507
+ const resolver = new WorkflowHostClient({
1508
+ databasePath: this.databasePath,
1509
+ ...(this.options.env === undefined ? {} : { env: this.options.env }),
1510
+ });
1511
+ const proposal = await resolver.resolveSettingsChange({
1512
+ cwd: projectPath,
1513
+ workflowRef: run.workflowSourceRef,
1514
+ definitionDigest: run.definitionDigest,
1515
+ mountPath: scope.mountPath,
1516
+ current: scope.settings,
1517
+ patch: request.patch as unknown as JsonValue,
1518
+ actorId: request.controllerResourceUid,
1519
+ });
1520
+ if (proposal.definitionDigest !== run.definitionDigest || !Array.isArray(proposal.paths)) {
1521
+ throw new Error("Workflow settings proposal does not match the durable source");
1522
+ }
1523
+ const proposedSettings = canonicalJson(proposal.settings);
1524
+ const definition: WorkflowSettingsDefinition<JsonValue> = {
1525
+ initial: proposal.settings,
1526
+ paths: proposal.paths as unknown as WorkflowSettingsPathRule[],
1527
+ parse: (value) => {
1528
+ if (canonicalJson(value) !== proposedSettings) {
1529
+ throw new Error("Workflow settings proposal changed before commit");
1530
+ }
1531
+ return proposal.settings;
1532
+ },
1533
+ };
1534
+ const result = await this.runStore.changeSettings(definition, {
1535
+ runId: request.runId,
1536
+ scopeId,
1537
+ requestId: request.actorRequestKey,
1538
+ expectedChangeNumber: request.expectedChangeNumber ?? scope.changeNumber,
1539
+ actor: { type: "controller", id: request.controllerResourceUid },
1540
+ source: "controller-request",
1541
+ patch: proposal.patch,
1542
+ });
1543
+ return {
1544
+ runId: request.runId,
1545
+ scopeId,
1546
+ changeNumber: result.scope.changeNumber,
1547
+ adopted: result.adopted,
1548
+ };
1549
+ }
1550
+
1551
+ private async ensureControllerWorkflow(
1552
+ active: ActiveController,
1553
+ request: WorkflowSchedulerRequest,
1554
+ ): Promise<WorkflowSchedulerResult> {
1555
+ const existing = this.queue.getWorkflowRun(request.runId);
1556
+ if (existing !== undefined) return controllerWorkflowResult(existing);
1557
+ const resolver = new WorkflowHostClient({
1558
+ databasePath: this.databasePath,
1559
+ ...(this.options.env === undefined ? {} : { env: this.options.env }),
1560
+ });
1561
+ const resolved = await resolver.resolveWorkflow({
1562
+ cwd: active.projectPath,
1563
+ workflowRef: request.workflow,
1564
+ });
1565
+ const token = randomUUID();
1566
+ const scoped = new SqliteControllerStore(this.databasePath, {
1567
+ state: this.state,
1568
+ projectPath: active.projectPath,
1569
+ });
1570
+ const prepared = scoped.prepareOrAdoptWorkflowRun({
1571
+ runId: request.runId,
1572
+ workflowName: resolved.workflowName,
1573
+ workflowSourceRef: resolved.workflowSourceRef,
1574
+ workflowSource: resolved.workflowSource,
1575
+ definitionDigest: resolved.definitionDigest,
1576
+ definitionSnapshot: resolved.definitionSnapshot,
1577
+ input: request.input,
1578
+ launchOptions: {},
1579
+ runnerId: this.hostId,
1580
+ claimToken: token,
1581
+ leaseMs: this.runClaimLeaseMs,
1582
+ originSessionId: `controller-${active.resource.metadata.uid}`,
1583
+ executionMode: "headless",
1584
+ });
1585
+ if (prepared.state === "claimed") {
1586
+ this.markPendingStart(request.runId, token);
1587
+ setImmediate(() => void this.activateRun(prepared.run, token));
1588
+ }
1589
+ return controllerWorkflowResult(prepared.run);
1590
+ }
1591
+
1592
+ private finishControllerSuccess(
1593
+ active: ActiveController,
1594
+ result: ReconcileResult<unknown>,
1595
+ ): void {
1596
+ if (active.settled) return;
1597
+ const now = new Date();
1598
+ const ref = { controller: active.claim.controller, key: active.claim.key };
1599
+ const status = applyStatusPatch(
1600
+ active.resource.status,
1601
+ result.status,
1602
+ active.resource.metadata.generation,
1603
+ now.toISOString(),
1604
+ );
1605
+ try {
1606
+ const updated = active.store.updateStatus({
1607
+ ref,
1608
+ expectedResourceVersion: active.claim.resourceVersion,
1609
+ claim: active.claim,
1610
+ status,
1611
+ ...(result.status?.finalizers === undefined
1612
+ ? {}
1613
+ : { finalizers: result.status.finalizers }),
1614
+ now: now.toISOString(),
1615
+ });
1616
+ active.store.recordEvent({
1617
+ ...ref,
1618
+ claim: active.claim,
1619
+ type: "reconcile_finished",
1620
+ payload: {
1621
+ reconcileId: active.reconcileId,
1622
+ result: result.kind,
1623
+ generation: active.resource.metadata.generation,
1624
+ durationMs: Math.max(0, now.getTime() - active.startedAt),
1625
+ ...(result.kind === "requeue" && result.afterMs !== undefined
1626
+ ? { requeueAfterMs: result.afterMs }
1627
+ : {}),
1628
+ },
1629
+ });
1630
+ if (
1631
+ result.kind === "settled" &&
1632
+ updated.metadata.deletionTimestamp !== undefined &&
1633
+ updated.metadata.finalizers.length === 0
1634
+ ) {
1635
+ active.store.recordEvent({
1636
+ ...ref,
1637
+ claim: active.claim,
1638
+ type: "resource_deleted",
1639
+ payload: { reconcileId: active.reconcileId },
1640
+ });
1641
+ active.store.deleteResource(ref, active.claim.resourceVersion, active.claim);
1642
+ } else if (result.kind === "settled") {
1643
+ active.store.settleClaim(active.claim, now.toISOString());
1644
+ } else {
1645
+ active.store.requeueClaim(
1646
+ active.claim,
1647
+ { availableAt: new Date(now.getTime() + (result.afterMs ?? 0)).toISOString() },
1648
+ now.toISOString(),
1649
+ );
1650
+ }
1651
+ active.settled = true;
1652
+ } catch (error) {
1653
+ if (error instanceof ResourceConflictError) {
1654
+ active.store.recordEvent({
1655
+ ...ref,
1656
+ claim: active.claim,
1657
+ type: "reconcile_conflict",
1658
+ payload: { reconcileId: active.reconcileId },
1659
+ });
1660
+ active.store.requeueClaim(
1661
+ active.claim,
1662
+ { availableAt: now.toISOString() },
1663
+ now.toISOString(),
1664
+ );
1665
+ active.settled = true;
1666
+ return;
1667
+ }
1668
+ throw error;
1669
+ }
1670
+ }
1671
+
1672
+ private finishControllerFailure(active: ActiveController, failure: string): void {
1673
+ if (active.settled) return;
1674
+ const now = new Date();
1675
+ const delay = Math.min(60_000, 1_000 * 2 ** Math.min(active.claim.consecutiveErrors, 6));
1676
+ const error = failure.slice(0, 8_192);
1677
+ active.store.recordEvent({
1678
+ controller: active.claim.controller,
1679
+ key: active.claim.key,
1680
+ claim: active.claim,
1681
+ type: "reconcile_failed",
1682
+ payload: {
1683
+ reconcileId: active.reconcileId,
1684
+ error,
1685
+ durationMs: Math.max(0, now.getTime() - active.startedAt),
1686
+ requeueAfterMs: delay,
1687
+ },
1688
+ });
1689
+ active.store.requeueClaim(
1690
+ active.claim,
1691
+ { availableAt: new Date(now.getTime() + delay).toISOString(), error },
1692
+ now.toISOString(),
1693
+ );
1694
+ active.settled = true;
1695
+ }
1696
+
1697
+ private async expireTimedOutDecision(): Promise<void> {
1698
+ if (this.stopping || this.claim === null || this.decisionTimeoutActive) return;
1699
+ this.decisionTimeoutActive = true;
1700
+ try {
1701
+ const now = new Date();
1702
+ const requests = await this.decisions.listExpiredDefaultRequests(now);
1703
+ const candidate = requests
1704
+ .map((request) => ({
1705
+ request,
1706
+ interaction: this.hostState.getInteraction(request.decisionId),
1707
+ }))
1708
+ .find(
1709
+ ({ request, interaction }) =>
1710
+ interaction?.kind === "decision" &&
1711
+ interaction.runId === request.runId &&
1712
+ ["pending", "presenting"].includes(interaction.status) &&
1713
+ !this.activeRuns.has(request.runId) &&
1714
+ !this.pendingStarts.has(request.runId),
1715
+ );
1716
+ if (
1717
+ candidate === undefined ||
1718
+ candidate.interaction === undefined ||
1719
+ candidate.request.defaultResponse === undefined
1720
+ ) {
1721
+ return;
1722
+ }
1723
+ const { request, interaction } = candidate;
1724
+ const claimToken = randomUUID();
1725
+ const claimed = this.queue.claimWorkflowRunForControl({
1726
+ runId: request.runId,
1727
+ runnerId: this.hostId,
1728
+ claimToken,
1729
+ leaseMs: this.runClaimLeaseMs,
1730
+ });
1731
+ if (claimed === undefined) return;
1732
+ this.controlClaims.set(request.runId, claimToken);
1733
+ let committed = false;
1734
+ const afterCommit: Array<() => void> = [];
1735
+ try {
1736
+ this.state.transaction(() => {
1737
+ const accepted = this.decisions.resolveTimeoutSync(request, new Date());
1738
+ const submissionId = `timeout-${request.decisionId}`;
1739
+ this.hostState.submitInteraction({
1740
+ requestId: interaction.requestId,
1741
+ submissionId,
1742
+ idempotencyKey: submissionId,
1743
+ expectedRevision: interaction.revision,
1744
+ payload: request.defaultResponse as JsonValue,
1745
+ accepted: true,
1746
+ receipt: accepted.decision as unknown as JsonValue,
1747
+ });
1748
+ if (!this.queue.parkWorkflowRun({ runId: request.runId, claimToken })) {
1749
+ throw new Error("Decision timeout could not release the parent run claim");
1750
+ }
1751
+ this.prepareDecisionContinuation(interaction, request, accepted.decision, afterCommit);
1752
+ });
1753
+ committed = true;
1754
+ for (const effect of afterCommit) setImmediate(effect);
1755
+ this.log(`human decision ${request.decisionId} resolved by its timeout policy`);
1756
+ } catch (error) {
1757
+ this.log(`human decision timeout failed for ${request.decisionId}: ${errorMessage(error)}`);
1758
+ } finally {
1759
+ this.controlClaims.delete(request.runId);
1760
+ if (!committed) this.queue.parkWorkflowRun({ runId: request.runId, claimToken });
1761
+ }
1762
+ } catch (error) {
1763
+ this.log(`human decision timeout scan failed: ${errorMessage(error)}`);
1764
+ } finally {
1765
+ this.decisionTimeoutActive = false;
1766
+ }
1767
+ }
1768
+
1769
+ private expireTimedOutInteraction(): void {
1770
+ if (this.stopping || this.claim === null) return;
1771
+ const runId = this.hostState
1772
+ .expiredInteractionRunIds()
1773
+ .find((candidate) => !this.activeRuns.has(candidate) && !this.pendingStarts.has(candidate));
1774
+ if (runId === undefined) return;
1775
+ const claimToken = randomUUID();
1776
+ const claimed = this.queue.claimWorkflowRunForControl({
1777
+ runId,
1778
+ runnerId: this.hostId,
1779
+ claimToken,
1780
+ leaseMs: this.runClaimLeaseMs,
1781
+ });
1782
+ if (claimed === undefined) return;
1783
+ let activated = false;
1784
+ try {
1785
+ activated = this.queue.beginWorkflowRunInteractionTimeout({ runId, claimToken });
1786
+ if (activated) {
1787
+ this.markPendingStart(runId, claimToken);
1788
+ setImmediate(() => void this.activateRun(claimed, claimToken));
1789
+ this.log(`run ${runId} is finishing its expired origin-session deadline`);
1790
+ }
1791
+ } catch (error) {
1792
+ this.log(`interaction timeout failed for run ${runId}: ${errorMessage(error)}`);
1793
+ } finally {
1794
+ if (!activated) this.queue.parkWorkflowRun({ runId, claimToken });
1795
+ }
1796
+ }
1797
+
1798
+ private markPendingStart(runId: string, claimToken: string): void {
1799
+ this.pendingStarts.add(runId);
1800
+ this.pendingRunClaims.set(runId, claimToken);
1801
+ }
1802
+
1803
+ private clearPendingStart(runId: string, claimToken: string): void {
1804
+ if (this.pendingRunClaims.get(runId) !== claimToken) return;
1805
+ this.pendingRunClaims.delete(runId);
1806
+ this.pendingStarts.delete(runId);
1807
+ }
1808
+
1809
+ private async claimOne(): Promise<void> {
1810
+ if (this.stopping || this.claim === null) return;
1811
+ const excluded = new Set([
1812
+ ...this.activeRuns.keys(),
1813
+ ...this.pendingStarts,
1814
+ ...this.blockedRuns,
1815
+ ]);
1816
+ const validatingRunId = this.hostState
1817
+ .validatingInteractionRunIds()
1818
+ .find((runId) => !excluded.has(runId));
1819
+ if (validatingRunId !== undefined) {
1820
+ const validationToken = randomUUID();
1821
+ const validationRun = this.queue.claimWorkflowRunForInteractionValidation({
1822
+ runId: validatingRunId,
1823
+ runnerId: this.hostId,
1824
+ claimToken: validationToken,
1825
+ leaseMs: this.runClaimLeaseMs,
1826
+ });
1827
+ if (validationRun !== undefined) {
1828
+ this.markPendingStart(validatingRunId, validationToken);
1829
+ await this.activateRun(validationRun, validationToken);
1830
+ return;
1831
+ }
1832
+ }
1833
+ const token = randomUUID();
1834
+ const claimed = this.queue.claimNextWorkflowRun({
1835
+ runnerId: this.hostId,
1836
+ claimToken: token,
1837
+ leaseMs: this.runClaimLeaseMs,
1838
+ excludeRunIds: [...excluded],
1839
+ });
1840
+ if (claimed === undefined) return;
1841
+ this.markPendingStart(claimed.runId, token);
1842
+ await this.activateRun(claimed, token);
1843
+ }
1844
+
1845
+ private activateRun(record: WorkflowRunQueueRecord, claimToken: string): Promise<void> {
1846
+ const existing = this.activationTasks.get(record.runId);
1847
+ if (existing !== undefined) return existing;
1848
+ const task = this.activateRunNow(record, claimToken).finally(() => {
1849
+ if (this.activationTasks.get(record.runId) === task) {
1850
+ this.activationTasks.delete(record.runId);
1851
+ }
1852
+ if (this.stopping) return;
1853
+ if (this.pendingResumes.delete(record.runId)) {
1854
+ setImmediate(() => void this.resumePendingRun(record.runId));
1855
+ } else {
1856
+ setImmediate(() => void this.claimOne());
1857
+ }
1858
+ });
1859
+ this.activationTasks.set(record.runId, task);
1860
+ return task;
1861
+ }
1862
+
1863
+ private async activateRunNow(record: WorkflowRunQueueRecord, claimToken: string): Promise<void> {
1864
+ const runId = record.runId;
1865
+ if (this.pendingRunClaims.get(runId) !== claimToken) return;
1866
+ if (this.stopping) {
1867
+ this.clearPendingStart(runId, claimToken);
1868
+ this.queue.parkWorkflowRun({ runId, claimToken });
1869
+ return;
1870
+ }
1871
+ const existingActive = this.activeRuns.get(runId);
1872
+ if (existingActive !== undefined) {
1873
+ this.clearPendingStart(runId, claimToken);
1874
+ if (existingActive.claimToken !== claimToken) {
1875
+ this.queue.parkWorkflowRun({ runId, claimToken });
1876
+ }
1877
+ return;
1878
+ }
1879
+ const generation = record.claimGeneration;
1880
+ const projectPath = this.queue.workflowRunProjectPath(runId);
1881
+ if (generation === null || projectPath === undefined || this.claim === null) {
1882
+ this.clearPendingStart(runId, claimToken);
1883
+ this.queue.parkWorkflowRun({ runId, claimToken });
1884
+ return;
1885
+ }
1886
+ const envelope: WorkerLaunchEnvelope = {
1887
+ schema: "pi-workflows.worker-launch.v1",
1888
+ runId,
1889
+ generation,
1890
+ workerEpoch: randomUUID(),
1891
+ projectPath,
1892
+ workflowSource: record.workflowSource as JsonValue,
1893
+ definitionDigest: record.definitionDigest,
1894
+ inputHash: `sha256:${createHash("sha256").update(canonicalJson(record.input)).digest("hex")}`,
1895
+ protocolVersion: 1,
1896
+ };
1897
+ this.hostState.recordWorkerStart(envelope, this.claim.epoch);
1898
+ let active: ActiveRun;
1899
+ const supervisor = new WorkflowWorkerSupervisor(envelope, {
1900
+ registry: this.registry,
1901
+ onMessage: async (message) => await this.handleWorkerMessage(message),
1902
+ ...(this.options.env === undefined ? {} : { env: this.options.env }),
1903
+ ...(this.options.workerEntryPath === undefined
1904
+ ? {}
1905
+ : { workerEntryPath: this.options.workerEntryPath }),
1906
+ ...(this.options.workerStartupTimeoutMs === undefined
1907
+ ? {}
1908
+ : { startupTimeoutMs: this.options.workerStartupTimeoutMs }),
1909
+ onSpawn: (identity) => {
1910
+ active.workerPid = identity.pid;
1911
+ this.hostState.attachWorkerProcess(
1912
+ envelope.workerEpoch,
1913
+ identity.pid,
1914
+ identity.startIdentity,
1915
+ );
1916
+ },
1917
+ onDiagnostic: (message) => this.log(`worker ${envelope.workerEpoch}: ${message}`),
1918
+ });
1919
+ active = { record, claimToken, generation, supervisor, exiting: false };
1920
+ this.activeRuns.set(runId, active);
1921
+ this.clearPendingStart(runId, claimToken);
1922
+ try {
1923
+ this.runStore.synchronizeRevision(runId);
1924
+ const effectRecovery = await this.runStore.recoverApplyingEffects(runId);
1925
+ if (effectRecovery === "ambiguous" || hasUncertainEffect(this.state, runId)) {
1926
+ if (!this.queue.parkWorkflowRunForAmbiguousEffect({ runId, claimToken })) {
1927
+ throw new Error("Run claim was lost during ambiguous-effect recovery");
1928
+ }
1929
+ this.hostState.finishWorker({
1930
+ workerEpoch: envelope.workerEpoch,
1931
+ outcome: "orphaned",
1932
+ diagnostic: "effect outcome is ambiguous",
1933
+ });
1934
+ return;
1935
+ }
1936
+ await supervisor.start();
1937
+ const result = await supervisor.wait();
1938
+ this.hostState.finishWorker({
1939
+ workerEpoch: envelope.workerEpoch,
1940
+ outcome: result.outcome,
1941
+ exitCode: result.exitCode,
1942
+ signal: result.signal,
1943
+ ...(result.diagnostic === undefined ? {} : { diagnostic: result.diagnostic }),
1944
+ });
1945
+ if (
1946
+ active.control === undefined &&
1947
+ (result.outcome !== "exited" ||
1948
+ this.hostState.validatingInteraction(active.record.runId) !== undefined)
1949
+ ) {
1950
+ await this.recoverWorkerExit(active, result.outcome);
1951
+ }
1952
+ } catch (error) {
1953
+ this.log(`worker ${envelope.workerEpoch} failed: ${errorMessage(error)}`);
1954
+ await supervisor.stop("crashed");
1955
+ const result = await supervisor.wait();
1956
+ try {
1957
+ this.hostState.finishWorker({
1958
+ workerEpoch: envelope.workerEpoch,
1959
+ outcome: "crashed",
1960
+ exitCode: result.exitCode,
1961
+ signal: result.signal,
1962
+ diagnostic: result.diagnostic ?? errorMessage(error),
1963
+ });
1964
+ } catch {
1965
+ // A completed worker record wins.
1966
+ }
1967
+ if (active.control === undefined) {
1968
+ await this.recoverWorkerExit(active, "crashed");
1969
+ }
1970
+ } finally {
1971
+ this.reapWorkerDescendants(envelope.workerEpoch);
1972
+ this.activeRuns.delete(runId);
1973
+ }
1974
+ }
1975
+
1976
+ private async resumePendingRun(runId: string): Promise<void> {
1977
+ if (this.stopping || this.claim === null) return;
1978
+ const token = randomUUID();
1979
+ const claimed = this.queue.claimWorkflowRunForInteractionValidation({
1980
+ runId,
1981
+ runnerId: this.hostId,
1982
+ claimToken: token,
1983
+ leaseMs: this.runClaimLeaseMs,
1984
+ });
1985
+ if (claimed === undefined) {
1986
+ await this.claimOne();
1987
+ return;
1988
+ }
1989
+ this.markPendingStart(runId, token);
1990
+ await this.activateRun(claimed, token);
1991
+ }
1992
+
1993
+ private async handleWorkerMessage(message: WorkerMessage): Promise<WorkerResponse> {
1994
+ const active = this.activeRuns.get(message.runId);
1995
+ if (
1996
+ active === undefined ||
1997
+ active.generation !== message.generation ||
1998
+ active.supervisor.envelope.workerEpoch !== message.workerEpoch
1999
+ ) {
2000
+ return workerResponse(message, "claimLost", undefined, "Worker epoch is stale");
2001
+ }
2002
+ const stored = this.hostState.readWorkerMessage(message);
2003
+ if (stored !== undefined) return stored;
2004
+ const response = await this.handleFreshWorkerMessage(active, message);
2005
+ return this.hostState.recordWorkerMessage(message, response);
2006
+ }
2007
+
2008
+ private async handleFreshWorkerMessage(
2009
+ active: ActiveRun,
2010
+ message: WorkerMessage,
2011
+ ): Promise<WorkerResponse> {
2012
+ if (message.operation === "worker.ready") {
2013
+ if (
2014
+ !this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
2015
+ ) {
2016
+ active.claimLost = true;
2017
+ setImmediate(() => void active.supervisor.stop("claimLost"));
2018
+ return workerResponse(message, "claimLost", undefined, "Run claim expired");
2019
+ }
2020
+ this.hostState.markWorkerReady(message.workerEpoch);
2021
+ this.queue.markWorkflowRunRunning({ runId: message.runId, claimToken: active.claimToken });
2022
+ const revision = this.runStore.synchronizeRevision(message.runId);
2023
+ const current = this.queue.getWorkflowRun(message.runId);
2024
+ const candidateInteraction =
2025
+ this.hostState.acceptedInteraction(message.runId) ??
2026
+ this.hostState.validatingInteraction(message.runId);
2027
+ const timedOutInteraction = this.hostState.timedOutInteraction(message.runId);
2028
+ const resumeInteractionAttemptId =
2029
+ candidateInteraction?.attemptId ?? timedOutInteraction?.attemptId;
2030
+ return workerResponse(
2031
+ message,
2032
+ "accepted",
2033
+ {
2034
+ initialized: current?.initialized ?? active.record.initialized,
2035
+ input: active.record.input,
2036
+ launchOptions: active.record.launchOptions,
2037
+ parentRunId: active.record.parentRunId,
2038
+ originSessionId: active.record.originSessionId,
2039
+ stateDirectory: this.stateDirectory,
2040
+ ...(resumeInteractionAttemptId === undefined ? {} : { resumeInteractionAttemptId }),
2041
+ ...(candidateInteraction === undefined ? {} : { candidateInteraction }),
2042
+ ...(this.options.piArgs === undefined ? {} : { piArgs: this.options.piArgs }),
2043
+ } as JsonValue,
2044
+ undefined,
2045
+ revision,
2046
+ );
2047
+ }
2048
+ if (message.operation === "worker.exiting") {
2049
+ const payload = requireRecord(message.payload, "worker exit payload");
2050
+ if (payload.status === "workflowLoadFailed") {
2051
+ active.workflowLoadFailure = requireString(payload.error, "workflow load error").slice(
2052
+ 0,
2053
+ 8_192,
2054
+ );
2055
+ }
2056
+ active.exiting = true;
2057
+ return workerResponse(message, "accepted", {});
2058
+ }
2059
+ if (message.operation === "process.register" || message.operation === "process.unregister") {
2060
+ return this.handleWorkerProcessOperation(active, message);
2061
+ }
2062
+ const currentRevision = runRevision(this.state, message.runId);
2063
+ if (message.expectedRevision !== currentRevision) {
2064
+ return workerResponse(
2065
+ message,
2066
+ "rejected",
2067
+ undefined,
2068
+ `Workflow run revision conflict: expected ${message.expectedRevision}, got ${currentRevision}`,
2069
+ currentRevision,
2070
+ );
2071
+ }
2072
+ if (
2073
+ !this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
2074
+ ) {
2075
+ active.claimLost = true;
2076
+ setImmediate(() => void active.supervisor.stop("claimLost"));
2077
+ return workerResponse(message, "claimLost", undefined, "Run claim expired");
2078
+ }
2079
+ try {
2080
+ const payload = requireRecord(message.payload, "worker payload");
2081
+ let result: unknown;
2082
+ switch (message.operation) {
2083
+ case "store.initializeRun":
2084
+ result = await this.runStore.initializeRunFromSnapshot(
2085
+ payload.snapshot as WorkflowDefinitionSnapshot,
2086
+ requireString(payload.workflowName, "workflowName"),
2087
+ payload.state as WorkflowRunState,
2088
+ payload.options as never,
2089
+ );
2090
+ break;
2091
+ case "store.prepareRunResume":
2092
+ await this.prepareInteractionResume(message.runId);
2093
+ result = await this.runStore.prepareRunResume(message.runId);
2094
+ break;
2095
+ case "store.readRun":
2096
+ result = this.runStore.readRun(
2097
+ requireString(payload.runId, "runId"),
2098
+ payload.options as never,
2099
+ );
2100
+ break;
2101
+ case "store.writeSnapshot":
2102
+ result = await this.runStore.writeSnapshot(
2103
+ message.runId,
2104
+ payload.state as WorkflowRunState,
2105
+ payload.event as WorkflowTraceEventDraft,
2106
+ );
2107
+ break;
2108
+ case "store.publishUpdate":
2109
+ result = await this.runStore.publishUpdate(
2110
+ message.runId,
2111
+ payload.state as WorkflowRunState,
2112
+ requireString(payload.nodeId, "nodeId"),
2113
+ requireString(payload.attemptId, "attemptId"),
2114
+ payload.update as WorkflowUpdateInput,
2115
+ );
2116
+ break;
2117
+ case "store.findSettingsScope":
2118
+ result = this.runStore.findSettingsScope(
2119
+ message.runId,
2120
+ requireString(payload.mountPath, "mountPath", true),
2121
+ requireNonNegativeInteger(payload.invocation, "invocation"),
2122
+ );
2123
+ break;
2124
+ case "store.ensureSettingsScope":
2125
+ result = this.runStore.ensureSettingsScope(payload.options as never);
2126
+ break;
2127
+ case "store.getSettingsScopeAtChange":
2128
+ result = this.runStore.getSettingsScopeAtChange(
2129
+ requireString(payload.scopeId, "scopeId"),
2130
+ requireNonNegativeInteger(payload.changeNumber, "changeNumber"),
2131
+ );
2132
+ break;
2133
+ case "store.createHumanDecisionRequest": {
2134
+ const decision = payload.request as HumanDecisionRequest;
2135
+ result = await this.runStore.createHumanDecisionRequest(decision);
2136
+ if (active.record.originSessionId !== null) {
2137
+ this.hostState.createInteractiveRequest({
2138
+ requestId: decision.decisionId,
2139
+ runId: decision.runId,
2140
+ attemptId: decision.attemptId,
2141
+ targetSessionId: active.record.originSessionId,
2142
+ kind: "decision",
2143
+ contract: decision as unknown as JsonValue,
2144
+ });
2145
+ }
2146
+ break;
2147
+ }
2148
+ case "store.readResolvedHumanDecision":
2149
+ result = await this.runStore.readResolvedHumanDecision(
2150
+ requireString(payload.decisionId, "decisionId"),
2151
+ );
2152
+ break;
2153
+ case "store.reserveEffect":
2154
+ result = await this.runStore.reserveEffect(payload.options as never);
2155
+ if (
2156
+ typeof result === "object" &&
2157
+ result !== null &&
2158
+ (result as { disposition?: unknown }).disposition === "ambiguous" &&
2159
+ !this.queue.parkWorkflowRunForAmbiguousEffect({
2160
+ runId: message.runId,
2161
+ claimToken: active.claimToken,
2162
+ })
2163
+ ) {
2164
+ throw new Error("Run claim was lost during ambiguous-effect recovery");
2165
+ }
2166
+ break;
2167
+ case "store.settleEffect": {
2168
+ const options = payload.options as {
2169
+ outcome?: unknown;
2170
+ };
2171
+ await this.runStore.settleEffect(payload.options as never);
2172
+ if (
2173
+ options.outcome === "ambiguous" &&
2174
+ !this.queue.parkWorkflowRunForAmbiguousEffect({
2175
+ runId: message.runId,
2176
+ claimToken: active.claimToken,
2177
+ })
2178
+ ) {
2179
+ throw new Error("Run claim was lost during ambiguous-effect recovery");
2180
+ }
2181
+ result = null;
2182
+ break;
2183
+ }
2184
+ case "interaction.accept":
2185
+ result = this.acceptWorkerInteraction(message, payload);
2186
+ break;
2187
+ case "interaction.reject":
2188
+ result = this.rejectWorkerInteraction(active, message, payload);
2189
+ break;
2190
+ case "notification.request":
2191
+ result = this.enqueueWorkerNotification(active, message, payload);
2192
+ break;
2193
+ case "presentation.request":
2194
+ result = this.requestWorkerPresentation(active, message, payload);
2195
+ break;
2196
+ case "interaction.request": {
2197
+ result = this.parkForInteraction(active, message, payload);
2198
+ const exitDeadline = setTimeout(() => {
2199
+ if (this.activeRuns.get(message.runId) === active && !active.exiting) {
2200
+ active.control = "handoff";
2201
+ void active.supervisor.stop("orphaned");
2202
+ }
2203
+ }, 1_000);
2204
+ exitDeadline.unref?.();
2205
+ break;
2206
+ }
2207
+ default:
2208
+ return workerResponse(message, "rejected", undefined, "Unknown worker operation");
2209
+ }
2210
+ return workerResponse(
2211
+ message,
2212
+ "accepted",
2213
+ result === undefined ? null : (result as JsonValue),
2214
+ undefined,
2215
+ runRevision(this.state, message.runId),
2216
+ );
2217
+ } catch (error) {
2218
+ return workerResponse(message, "rejected", undefined, errorMessage(error));
2219
+ }
2220
+ }
2221
+
2222
+ private handleWorkerProcessOperation(active: ActiveRun, message: WorkerMessage): WorkerResponse {
2223
+ if (
2224
+ message.operation === "process.register" &&
2225
+ !this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
2226
+ ) {
2227
+ active.claimLost = true;
2228
+ setImmediate(() => void active.supervisor.stop("claimLost"));
2229
+ return workerResponse(message, "claimLost", undefined, "Run claim expired");
2230
+ }
2231
+ try {
2232
+ const payload = requireRecord(message.payload, "worker process payload");
2233
+ const pid = requireNonNegativeInteger(payload.pid, "pid");
2234
+ if (pid === 0) throw new Error("Worker process PID must be positive");
2235
+ const owned = this.workerDescendants.get(message.workerEpoch);
2236
+ if (message.operation === "process.unregister") {
2237
+ if (owned?.has(pid) !== true) {
2238
+ throw new Error("Worker process is not registered to this worker epoch");
2239
+ }
2240
+ this.registry.unregister(pid);
2241
+ owned.delete(pid);
2242
+ if (owned.size === 0) this.workerDescendants.delete(message.workerEpoch);
2243
+ return workerResponse(message, "accepted", { pid });
2244
+ }
2245
+ if (active.workerPid === undefined || processParentPid(pid) !== active.workerPid) {
2246
+ throw new Error("Worker process is not a direct child of the active worker");
2247
+ }
2248
+ for (const [workerEpoch, pids] of this.workerDescendants) {
2249
+ if (workerEpoch !== message.workerEpoch && pids.has(pid)) {
2250
+ throw new Error("Worker process is already registered to another worker epoch");
2251
+ }
2252
+ }
2253
+ this.registry.register(pid);
2254
+ const descendants = owned ?? new Set<number>();
2255
+ descendants.add(pid);
2256
+ this.workerDescendants.set(message.workerEpoch, descendants);
2257
+ return workerResponse(message, "accepted", { pid });
2258
+ } catch (error) {
2259
+ return workerResponse(message, "rejected", undefined, errorMessage(error));
2260
+ }
2261
+ }
2262
+
2263
+ private reapWorkerDescendants(workerEpoch: string): void {
2264
+ const descendants = this.workerDescendants.get(workerEpoch);
2265
+ if (descendants === undefined) return;
2266
+ this.workerDescendants.delete(workerEpoch);
2267
+ for (const pid of descendants) this.registry.kill(pid);
2268
+ }
2269
+
2270
+ private acceptWorkerInteraction(
2271
+ message: WorkerMessage,
2272
+ payload: Record<string, unknown>,
2273
+ ): InteractiveSubmissionRecord {
2274
+ const candidate = this.requireWorkerInteraction(message, payload, true);
2275
+ return this.hostState.finishInteractionValidation({
2276
+ requestId: candidate.requestId,
2277
+ submissionId: candidate.submissionId,
2278
+ accepted: true,
2279
+ receipt: { status: "accepted" },
2280
+ });
2281
+ }
2282
+
2283
+ private rejectWorkerInteraction(
2284
+ active: ActiveRun,
2285
+ message: WorkerMessage,
2286
+ payload: Record<string, unknown>,
2287
+ ): InteractiveSubmissionRecord {
2288
+ const candidate = this.requireWorkerInteraction(message, payload, false);
2289
+ return this.settleRejectedInteraction(
2290
+ active,
2291
+ candidate,
2292
+ requireString(payload.error, "interaction validation error"),
2293
+ );
2294
+ }
2295
+
2296
+ private requireWorkerInteraction(
2297
+ message: WorkerMessage,
2298
+ payload: Record<string, unknown>,
2299
+ acceptSettled: boolean,
2300
+ ) {
2301
+ const candidate =
2302
+ this.hostState.validatingInteraction(message.runId) ??
2303
+ (acceptSettled ? this.hostState.acceptedInteraction(message.runId) : undefined);
2304
+ if (
2305
+ candidate === undefined ||
2306
+ candidate.requestId !== requireString(payload.requestId, "interaction requestId") ||
2307
+ candidate.submissionId !== requireString(payload.submissionId, "interaction submissionId") ||
2308
+ candidate.attemptId !== message.attemptId ||
2309
+ candidate.attemptId !== requireString(payload.attemptId, "interaction attemptId")
2310
+ ) {
2311
+ throw new Error("Interactive submission does not match the worker candidate");
2312
+ }
2313
+ return candidate;
2314
+ }
2315
+
2316
+ private settleRejectedInteraction(
2317
+ active: ActiveRun,
2318
+ candidate: {
2319
+ requestId: string;
2320
+ submissionId: string;
2321
+ attemptId: string;
2322
+ },
2323
+ error: string,
2324
+ ): InteractiveSubmissionRecord {
2325
+ const result = this.state.transaction(() => {
2326
+ const submission = this.hostState.finishInteractionValidation({
2327
+ requestId: candidate.requestId,
2328
+ submissionId: candidate.submissionId,
2329
+ accepted: false,
2330
+ receipt: { status: "rejected", error },
2331
+ });
2332
+ const now = Date.now();
2333
+ this.state.connection
2334
+ .prepare(
2335
+ `UPDATE node_attempts SET status = 'waiting', updated_at = ?
2336
+ WHERE attempt_id = ? AND run_id = ? AND status IN ('running', 'interrupted')`,
2337
+ )
2338
+ .run(now, candidate.attemptId, active.record.runId);
2339
+ this.state.connection
2340
+ .prepare(
2341
+ `UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
2342
+ WHERE run_id = ? AND status = 'running'`,
2343
+ )
2344
+ .run(error, now, now, active.record.runId);
2345
+ if (
2346
+ !this.queue.parkWorkflowRun({
2347
+ runId: active.record.runId,
2348
+ claimToken: active.claimToken,
2349
+ })
2350
+ ) {
2351
+ throw new Error("Rejected interaction could not release the run claim");
2352
+ }
2353
+ recordViewerDeltas(
2354
+ this.state,
2355
+ active.record.runId,
2356
+ [{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
2357
+ now,
2358
+ );
2359
+ return submission;
2360
+ });
2361
+ active.control = "handoff";
2362
+ return result;
2363
+ }
2364
+
2365
+ private enqueueWorkerNotification(
2366
+ active: ActiveRun,
2367
+ message: WorkerMessage,
2368
+ payload: Record<string, unknown>,
2369
+ ): JsonValue {
2370
+ if (active.record.originSessionId === null) {
2371
+ throw new Error("Workflow notification has no origin Pi session");
2372
+ }
2373
+ const request = requireRecord(payload.request, "notification request");
2374
+ if (
2375
+ requireString(request.runId, "notification.runId") !== message.runId ||
2376
+ requireString(request.workflowName, "notification.workflowName") !==
2377
+ active.record.workflowName
2378
+ ) {
2379
+ throw new Error("Workflow notification provenance does not match the active run");
2380
+ }
2381
+ const notificationIndex = requireNonNegativeInteger(
2382
+ request.notificationIndex,
2383
+ "notificationIndex",
2384
+ );
2385
+ if (notificationIndex === 0) throw new Error("notificationIndex must be positive");
2386
+ const kind = request.kind;
2387
+ if (kind !== "progress" && kind !== "final") {
2388
+ throw new Error("Workflow notification kind is invalid");
2389
+ }
2390
+ const record = this.queue.enqueueWorkflowNotification({
2391
+ runId: message.runId,
2392
+ nodeId: requireString(request.nodeId, "notification.nodeId"),
2393
+ attemptId: requireString(request.attemptId, "notification.attemptId"),
2394
+ notificationIndex,
2395
+ targetSessionId: active.record.originSessionId,
2396
+ kind,
2397
+ content: requireString(request.content, "notification.content"),
2398
+ });
2399
+ return {
2400
+ notificationId: record.notificationId,
2401
+ targetSessionId: record.targetSessionId,
2402
+ };
2403
+ }
2404
+
2405
+ private requestWorkerPresentation(
2406
+ active: ActiveRun,
2407
+ message: WorkerMessage,
2408
+ payload: Record<string, unknown>,
2409
+ ): JsonValue {
2410
+ if (active.record.originSessionId === null) {
2411
+ throw new Error("Workflow presentation has no origin Pi session");
2412
+ }
2413
+ const instructions = requireString(payload.instructions, "presentation instructions").trim();
2414
+ if (instructions.length === 0) throw new Error("Presentation instructions must not be empty");
2415
+ const facts: WorkflowTurnIntentFacts & { presentationPrompt: string } = {
2416
+ schema: "pi-workflows.deferred-turn-facts.v1",
2417
+ workflowName: active.record.workflowName,
2418
+ runId: message.runId,
2419
+ observedState: "completed",
2420
+ cause: "terminal",
2421
+ nodeId: null,
2422
+ attemptId: null,
2423
+ reason: null,
2424
+ handoff: false,
2425
+ presentationPrompt: instructions,
2426
+ };
2427
+ const intent = this.queue.ensureWorkflowTurnIntent({
2428
+ intentId: `presentation-${message.runId}`,
2429
+ sourceEventId: `presentation-request-${message.runId}`,
2430
+ runId: message.runId,
2431
+ workflowRef: active.record.workflowSourceRef,
2432
+ targetSessionId: active.record.originSessionId,
2433
+ cause: "terminal",
2434
+ fallbackFacts: facts,
2435
+ eligible: false,
2436
+ });
2437
+ return intent as unknown as JsonValue;
2438
+ }
2439
+
2440
+ private parkForInteraction(
2441
+ active: ActiveRun,
2442
+ message: WorkerMessage,
2443
+ payload: Record<string, unknown>,
2444
+ ): JsonValue {
2445
+ if (active.record.originSessionId === null) {
2446
+ throw new Error("Interactive request has no origin Pi session");
2447
+ }
2448
+ const attemptId = requireString(payload.attemptId, "attemptId");
2449
+ const requestId = `interaction-${message.runId}-${attemptId}`;
2450
+ return this.state.transaction(() => {
2451
+ const request = this.hostState.createInteractiveRequest({
2452
+ requestId,
2453
+ runId: message.runId,
2454
+ attemptId,
2455
+ targetSessionId: active.record.originSessionId as string,
2456
+ kind: payload.kind === "assistant" ? "assistant" : "agent",
2457
+ contract: payload.contract as JsonValue,
2458
+ });
2459
+ const now = Date.now();
2460
+ this.state.connection
2461
+ .prepare(
2462
+ `UPDATE node_attempts SET status = 'waiting', updated_at = ?
2463
+ WHERE attempt_id = ? AND run_id = ? AND status = 'running'`,
2464
+ )
2465
+ .run(now, attemptId, message.runId);
2466
+ this.state.connection
2467
+ .prepare(
2468
+ `UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
2469
+ WHERE run_id = ? AND status = 'running'`,
2470
+ )
2471
+ .run("waiting for origin Pi session", now, now, message.runId);
2472
+ if (!this.queue.parkWorkflowRun({ runId: message.runId, claimToken: active.claimToken })) {
2473
+ throw new Error("Interactive request could not release the run claim");
2474
+ }
2475
+ active.control = "handoff";
2476
+ recordViewerDeltas(
2477
+ this.state,
2478
+ message.runId,
2479
+ [{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
2480
+ now,
2481
+ );
2482
+ return request as unknown as JsonValue;
2483
+ });
2484
+ }
2485
+
2486
+ private async prepareInteractionResume(runId: string): Promise<void> {
2487
+ const accepted = this.hostState.acceptedInteraction(runId);
2488
+ const candidate = accepted ?? this.hostState.validatingInteraction(runId);
2489
+ const timedOut = this.hostState.timedOutInteraction(runId);
2490
+ if (candidate === undefined && timedOut === undefined) return;
2491
+ const loaded = this.runStore.readRun(runId);
2492
+ if (loaded === null) return;
2493
+ if (timedOut !== undefined) {
2494
+ if (loaded.state.status !== "running") {
2495
+ throw new Error(`Timed-out interaction run ${runId} is not running`);
2496
+ }
2497
+ return;
2498
+ }
2499
+ if (candidate === undefined || loaded.state.status !== "waiting") return;
2500
+ loaded.state.status = "running";
2501
+ delete loaded.state.statusDetail;
2502
+ delete loaded.state.finishedAt;
2503
+ await this.runStore.writeSnapshot(runId, loaded.state, {
2504
+ scope: "node",
2505
+ type: accepted === undefined ? "interaction_validation_started" : "interaction_accepted",
2506
+ nodeId: candidate.nodeId,
2507
+ attemptId: candidate.attemptId,
2508
+ payload: {
2509
+ requestId: candidate.requestId,
2510
+ submissionId: candidate.submissionId,
2511
+ },
2512
+ });
2513
+ }
2514
+
2515
+ private applyLifecycleProjection(context: {
2516
+ runId: string;
2517
+ state: WorkflowRunState;
2518
+ event: { type: string; attemptId?: string; payload?: Record<string, unknown> };
2519
+ database: StateDatabase;
2520
+ now: number;
2521
+ }): void {
2522
+ const active = this.activeRuns.get(context.runId);
2523
+ if (active === undefined) return;
2524
+ if (context.event.type === "node_finished" && context.event.attemptId !== undefined) {
2525
+ this.hostState.consumeAcceptedInteraction(
2526
+ context.runId,
2527
+ context.event.attemptId,
2528
+ context.now,
2529
+ );
2530
+ }
2531
+ if (context.state.status === "running") {
2532
+ if (context.event.type === "run_started" || context.event.type === "run_resumed") {
2533
+ context.database.connection
2534
+ .prepare(
2535
+ `UPDATE run_queue SET status = 'running', started_at = COALESCE(started_at, ?), updated_at = ?
2536
+ WHERE run_id = ? AND status IN ('starting', 'parked')`,
2537
+ )
2538
+ .run(context.now, context.now, context.runId);
2539
+ }
2540
+ return;
2541
+ }
2542
+ this.completeControllerWorkflow(context.runId, context.state);
2543
+ const queueStatus =
2544
+ context.state.status === "completed"
2545
+ ? "done"
2546
+ : context.state.status === "waiting"
2547
+ ? "parked"
2548
+ : context.state.status === "cancelled"
2549
+ ? "cancelled"
2550
+ : "failed";
2551
+ const run = context.database.connection
2552
+ .prepare("SELECT error_hash AS errorHash FROM runs WHERE run_id = ?")
2553
+ .get(context.runId) as { errorHash?: Buffer | null } | undefined;
2554
+ context.database.connection
2555
+ .prepare(
2556
+ `UPDATE run_queue
2557
+ SET status = ?, error_code = ?, error_hash = ?, updated_at = ?, finished_at = ?
2558
+ WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
2559
+ )
2560
+ .run(
2561
+ queueStatus,
2562
+ queueStatus === "failed" ? context.state.status : null,
2563
+ run?.errorHash ?? null,
2564
+ context.now,
2565
+ ["done", "failed", "cancelled"].includes(queueStatus) ? context.now : null,
2566
+ context.runId,
2567
+ );
2568
+ if (context.state.status === "completed") {
2569
+ if (
2570
+ context.event.payload?.presentationRequired === true &&
2571
+ active.record.originSessionId !== null &&
2572
+ this.queue.findPendingWorkflowTurnIntent({
2573
+ runId: context.runId,
2574
+ targetSessionId: active.record.originSessionId,
2575
+ }) === undefined
2576
+ ) {
2577
+ const facts: WorkflowTurnIntentFacts & { presentationPrompt: string } = {
2578
+ schema: "pi-workflows.deferred-turn-facts.v1",
2579
+ workflowName: active.record.workflowName,
2580
+ runId: context.runId,
2581
+ observedState: "completed",
2582
+ cause: "terminal",
2583
+ nodeId: null,
2584
+ attemptId: null,
2585
+ reason: "workflow presentation instructions were unavailable",
2586
+ handoff: false,
2587
+ presentationPrompt:
2588
+ "Summarize the completed workflow result for the user in a normal response.",
2589
+ };
2590
+ this.queue.ensureWorkflowTurnIntent({
2591
+ intentId: `presentation-${context.runId}`,
2592
+ sourceEventId: `presentation-request-${context.runId}`,
2593
+ runId: context.runId,
2594
+ workflowRef: active.record.workflowSourceRef,
2595
+ targetSessionId: active.record.originSessionId,
2596
+ cause: "terminal",
2597
+ fallbackFacts: facts,
2598
+ eligible: false,
2599
+ });
2600
+ }
2601
+ context.database.connection
2602
+ .prepare(
2603
+ `UPDATE turn_intents SET eligible_at = ?
2604
+ WHERE run_id = ? AND resolved_at IS NULL AND eligible_at IS NULL`,
2605
+ )
2606
+ .run(context.now, context.runId);
2607
+ }
2608
+ context.database.connection
2609
+ .prepare(
2610
+ `UPDATE leases
2611
+ SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
2612
+ acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
2613
+ WHERE resource_id = (SELECT resource_id FROM runs WHERE run_id = ?)
2614
+ AND generation = ?`,
2615
+ )
2616
+ .run(context.runId, active.generation);
2617
+ }
2618
+
2619
+ private completeControllerWorkflow(runId: string, state: WorkflowRunState): void {
2620
+ const row = this.state.connection
2621
+ .prepare(
2622
+ `SELECT w.request_id AS requestId, c.controller_name AS controller,
2623
+ c.resource_key AS resourceKey, p.canonical_path AS projectPath
2624
+ FROM controller_workflows w
2625
+ JOIN controller_resources c ON c.controller_resource_id = w.controller_resource_id
2626
+ JOIN projects p ON p.project_id = c.project_id
2627
+ WHERE w.run_id = ? LIMIT 1`,
2628
+ )
2629
+ .get(runId) as
2630
+ | {
2631
+ requestId?: unknown;
2632
+ controller?: unknown;
2633
+ resourceKey?: unknown;
2634
+ projectPath?: unknown;
2635
+ }
2636
+ | undefined;
2637
+ if (
2638
+ typeof row?.requestId !== "string" ||
2639
+ typeof row.controller !== "string" ||
2640
+ typeof row.resourceKey !== "string" ||
2641
+ typeof row.projectPath !== "string"
2642
+ ) {
2643
+ return;
2644
+ }
2645
+ const store = new SqliteControllerStore(this.databasePath, {
2646
+ state: this.state,
2647
+ projectPath: row.projectPath,
2648
+ });
2649
+ const childState =
2650
+ state.status === "completed"
2651
+ ? "succeeded"
2652
+ : state.status === "waiting"
2653
+ ? "waiting"
2654
+ : "failed";
2655
+ store.completeWorkflow(row.requestId, {
2656
+ state: childState,
2657
+ runId,
2658
+ ...(state.error === undefined ? {} : { error: state.error }),
2659
+ });
2660
+ store.enqueue({ controller: row.controller, key: row.resourceKey });
2661
+ }
2662
+
2663
+ private commitActivePause(active: ActiveRun): void {
2664
+ const now = Date.now();
2665
+ this.state.transaction(() => {
2666
+ this.state.connection
2667
+ .prepare("UPDATE runs SET paused = 1, status_detail = ?, updated_at = ? WHERE run_id = ?")
2668
+ .run("paused", now, active.record.runId);
2669
+ if (
2670
+ !this.queue.parkWorkflowRun({
2671
+ runId: active.record.runId,
2672
+ claimToken: active.claimToken,
2673
+ })
2674
+ ) {
2675
+ throw new Error("Run claim was lost before pause committed");
2676
+ }
2677
+ });
2678
+ }
2679
+
2680
+ private async recoverWorkerExit(active: ActiveRun, outcome: string): Promise<void> {
2681
+ if (
2682
+ !this.queue.verifyWorkflowRunClaim({
2683
+ runId: active.record.runId,
2684
+ claimToken: active.claimToken,
2685
+ })
2686
+ ) {
2687
+ return;
2688
+ }
2689
+ const recovery = await this.runStore.recoverApplyingEffects(active.record.runId);
2690
+ if (recovery === "ambiguous" || hasUncertainEffect(this.state, active.record.runId)) {
2691
+ this.blockedRuns.add(active.record.runId);
2692
+ if (
2693
+ !this.queue.parkWorkflowRunForAmbiguousEffect({
2694
+ runId: active.record.runId,
2695
+ claimToken: active.claimToken,
2696
+ })
2697
+ ) {
2698
+ return;
2699
+ }
2700
+ this.log(`run ${active.record.runId} parked for ambiguous effect recovery`);
2701
+ return;
2702
+ }
2703
+ const validating = this.hostState.validatingInteraction(active.record.runId);
2704
+ if (validating !== undefined) {
2705
+ this.settleRejectedInteraction(
2706
+ active,
2707
+ validating,
2708
+ `Workflow worker ${outcome} before it completed interaction validation`,
2709
+ );
2710
+ this.log(`run ${active.record.runId} rejected an interrupted interaction validation`);
2711
+ return;
2712
+ }
2713
+ if (active.workflowLoadFailure !== undefined) {
2714
+ if (
2715
+ this.queue.parkWorkflowRunForSourceChange({
2716
+ runId: active.record.runId,
2717
+ claimToken: active.claimToken,
2718
+ detail: active.workflowLoadFailure,
2719
+ })
2720
+ ) {
2721
+ this.log(`run ${active.record.runId} parked because its workflow source could not load`);
2722
+ }
2723
+ return;
2724
+ }
2725
+ this.queue.parkWorkflowRun({ runId: active.record.runId, claimToken: active.claimToken });
2726
+ this.log(`run ${active.record.runId} parked after worker ${outcome}`);
2727
+ }
2728
+
2729
+ private releaseLock(): void {
2730
+ try {
2731
+ const current = JSON.parse(fs.readFileSync(this.lockPath, "utf8")) as unknown;
2732
+ if (isLockRecord(current) && current.hostId === this.hostId) fs.rmSync(this.lockPath);
2733
+ } catch {
2734
+ // The lock is already gone.
2735
+ }
2736
+ }
2737
+
2738
+ private log(message: string): void {
2739
+ this.options.onLog?.(message);
2740
+ }
2741
+ }
2742
+
2743
+ function acquireHostLock(
2744
+ lockPath: string,
2745
+ record: { pid: number; startIdentity: string; hostId: string },
2746
+ ): void {
2747
+ try {
2748
+ const existing = JSON.parse(fs.readFileSync(lockPath, "utf8")) as unknown;
2749
+ if (isLockRecord(existing) && matchesProcessIdentity(existing)) {
2750
+ throw new Error(`A workflow host is already running with PID ${existing.pid}`);
2751
+ }
2752
+ fs.rmSync(lockPath, { force: true });
2753
+ } catch (error) {
2754
+ if (error instanceof Error && error.message.startsWith("A workflow host is already"))
2755
+ throw error;
2756
+ }
2757
+ fs.writeFileSync(
2758
+ lockPath,
2759
+ `${JSON.stringify({ schema: "pi-workflows.host-lock.v1", ...record })}\n`,
2760
+ { encoding: "utf8", mode: 0o600, flag: "wx" },
2761
+ );
2762
+ }
2763
+
2764
+ function workerResponse(
2765
+ message: WorkerMessage,
2766
+ outcome: WorkerResponse["outcome"],
2767
+ result?: JsonValue,
2768
+ error?: string,
2769
+ revision?: number,
2770
+ ): WorkerResponse {
2771
+ return {
2772
+ schema: "pi-workflows.worker-response.v1",
2773
+ messageId: message.messageId,
2774
+ outcome,
2775
+ ...(revision === undefined ? {} : { revision }),
2776
+ ...(result === undefined ? {} : { result }),
2777
+ ...(error === undefined ? {} : { error }),
2778
+ };
2779
+ }
2780
+
2781
+ function controllerWorkerResponse(
2782
+ message: ControllerWorkerMessage,
2783
+ outcome: ControllerWorkerResponse["outcome"],
2784
+ result?: JsonValue,
2785
+ error?: string,
2786
+ ): ControllerWorkerResponse {
2787
+ return {
2788
+ schema: "pi-workflows.controller-worker-response.v1",
2789
+ messageId: message.messageId,
2790
+ outcome,
2791
+ ...(result === undefined ? {} : { result }),
2792
+ ...(error === undefined ? {} : { error }),
2793
+ };
2794
+ }
2795
+
2796
+ function controllerWorkflowResult(record: WorkflowRunQueueRecord): WorkflowSchedulerResult {
2797
+ switch (record.status) {
2798
+ case "queued":
2799
+ case "starting":
2800
+ case "running":
2801
+ return { state: "running", runId: record.runId };
2802
+ case "parked":
2803
+ return { state: "waiting", runId: record.runId };
2804
+ case "done":
2805
+ return { state: "succeeded", runId: record.runId };
2806
+ case "failed":
2807
+ case "cancelled":
2808
+ return {
2809
+ state: "failed",
2810
+ runId: record.runId,
2811
+ ...(record.errorMessage === null ? {} : { error: record.errorMessage }),
2812
+ };
2813
+ }
2814
+ }
2815
+
2816
+ function runRevision(state: StateDatabase, runId: string): number {
2817
+ const row = state.connection
2818
+ .prepare(
2819
+ `SELECT resources.revision FROM resources
2820
+ JOIN runs ON runs.resource_id = resources.resource_id WHERE runs.run_id = ?`,
2821
+ )
2822
+ .get(runId);
2823
+ if (!isRevisionRow(row)) throw new Error(`Workflow run not found: ${runId}`);
2824
+ return row.revision;
2825
+ }
2826
+
2827
+ function hasUncertainEffect(state: StateDatabase, runId: string): boolean {
2828
+ const row = state.connection
2829
+ .prepare(
2830
+ `SELECT 1 AS present FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
2831
+ WHERE r.run_id = ? AND e.status IN ('applying', 'ambiguous') LIMIT 1`,
2832
+ )
2833
+ .get(runId);
2834
+ return row !== undefined;
2835
+ }
2836
+
2837
+ function controllerPathAllowed(
2838
+ projectPath: string,
2839
+ controllerName: string,
2840
+ controllerPath: string,
2841
+ ): boolean {
2842
+ if (
2843
+ controllerFileStem(controllerPath) !== controllerName ||
2844
+ !/\.controller\.(?:ts|js|mts|mjs)$/u.test(controllerPath)
2845
+ ) {
2846
+ return false;
2847
+ }
2848
+ return controllerSearchDirs({ cwd: projectPath }).some(
2849
+ ({ dir }) => path.dirname(controllerPath) === path.resolve(dir),
2850
+ );
2851
+ }
2852
+
2853
+ function payloadLimit(payload: JsonValue): number {
2854
+ if (typeof payload !== "object" || payload === null || Array.isArray(payload)) return 100;
2855
+ const limit = (payload as Record<string, unknown>).limit;
2856
+ return typeof limit === "number" && Number.isSafeInteger(limit) && limit > 0 ? limit : 100;
2857
+ }
2858
+
2859
+ function requireRunId(request: HostRequest): string {
2860
+ if (request.runId === undefined) throw new Error(`${request.operation} requires runId`);
2861
+ return request.runId;
2862
+ }
2863
+
2864
+ function requireRecord(value: unknown, name: string): Record<string, unknown> {
2865
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
2866
+ throw new Error(`${name} must be an object`);
2867
+ }
2868
+ return value as Record<string, unknown>;
2869
+ }
2870
+
2871
+ function requireString(value: unknown, name: string, allowEmpty = false): string {
2872
+ if (typeof value !== "string" || (!allowEmpty && value.length === 0)) {
2873
+ throw new Error(`${name} must be text`);
2874
+ }
2875
+ return value;
2876
+ }
2877
+
2878
+ function requireAbsolutePath(value: unknown, name: string): string {
2879
+ const parsed = requireString(value, name);
2880
+ if (!path.isAbsolute(parsed)) throw new Error(`${name} must be an absolute path`);
2881
+ return parsed;
2882
+ }
2883
+
2884
+ function requireNonNegativeInteger(value: unknown, name: string): number {
2885
+ if (!Number.isSafeInteger(value) || (value as number) < 0) {
2886
+ throw new Error(`${name} must be a non-negative integer`);
2887
+ }
2888
+ return value as number;
2889
+ }
2890
+
2891
+ function isRevisionRow(value: unknown): value is { revision: number } {
2892
+ return (
2893
+ typeof value === "object" &&
2894
+ value !== null &&
2895
+ typeof (value as { revision?: unknown }).revision === "number"
2896
+ );
2897
+ }
2898
+
2899
+ function isLockRecord(
2900
+ value: unknown,
2901
+ ): value is { schema: string; pid: number; startIdentity: string; hostId: string } {
2902
+ return (
2903
+ typeof value === "object" &&
2904
+ value !== null &&
2905
+ (value as { schema?: unknown }).schema === "pi-workflows.host-lock.v1" &&
2906
+ typeof (value as { pid?: unknown }).pid === "number" &&
2907
+ typeof (value as { startIdentity?: unknown }).startIdentity === "string" &&
2908
+ typeof (value as { hostId?: unknown }).hostId === "string"
2909
+ );
519
2910
  }