@osolmaz/pi-workflows 0.16.2 → 0.16.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -41
- package/dist/builtins/monitor.workflow.d.ts +1 -1
- package/dist/builtins/monitor.workflow.js +8 -8
- package/dist/builtins/pi-agent-group.js +3 -3
- package/dist/builtins/pi-agent-group.js.map +1 -1
- package/dist/channels/adapter-entry.js +14 -14
- package/dist/channels/adapter-entry.js.map +1 -1
- package/dist/client/client.d.ts +6 -6
- package/dist/client/client.js +27 -27
- package/dist/client/client.js.map +1 -1
- package/dist/client/protocol.d.ts +4 -1
- package/dist/client/protocol.js +17 -11
- package/dist/client/protocol.js.map +1 -1
- package/dist/client/resolver.d.ts +4 -4
- package/dist/client/view.d.ts +1 -1
- package/dist/extension/index.d.ts +1 -1
- package/dist/extension/index.js +32 -32
- package/dist/extension/index.js.map +1 -1
- package/dist/extension/recorder.d.ts +1 -1
- package/dist/extension/recorder.js +1 -1
- package/dist/extension/recorder.js.map +1 -1
- package/dist/extension/remote-recorder-store.d.ts +1 -1
- package/dist/extension/remote-recorder-store.js +2 -2
- package/dist/extension/remote-recorder-store.js.map +1 -1
- package/dist/extension/{controller-command.d.ts → resource-manager-command.d.ts} +6 -6
- package/dist/extension/{controller-command.js → resource-manager-command.js} +7 -7
- package/dist/extension/resource-manager-command.js.map +1 -0
- package/dist/extension/session-delivery.d.ts +1 -1
- package/dist/extension/session-delivery.js +2 -2
- package/dist/extension/session-delivery.js.map +1 -1
- package/dist/extension/session-view.d.ts +1 -1
- package/dist/extension/session-view.js +1 -1
- package/dist/extension/session-view.js.map +1 -1
- package/dist/extension/workflow-message-coordinator.d.ts +1 -1
- package/dist/extension/workflow-message-coordinator.js +6 -6
- package/dist/extension/workflow-message-coordinator.js.map +1 -1
- package/dist/resource-managers/conditions.d.ts +6 -0
- package/dist/{controllers → resource-managers}/conditions.js +3 -3
- package/dist/resource-managers/conditions.js.map +1 -0
- package/dist/resource-managers/definition.d.ts +6 -0
- package/dist/resource-managers/definition.js +45 -0
- package/dist/resource-managers/definition.js.map +1 -0
- package/dist/resource-managers/effects.d.ts +15 -0
- package/dist/{controllers → resource-managers}/effects.js +2 -2
- package/dist/resource-managers/effects.js.map +1 -0
- package/dist/resource-managers/errors.d.ts +12 -0
- package/dist/resource-managers/errors.js +25 -0
- package/dist/resource-managers/errors.js.map +1 -0
- package/dist/resource-managers/index.d.ts +12 -0
- package/dist/resource-managers/index.js +12 -0
- package/dist/resource-managers/index.js.map +1 -0
- package/dist/resource-managers/json.js.map +1 -0
- package/dist/resource-managers/loader.d.ts +23 -0
- package/dist/resource-managers/loader.js +75 -0
- package/dist/resource-managers/loader.js.map +1 -0
- package/dist/resource-managers/results.d.ts +5 -0
- package/dist/resource-managers/results.js.map +1 -0
- package/dist/resource-managers/runtime.d.ts +59 -0
- package/dist/{controllers/manager.js → resource-managers/runtime.js} +57 -57
- package/dist/resource-managers/runtime.js.map +1 -0
- package/dist/{controllers → resource-managers}/sqlite.d.ts +44 -44
- package/dist/{controllers → resource-managers}/sqlite.js +139 -130
- package/dist/resource-managers/sqlite.js.map +1 -0
- package/dist/{controllers → resource-managers}/store.d.ts +33 -33
- package/dist/{controllers → resource-managers}/store.js.map +1 -1
- package/dist/{controllers → resource-managers}/types.d.ts +41 -41
- package/dist/{controllers → resource-managers}/types.js.map +1 -1
- package/dist/resource-managers/workflows.d.ts +34 -0
- package/dist/{controllers → resource-managers}/workflows.js +15 -15
- package/dist/resource-managers/workflows.js.map +1 -0
- package/dist/server/channel-effects.js.map +1 -0
- package/dist/{host → server}/channel-supervisor.d.ts +6 -6
- package/dist/{host → server}/channel-supervisor.js +3 -3
- package/dist/server/channel-supervisor.js.map +1 -0
- package/dist/{host/child-worker-supervisor.d.ts → server/child-runner-supervisor.d.ts} +15 -13
- package/dist/{host/child-worker-supervisor.js → server/child-runner-supervisor.js} +6 -5
- package/dist/server/child-runner-supervisor.js.map +1 -0
- package/dist/{host → server}/processes.d.ts +3 -3
- package/dist/{host → server}/processes.js +3 -3
- package/dist/server/processes.js.map +1 -0
- package/dist/{host → server}/resolver-entry.d.ts +6 -6
- package/dist/{host → server}/resolver-entry.js +14 -14
- package/dist/server/resolver-entry.js.map +1 -0
- package/dist/server/resource-runner-entry.d.ts +2 -0
- package/dist/{host/controller-worker-entry.js → server/resource-runner-entry.js} +33 -33
- package/dist/server/resource-runner-entry.js.map +1 -0
- package/dist/server/resource-runner-protocol.d.ts +36 -0
- package/dist/server/resource-runner-protocol.js +49 -0
- package/dist/server/resource-runner-protocol.js.map +1 -0
- package/dist/server/resource-runner-supervisor.d.ts +21 -0
- package/dist/{host/worker-supervisor.js → server/resource-runner-supervisor.js} +17 -17
- package/dist/server/resource-runner-supervisor.js.map +1 -0
- package/dist/{host → server}/rpc-bridge.d.ts +3 -3
- package/dist/{host → server}/rpc-bridge.js +3 -3
- package/dist/server/rpc-bridge.js.map +1 -0
- package/dist/{host → server}/rpc-executor.d.ts +1 -1
- package/dist/{host → server}/rpc-executor.js +3 -3
- package/dist/server/rpc-executor.js.map +1 -0
- package/dist/{host/host-entry.js → server/server-entry.js} +6 -6
- package/dist/server/server-entry.js.map +1 -0
- package/dist/{host/runner.d.ts → server/server.d.ts} +39 -38
- package/dist/{host/runner.js → server/server.js} +408 -374
- package/dist/server/server.js.map +1 -0
- package/dist/{host → server}/state.d.ts +25 -25
- package/dist/{host → server}/state.js +49 -49
- package/dist/server/state.js.map +1 -0
- package/dist/{host → server}/view.d.ts +7 -7
- package/dist/{host → server}/view.js +13 -13
- package/dist/server/view.js.map +1 -0
- package/dist/server/workflow-runner-content.d.ts +6 -0
- package/dist/server/workflow-runner-content.js +96 -0
- package/dist/server/workflow-runner-content.js.map +1 -0
- package/dist/{host/worker-entry.d.ts → server/workflow-runner-entry.d.ts} +3 -3
- package/dist/{host/worker-entry.js → server/workflow-runner-entry.js} +68 -35
- package/dist/server/workflow-runner-entry.js.map +1 -0
- package/dist/server/workflow-runner-protocol.d.ts +69 -0
- package/dist/server/workflow-runner-protocol.js +176 -0
- package/dist/server/workflow-runner-protocol.js.map +1 -0
- package/dist/{host/worker-store.d.ts → server/workflow-runner-store.d.ts} +11 -11
- package/dist/{host/worker-store.js → server/workflow-runner-store.js} +8 -8
- package/dist/server/workflow-runner-store.js.map +1 -0
- package/dist/server/workflow-runner-supervisor.d.ts +22 -0
- package/dist/server/workflow-runner-supervisor.js +56 -0
- package/dist/server/workflow-runner-supervisor.js.map +1 -0
- package/dist/state/prune.d.ts +1 -1
- package/dist/state/prune.js +17 -10
- package/dist/state/prune.js.map +1 -1
- package/dist/state/schema.js +2 -2
- package/dist/state/workflow-messages.d.ts +1 -1
- package/dist/state/workflow-messages.js +1 -1
- package/dist/state/workflow-messages.js.map +1 -1
- package/dist/viewer/backup.js +1 -1
- package/dist/viewer/backup.js.map +1 -1
- package/dist/viewer/cli.d.ts +2 -2
- package/dist/viewer/cli.js +46 -32
- package/dist/viewer/cli.js.map +1 -1
- package/dist/viewer/tui.d.ts +1 -1
- package/dist/viewer/tui.js +1 -1
- package/dist/viewer/tui.js.map +1 -1
- package/dist/workflows/command-batch.js +2 -2
- package/dist/workflows/engine.js +14 -15
- package/dist/workflows/engine.js.map +1 -1
- package/dist/workflows/store.d.ts +12 -11
- package/dist/workflows/store.js +24 -18
- package/dist/workflows/store.js.map +1 -1
- package/dist/workflows/types.d.ts +2 -2
- package/docs/2026-08-20-durable-workflow-launch-plan.md +12 -12
- package/docs/2026-08-25-workflow-follow-ups.md +7 -7
- package/docs/2026-08-25-workflow-settings.md +3 -3
- package/docs/2026-08-30-out-of-process-workflow-host-plan.md +3 -3
- package/docs/2026-09-01-restore-session-delivery-controls-plan.md +5 -5
- package/docs/2026-09-01-unified-workflow-client-plan.md +6 -6
- package/docs/2026-09-02-installed-live-e2e-plan.md +3 -3
- package/docs/2026-09-02-unify-workflow-messages-plan.md +8 -8
- package/docs/2026-09-04-workflow-run-state-plan.md +6 -6
- package/docs/DEFERRED_TURNS.md +7 -7
- package/docs/DESIGN_PHILOSOPHY.md +1 -1
- package/docs/HUMAN_DECISIONS.md +11 -11
- package/docs/HUMAN_DECISION_PRESENTATIONS.md +4 -4
- package/docs/MONITOR.md +1 -1
- package/docs/RESOURCE_MANAGERS.md +221 -0
- package/docs/SQLITE_STATE.md +35 -33
- package/docs/WORKFLOW_COMPOSITION.md +1 -1
- package/docs/{WORKFLOW_HOST.md → WORKFLOW_SERVER.md} +125 -119
- package/docs/WORKFLOW_STEP_MESSAGES.md +18 -18
- package/docs/WORKFLOW_UPDATES.md +16 -16
- package/docs/development.md +15 -14
- package/docs/live-replay-protocol.md +20 -20
- package/docs/plans/2026-08-04-controller-runtime-plan.md +25 -25
- package/docs/plans/2026-08-05-always-on-workflows-plan.md +8 -8
- package/docs/plans/2026-08-10-agent-managed-monitor-workflows-plan.md +2 -2
- package/docs/plans/2026-08-13-session-addressed-workflow-notifications-plan.md +1 -1
- package/docs/plans/2026-08-16-workflow-updates-plan.md +5 -5
- package/docs/plans/2026-08-19-human-decision-gates-plan.md +1 -1
- package/docs/plans/2026-08-20-bounded-command-batches-plan.md +1 -1
- package/docs/plans/2026-08-21-deferred-turn-intents-plan.md +3 -3
- package/docs/plans/2026-08-21-sanity-check-plan.md +2 -2
- package/docs/plans/2026-08-23-assistant-agent-completion-plan.md +5 -5
- package/docs/plans/2026-08-23-sqlite-state-plan.md +25 -25
- package/docs/plans/2026-08-25-live-workflow-settings-plan.md +4 -4
- package/docs/plans/2026-08-27-workflow-terminal-restart-plan.md +3 -3
- package/docs/plans/2026-09-04-workflow-runner-resume-state-plan.md +318 -0
- package/docs/plans/piw-viewer-experience-implementation-plan.md +1 -1
- package/docs/tui-viewer.md +9 -9
- package/docs/workflows.md +72 -66
- package/examples/{controllers/pull-request.controller.ts → resource-managers/pull-request.resource-manager.ts} +12 -12
- package/herdr-plugin.toml +1 -1
- package/package.json +12 -11
- package/protocol/client.v1.schema.json +7 -7
- package/protocol/fixtures/client-v1.json +1 -1
- package/src/builtins/monitor.workflow.ts +9 -9
- package/src/builtins/pi-agent-group.ts +3 -3
- package/src/channels/adapter-entry.ts +14 -14
- package/src/client/client.ts +33 -32
- package/src/client/protocol.ts +16 -11
- package/src/client/resolver.ts +4 -4
- package/src/client/view.ts +1 -1
- package/src/extension/index.ts +44 -34
- package/src/extension/recorder.ts +1 -1
- package/src/extension/remote-recorder-store.ts +2 -2
- package/src/extension/resource-manager-command.ts +45 -0
- package/src/extension/session-delivery.ts +2 -2
- package/src/extension/session-view.ts +1 -1
- package/src/extension/workflow-message-coordinator.ts +6 -6
- package/src/{controllers → resource-managers}/conditions.ts +23 -23
- package/src/resource-managers/definition.ts +71 -0
- package/src/{controllers → resource-managers}/effects.ts +9 -9
- package/src/resource-managers/errors.ts +27 -0
- package/src/resource-managers/index.ts +88 -0
- package/src/resource-managers/loader.ts +111 -0
- package/src/{controllers → resource-managers}/results.ts +4 -4
- package/src/{controllers/manager.ts → resource-managers/runtime.ts} +95 -92
- package/src/{controllers → resource-managers}/sqlite.ts +198 -186
- package/src/{controllers → resource-managers}/store.ts +46 -38
- package/src/{controllers → resource-managers}/types.ts +50 -41
- package/src/{controllers → resource-managers}/workflows.ts +43 -41
- package/src/{host → server}/channel-supervisor.ts +8 -8
- package/src/{host/child-worker-supervisor.ts → server/child-runner-supervisor.ts} +22 -18
- package/src/{host → server}/processes.ts +3 -3
- package/src/{host → server}/resolver-entry.ts +28 -25
- package/src/{host/controller-worker-entry.ts → server/resource-runner-entry.ts} +67 -65
- package/src/server/resource-runner-protocol.ts +103 -0
- package/src/server/resource-runner-supervisor.ts +76 -0
- package/src/{host → server}/rpc-bridge.ts +3 -3
- package/src/{host → server}/rpc-executor.ts +4 -4
- package/src/{host/host-entry.ts → server/server-entry.ts} +5 -5
- package/src/{host/runner.ts → server/server.ts} +526 -461
- package/src/{host → server}/state.ts +78 -67
- package/src/{host → server}/view.ts +15 -15
- package/src/server/workflow-runner-content.ts +119 -0
- package/src/{host/worker-entry.ts → server/workflow-runner-entry.ts} +98 -57
- package/src/server/workflow-runner-protocol.ts +241 -0
- package/src/{host/worker-store.ts → server/workflow-runner-store.ts} +18 -20
- package/src/server/workflow-runner-supervisor.ts +79 -0
- package/src/state/prune.ts +27 -9
- package/src/state/schema.ts +2 -2
- package/src/state/workflow-messages.ts +1 -1
- package/src/viewer/backup.ts +1 -1
- package/src/viewer/cli.ts +50 -36
- package/src/viewer/tui.ts +1 -1
- package/src/workflows/command-batch.ts +2 -2
- package/src/workflows/engine.ts +14 -15
- package/src/workflows/store.ts +31 -28
- package/src/workflows/types.ts +2 -2
- package/dist/controllers/conditions.d.ts +0 -6
- package/dist/controllers/conditions.js.map +0 -1
- package/dist/controllers/definition.d.ts +0 -6
- package/dist/controllers/definition.js +0 -45
- package/dist/controllers/definition.js.map +0 -1
- package/dist/controllers/effects.d.ts +0 -15
- package/dist/controllers/effects.js.map +0 -1
- package/dist/controllers/errors.d.ts +0 -12
- package/dist/controllers/errors.js +0 -25
- package/dist/controllers/errors.js.map +0 -1
- package/dist/controllers/index.d.ts +0 -12
- package/dist/controllers/index.js +0 -12
- package/dist/controllers/index.js.map +0 -1
- package/dist/controllers/json.js.map +0 -1
- package/dist/controllers/loader.d.ts +0 -23
- package/dist/controllers/loader.js +0 -74
- package/dist/controllers/loader.js.map +0 -1
- package/dist/controllers/manager.d.ts +0 -59
- package/dist/controllers/manager.js.map +0 -1
- package/dist/controllers/results.d.ts +0 -5
- package/dist/controllers/results.js.map +0 -1
- package/dist/controllers/sqlite.js.map +0 -1
- package/dist/controllers/workflows.d.ts +0 -34
- package/dist/controllers/workflows.js.map +0 -1
- package/dist/extension/controller-command.js.map +0 -1
- package/dist/host/channel-effects.js.map +0 -1
- package/dist/host/channel-supervisor.js.map +0 -1
- package/dist/host/child-worker-supervisor.js.map +0 -1
- package/dist/host/controller-worker-entry.d.ts +0 -2
- package/dist/host/controller-worker-entry.js.map +0 -1
- package/dist/host/controller-worker-protocol.d.ts +0 -36
- package/dist/host/controller-worker-protocol.js +0 -49
- package/dist/host/controller-worker-protocol.js.map +0 -1
- package/dist/host/controller-worker-supervisor.d.ts +0 -21
- package/dist/host/controller-worker-supervisor.js +0 -54
- package/dist/host/controller-worker-supervisor.js.map +0 -1
- package/dist/host/host-entry.js.map +0 -1
- package/dist/host/processes.js.map +0 -1
- package/dist/host/resolver-entry.js.map +0 -1
- package/dist/host/rpc-bridge.js.map +0 -1
- package/dist/host/rpc-executor.js.map +0 -1
- package/dist/host/runner.js.map +0 -1
- package/dist/host/state.js.map +0 -1
- package/dist/host/view.js.map +0 -1
- package/dist/host/worker-entry.js.map +0 -1
- package/dist/host/worker-protocol.d.ts +0 -46
- package/dist/host/worker-protocol.js +0 -122
- package/dist/host/worker-protocol.js.map +0 -1
- package/dist/host/worker-store.js.map +0 -1
- package/dist/host/worker-supervisor.d.ts +0 -22
- package/dist/host/worker-supervisor.js.map +0 -1
- package/docs/CONTROLLERS.md +0 -217
- package/src/controllers/definition.ts +0 -65
- package/src/controllers/errors.ts +0 -27
- package/src/controllers/index.ts +0 -88
- package/src/controllers/loader.ts +0 -104
- package/src/extension/controller-command.ts +0 -45
- package/src/host/controller-worker-protocol.ts +0 -104
- package/src/host/controller-worker-supervisor.ts +0 -79
- package/src/host/worker-protocol.ts +0 -176
- package/src/host/worker-supervisor.ts +0 -74
- /package/dist/{controllers → resource-managers}/json.d.ts +0 -0
- /package/dist/{controllers → resource-managers}/json.js +0 -0
- /package/dist/{controllers → resource-managers}/results.js +0 -0
- /package/dist/{controllers → resource-managers}/store.js +0 -0
- /package/dist/{controllers → resource-managers}/types.js +0 -0
- /package/dist/{host → server}/channel-effects.d.ts +0 -0
- /package/dist/{host → server}/channel-effects.js +0 -0
- /package/dist/{host/host-entry.d.ts → server/server-entry.d.ts} +0 -0
- /package/src/{controllers → resource-managers}/json.ts +0 -0
- /package/src/{host → server}/channel-effects.ts +0 -0
|
@@ -7,11 +7,11 @@ import { channelResponse, } from "../channels/protocol.js";
|
|
|
7
7
|
import { WorkflowClient } from "../client/client.js";
|
|
8
8
|
import { CLIENT_PROTOCOL_SCHEMA, encodeProtocolLine, clientSocketPath, NdjsonFrameDecoder, parseClientRequest, } from "../client/protocol.js";
|
|
9
9
|
import { WORKFLOW_TURN_REPORT_RECEIPT_SCHEMA, } from "../client/view.js";
|
|
10
|
-
import { applyStatusPatch } from "../
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
10
|
+
import { applyStatusPatch } from "../resource-managers/conditions.js";
|
|
11
|
+
import { ManagedResourceConflictError } from "../resource-managers/errors.js";
|
|
12
|
+
import { resourceManagerFileStem, resourceManagerSearchDirs, discoverResourceManagers, } from "../resource-managers/loader.js";
|
|
13
|
+
import { SqliteResourceManagerStore, } from "../resource-managers/sqlite.js";
|
|
14
|
+
import { ResourceManagerWorkflowCoordinator, } from "../resource-managers/workflows.js";
|
|
15
15
|
import { StateDatabase, workflowStatePath } from "../state/database.js";
|
|
16
16
|
import { canonicalJson } from "../state/json.js";
|
|
17
17
|
import { resourceIdFor } from "../state/mutation.js";
|
|
@@ -27,31 +27,32 @@ import { validateWorkflowUpdate } from "../workflows/updates.js";
|
|
|
27
27
|
import { notificationWorkflowMessageContent, terminalWorkflowMessageContent, } from "../workflows/workflow-message-content.js";
|
|
28
28
|
import { ChannelEffectStore, channelEffectAttemptId, channelEffectId, } from "./channel-effects.js";
|
|
29
29
|
import { ChannelAdapterSupervisor } from "./channel-supervisor.js";
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
|
|
36
|
-
const
|
|
30
|
+
import { ServerProcessRegistry, matchesProcessIdentity, processParentPid, processStartIdentity, } from "./processes.js";
|
|
31
|
+
import { ResourceRunnerSupervisor } from "./resource-runner-supervisor.js";
|
|
32
|
+
import { ServerStateStore, } from "./state.js";
|
|
33
|
+
import { ServerViewStore, WORKFLOW_PAGE_KINDS } from "./view.js";
|
|
34
|
+
import { boundRunnerResponse, readRunnerContentChunk } from "./workflow-runner-content.js";
|
|
35
|
+
import { WorkflowRunnerSupervisor } from "./workflow-runner-supervisor.js";
|
|
36
|
+
const SERVER_LEASE_MS = 30_000;
|
|
37
|
+
const SERVER_RENEW_MS = 10_000;
|
|
37
38
|
const PACKAGE_VERSION = runtimePackageVersion();
|
|
38
39
|
const CLAIM_POLL_MS = 2_000;
|
|
39
40
|
const TERMINAL_MESSAGE_RECONCILE_MS = 1_000;
|
|
40
41
|
const RUN_CLAIM_LEASE_MS = 30_000;
|
|
41
|
-
const
|
|
42
|
-
const
|
|
43
|
-
const
|
|
44
|
-
const
|
|
45
|
-
/** Global package-owned
|
|
46
|
-
export class
|
|
42
|
+
const RESOURCE_MANAGER_CLAIM_LEASE_MS = 120_000;
|
|
43
|
+
const RESOURCE_MANAGER_RENEW_MS = 30_000;
|
|
44
|
+
const MAX_RESOURCE_RUNNERS = 4;
|
|
45
|
+
const DEFAULT_RESOURCE_MANAGER_TIMEOUT_MS = 60_000;
|
|
46
|
+
/** Global package-owned server. It writes state and supervises code-only runners. */
|
|
47
|
+
export class WorkflowServer {
|
|
47
48
|
options;
|
|
48
|
-
|
|
49
|
+
serverId;
|
|
49
50
|
databasePath;
|
|
50
51
|
stateDirectory;
|
|
51
52
|
socketPath;
|
|
52
53
|
lockPath;
|
|
53
54
|
state;
|
|
54
|
-
|
|
55
|
+
serverState;
|
|
55
56
|
queue;
|
|
56
57
|
decisions;
|
|
57
58
|
channelEffects;
|
|
@@ -59,8 +60,8 @@ export class WorkflowHost {
|
|
|
59
60
|
views;
|
|
60
61
|
registry;
|
|
61
62
|
activeRuns = new Map();
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
runnerDescendants = new Map();
|
|
64
|
+
activeResourceManagers = new Map();
|
|
64
65
|
activeChannels = new Map();
|
|
65
66
|
controlClaims = new Map();
|
|
66
67
|
activationTasks = new Map();
|
|
@@ -80,7 +81,7 @@ export class WorkflowHost {
|
|
|
80
81
|
viewTimer = null;
|
|
81
82
|
stopping = false;
|
|
82
83
|
started = false;
|
|
83
|
-
|
|
84
|
+
resourceManagerPollActive = false;
|
|
84
85
|
decisionTimeoutActive = false;
|
|
85
86
|
decisionChannelConfig = null;
|
|
86
87
|
decisionChannelError = null;
|
|
@@ -88,14 +89,17 @@ export class WorkflowHost {
|
|
|
88
89
|
nextTerminalMessageReconciliationAt = 0;
|
|
89
90
|
constructor(options = {}) {
|
|
90
91
|
this.options = options;
|
|
91
|
-
this.
|
|
92
|
+
this.serverId = options.runnerId ?? `host-${randomUUID()}`;
|
|
92
93
|
this.databasePath = path.resolve(options.databasePath ?? workflowStatePath());
|
|
93
94
|
this.stateDirectory = path.join(path.dirname(this.databasePath), "host");
|
|
94
95
|
this.socketPath = clientSocketPath(this.databasePath);
|
|
95
96
|
this.lockPath = path.join(this.stateDirectory, "host.lock.json");
|
|
96
97
|
this.state = new StateDatabase({ filePath: this.databasePath });
|
|
97
|
-
this.
|
|
98
|
-
this.queue = new
|
|
98
|
+
this.serverState = new ServerStateStore(this.databasePath, { state: this.state });
|
|
99
|
+
this.queue = new SqliteResourceManagerStore(this.databasePath, {
|
|
100
|
+
state: this.state,
|
|
101
|
+
global: true,
|
|
102
|
+
});
|
|
99
103
|
this.decisions = new HumanDecisionStore(this.databasePath, {
|
|
100
104
|
state: this.state,
|
|
101
105
|
authorityProvider: (runId) => {
|
|
@@ -111,13 +115,13 @@ export class WorkflowHost {
|
|
|
111
115
|
return token === undefined ? undefined : this.queue.workflowRunAuthority(runId, token);
|
|
112
116
|
},
|
|
113
117
|
snapshotLifecycle: (context) => this.applyLifecycleProjection(context),
|
|
114
|
-
|
|
118
|
+
allowServerSessionRecording: true,
|
|
115
119
|
});
|
|
116
|
-
this.views = new
|
|
120
|
+
this.views = new ServerViewStore(this.state, this.queue, this.serverState, this.runStore, (runId) => this.activeRuns.has(runId), (targetSessionId) => {
|
|
117
121
|
const coordinator = this.sessionCoordinators.get(targetSessionId);
|
|
118
122
|
return coordinator?.branchReported === true && coordinator.modelTurnActive;
|
|
119
123
|
});
|
|
120
|
-
this.registry = options.registry ?? new
|
|
124
|
+
this.registry = options.registry ?? new ServerProcessRegistry(this.stateDirectory);
|
|
121
125
|
}
|
|
122
126
|
get endpoint() {
|
|
123
127
|
return this.socketPath;
|
|
@@ -132,23 +136,23 @@ export class WorkflowHost {
|
|
|
132
136
|
fs.chmodSync(this.stateDirectory, 0o700);
|
|
133
137
|
const startIdentity = processStartIdentity(process.pid);
|
|
134
138
|
if (startIdentity === undefined) {
|
|
135
|
-
throw new Error("Cannot attest the workflow
|
|
139
|
+
throw new Error("Cannot attest the workflow server process start identity");
|
|
136
140
|
}
|
|
137
|
-
|
|
141
|
+
acquireServerLock(this.lockPath, { pid: process.pid, startIdentity, serverId: this.serverId });
|
|
138
142
|
try {
|
|
139
143
|
const reaped = this.registry.reapOrphans();
|
|
140
144
|
if (reaped.length > 0)
|
|
141
145
|
this.log(`reaped ${reaped.length} exact orphan process(es)`);
|
|
142
|
-
const
|
|
143
|
-
this.claim = this.
|
|
144
|
-
|
|
146
|
+
const previousServer = this.serverState.serverStatus();
|
|
147
|
+
this.claim = this.serverState.acquireServer({
|
|
148
|
+
serverId: this.serverId,
|
|
145
149
|
pid: process.pid,
|
|
146
150
|
processStartIdentity: startIdentity,
|
|
147
|
-
leaseMs: this.options.
|
|
151
|
+
leaseMs: this.options.serverLeaseMs ?? SERVER_LEASE_MS,
|
|
148
152
|
});
|
|
149
|
-
this.
|
|
150
|
-
const previousHeartbeatAt =
|
|
151
|
-
this.
|
|
153
|
+
this.recoverPreviousServer(previousServer.serverId, this.claim.epoch);
|
|
154
|
+
const previousHeartbeatAt = previousServer.heartbeatAt === null ? undefined : Date.parse(previousServer.heartbeatAt);
|
|
155
|
+
this.serverState.recoverInactiveModelTurns(previousHeartbeatAt);
|
|
152
156
|
this.discoverMissingTerminalWorkflowMessages();
|
|
153
157
|
this.reconcilePendingTerminalWorkflowMessages(Date.now(), true);
|
|
154
158
|
await this.listen();
|
|
@@ -158,7 +162,7 @@ export class WorkflowHost {
|
|
|
158
162
|
this.expireTimedOutInteraction();
|
|
159
163
|
void this.expireTimedOutDecision();
|
|
160
164
|
void this.claimOne();
|
|
161
|
-
void this.
|
|
165
|
+
void this.claimResourceManagerOne();
|
|
162
166
|
void this.reloadDecisionChannels().catch((error) => {
|
|
163
167
|
this.log(`decision channel startup failed: ${errorMessage(error)}`);
|
|
164
168
|
});
|
|
@@ -169,7 +173,7 @@ export class WorkflowHost {
|
|
|
169
173
|
await this.closeServer(server);
|
|
170
174
|
if (this.claim !== null) {
|
|
171
175
|
try {
|
|
172
|
-
this.
|
|
176
|
+
this.serverState.releaseServer(this.claim);
|
|
173
177
|
}
|
|
174
178
|
catch {
|
|
175
179
|
// Preserve the startup error when cleanup cannot release an already-lost claim.
|
|
@@ -209,11 +213,11 @@ export class WorkflowHost {
|
|
|
209
213
|
await active.supervisor.stop("orphaned");
|
|
210
214
|
this.queue.parkWorkflowRun({ runId: active.record.runId, claimToken: active.claimToken });
|
|
211
215
|
}));
|
|
212
|
-
await Promise.allSettled([...this.
|
|
216
|
+
await Promise.allSettled([...this.activeResourceManagers.values()].map(async (active) => {
|
|
213
217
|
clearInterval(active.renewTimer);
|
|
214
218
|
await active.supervisor.stop("orphaned");
|
|
215
219
|
if (!active.settled) {
|
|
216
|
-
active.store.requeueClaim(active.claim, { availableAt: new Date().toISOString(), error: "Workflow
|
|
220
|
+
active.store.requeueClaim(active.claim, { availableAt: new Date().toISOString(), error: "Workflow server stopped" }, new Date().toISOString());
|
|
217
221
|
}
|
|
218
222
|
}));
|
|
219
223
|
await Promise.allSettled([...this.activeChannels.values()].map(async (active) => {
|
|
@@ -225,14 +229,14 @@ export class WorkflowHost {
|
|
|
225
229
|
await Promise.allSettled(this.maintenanceCommands.values());
|
|
226
230
|
this.registry.killAll();
|
|
227
231
|
if (this.claim !== null)
|
|
228
|
-
this.
|
|
232
|
+
this.serverState.releaseServer(this.claim);
|
|
229
233
|
this.claim = null;
|
|
230
234
|
if (process.platform !== "win32")
|
|
231
235
|
fs.rmSync(this.socketPath, { force: true });
|
|
232
236
|
this.releaseLock();
|
|
233
237
|
this.runStore.close();
|
|
234
238
|
this.queue.close();
|
|
235
|
-
this.
|
|
239
|
+
this.serverState.close();
|
|
236
240
|
this.state.close();
|
|
237
241
|
this.started = false;
|
|
238
242
|
}
|
|
@@ -254,19 +258,19 @@ export class WorkflowHost {
|
|
|
254
258
|
await closed;
|
|
255
259
|
this.sockets.clear();
|
|
256
260
|
}
|
|
257
|
-
|
|
258
|
-
if (
|
|
261
|
+
recoverPreviousServer(previousServerId, serverEpoch) {
|
|
262
|
+
if (previousServerId === null || previousServerId === this.serverId)
|
|
259
263
|
return;
|
|
260
264
|
const now = Date.now();
|
|
261
265
|
this.state.transaction(() => {
|
|
262
266
|
this.state.connection
|
|
263
267
|
.prepare(`UPDATE leases SET expires_at = ?
|
|
264
268
|
WHERE owner_id = ? AND owner_type IN ('host', 'controller') AND expires_at > ?`)
|
|
265
|
-
.run(now,
|
|
269
|
+
.run(now, previousServerId, now);
|
|
266
270
|
this.state.connection
|
|
267
271
|
.prepare(`UPDATE run_workers SET status = 'orphaned', finished_at = ?
|
|
268
272
|
WHERE host_epoch < ? AND status IN ('starting', 'ready', 'running')`)
|
|
269
|
-
.run(now,
|
|
273
|
+
.run(now, serverEpoch);
|
|
270
274
|
});
|
|
271
275
|
}
|
|
272
276
|
startTimers() {
|
|
@@ -274,7 +278,7 @@ export class WorkflowHost {
|
|
|
274
278
|
try {
|
|
275
279
|
if (this.claim === null)
|
|
276
280
|
return;
|
|
277
|
-
this.claim = this.
|
|
281
|
+
this.claim = this.serverState.renewServer(this.claim, this.options.serverLeaseMs ?? SERVER_LEASE_MS);
|
|
278
282
|
for (const active of this.activeRuns.values()) {
|
|
279
283
|
if (active.control !== undefined || active.exiting)
|
|
280
284
|
continue;
|
|
@@ -289,16 +293,16 @@ export class WorkflowHost {
|
|
|
289
293
|
}
|
|
290
294
|
}
|
|
291
295
|
catch (error) {
|
|
292
|
-
this.log(`
|
|
296
|
+
this.log(`server claim lost: ${errorMessage(error)}`);
|
|
293
297
|
void this.stop();
|
|
294
298
|
}
|
|
295
|
-
}, this.options.
|
|
299
|
+
}, this.options.serverRenewMs ?? SERVER_RENEW_MS);
|
|
296
300
|
this.heartbeatTimer.unref?.();
|
|
297
301
|
this.pollTimer = setInterval(() => {
|
|
298
302
|
this.expireTimedOutInteraction();
|
|
299
303
|
void this.expireTimedOutDecision();
|
|
300
304
|
void this.claimOne();
|
|
301
|
-
void this.
|
|
305
|
+
void this.claimResourceManagerOne();
|
|
302
306
|
this.reconcilePendingTerminalWorkflowMessages();
|
|
303
307
|
}, this.options.claimPollMs ?? CLAIM_POLL_MS);
|
|
304
308
|
this.pollTimer.unref?.();
|
|
@@ -393,7 +397,7 @@ export class WorkflowHost {
|
|
|
393
397
|
for (const [sessionId, coordinator] of this.sessionCoordinators) {
|
|
394
398
|
if (connectionId !== undefined && coordinator.connectionId !== connectionId)
|
|
395
399
|
continue;
|
|
396
|
-
this.
|
|
400
|
+
this.serverState.markSessionModelTurnsInactive(sessionId);
|
|
397
401
|
this.sessionCoordinators.delete(sessionId);
|
|
398
402
|
changed = true;
|
|
399
403
|
}
|
|
@@ -437,7 +441,7 @@ export class WorkflowHost {
|
|
|
437
441
|
if (payload.coordinator === true) {
|
|
438
442
|
const previous = this.sessionCoordinators.get(sessionId);
|
|
439
443
|
if (previous !== undefined && previous.connectionId !== connection.id) {
|
|
440
|
-
this.
|
|
444
|
+
this.serverState.markSessionModelTurnsInactive(sessionId);
|
|
441
445
|
}
|
|
442
446
|
const coordinator = {
|
|
443
447
|
connectionId: connection.id,
|
|
@@ -530,7 +534,7 @@ export class WorkflowHost {
|
|
|
530
534
|
before,
|
|
531
535
|
apply,
|
|
532
536
|
...(backupPath === undefined ? {} : { backupPath }),
|
|
533
|
-
});
|
|
537
|
+
}, () => this.activeRunnerContentHashes());
|
|
534
538
|
return toJsonValue(report);
|
|
535
539
|
});
|
|
536
540
|
default:
|
|
@@ -542,18 +546,18 @@ export class WorkflowHost {
|
|
|
542
546
|
}
|
|
543
547
|
}
|
|
544
548
|
async executeMaintenanceCommand(request, operation) {
|
|
545
|
-
const adopted = this.
|
|
549
|
+
const adopted = this.serverState.adoptCommand(request);
|
|
546
550
|
if (adopted !== undefined)
|
|
547
551
|
return adopted;
|
|
548
552
|
const claim = this.claim;
|
|
549
553
|
if (claim === null || this.stopping) {
|
|
550
|
-
return clientResponse(request.requestId, "unavailable", undefined, "Workflow
|
|
554
|
+
return clientResponse(request.requestId, "unavailable", undefined, "Workflow server is stopping");
|
|
551
555
|
}
|
|
552
556
|
const key = canonicalJson([request.clientId, request.idempotencyKey]);
|
|
553
557
|
const active = this.maintenanceCommands.get(key);
|
|
554
558
|
if (active !== undefined) {
|
|
555
559
|
await active;
|
|
556
|
-
return (this.
|
|
560
|
+
return (this.serverState.adoptCommand(request) ??
|
|
557
561
|
clientResponse(request.requestId, "unavailable", undefined, "Workflow maintenance command did not complete durably"));
|
|
558
562
|
}
|
|
559
563
|
const execution = (async () => {
|
|
@@ -564,7 +568,7 @@ export class WorkflowHost {
|
|
|
564
568
|
catch (error) {
|
|
565
569
|
result = { outcome: "rejected", error: errorMessage(error) };
|
|
566
570
|
}
|
|
567
|
-
return this.
|
|
571
|
+
return this.serverState.executeCommand(request, claim.epoch, () => result);
|
|
568
572
|
})();
|
|
569
573
|
this.maintenanceCommands.set(key, execution);
|
|
570
574
|
try {
|
|
@@ -625,16 +629,16 @@ export class WorkflowHost {
|
|
|
625
629
|
reportWorkflowBranch(connection, request) {
|
|
626
630
|
const report = parseWorkflowBranchReport(request.payload);
|
|
627
631
|
const coordinator = this.requireSessionCoordinator(connection, report.targetSessionId, report.coordinatorEpoch);
|
|
628
|
-
const messages = this.
|
|
632
|
+
const messages = this.serverState.workflowMessages.listSession(report.targetSessionId);
|
|
629
633
|
const allowed = new Set(messages.map((message) => message.workflowMessageId));
|
|
630
634
|
const reconciledRunIds = new Set();
|
|
631
635
|
this.state.transaction(() => {
|
|
632
636
|
if (coordinator.needsTimerResume) {
|
|
633
|
-
this.
|
|
637
|
+
this.serverState.resumeSessionModelTurns(report.targetSessionId);
|
|
634
638
|
}
|
|
635
|
-
this.
|
|
639
|
+
this.serverState.workflowMessages.adoptBranch(report.targetSessionId, report.entries, allowed);
|
|
636
640
|
if (report.isIdle && !report.hasPendingMessages) {
|
|
637
|
-
for (const turn of this.
|
|
641
|
+
for (const turn of this.serverState.workflowMessages.openTurnsForSession(report.targetSessionId)) {
|
|
638
642
|
reconciledRunIds.add(turn.runId);
|
|
639
643
|
this.applyWorkflowTurnEnd({
|
|
640
644
|
state: "ended",
|
|
@@ -648,13 +652,13 @@ export class WorkflowHost {
|
|
|
648
652
|
});
|
|
649
653
|
}
|
|
650
654
|
const branchIds = new Set(report.entries.map((entry) => entry.workflowMessageId));
|
|
651
|
-
const refreshed = this.
|
|
652
|
-
for (const interaction of this.
|
|
655
|
+
const refreshed = this.serverState.workflowMessages.listSession(report.targetSessionId);
|
|
656
|
+
for (const interaction of this.serverState.listPendingInteractions(report.targetSessionId)) {
|
|
653
657
|
const sourceMessages = refreshed.filter((message) => message.sourceId === interaction.requestId);
|
|
654
658
|
if (sourceMessages.some((message) => message.status === "sent") &&
|
|
655
659
|
!sourceMessages.some((message) => branchIds.has(message.workflowMessageId))) {
|
|
656
|
-
this.
|
|
657
|
-
this.
|
|
660
|
+
this.serverState.workflowMessages.cancelPendingForSource(interaction.requestId);
|
|
661
|
+
this.serverState.ensureInteractionMessage(interaction, interaction.kind === "decision" ? "initial" : "resumed");
|
|
658
662
|
}
|
|
659
663
|
}
|
|
660
664
|
}
|
|
@@ -679,7 +683,7 @@ export class WorkflowHost {
|
|
|
679
683
|
}
|
|
680
684
|
let outcome = "accepted";
|
|
681
685
|
const receipt = this.state.transaction(() => {
|
|
682
|
-
const existing = this.
|
|
686
|
+
const existing = this.serverState.workflowMessages.getTurn(report.workflowTurnId);
|
|
683
687
|
if (report.state === "ended") {
|
|
684
688
|
if (existing === undefined) {
|
|
685
689
|
outcome = "adopted";
|
|
@@ -701,7 +705,7 @@ export class WorkflowHost {
|
|
|
701
705
|
return workflowTurnReceipt("settled", turn);
|
|
702
706
|
}
|
|
703
707
|
if (existing !== undefined) {
|
|
704
|
-
const turn = this.
|
|
708
|
+
const turn = this.serverState.workflowMessages.startTurn(report);
|
|
705
709
|
outcome = "adopted";
|
|
706
710
|
return workflowTurnReceipt(turn.state === "started" ? "active" : "settled", turn);
|
|
707
711
|
}
|
|
@@ -715,7 +719,7 @@ export class WorkflowHost {
|
|
|
715
719
|
outcome = "adopted";
|
|
716
720
|
return workflowTurnReceipt("absent", null);
|
|
717
721
|
}
|
|
718
|
-
const message = this.
|
|
722
|
+
const message = this.serverState.workflowMessages.require(report.workflowMessageId);
|
|
719
723
|
const queuedRun = this.queue.getWorkflowRun(report.runId);
|
|
720
724
|
if (message.kind === "step" &&
|
|
721
725
|
(queuedRun === undefined || ["done", "failed", "cancelled"].includes(queuedRun.status))) {
|
|
@@ -724,11 +728,11 @@ export class WorkflowHost {
|
|
|
724
728
|
}
|
|
725
729
|
if (message.kind === "step") {
|
|
726
730
|
const now = Date.now();
|
|
727
|
-
this.
|
|
728
|
-
this.
|
|
729
|
-
return workflowTurnReceipt("active", this.
|
|
731
|
+
this.serverState.beginInteractionModelTurn(message.sourceId, now);
|
|
732
|
+
this.serverState.workflowMessages.cancelPendingForSource(message.sourceId, "step", now);
|
|
733
|
+
return workflowTurnReceipt("active", this.serverState.workflowMessages.startTurn({ ...report, now }));
|
|
730
734
|
}
|
|
731
|
-
return workflowTurnReceipt("active", this.
|
|
735
|
+
return workflowTurnReceipt("active", this.serverState.workflowMessages.startTurn(report));
|
|
732
736
|
});
|
|
733
737
|
coordinator.modelTurnActive = receipt.ownership === "active";
|
|
734
738
|
this.views.noteOriginActivityChange();
|
|
@@ -738,27 +742,27 @@ export class WorkflowHost {
|
|
|
738
742
|
return clientResponse(request.requestId, outcome, receipt);
|
|
739
743
|
}
|
|
740
744
|
applyWorkflowTurnEnd(report) {
|
|
741
|
-
const current = this.
|
|
745
|
+
const current = this.serverState.workflowMessages.requireTurn(report.workflowTurnId);
|
|
742
746
|
if (current.state === "ended") {
|
|
743
|
-
return this.
|
|
747
|
+
return this.serverState.workflowMessages.endTurn(report);
|
|
744
748
|
}
|
|
745
|
-
const turn = this.
|
|
746
|
-
const message = this.
|
|
749
|
+
const turn = this.serverState.workflowMessages.endTurn(report);
|
|
750
|
+
const message = this.serverState.workflowMessages.require(report.workflowMessageId);
|
|
747
751
|
if (message.kind !== "step")
|
|
748
752
|
return turn;
|
|
749
|
-
const interaction = this.
|
|
753
|
+
const interaction = this.serverState.getInteraction(message.sourceId);
|
|
750
754
|
if (interaction === undefined || interaction.status !== "pending")
|
|
751
755
|
return turn;
|
|
752
756
|
if (report.stopReason === "aborted") {
|
|
753
757
|
if (!this.queue.pauseParkedWorkflowRun({ runId: report.runId })) {
|
|
754
758
|
throw new Error("Interrupted workflow turn could not pause its exact parked run");
|
|
755
759
|
}
|
|
756
|
-
this.
|
|
760
|
+
this.serverState.workflowMessages.cancelPendingForSource(interaction.requestId, "step");
|
|
757
761
|
return turn;
|
|
758
762
|
}
|
|
759
763
|
if (this.queue.isWorkflowRunPaused(report.runId))
|
|
760
764
|
return turn;
|
|
761
|
-
if (this.
|
|
765
|
+
if (this.serverState.validatingInteraction(report.runId) !== undefined)
|
|
762
766
|
return turn;
|
|
763
767
|
const changed = this.state.connection
|
|
764
768
|
.prepare(`UPDATE interactive_requests
|
|
@@ -768,14 +772,14 @@ export class WorkflowHost {
|
|
|
768
772
|
.run(Date.now(), interaction.requestId);
|
|
769
773
|
if (changed.changes !== 1)
|
|
770
774
|
return turn;
|
|
771
|
-
const updated = this.
|
|
775
|
+
const updated = this.serverState.getInteraction(interaction.requestId);
|
|
772
776
|
if (updated === undefined)
|
|
773
777
|
throw new Error("Workflow interaction disappeared after turn end");
|
|
774
778
|
if (updated.unproductiveTurnEnds <= 2) {
|
|
775
|
-
this.
|
|
779
|
+
this.serverState.ensureInteractionMessage(updated, "reminder");
|
|
776
780
|
return turn;
|
|
777
781
|
}
|
|
778
|
-
this.
|
|
782
|
+
this.serverState.workflowMessages.cancelPendingForSource(updated.requestId, "step");
|
|
779
783
|
this.state.connection
|
|
780
784
|
.prepare(`UPDATE interactive_requests SET status = 'cancelled', revision = revision + 1, updated_at = ?
|
|
781
785
|
WHERE request_id = ? AND status = 'pending'`)
|
|
@@ -854,9 +858,9 @@ export class WorkflowHost {
|
|
|
854
858
|
const submissionId = requireString(startedReceipt.submissionId, "submissionId");
|
|
855
859
|
for (;;) {
|
|
856
860
|
if (this.stopping) {
|
|
857
|
-
return clientResponse(request.requestId, "unavailable", undefined, "Workflow
|
|
861
|
+
return clientResponse(request.requestId, "unavailable", undefined, "Workflow server stopped while validating the submission");
|
|
858
862
|
}
|
|
859
|
-
const submission = this.
|
|
863
|
+
const submission = this.serverState.interactionSubmission(requestId, submissionId);
|
|
860
864
|
if (submission?.outcome === "accepted" || submission?.outcome === "adopted") {
|
|
861
865
|
return clientResponse(request.requestId, started.outcome, submission.receipt ?? { requestId, submissionId });
|
|
862
866
|
}
|
|
@@ -867,7 +871,7 @@ export class WorkflowHost {
|
|
|
867
871
|
: "Workflow step output failed validation";
|
|
868
872
|
return clientResponse(request.requestId, "rejected", receipt, detail);
|
|
869
873
|
}
|
|
870
|
-
await
|
|
874
|
+
await serverDelay(25);
|
|
871
875
|
}
|
|
872
876
|
}
|
|
873
877
|
handleRequest(request, connection) {
|
|
@@ -877,13 +881,13 @@ export class WorkflowHost {
|
|
|
877
881
|
type: "response",
|
|
878
882
|
requestId: request.requestId,
|
|
879
883
|
outcome: "unavailable",
|
|
880
|
-
error: "Workflow
|
|
884
|
+
error: "Workflow server is stopping",
|
|
881
885
|
};
|
|
882
886
|
}
|
|
883
887
|
const afterCommit = [];
|
|
884
888
|
let response;
|
|
885
889
|
try {
|
|
886
|
-
response = this.
|
|
890
|
+
response = this.serverState.executeCommand(request, this.claim.epoch, () => this.executeOperation(request, afterCommit, connection));
|
|
887
891
|
}
|
|
888
892
|
catch (error) {
|
|
889
893
|
response = {
|
|
@@ -900,9 +904,9 @@ export class WorkflowHost {
|
|
|
900
904
|
}
|
|
901
905
|
executeOperation(request, afterCommit, connection) {
|
|
902
906
|
switch (request.operation) {
|
|
903
|
-
case "
|
|
907
|
+
case "server.status":
|
|
904
908
|
return { outcome: "accepted", receipt: this.statusReceipt() };
|
|
905
|
-
case "
|
|
909
|
+
case "server.stop":
|
|
906
910
|
afterCommit.push(() => void this.stop());
|
|
907
911
|
return { outcome: "accepted", receipt: { stopping: true } };
|
|
908
912
|
case "run.list":
|
|
@@ -962,7 +966,7 @@ export class WorkflowHost {
|
|
|
962
966
|
status: "parked",
|
|
963
967
|
paused: true,
|
|
964
968
|
alreadyPaused: true,
|
|
965
|
-
detail: "The supervised
|
|
969
|
+
detail: "The supervised runner is already stopping at the pause boundary.",
|
|
966
970
|
},
|
|
967
971
|
};
|
|
968
972
|
}
|
|
@@ -994,7 +998,7 @@ export class WorkflowHost {
|
|
|
994
998
|
status: "parked",
|
|
995
999
|
paused: true,
|
|
996
1000
|
alreadyPaused: false,
|
|
997
|
-
detail: "The run is durably paused; the supervised
|
|
1001
|
+
detail: "The run is durably paused; the supervised runner is stopping.",
|
|
998
1002
|
},
|
|
999
1003
|
};
|
|
1000
1004
|
}
|
|
@@ -1016,12 +1020,12 @@ export class WorkflowHost {
|
|
|
1016
1020
|
const runId = requireRunId(request);
|
|
1017
1021
|
const resumedInteraction = this.state.transaction(() => {
|
|
1018
1022
|
const now = Date.now();
|
|
1019
|
-
const pausedAt = this.
|
|
1023
|
+
const pausedAt = this.serverState.interactionPauseStartedAt(runId);
|
|
1020
1024
|
if (!this.queue.resumePausedInteraction({ runId, now: new Date(now).toISOString() })) {
|
|
1021
1025
|
return false;
|
|
1022
1026
|
}
|
|
1023
|
-
this.
|
|
1024
|
-
this.
|
|
1027
|
+
this.serverState.resumeInteractionModelTurn(runId, pausedAt, now);
|
|
1028
|
+
this.serverState.resumePendingInteraction(runId, now);
|
|
1025
1029
|
return true;
|
|
1026
1030
|
});
|
|
1027
1031
|
if (resumedInteraction) {
|
|
@@ -1053,7 +1057,7 @@ export class WorkflowHost {
|
|
|
1053
1057
|
const token = randomUUID();
|
|
1054
1058
|
const claimed = this.queue.claimWorkflowRun({
|
|
1055
1059
|
runId,
|
|
1056
|
-
runnerId: this.
|
|
1060
|
+
runnerId: this.serverId,
|
|
1057
1061
|
claimToken: token,
|
|
1058
1062
|
leaseMs: this.runClaimLeaseMs,
|
|
1059
1063
|
});
|
|
@@ -1090,12 +1094,12 @@ export class WorkflowHost {
|
|
|
1090
1094
|
return this.startRun(request, afterCommit);
|
|
1091
1095
|
case "checkpoint.answer":
|
|
1092
1096
|
return this.answerCheckpoint(request, afterCommit);
|
|
1093
|
-
case "
|
|
1094
|
-
case "
|
|
1095
|
-
case "
|
|
1096
|
-
case "
|
|
1097
|
-
case "
|
|
1098
|
-
return this.
|
|
1097
|
+
case "resourceManager.list":
|
|
1098
|
+
case "resourceManager.get":
|
|
1099
|
+
case "resourceManager.apply":
|
|
1100
|
+
case "resourceManager.reconcile":
|
|
1101
|
+
case "resourceManager.delete":
|
|
1102
|
+
return this.executeResourceManagerOperation(request);
|
|
1099
1103
|
case "interaction.update":
|
|
1100
1104
|
return this.publishInteractionUpdate(request);
|
|
1101
1105
|
case "interaction.submit":
|
|
@@ -1142,7 +1146,7 @@ export class WorkflowHost {
|
|
|
1142
1146
|
}
|
|
1143
1147
|
const payload = requireRecord(request.payload, "run.restart payload");
|
|
1144
1148
|
const workflowMessageId = requireString(payload.workflowMessageId, "workflowMessageId");
|
|
1145
|
-
const terminal = this.
|
|
1149
|
+
const terminal = this.serverState.workflowMessages.latestForSource("terminal", `terminal:${sourceRunId}`);
|
|
1146
1150
|
if (terminal === undefined ||
|
|
1147
1151
|
terminal.workflowMessageId !== workflowMessageId ||
|
|
1148
1152
|
terminal.targetSessionId !== session.targetSessionId ||
|
|
@@ -1153,7 +1157,7 @@ export class WorkflowHost {
|
|
|
1153
1157
|
if (sessionView.run?.runId !== sourceRunId) {
|
|
1154
1158
|
return { outcome: "rejected", error: "Workflow terminal result is no longer current" };
|
|
1155
1159
|
}
|
|
1156
|
-
const turn = this.
|
|
1160
|
+
const turn = this.serverState.workflowMessages.latestTurnForMessage(workflowMessageId);
|
|
1157
1161
|
if (turn === undefined) {
|
|
1158
1162
|
return { outcome: "rejected", error: "Workflow restart requires a terminal model turn" };
|
|
1159
1163
|
}
|
|
@@ -1199,7 +1203,7 @@ export class WorkflowHost {
|
|
|
1199
1203
|
}
|
|
1200
1204
|
const runId = `restart-${createHash("sha256").update(workflowMessageId).digest("hex").slice(0, 40)}`;
|
|
1201
1205
|
const claimToken = randomUUID();
|
|
1202
|
-
const scoped = new
|
|
1206
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
1203
1207
|
state: this.state,
|
|
1204
1208
|
projectPath,
|
|
1205
1209
|
});
|
|
@@ -1212,7 +1216,7 @@ export class WorkflowHost {
|
|
|
1212
1216
|
definitionSnapshot: this.state.readJson(definition.definitionHash),
|
|
1213
1217
|
input: source.input,
|
|
1214
1218
|
launchOptions: source.launchOptions,
|
|
1215
|
-
runnerId: this.
|
|
1219
|
+
runnerId: this.serverId,
|
|
1216
1220
|
claimToken,
|
|
1217
1221
|
leaseMs: this.runClaimLeaseMs,
|
|
1218
1222
|
originSessionId: session.targetSessionId,
|
|
@@ -1399,56 +1403,68 @@ export class WorkflowHost {
|
|
|
1399
1403
|
...(entrySequence === undefined ? {} : { entrySequence }),
|
|
1400
1404
|
};
|
|
1401
1405
|
}
|
|
1402
|
-
|
|
1403
|
-
const payload = requireRecord(request.payload, "
|
|
1406
|
+
executeResourceManagerOperation(request) {
|
|
1407
|
+
const payload = requireRecord(request.payload, "resource manager payload");
|
|
1404
1408
|
const projectPath = path.resolve(requireString(payload.projectPath, "projectPath"));
|
|
1405
1409
|
if (payload.projectPath !== projectPath) {
|
|
1406
1410
|
return {
|
|
1407
1411
|
outcome: "rejected",
|
|
1408
|
-
error: "
|
|
1412
|
+
error: "ResourceManager projectPath must be absolute and normalized",
|
|
1409
1413
|
};
|
|
1410
1414
|
}
|
|
1411
|
-
const store = new
|
|
1415
|
+
const store = new SqliteResourceManagerStore(this.databasePath, {
|
|
1412
1416
|
state: this.state,
|
|
1413
1417
|
projectPath,
|
|
1414
1418
|
});
|
|
1415
|
-
if (request.operation === "
|
|
1419
|
+
if (request.operation === "resourceManager.list") {
|
|
1416
1420
|
return { outcome: "accepted", receipt: store.listResources() };
|
|
1417
1421
|
}
|
|
1418
|
-
const
|
|
1422
|
+
const resourceManager = requireString(payload.resourceManager, "resourceManager");
|
|
1419
1423
|
const key = requireString(payload.key, "key");
|
|
1420
|
-
const ref = {
|
|
1421
|
-
if (request.operation === "
|
|
1424
|
+
const ref = { resourceManager, key };
|
|
1425
|
+
if (request.operation === "resourceManager.get") {
|
|
1422
1426
|
const resource = store.getResource(ref);
|
|
1423
1427
|
return resource === undefined
|
|
1424
|
-
? { outcome: "notFound", error: `
|
|
1428
|
+
? { outcome: "notFound", error: `Managed resource not found: ${resourceManager}/${key}` }
|
|
1425
1429
|
: { outcome: "accepted", receipt: resource };
|
|
1426
1430
|
}
|
|
1427
|
-
if (request.operation === "
|
|
1431
|
+
if (request.operation === "resourceManager.apply") {
|
|
1428
1432
|
if (!Object.hasOwn(payload, "spec") || !Object.hasOwn(payload, "initialStatus")) {
|
|
1429
|
-
return {
|
|
1433
|
+
return {
|
|
1434
|
+
outcome: "rejected",
|
|
1435
|
+
error: "ResourceManager apply requires spec and initialStatus",
|
|
1436
|
+
};
|
|
1430
1437
|
}
|
|
1431
1438
|
const spec = payload.spec;
|
|
1432
1439
|
const initialStatus = payload.initialStatus;
|
|
1433
|
-
const
|
|
1440
|
+
const resourceManagerPath = path.resolve(requireString(payload.resourceManagerPath, "resourceManagerPath"));
|
|
1434
1441
|
const sourceHash = requireString(payload.sourceHash, "sourceHash");
|
|
1435
|
-
if (!
|
|
1436
|
-
return {
|
|
1442
|
+
if (!resourceManagerPathAllowed(projectPath, resourceManager, resourceManagerPath)) {
|
|
1443
|
+
return {
|
|
1444
|
+
outcome: "rejected",
|
|
1445
|
+
error: "ResourceManager source does not match discovery rules",
|
|
1446
|
+
};
|
|
1437
1447
|
}
|
|
1438
1448
|
const observedHash = createHash("sha256")
|
|
1439
|
-
.update(fs.readFileSync(
|
|
1449
|
+
.update(fs.readFileSync(resourceManagerPath))
|
|
1440
1450
|
.digest("hex");
|
|
1441
1451
|
if (observedHash !== sourceHash) {
|
|
1442
|
-
return {
|
|
1452
|
+
return {
|
|
1453
|
+
outcome: "conflict",
|
|
1454
|
+
error: "ResourceManager source changed before apply committed",
|
|
1455
|
+
};
|
|
1443
1456
|
}
|
|
1444
|
-
const resource = store.putResource({
|
|
1457
|
+
const resource = store.putResource({ resourceManager, key, spec, initialStatus });
|
|
1445
1458
|
return { outcome: "accepted", receipt: resource };
|
|
1446
1459
|
}
|
|
1447
1460
|
const existing = store.getResource(ref);
|
|
1448
1461
|
if (existing === undefined) {
|
|
1449
|
-
return {
|
|
1462
|
+
return {
|
|
1463
|
+
outcome: "notFound",
|
|
1464
|
+
error: `Managed resource not found: ${resourceManager}/${key}`,
|
|
1465
|
+
};
|
|
1450
1466
|
}
|
|
1451
|
-
if (request.operation === "
|
|
1467
|
+
if (request.operation === "resourceManager.reconcile") {
|
|
1452
1468
|
store.enqueue(ref);
|
|
1453
1469
|
return { outcome: "accepted", receipt: existing };
|
|
1454
1470
|
}
|
|
@@ -1457,20 +1473,20 @@ export class WorkflowHost {
|
|
|
1457
1473
|
return { outcome: "accepted", receipt: deleting };
|
|
1458
1474
|
}
|
|
1459
1475
|
statusReceipt() {
|
|
1460
|
-
const
|
|
1476
|
+
const serverStatus = this.serverState.serverStatus();
|
|
1461
1477
|
const count = (sql, ...params) => {
|
|
1462
1478
|
const row = this.state.connection.prepare(sql).get(...params);
|
|
1463
1479
|
return typeof row?.count === "number" ? row.count : 0;
|
|
1464
1480
|
};
|
|
1465
1481
|
const now = Date.now();
|
|
1466
1482
|
return {
|
|
1467
|
-
state:
|
|
1468
|
-
epoch:
|
|
1483
|
+
state: serverStatus.live ? "running" : "stale",
|
|
1484
|
+
epoch: serverStatus.epoch,
|
|
1469
1485
|
socketAvailable: this.server?.listening === true,
|
|
1470
|
-
startedAt:
|
|
1471
|
-
heartbeatAt:
|
|
1472
|
-
expiresAt:
|
|
1473
|
-
|
|
1486
|
+
startedAt: serverStatus.startedAt,
|
|
1487
|
+
heartbeatAt: serverStatus.heartbeatAt,
|
|
1488
|
+
expiresAt: serverStatus.expiresAt,
|
|
1489
|
+
activeRunners: this.activeRuns.size,
|
|
1474
1490
|
connectedClients: this.sockets.size,
|
|
1475
1491
|
queuedRuns: count("SELECT COUNT(*) AS count FROM run_queue WHERE status = 'queued'"),
|
|
1476
1492
|
runningRuns: count("SELECT COUNT(*) AS count FROM run_queue WHERE status IN ('starting', 'running')"),
|
|
@@ -1483,8 +1499,8 @@ export class WorkflowHost {
|
|
|
1483
1499
|
AND l.owner_id IS NOT NULL AND l.expires_at <= ?`, now),
|
|
1484
1500
|
pendingInteractions: count("SELECT COUNT(*) AS count FROM interactive_requests WHERE status = 'pending'"),
|
|
1485
1501
|
ambiguousEffects: count("SELECT COUNT(*) AS count FROM effects WHERE status = 'ambiguous'"),
|
|
1486
|
-
|
|
1487
|
-
|
|
1502
|
+
pendingResourceManagers: count("SELECT COUNT(*) AS count FROM controller_queue"),
|
|
1503
|
+
activeResourceRunners: this.activeResourceManagers.size,
|
|
1488
1504
|
lifecycleContradictions: count(`SELECT COUNT(*) AS count
|
|
1489
1505
|
FROM runs r JOIN run_queue q ON q.run_id = r.run_id
|
|
1490
1506
|
WHERE NOT (
|
|
@@ -1497,6 +1513,14 @@ export class WorkflowHost {
|
|
|
1497
1513
|
)`),
|
|
1498
1514
|
};
|
|
1499
1515
|
}
|
|
1516
|
+
activeRunnerContentHashes() {
|
|
1517
|
+
const digests = new Set();
|
|
1518
|
+
for (const active of this.activeRuns.values()) {
|
|
1519
|
+
for (const digest of active.contentDigests)
|
|
1520
|
+
digests.add(digest);
|
|
1521
|
+
}
|
|
1522
|
+
return [...digests].map((digest) => Buffer.from(digest, "hex"));
|
|
1523
|
+
}
|
|
1500
1524
|
stateStatusReceipt() {
|
|
1501
1525
|
const count = (sql, ...params) => {
|
|
1502
1526
|
const row = this.state.connection.prepare(sql).get(...params);
|
|
@@ -1508,7 +1532,7 @@ export class WorkflowHost {
|
|
|
1508
1532
|
counts: {
|
|
1509
1533
|
resources: count("SELECT COUNT(*) AS count FROM resources"),
|
|
1510
1534
|
runs: count("SELECT COUNT(*) AS count FROM runs"),
|
|
1511
|
-
|
|
1535
|
+
resourceManagers: count("SELECT COUNT(*) AS count FROM controller_resources"),
|
|
1512
1536
|
decisions: count("SELECT COUNT(*) AS count FROM human_decisions"),
|
|
1513
1537
|
settingsScopes: count("SELECT COUNT(*) AS count FROM workflow_settings"),
|
|
1514
1538
|
pendingInteractions: count("SELECT COUNT(*) AS count FROM interactive_requests WHERE status = 'pending'"),
|
|
@@ -1576,7 +1600,7 @@ export class WorkflowHost {
|
|
|
1576
1600
|
answerDecision(command, afterCommit) {
|
|
1577
1601
|
const payload = requireRecord(command.payload, "decision answer payload");
|
|
1578
1602
|
const requestId = requireString(payload.requestId, "requestId");
|
|
1579
|
-
const interaction = this.
|
|
1603
|
+
const interaction = this.serverState.getInteraction(requestId);
|
|
1580
1604
|
if (interaction === undefined || interaction.kind !== "decision") {
|
|
1581
1605
|
return { outcome: "notFound", error: `Decision request not found: ${requestId}` };
|
|
1582
1606
|
}
|
|
@@ -1611,7 +1635,7 @@ export class WorkflowHost {
|
|
|
1611
1635
|
if (accepted.status === "conflict") {
|
|
1612
1636
|
return { outcome: "conflict", error: "Another human decision answer already won" };
|
|
1613
1637
|
}
|
|
1614
|
-
this.
|
|
1638
|
+
this.serverState.submitInteraction({
|
|
1615
1639
|
requestId: options.interaction.requestId,
|
|
1616
1640
|
submissionId: options.submissionId,
|
|
1617
1641
|
idempotencyKey: options.idempotencyKey,
|
|
@@ -1655,7 +1679,7 @@ export class WorkflowHost {
|
|
|
1655
1679
|
}
|
|
1656
1680
|
prepareContinuation(options, afterCommit) {
|
|
1657
1681
|
const token = randomUUID();
|
|
1658
|
-
const scoped = new
|
|
1682
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
1659
1683
|
state: this.state,
|
|
1660
1684
|
projectPath: options.projectPath,
|
|
1661
1685
|
});
|
|
@@ -1668,7 +1692,7 @@ export class WorkflowHost {
|
|
|
1668
1692
|
definitionSnapshot: options.definitionSnapshot,
|
|
1669
1693
|
input: options.input,
|
|
1670
1694
|
launchOptions: options.launchOptions,
|
|
1671
|
-
runnerId: this.
|
|
1695
|
+
runnerId: this.serverId,
|
|
1672
1696
|
claimToken: token,
|
|
1673
1697
|
leaseMs: this.runClaimLeaseMs,
|
|
1674
1698
|
originSessionId: options.originSessionId,
|
|
@@ -1691,7 +1715,7 @@ export class WorkflowHost {
|
|
|
1691
1715
|
const originSessionId = requireString(payload.originSessionId, "originSessionId");
|
|
1692
1716
|
const executionMode = payload.executionMode === "headless" ? "headless" : "interactive";
|
|
1693
1717
|
const token = randomUUID();
|
|
1694
|
-
const scoped = new
|
|
1718
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
1695
1719
|
state: this.state,
|
|
1696
1720
|
projectPath,
|
|
1697
1721
|
});
|
|
@@ -1704,7 +1728,7 @@ export class WorkflowHost {
|
|
|
1704
1728
|
definitionSnapshot: payload.definitionSnapshot,
|
|
1705
1729
|
input: payload.input,
|
|
1706
1730
|
launchOptions: payload.launchOptions ?? {},
|
|
1707
|
-
runnerId: this.
|
|
1731
|
+
runnerId: this.serverId,
|
|
1708
1732
|
claimToken: token,
|
|
1709
1733
|
leaseMs: this.runClaimLeaseMs,
|
|
1710
1734
|
originSessionId,
|
|
@@ -1729,7 +1753,7 @@ export class WorkflowHost {
|
|
|
1729
1753
|
const runId = requireRunId(request);
|
|
1730
1754
|
const payload = requireRecord(request.payload, "interaction update payload");
|
|
1731
1755
|
const requestId = requireString(payload.requestId, "requestId");
|
|
1732
|
-
const interaction = this.
|
|
1756
|
+
const interaction = this.serverState.getInteraction(requestId);
|
|
1733
1757
|
if (interaction === undefined || interaction.runId !== runId) {
|
|
1734
1758
|
return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
|
|
1735
1759
|
}
|
|
@@ -1754,7 +1778,7 @@ export class WorkflowHost {
|
|
|
1754
1778
|
const token = randomUUID();
|
|
1755
1779
|
const claimed = this.queue.claimWorkflowRunForControl({
|
|
1756
1780
|
runId,
|
|
1757
|
-
runnerId: this.
|
|
1781
|
+
runnerId: this.serverId,
|
|
1758
1782
|
claimToken: token,
|
|
1759
1783
|
leaseMs: this.runClaimLeaseMs,
|
|
1760
1784
|
});
|
|
@@ -1785,7 +1809,7 @@ export class WorkflowHost {
|
|
|
1785
1809
|
submitInteraction(request) {
|
|
1786
1810
|
const payload = requireRecord(request.payload, "interaction payload");
|
|
1787
1811
|
const requestId = requireString(payload.requestId, "requestId");
|
|
1788
|
-
const current = this.
|
|
1812
|
+
const current = this.serverState.getInteraction(requestId);
|
|
1789
1813
|
if (current === undefined || current.runId !== requireRunId(request)) {
|
|
1790
1814
|
return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
|
|
1791
1815
|
}
|
|
@@ -1801,7 +1825,7 @@ export class WorkflowHost {
|
|
|
1801
1825
|
return { outcome: "conflict", error: "Interactive request attempt is stale" };
|
|
1802
1826
|
}
|
|
1803
1827
|
const submissionId = requireString(payload.submissionId, "submissionId");
|
|
1804
|
-
const submission = this.
|
|
1828
|
+
const submission = this.serverState.beginInteractionValidation({
|
|
1805
1829
|
requestId,
|
|
1806
1830
|
submissionId,
|
|
1807
1831
|
idempotencyKey: request.idempotencyKey,
|
|
@@ -1825,7 +1849,7 @@ export class WorkflowHost {
|
|
|
1825
1849
|
const token = randomUUID();
|
|
1826
1850
|
const claimed = this.queue.claimWorkflowRunForInteractionValidation({
|
|
1827
1851
|
runId: interaction.runId,
|
|
1828
|
-
runnerId: this.
|
|
1852
|
+
runnerId: this.serverId,
|
|
1829
1853
|
claimToken: token,
|
|
1830
1854
|
leaseMs: this.runClaimLeaseMs,
|
|
1831
1855
|
});
|
|
@@ -1863,7 +1887,7 @@ export class WorkflowHost {
|
|
|
1863
1887
|
...(sourceResourceId === undefined ? {} : { sourceResourceId }),
|
|
1864
1888
|
...(stableMessageIds === undefined ? {} : { stableMessageIds }),
|
|
1865
1889
|
error: errorCode,
|
|
1866
|
-
actorId: this.
|
|
1890
|
+
actorId: this.serverId,
|
|
1867
1891
|
});
|
|
1868
1892
|
for (const effect of effects) {
|
|
1869
1893
|
this.recordChannelEffectResult(effect, "unknown", errorCode);
|
|
@@ -1878,13 +1902,13 @@ export class WorkflowHost {
|
|
|
1878
1902
|
throw new Error("Channel recovery action must be confirm or retry");
|
|
1879
1903
|
}
|
|
1880
1904
|
if (this.claim === null)
|
|
1881
|
-
throw new Error("Workflow
|
|
1905
|
+
throw new Error("Workflow server claim is unavailable");
|
|
1882
1906
|
const effect = this.state.transaction(() => {
|
|
1883
1907
|
const recovered = this.channelEffects.recover({
|
|
1884
1908
|
stableMessageId: messageId,
|
|
1885
1909
|
action,
|
|
1886
1910
|
actorId,
|
|
1887
|
-
ownerId: this.
|
|
1911
|
+
ownerId: this.serverId,
|
|
1888
1912
|
leaseGeneration: this.claim?.epoch ?? 0,
|
|
1889
1913
|
});
|
|
1890
1914
|
const attemptId = channelEffectAttemptId(recovered.effectId, recovered.attemptNumber);
|
|
@@ -2102,9 +2126,9 @@ export class WorkflowHost {
|
|
|
2102
2126
|
if (config === undefined ||
|
|
2103
2127
|
!config.allowedUserIds.includes(message.actorId) ||
|
|
2104
2128
|
!config.allowedChatIds.includes(message.chatId)) {
|
|
2105
|
-
throw new Error("Telegram answer source is not authorized by the
|
|
2129
|
+
throw new Error("Telegram answer source is not authorized by the server profile");
|
|
2106
2130
|
}
|
|
2107
|
-
const interaction = this.
|
|
2131
|
+
const interaction = this.serverState.listPendingDecisionInteractions().find((candidate) => {
|
|
2108
2132
|
const request = candidate.contract;
|
|
2109
2133
|
return request.decisionId === message.decisionId;
|
|
2110
2134
|
});
|
|
@@ -2138,7 +2162,7 @@ export class WorkflowHost {
|
|
|
2138
2162
|
setImmediate(effect);
|
|
2139
2163
|
}
|
|
2140
2164
|
async nextChannelCommand(active) {
|
|
2141
|
-
const interactions = this.
|
|
2165
|
+
const interactions = this.serverState.listPendingDecisionInteractions();
|
|
2142
2166
|
for (const interaction of interactions) {
|
|
2143
2167
|
const request = interaction.contract;
|
|
2144
2168
|
if (!audienceChannels(this.decisionChannelConfig, request.audience).includes(active.channelId)) {
|
|
@@ -2214,7 +2238,7 @@ export class WorkflowHost {
|
|
|
2214
2238
|
}
|
|
2215
2239
|
ensureApplyingChannelEffect(active, request, purpose) {
|
|
2216
2240
|
if (this.claim === null)
|
|
2217
|
-
throw new Error("Workflow
|
|
2241
|
+
throw new Error("Workflow server claim is unavailable");
|
|
2218
2242
|
return this.state.transaction(() => {
|
|
2219
2243
|
const effect = this.channelEffects.ensureApplying({
|
|
2220
2244
|
channelResourceId: active.resourceId,
|
|
@@ -2223,7 +2247,7 @@ export class WorkflowHost {
|
|
|
2223
2247
|
decisionId: request.decisionId,
|
|
2224
2248
|
purpose,
|
|
2225
2249
|
request,
|
|
2226
|
-
ownerId: this.
|
|
2250
|
+
ownerId: this.serverId,
|
|
2227
2251
|
leaseGeneration: this.claim?.epoch ?? 0,
|
|
2228
2252
|
maxAutomaticAttempts: 3,
|
|
2229
2253
|
});
|
|
@@ -2323,14 +2347,14 @@ export class WorkflowHost {
|
|
|
2323
2347
|
}
|
|
2324
2348
|
return row.revision;
|
|
2325
2349
|
}
|
|
2326
|
-
async
|
|
2350
|
+
async claimResourceManagerOne() {
|
|
2327
2351
|
if (this.stopping ||
|
|
2328
2352
|
this.claim === null ||
|
|
2329
|
-
this.
|
|
2330
|
-
this.
|
|
2353
|
+
this.resourceManagerPollActive ||
|
|
2354
|
+
this.activeResourceManagers.size >= MAX_RESOURCE_RUNNERS) {
|
|
2331
2355
|
return;
|
|
2332
2356
|
}
|
|
2333
|
-
this.
|
|
2357
|
+
this.resourceManagerPollActive = true;
|
|
2334
2358
|
try {
|
|
2335
2359
|
const rows = this.state.connection
|
|
2336
2360
|
.prepare(`SELECT DISTINCT p.canonical_path AS projectPath
|
|
@@ -2344,84 +2368,87 @@ export class WorkflowHost {
|
|
|
2344
2368
|
for (const row of rows) {
|
|
2345
2369
|
if (typeof row.projectPath !== "string")
|
|
2346
2370
|
continue;
|
|
2347
|
-
const discovered = await
|
|
2371
|
+
const discovered = await discoverResourceManagers({ cwd: row.projectPath });
|
|
2348
2372
|
if (discovered.length === 0)
|
|
2349
2373
|
continue;
|
|
2350
2374
|
const byName = new Map(discovered.map((item) => [item.name, item.path]));
|
|
2351
|
-
const store = new
|
|
2375
|
+
const store = new SqliteResourceManagerStore(this.databasePath, {
|
|
2352
2376
|
state: this.state,
|
|
2353
2377
|
projectPath: row.projectPath,
|
|
2354
2378
|
});
|
|
2355
2379
|
const claim = store.claimNext({
|
|
2356
|
-
|
|
2357
|
-
ownerId: this.
|
|
2358
|
-
leaseMs:
|
|
2359
|
-
exclude: [...this.
|
|
2380
|
+
resourceManagers: [...byName.keys()],
|
|
2381
|
+
ownerId: this.serverId,
|
|
2382
|
+
leaseMs: RESOURCE_MANAGER_CLAIM_LEASE_MS,
|
|
2383
|
+
exclude: [...this.activeResourceManagers.values()]
|
|
2360
2384
|
.filter((active) => active.projectPath === row.projectPath)
|
|
2361
2385
|
.map((active) => ({
|
|
2362
|
-
|
|
2386
|
+
resourceManager: active.claim.resourceManager,
|
|
2363
2387
|
key: active.claim.key,
|
|
2364
2388
|
})),
|
|
2365
2389
|
});
|
|
2366
2390
|
if (claim === undefined)
|
|
2367
2391
|
continue;
|
|
2368
|
-
const definitionPath = byName.get(claim.
|
|
2369
|
-
const resource = store.getResource({
|
|
2392
|
+
const definitionPath = byName.get(claim.resourceManager);
|
|
2393
|
+
const resource = store.getResource({
|
|
2394
|
+
resourceManager: claim.resourceManager,
|
|
2395
|
+
key: claim.key,
|
|
2396
|
+
});
|
|
2370
2397
|
if (definitionPath === undefined || resource === undefined) {
|
|
2371
2398
|
store.requeueClaim(claim, {
|
|
2372
2399
|
availableAt: new Date().toISOString(),
|
|
2373
|
-
error: "
|
|
2400
|
+
error: "ResourceManager source is unavailable",
|
|
2374
2401
|
});
|
|
2375
2402
|
continue;
|
|
2376
2403
|
}
|
|
2377
|
-
void this.
|
|
2404
|
+
void this.activateResourceManager(row.projectPath, definitionPath, store, claim, resource);
|
|
2378
2405
|
break;
|
|
2379
2406
|
}
|
|
2380
2407
|
}
|
|
2381
2408
|
catch (error) {
|
|
2382
|
-
this.log(`
|
|
2409
|
+
this.log(`resource manager scheduling failed: ${errorMessage(error)}`);
|
|
2383
2410
|
}
|
|
2384
2411
|
finally {
|
|
2385
|
-
this.
|
|
2412
|
+
this.resourceManagerPollActive = false;
|
|
2386
2413
|
}
|
|
2387
2414
|
}
|
|
2388
|
-
async
|
|
2415
|
+
async activateResourceManager(projectPath, definitionPath, store, claim, resource) {
|
|
2389
2416
|
if (this.claim === null || this.stopping) {
|
|
2390
2417
|
store.requeueClaim(claim, { availableAt: new Date().toISOString() });
|
|
2391
2418
|
return;
|
|
2392
2419
|
}
|
|
2393
|
-
const
|
|
2394
|
-
const key = `${projectPath}\u0000${claim.
|
|
2420
|
+
const runnerEpoch = randomUUID();
|
|
2421
|
+
const key = `${projectPath}\u0000${claim.resourceManager}\u0000${claim.key}`;
|
|
2395
2422
|
const reconcileId = randomUUID();
|
|
2396
2423
|
const envelope = {
|
|
2397
2424
|
schema: "pi-workflows.controller-worker-launch.v1",
|
|
2398
|
-
|
|
2399
|
-
|
|
2425
|
+
runnerEpoch,
|
|
2426
|
+
serverEpoch: this.claim.epoch,
|
|
2400
2427
|
generation: claim.generation,
|
|
2401
2428
|
projectPath,
|
|
2402
2429
|
definitionPath,
|
|
2403
|
-
|
|
2430
|
+
resourceManagerName: claim.resourceManager,
|
|
2404
2431
|
resource,
|
|
2405
|
-
timeoutMs:
|
|
2432
|
+
timeoutMs: DEFAULT_RESOURCE_MANAGER_TIMEOUT_MS,
|
|
2406
2433
|
};
|
|
2407
2434
|
let active;
|
|
2408
|
-
const supervisor = new
|
|
2435
|
+
const supervisor = new ResourceRunnerSupervisor(envelope, {
|
|
2409
2436
|
registry: this.registry,
|
|
2410
|
-
onMessage: async (message) => await this.
|
|
2437
|
+
onMessage: async (message) => await this.handleResourceRunnerMessage(active, message),
|
|
2411
2438
|
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
2412
|
-
...(this.options.
|
|
2439
|
+
...(this.options.resourceRunnerEntryPath === undefined
|
|
2413
2440
|
? {}
|
|
2414
|
-
: {
|
|
2415
|
-
...(this.options.
|
|
2441
|
+
: { runnerEntryPath: this.options.resourceRunnerEntryPath }),
|
|
2442
|
+
...(this.options.runnerStartupTimeoutMs === undefined
|
|
2416
2443
|
? {}
|
|
2417
|
-
: { startupTimeoutMs: this.options.
|
|
2418
|
-
onDiagnostic: (message) => this.log(`
|
|
2444
|
+
: { startupTimeoutMs: this.options.runnerStartupTimeoutMs }),
|
|
2445
|
+
onDiagnostic: (message) => this.log(`resource runner ${runnerEpoch}: ${message}`),
|
|
2419
2446
|
});
|
|
2420
2447
|
const renewTimer = setInterval(() => {
|
|
2421
|
-
if (!store.renewClaim(claim,
|
|
2448
|
+
if (!store.renewClaim(claim, RESOURCE_MANAGER_CLAIM_LEASE_MS)) {
|
|
2422
2449
|
void supervisor.stop("claimLost");
|
|
2423
2450
|
}
|
|
2424
|
-
},
|
|
2451
|
+
}, RESOURCE_MANAGER_RENEW_MS);
|
|
2425
2452
|
renewTimer.unref?.();
|
|
2426
2453
|
active = {
|
|
2427
2454
|
key,
|
|
@@ -2435,9 +2462,9 @@ export class WorkflowHost {
|
|
|
2435
2462
|
renewTimer,
|
|
2436
2463
|
settled: false,
|
|
2437
2464
|
};
|
|
2438
|
-
this.
|
|
2465
|
+
this.activeResourceManagers.set(key, active);
|
|
2439
2466
|
store.recordEvent({
|
|
2440
|
-
|
|
2467
|
+
resourceManager: claim.resourceManager,
|
|
2441
2468
|
key: claim.key,
|
|
2442
2469
|
claim,
|
|
2443
2470
|
type: "reconcile_started",
|
|
@@ -2447,36 +2474,36 @@ export class WorkflowHost {
|
|
|
2447
2474
|
await supervisor.start();
|
|
2448
2475
|
const result = await supervisor.wait();
|
|
2449
2476
|
if (result.outcome !== "exited" && !active.settled) {
|
|
2450
|
-
this.
|
|
2477
|
+
this.finishResourceManagerFailure(active, result.diagnostic ?? `Resource runner ${result.outcome}`);
|
|
2451
2478
|
}
|
|
2452
2479
|
}
|
|
2453
2480
|
catch (error) {
|
|
2454
2481
|
await supervisor.stop("crashed");
|
|
2455
2482
|
if (!active.settled)
|
|
2456
|
-
this.
|
|
2483
|
+
this.finishResourceManagerFailure(active, errorMessage(error));
|
|
2457
2484
|
}
|
|
2458
2485
|
finally {
|
|
2459
2486
|
clearInterval(renewTimer);
|
|
2460
|
-
this.
|
|
2487
|
+
this.activeResourceManagers.delete(key);
|
|
2461
2488
|
if (!this.stopping)
|
|
2462
|
-
setImmediate(() => void this.
|
|
2489
|
+
setImmediate(() => void this.claimResourceManagerOne());
|
|
2463
2490
|
}
|
|
2464
2491
|
}
|
|
2465
|
-
async
|
|
2466
|
-
const current = this.
|
|
2492
|
+
async handleResourceRunnerMessage(active, message) {
|
|
2493
|
+
const current = this.activeResourceManagers.get(active.key);
|
|
2467
2494
|
if (current !== active ||
|
|
2468
|
-
message.
|
|
2495
|
+
message.runnerEpoch !== active.supervisor.envelope.runnerEpoch ||
|
|
2469
2496
|
message.generation !== active.claim.generation) {
|
|
2470
|
-
return
|
|
2497
|
+
return resourceRunnerResponse(message, "claimLost", undefined, "Resource runner is stale");
|
|
2471
2498
|
}
|
|
2472
|
-
if (!active.store.renewClaim(active.claim,
|
|
2473
|
-
return
|
|
2499
|
+
if (!active.store.renewClaim(active.claim, RESOURCE_MANAGER_CLAIM_LEASE_MS)) {
|
|
2500
|
+
return resourceRunnerResponse(message, "claimLost", undefined, "ResourceManager claim expired");
|
|
2474
2501
|
}
|
|
2475
2502
|
try {
|
|
2476
|
-
const payload = requireRecord(message.payload, "
|
|
2503
|
+
const payload = requireRecord(message.payload, "resource runner payload");
|
|
2477
2504
|
switch (message.operation) {
|
|
2478
|
-
case "
|
|
2479
|
-
return
|
|
2505
|
+
case "runner.ready":
|
|
2506
|
+
return resourceRunnerResponse(message, "accepted", {});
|
|
2480
2507
|
case "effect.reserve": {
|
|
2481
2508
|
const reservation = active.store.reserveEffect({
|
|
2482
2509
|
key: requireString(payload.key, "key"),
|
|
@@ -2486,12 +2513,12 @@ export class WorkflowHost {
|
|
|
2486
2513
|
kind: requireString(payload.kind, "kind"),
|
|
2487
2514
|
requestFingerprint: requireString(payload.requestFingerprint, "requestFingerprint"),
|
|
2488
2515
|
});
|
|
2489
|
-
return
|
|
2516
|
+
return resourceRunnerResponse(message, "accepted", reservation);
|
|
2490
2517
|
}
|
|
2491
2518
|
case "effect.settle": {
|
|
2492
2519
|
const state = requireString(payload.state, "state");
|
|
2493
2520
|
if (!["applied", "rejected", "indeterminate"].includes(state)) {
|
|
2494
|
-
throw new Error("
|
|
2521
|
+
throw new Error("ResourceManager effect state is invalid");
|
|
2495
2522
|
}
|
|
2496
2523
|
const record = active.store.updateEffect({
|
|
2497
2524
|
resourceUid: active.resource.metadata.uid,
|
|
@@ -2504,19 +2531,19 @@ export class WorkflowHost {
|
|
|
2504
2531
|
...(typeof payload.error === "string" ? { error: payload.error } : {}),
|
|
2505
2532
|
});
|
|
2506
2533
|
active.store.recordEvent({
|
|
2507
|
-
|
|
2534
|
+
resourceManager: active.claim.resourceManager,
|
|
2508
2535
|
key: active.claim.key,
|
|
2509
2536
|
claim: active.claim,
|
|
2510
2537
|
type: `effect_${state}`,
|
|
2511
2538
|
payload: { effectKey: record.key },
|
|
2512
2539
|
});
|
|
2513
|
-
return
|
|
2540
|
+
return resourceRunnerResponse(message, "accepted", record);
|
|
2514
2541
|
}
|
|
2515
2542
|
case "workflow.ensure":
|
|
2516
2543
|
case "workflow.changeSettings":
|
|
2517
2544
|
case "workflow.queueFollowUp":
|
|
2518
2545
|
case "workflow.removeFollowUp": {
|
|
2519
|
-
const workflows = new
|
|
2546
|
+
const workflows = new ResourceManagerWorkflowCoordinator(active.store, this.resourceManagerWorkflowScheduler(active)).forResource(active.resource, active.claim, new AbortController().signal);
|
|
2520
2547
|
const result = message.operation === "workflow.ensure"
|
|
2521
2548
|
? await workflows.ensure(payload)
|
|
2522
2549
|
: message.operation === "workflow.changeSettings"
|
|
@@ -2524,34 +2551,34 @@ export class WorkflowHost {
|
|
|
2524
2551
|
: message.operation === "workflow.queueFollowUp"
|
|
2525
2552
|
? await workflows.queueFollowUp(payload)
|
|
2526
2553
|
: await workflows.removeFollowUp(payload);
|
|
2527
|
-
return
|
|
2554
|
+
return resourceRunnerResponse(message, "accepted", result);
|
|
2528
2555
|
}
|
|
2529
|
-
case "
|
|
2530
|
-
this.
|
|
2531
|
-
return
|
|
2532
|
-
case "
|
|
2533
|
-
this.
|
|
2534
|
-
return
|
|
2556
|
+
case "runner.finished":
|
|
2557
|
+
this.finishResourceManagerSuccess(active, message.payload);
|
|
2558
|
+
return resourceRunnerResponse(message, "accepted", {});
|
|
2559
|
+
case "runner.failed":
|
|
2560
|
+
this.finishResourceManagerFailure(active, requireString(payload.error, "error"));
|
|
2561
|
+
return resourceRunnerResponse(message, "accepted", {});
|
|
2535
2562
|
}
|
|
2536
2563
|
}
|
|
2537
2564
|
catch (error) {
|
|
2538
|
-
return
|
|
2565
|
+
return resourceRunnerResponse(message, "rejected", undefined, errorMessage(error));
|
|
2539
2566
|
}
|
|
2540
2567
|
}
|
|
2541
|
-
|
|
2568
|
+
resourceManagerWorkflowScheduler(active) {
|
|
2542
2569
|
return {
|
|
2543
|
-
ensure: async (request, _signal, _onComplete) => await this.
|
|
2544
|
-
changeSettings: async (request) => await this.
|
|
2570
|
+
ensure: async (request, _signal, _onComplete) => await this.ensureResourceManagerWorkflow(active, request),
|
|
2571
|
+
changeSettings: async (request) => await this.changeResourceManagerWorkflowSettings(active, request),
|
|
2545
2572
|
queueFollowUp: async (request) => {
|
|
2546
2573
|
const targetSessionId = this.runStore.originSessionId(request.runId);
|
|
2547
2574
|
if (targetSessionId === undefined) {
|
|
2548
|
-
throw new Error("
|
|
2575
|
+
throw new Error("ResourceManager follow-up requires an origin Pi session");
|
|
2549
2576
|
}
|
|
2550
2577
|
const result = this.runStore.queueFollowUp({
|
|
2551
2578
|
runId: request.runId,
|
|
2552
2579
|
requestId: request.actorRequestKey,
|
|
2553
2580
|
targetSessionId,
|
|
2554
|
-
actor: { type: "controller", id: request.
|
|
2581
|
+
actor: { type: "controller", id: request.managedResourceUid },
|
|
2555
2582
|
source: "controller-request",
|
|
2556
2583
|
prompt: request.prompt,
|
|
2557
2584
|
});
|
|
@@ -2567,7 +2594,7 @@ export class WorkflowHost {
|
|
|
2567
2594
|
const result = this.runStore.removeFollowUp({
|
|
2568
2595
|
runId: request.runId,
|
|
2569
2596
|
followUpId: request.followUpId,
|
|
2570
|
-
actor: { type: "controller", id: request.
|
|
2597
|
+
actor: { type: "controller", id: request.managedResourceUid },
|
|
2571
2598
|
source: "controller-request",
|
|
2572
2599
|
});
|
|
2573
2600
|
return {
|
|
@@ -2580,14 +2607,14 @@ export class WorkflowHost {
|
|
|
2580
2607
|
},
|
|
2581
2608
|
};
|
|
2582
2609
|
}
|
|
2583
|
-
async
|
|
2610
|
+
async changeResourceManagerWorkflowSettings(_active, request) {
|
|
2584
2611
|
const run = this.queue.getWorkflowRun(request.runId);
|
|
2585
2612
|
if (run === undefined)
|
|
2586
2613
|
throw new Error(`Workflow run not found: ${request.runId}`);
|
|
2587
2614
|
const scopes = this.runStore.listSettingsScopes(request.runId);
|
|
2588
2615
|
const scopeId = request.scopeId ?? (scopes.length === 1 ? scopes[0]?.scopeId : undefined);
|
|
2589
2616
|
if (scopeId === undefined) {
|
|
2590
|
-
throw new Error("
|
|
2617
|
+
throw new Error("ResourceManager settings changes require scopeId when several scopes exist");
|
|
2591
2618
|
}
|
|
2592
2619
|
const scope = this.runStore.getSettingsScope(scopeId);
|
|
2593
2620
|
if (scope === undefined || scope.activeRunId !== request.runId) {
|
|
@@ -2614,7 +2641,7 @@ export class WorkflowHost {
|
|
|
2614
2641
|
mountPath: scope.mountPath,
|
|
2615
2642
|
current: scope.settings,
|
|
2616
2643
|
patch: request.patch,
|
|
2617
|
-
actorId: request.
|
|
2644
|
+
actorId: request.managedResourceUid,
|
|
2618
2645
|
});
|
|
2619
2646
|
if (proposal.definitionDigest !== run.definitionDigest || !Array.isArray(proposal.paths)) {
|
|
2620
2647
|
throw new Error("Workflow settings proposal does not match the durable source");
|
|
@@ -2635,7 +2662,7 @@ export class WorkflowHost {
|
|
|
2635
2662
|
scopeId,
|
|
2636
2663
|
requestId: request.actorRequestKey,
|
|
2637
2664
|
expectedChangeNumber: request.expectedChangeNumber ?? scope.changeNumber,
|
|
2638
|
-
actor: { type: "controller", id: request.
|
|
2665
|
+
actor: { type: "controller", id: request.managedResourceUid },
|
|
2639
2666
|
source: "controller-request",
|
|
2640
2667
|
patch: proposal.patch,
|
|
2641
2668
|
});
|
|
@@ -2646,10 +2673,10 @@ export class WorkflowHost {
|
|
|
2646
2673
|
adopted: result.adopted,
|
|
2647
2674
|
};
|
|
2648
2675
|
}
|
|
2649
|
-
async
|
|
2676
|
+
async ensureResourceManagerWorkflow(active, request) {
|
|
2650
2677
|
const existing = this.queue.getWorkflowRun(request.runId);
|
|
2651
2678
|
if (existing !== undefined)
|
|
2652
|
-
return
|
|
2679
|
+
return resourceManagerWorkflowResult(existing);
|
|
2653
2680
|
const resolver = new WorkflowClient({
|
|
2654
2681
|
databasePath: this.databasePath,
|
|
2655
2682
|
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
@@ -2659,7 +2686,7 @@ export class WorkflowHost {
|
|
|
2659
2686
|
workflowRef: request.workflow,
|
|
2660
2687
|
});
|
|
2661
2688
|
const token = randomUUID();
|
|
2662
|
-
const scoped = new
|
|
2689
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
2663
2690
|
state: this.state,
|
|
2664
2691
|
projectPath: active.projectPath,
|
|
2665
2692
|
});
|
|
@@ -2672,23 +2699,23 @@ export class WorkflowHost {
|
|
|
2672
2699
|
definitionSnapshot: resolved.definitionSnapshot,
|
|
2673
2700
|
input: request.input,
|
|
2674
2701
|
launchOptions: {},
|
|
2675
|
-
runnerId: this.
|
|
2702
|
+
runnerId: this.serverId,
|
|
2676
2703
|
claimToken: token,
|
|
2677
2704
|
leaseMs: this.runClaimLeaseMs,
|
|
2678
|
-
originSessionId: `
|
|
2705
|
+
originSessionId: `resource-manager-${active.resource.metadata.uid}`,
|
|
2679
2706
|
executionMode: "headless",
|
|
2680
2707
|
});
|
|
2681
2708
|
if (prepared.state === "claimed") {
|
|
2682
2709
|
this.markPendingStart(request.runId, token);
|
|
2683
2710
|
setImmediate(() => void this.activateRun(prepared.run, token));
|
|
2684
2711
|
}
|
|
2685
|
-
return
|
|
2712
|
+
return resourceManagerWorkflowResult(prepared.run);
|
|
2686
2713
|
}
|
|
2687
|
-
|
|
2714
|
+
finishResourceManagerSuccess(active, result) {
|
|
2688
2715
|
if (active.settled)
|
|
2689
2716
|
return;
|
|
2690
2717
|
const now = new Date();
|
|
2691
|
-
const ref = {
|
|
2718
|
+
const ref = { resourceManager: active.claim.resourceManager, key: active.claim.key };
|
|
2692
2719
|
const status = applyStatusPatch(active.resource.status, result.status, active.resource.metadata.generation, now.toISOString());
|
|
2693
2720
|
try {
|
|
2694
2721
|
const updated = active.store.updateStatus({
|
|
@@ -2735,7 +2762,7 @@ export class WorkflowHost {
|
|
|
2735
2762
|
active.settled = true;
|
|
2736
2763
|
}
|
|
2737
2764
|
catch (error) {
|
|
2738
|
-
if (error instanceof
|
|
2765
|
+
if (error instanceof ManagedResourceConflictError) {
|
|
2739
2766
|
active.store.recordEvent({
|
|
2740
2767
|
...ref,
|
|
2741
2768
|
claim: active.claim,
|
|
@@ -2749,14 +2776,14 @@ export class WorkflowHost {
|
|
|
2749
2776
|
throw error;
|
|
2750
2777
|
}
|
|
2751
2778
|
}
|
|
2752
|
-
|
|
2779
|
+
finishResourceManagerFailure(active, failure) {
|
|
2753
2780
|
if (active.settled)
|
|
2754
2781
|
return;
|
|
2755
2782
|
const now = new Date();
|
|
2756
2783
|
const delay = Math.min(60_000, 1_000 * 2 ** Math.min(active.claim.consecutiveErrors, 6));
|
|
2757
2784
|
const error = failure.slice(0, 8_192);
|
|
2758
2785
|
active.store.recordEvent({
|
|
2759
|
-
|
|
2786
|
+
resourceManager: active.claim.resourceManager,
|
|
2760
2787
|
key: active.claim.key,
|
|
2761
2788
|
claim: active.claim,
|
|
2762
2789
|
type: "reconcile_failed",
|
|
@@ -2780,7 +2807,7 @@ export class WorkflowHost {
|
|
|
2780
2807
|
const candidate = requests
|
|
2781
2808
|
.map((request) => ({
|
|
2782
2809
|
request,
|
|
2783
|
-
interaction: this.
|
|
2810
|
+
interaction: this.serverState.getInteraction(request.decisionId),
|
|
2784
2811
|
}))
|
|
2785
2812
|
.find(({ request, interaction }) => interaction?.kind === "decision" &&
|
|
2786
2813
|
interaction.runId === request.runId &&
|
|
@@ -2796,7 +2823,7 @@ export class WorkflowHost {
|
|
|
2796
2823
|
const claimToken = randomUUID();
|
|
2797
2824
|
const claimed = this.queue.claimWorkflowRunForControl({
|
|
2798
2825
|
runId: request.runId,
|
|
2799
|
-
runnerId: this.
|
|
2826
|
+
runnerId: this.serverId,
|
|
2800
2827
|
claimToken,
|
|
2801
2828
|
leaseMs: this.runClaimLeaseMs,
|
|
2802
2829
|
});
|
|
@@ -2809,7 +2836,7 @@ export class WorkflowHost {
|
|
|
2809
2836
|
this.state.transaction(() => {
|
|
2810
2837
|
const accepted = this.decisions.resolveTimeoutSync(request, new Date());
|
|
2811
2838
|
const submissionId = `timeout-${request.decisionId}`;
|
|
2812
|
-
this.
|
|
2839
|
+
this.serverState.submitInteraction({
|
|
2813
2840
|
requestId: interaction.requestId,
|
|
2814
2841
|
submissionId,
|
|
2815
2842
|
idempotencyKey: submissionId,
|
|
@@ -2850,7 +2877,7 @@ export class WorkflowHost {
|
|
|
2850
2877
|
const activeSessionIds = new Set([...this.sessionCoordinators.entries()]
|
|
2851
2878
|
.filter(([, coordinator]) => coordinator.branchReported && coordinator.modelTurnActive)
|
|
2852
2879
|
.map(([sessionId]) => sessionId));
|
|
2853
|
-
const expired = this.
|
|
2880
|
+
const expired = this.serverState
|
|
2854
2881
|
.expiredInteractionRuns()
|
|
2855
2882
|
.find((candidate) => activeSessionIds.has(candidate.targetSessionId) &&
|
|
2856
2883
|
!this.activeRuns.has(candidate.runId) &&
|
|
@@ -2861,7 +2888,7 @@ export class WorkflowHost {
|
|
|
2861
2888
|
const claimToken = randomUUID();
|
|
2862
2889
|
const claimed = this.queue.claimWorkflowRunForControl({
|
|
2863
2890
|
runId,
|
|
2864
|
-
runnerId: this.
|
|
2891
|
+
runnerId: this.serverId,
|
|
2865
2892
|
claimToken,
|
|
2866
2893
|
leaseMs: this.runClaimLeaseMs,
|
|
2867
2894
|
});
|
|
@@ -2906,14 +2933,14 @@ export class WorkflowHost {
|
|
|
2906
2933
|
...this.pendingStarts,
|
|
2907
2934
|
...this.blockedRuns,
|
|
2908
2935
|
]);
|
|
2909
|
-
const validatingRunId = this.
|
|
2936
|
+
const validatingRunId = this.serverState
|
|
2910
2937
|
.validatingInteractionRunIds()
|
|
2911
2938
|
.find((runId) => !excluded.has(runId));
|
|
2912
2939
|
if (validatingRunId !== undefined) {
|
|
2913
2940
|
const validationToken = randomUUID();
|
|
2914
2941
|
const validationRun = this.queue.claimWorkflowRunForInteractionValidation({
|
|
2915
2942
|
runId: validatingRunId,
|
|
2916
|
-
runnerId: this.
|
|
2943
|
+
runnerId: this.serverId,
|
|
2917
2944
|
claimToken: validationToken,
|
|
2918
2945
|
leaseMs: this.runClaimLeaseMs,
|
|
2919
2946
|
});
|
|
@@ -2925,7 +2952,7 @@ export class WorkflowHost {
|
|
|
2925
2952
|
}
|
|
2926
2953
|
const token = randomUUID();
|
|
2927
2954
|
const claimed = this.queue.claimNextWorkflowRun({
|
|
2928
|
-
runnerId: this.
|
|
2955
|
+
runnerId: this.serverId,
|
|
2929
2956
|
claimToken: token,
|
|
2930
2957
|
leaseMs: this.runClaimLeaseMs,
|
|
2931
2958
|
excludeRunIds: [...excluded],
|
|
@@ -2983,30 +3010,30 @@ export class WorkflowHost {
|
|
|
2983
3010
|
schema: "pi-workflows.worker-launch.v1",
|
|
2984
3011
|
runId,
|
|
2985
3012
|
generation,
|
|
2986
|
-
|
|
3013
|
+
runnerEpoch: randomUUID(),
|
|
2987
3014
|
projectPath,
|
|
2988
3015
|
workflowSource: record.workflowSource,
|
|
2989
3016
|
definitionDigest: record.definitionDigest,
|
|
2990
3017
|
inputHash: `sha256:${createHash("sha256").update(canonicalJson(record.input)).digest("hex")}`,
|
|
2991
3018
|
protocolVersion: 1,
|
|
2992
3019
|
};
|
|
2993
|
-
this.
|
|
3020
|
+
this.serverState.recordRunnerStart(envelope, this.claim.epoch);
|
|
2994
3021
|
let active;
|
|
2995
|
-
const supervisor = new
|
|
3022
|
+
const supervisor = new WorkflowRunnerSupervisor(envelope, {
|
|
2996
3023
|
registry: this.registry,
|
|
2997
|
-
onMessage: async (message) => await this.
|
|
3024
|
+
onMessage: async (message) => await this.handleRunnerMessage(message),
|
|
2998
3025
|
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
2999
|
-
...(this.options.
|
|
3026
|
+
...(this.options.runnerEntryPath === undefined
|
|
3000
3027
|
? {}
|
|
3001
|
-
: {
|
|
3002
|
-
...(this.options.
|
|
3028
|
+
: { runnerEntryPath: this.options.runnerEntryPath }),
|
|
3029
|
+
...(this.options.runnerStartupTimeoutMs === undefined
|
|
3003
3030
|
? {}
|
|
3004
|
-
: { startupTimeoutMs: this.options.
|
|
3031
|
+
: { startupTimeoutMs: this.options.runnerStartupTimeoutMs }),
|
|
3005
3032
|
onSpawn: (identity) => {
|
|
3006
|
-
active.
|
|
3007
|
-
this.
|
|
3033
|
+
active.runnerPid = identity.pid;
|
|
3034
|
+
this.serverState.attachRunnerProcess(envelope.runnerEpoch, identity.pid, identity.startIdentity);
|
|
3008
3035
|
},
|
|
3009
|
-
onDiagnostic: (message) => this.log(`
|
|
3036
|
+
onDiagnostic: (message) => this.log(`runner ${envelope.runnerEpoch}: ${message}`),
|
|
3010
3037
|
});
|
|
3011
3038
|
active = {
|
|
3012
3039
|
record,
|
|
@@ -3015,6 +3042,7 @@ export class WorkflowHost {
|
|
|
3015
3042
|
supervisor,
|
|
3016
3043
|
launchProgressRevision: runRevision(this.state, runId),
|
|
3017
3044
|
exiting: false,
|
|
3045
|
+
contentDigests: new Set(),
|
|
3018
3046
|
};
|
|
3019
3047
|
this.activeRuns.set(runId, active);
|
|
3020
3048
|
this.clearPendingStart(runId, claimToken);
|
|
@@ -3025,8 +3053,8 @@ export class WorkflowHost {
|
|
|
3025
3053
|
if (!this.queue.parkWorkflowRunForAmbiguousEffect({ runId, claimToken })) {
|
|
3026
3054
|
throw new Error("Run claim was lost during ambiguous-effect recovery");
|
|
3027
3055
|
}
|
|
3028
|
-
this.
|
|
3029
|
-
|
|
3056
|
+
this.serverState.finishRunner({
|
|
3057
|
+
runnerEpoch: envelope.runnerEpoch,
|
|
3030
3058
|
outcome: "orphaned",
|
|
3031
3059
|
diagnostic: "effect outcome is ambiguous",
|
|
3032
3060
|
});
|
|
@@ -3034,8 +3062,8 @@ export class WorkflowHost {
|
|
|
3034
3062
|
}
|
|
3035
3063
|
await supervisor.start();
|
|
3036
3064
|
const result = await supervisor.wait();
|
|
3037
|
-
this.
|
|
3038
|
-
|
|
3065
|
+
this.serverState.finishRunner({
|
|
3066
|
+
runnerEpoch: envelope.runnerEpoch,
|
|
3039
3067
|
outcome: result.outcome,
|
|
3040
3068
|
exitCode: result.exitCode,
|
|
3041
3069
|
signal: result.signal,
|
|
@@ -3043,17 +3071,17 @@ export class WorkflowHost {
|
|
|
3043
3071
|
});
|
|
3044
3072
|
if (active.control === undefined &&
|
|
3045
3073
|
(result.outcome !== "exited" ||
|
|
3046
|
-
this.
|
|
3047
|
-
await this.
|
|
3074
|
+
this.serverState.validatingInteraction(active.record.runId) !== undefined)) {
|
|
3075
|
+
await this.recoverRunnerExit(active, result.outcome);
|
|
3048
3076
|
}
|
|
3049
3077
|
}
|
|
3050
3078
|
catch (error) {
|
|
3051
|
-
this.log(`
|
|
3079
|
+
this.log(`runner ${envelope.runnerEpoch} failed: ${errorMessage(error)}`);
|
|
3052
3080
|
await supervisor.stop("crashed");
|
|
3053
3081
|
const result = await supervisor.wait();
|
|
3054
3082
|
try {
|
|
3055
|
-
this.
|
|
3056
|
-
|
|
3083
|
+
this.serverState.finishRunner({
|
|
3084
|
+
runnerEpoch: envelope.runnerEpoch,
|
|
3057
3085
|
outcome: "crashed",
|
|
3058
3086
|
exitCode: result.exitCode,
|
|
3059
3087
|
signal: result.signal,
|
|
@@ -3061,14 +3089,14 @@ export class WorkflowHost {
|
|
|
3061
3089
|
});
|
|
3062
3090
|
}
|
|
3063
3091
|
catch {
|
|
3064
|
-
// A completed
|
|
3092
|
+
// A completed runner record wins.
|
|
3065
3093
|
}
|
|
3066
3094
|
if (active.control === undefined) {
|
|
3067
|
-
await this.
|
|
3095
|
+
await this.recoverRunnerExit(active, "crashed");
|
|
3068
3096
|
}
|
|
3069
3097
|
}
|
|
3070
3098
|
finally {
|
|
3071
|
-
this.
|
|
3099
|
+
this.reapRunnerDescendants(envelope.runnerEpoch);
|
|
3072
3100
|
this.activeRuns.delete(runId);
|
|
3073
3101
|
}
|
|
3074
3102
|
}
|
|
@@ -3078,7 +3106,7 @@ export class WorkflowHost {
|
|
|
3078
3106
|
const token = randomUUID();
|
|
3079
3107
|
const claimed = this.queue.claimWorkflowRunForInteractionValidation({
|
|
3080
3108
|
runId,
|
|
3081
|
-
runnerId: this.
|
|
3109
|
+
runnerId: this.serverId,
|
|
3082
3110
|
claimToken: token,
|
|
3083
3111
|
leaseMs: this.runClaimLeaseMs,
|
|
3084
3112
|
});
|
|
@@ -3089,66 +3117,68 @@ export class WorkflowHost {
|
|
|
3089
3117
|
this.markPendingStart(runId, token);
|
|
3090
3118
|
await this.activateRun(claimed, token);
|
|
3091
3119
|
}
|
|
3092
|
-
async
|
|
3120
|
+
async handleRunnerMessage(message) {
|
|
3093
3121
|
const active = this.activeRuns.get(message.runId);
|
|
3094
3122
|
if (active === undefined ||
|
|
3095
3123
|
active.generation !== message.generation ||
|
|
3096
|
-
active.supervisor.envelope.
|
|
3097
|
-
return
|
|
3098
|
-
}
|
|
3099
|
-
const stored = this.
|
|
3100
|
-
if (stored !== undefined)
|
|
3101
|
-
return stored;
|
|
3102
|
-
|
|
3103
|
-
|
|
3124
|
+
active.supervisor.envelope.runnerEpoch !== message.runnerEpoch) {
|
|
3125
|
+
return runnerResponse(message, "claimLost", undefined, "Runner epoch is stale");
|
|
3126
|
+
}
|
|
3127
|
+
const stored = this.serverState.readRunnerMessage(message);
|
|
3128
|
+
if (stored !== undefined) {
|
|
3129
|
+
return boundRunnerResponse(this.state, active.contentDigests, stored);
|
|
3130
|
+
}
|
|
3131
|
+
const response = await this.handleFreshRunnerMessage(active, message);
|
|
3132
|
+
const recorded = this.serverState.recordRunnerMessage(message, response);
|
|
3133
|
+
return boundRunnerResponse(this.state, active.contentDigests, recorded);
|
|
3104
3134
|
}
|
|
3105
|
-
async
|
|
3106
|
-
if (message.operation === "
|
|
3135
|
+
async handleFreshRunnerMessage(active, message) {
|
|
3136
|
+
if (message.operation === "runner.ready") {
|
|
3107
3137
|
if (!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })) {
|
|
3108
3138
|
active.claimLost = true;
|
|
3109
3139
|
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
3110
|
-
return
|
|
3140
|
+
return runnerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
3111
3141
|
}
|
|
3112
|
-
this.
|
|
3142
|
+
this.serverState.markRunnerReady(message.runnerEpoch);
|
|
3113
3143
|
this.queue.markWorkflowRunRunning({ runId: message.runId, claimToken: active.claimToken });
|
|
3114
3144
|
const revision = this.runStore.synchronizeRevision(message.runId);
|
|
3115
3145
|
active.launchProgressRevision = revision;
|
|
3116
3146
|
const current = this.queue.getWorkflowRun(message.runId);
|
|
3117
|
-
const candidateInteraction = this.
|
|
3118
|
-
this.
|
|
3119
|
-
const timedOutInteraction = this.
|
|
3147
|
+
const candidateInteraction = this.serverState.acceptedInteraction(message.runId) ??
|
|
3148
|
+
this.serverState.validatingInteraction(message.runId);
|
|
3149
|
+
const timedOutInteraction = this.serverState.timedOutInteraction(message.runId);
|
|
3120
3150
|
const resumeInteractionAttemptId = candidateInteraction?.attemptId ?? timedOutInteraction?.attemptId;
|
|
3121
3151
|
const record = current ?? active.record;
|
|
3122
|
-
return
|
|
3123
|
-
command:
|
|
3152
|
+
return runnerResponse(message, "accepted", {
|
|
3153
|
+
command: runnerRunCommand(record, resumeInteractionAttemptId),
|
|
3124
3154
|
originSessionId: record.originSessionId,
|
|
3125
3155
|
stateDirectory: this.stateDirectory,
|
|
3126
3156
|
...(candidateInteraction === undefined ? {} : { candidateInteraction }),
|
|
3127
3157
|
...(this.options.piArgs === undefined ? {} : { piArgs: this.options.piArgs }),
|
|
3128
3158
|
}, undefined, revision);
|
|
3129
3159
|
}
|
|
3130
|
-
if (message.operation === "
|
|
3131
|
-
const payload = requireRecord(message.payload, "
|
|
3160
|
+
if (message.operation === "runner.exiting") {
|
|
3161
|
+
const payload = requireRecord(message.payload, "runner exit payload");
|
|
3132
3162
|
if (payload.status === "workflowLoadFailed") {
|
|
3133
3163
|
active.workflowLoadFailure = requireString(payload.error, "workflow load error").slice(0, 8_192);
|
|
3134
3164
|
}
|
|
3135
3165
|
active.exiting = true;
|
|
3136
|
-
return
|
|
3166
|
+
return runnerResponse(message, "accepted", {});
|
|
3137
3167
|
}
|
|
3138
3168
|
if (message.operation === "process.register" || message.operation === "process.unregister") {
|
|
3139
|
-
return this.
|
|
3169
|
+
return this.handleRunnerProcessOperation(active, message);
|
|
3140
3170
|
}
|
|
3141
3171
|
const currentRevision = runRevision(this.state, message.runId);
|
|
3142
3172
|
if (message.expectedRevision !== currentRevision) {
|
|
3143
|
-
return
|
|
3173
|
+
return runnerResponse(message, "rejected", undefined, `Workflow run revision conflict: expected ${message.expectedRevision}, got ${currentRevision}`, currentRevision);
|
|
3144
3174
|
}
|
|
3145
3175
|
if (!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })) {
|
|
3146
3176
|
active.claimLost = true;
|
|
3147
3177
|
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
3148
|
-
return
|
|
3178
|
+
return runnerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
3149
3179
|
}
|
|
3150
3180
|
try {
|
|
3151
|
-
const payload = requireRecord(message.payload, "
|
|
3181
|
+
const payload = requireRecord(message.payload, "runner payload");
|
|
3152
3182
|
let result;
|
|
3153
3183
|
switch (message.operation) {
|
|
3154
3184
|
case "store.initializeRun":
|
|
@@ -3158,8 +3188,8 @@ export class WorkflowHost {
|
|
|
3158
3188
|
await this.prepareInteractionResume(message.runId);
|
|
3159
3189
|
result = await this.runStore.prepareRunResume(message.runId);
|
|
3160
3190
|
break;
|
|
3161
|
-
case "store.
|
|
3162
|
-
result = this.runStore.
|
|
3191
|
+
case "store.readRunState":
|
|
3192
|
+
result = this.runStore.readRunState(requireString(payload.runId, "runId"));
|
|
3163
3193
|
break;
|
|
3164
3194
|
case "store.writeSnapshot": {
|
|
3165
3195
|
const state = payload.state;
|
|
@@ -3185,7 +3215,7 @@ export class WorkflowHost {
|
|
|
3185
3215
|
const decision = payload.request;
|
|
3186
3216
|
result = await this.runStore.createHumanDecisionRequest(decision);
|
|
3187
3217
|
if (active.record.originSessionId !== null) {
|
|
3188
|
-
this.
|
|
3218
|
+
this.serverState.createInteractiveRequest({
|
|
3189
3219
|
requestId: decision.decisionId,
|
|
3190
3220
|
runId: decision.runId,
|
|
3191
3221
|
attemptId: decision.attemptId,
|
|
@@ -3211,6 +3241,9 @@ export class WorkflowHost {
|
|
|
3211
3241
|
throw new Error("Run claim was lost during ambiguous-effect recovery");
|
|
3212
3242
|
}
|
|
3213
3243
|
break;
|
|
3244
|
+
case "content.read":
|
|
3245
|
+
result = readRunnerContentChunk(this.state, active.contentDigests, requireString(payload.sha256, "runner content digest"), requireNonNegativeInteger(payload.offset, "runner content offset"));
|
|
3246
|
+
break;
|
|
3214
3247
|
case "store.settleEffect": {
|
|
3215
3248
|
const options = payload.options;
|
|
3216
3249
|
await this.runStore.settleEffect(payload.options);
|
|
@@ -3225,16 +3258,16 @@ export class WorkflowHost {
|
|
|
3225
3258
|
break;
|
|
3226
3259
|
}
|
|
3227
3260
|
case "interaction.accept":
|
|
3228
|
-
result = this.
|
|
3261
|
+
result = this.acceptRunnerInteraction(message, payload);
|
|
3229
3262
|
break;
|
|
3230
3263
|
case "interaction.reject":
|
|
3231
|
-
result = this.
|
|
3264
|
+
result = this.rejectRunnerInteraction(active, message, payload);
|
|
3232
3265
|
break;
|
|
3233
3266
|
case "notification.request":
|
|
3234
|
-
result = this.
|
|
3267
|
+
result = this.enqueueRunnerNotification(active, message, payload);
|
|
3235
3268
|
break;
|
|
3236
3269
|
case "presentation.request":
|
|
3237
|
-
result = this.
|
|
3270
|
+
result = this.requestRunnerPresentation(active, message, payload);
|
|
3238
3271
|
break;
|
|
3239
3272
|
case "interaction.request": {
|
|
3240
3273
|
result = this.parkForInteraction(active, message, payload);
|
|
@@ -3248,91 +3281,91 @@ export class WorkflowHost {
|
|
|
3248
3281
|
break;
|
|
3249
3282
|
}
|
|
3250
3283
|
default:
|
|
3251
|
-
return
|
|
3284
|
+
return runnerResponse(message, "rejected", undefined, "Unknown runner operation");
|
|
3252
3285
|
}
|
|
3253
|
-
return
|
|
3286
|
+
return runnerResponse(message, "accepted", result === undefined ? null : result, undefined, runRevision(this.state, message.runId));
|
|
3254
3287
|
}
|
|
3255
3288
|
catch (error) {
|
|
3256
|
-
return
|
|
3289
|
+
return runnerResponse(message, "rejected", undefined, errorMessage(error));
|
|
3257
3290
|
}
|
|
3258
3291
|
}
|
|
3259
|
-
|
|
3292
|
+
handleRunnerProcessOperation(active, message) {
|
|
3260
3293
|
if (message.operation === "process.register" &&
|
|
3261
3294
|
!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })) {
|
|
3262
3295
|
active.claimLost = true;
|
|
3263
3296
|
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
3264
|
-
return
|
|
3297
|
+
return runnerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
3265
3298
|
}
|
|
3266
3299
|
try {
|
|
3267
|
-
const payload = requireRecord(message.payload, "
|
|
3300
|
+
const payload = requireRecord(message.payload, "runner process payload");
|
|
3268
3301
|
const pid = requireNonNegativeInteger(payload.pid, "pid");
|
|
3269
3302
|
if (pid === 0)
|
|
3270
|
-
throw new Error("
|
|
3271
|
-
const owned = this.
|
|
3303
|
+
throw new Error("Runner process PID must be positive");
|
|
3304
|
+
const owned = this.runnerDescendants.get(message.runnerEpoch);
|
|
3272
3305
|
if (message.operation === "process.unregister") {
|
|
3273
3306
|
if (owned?.has(pid) !== true) {
|
|
3274
|
-
throw new Error("
|
|
3307
|
+
throw new Error("Runner process is not registered to this runner epoch");
|
|
3275
3308
|
}
|
|
3276
3309
|
this.registry.unregister(pid);
|
|
3277
3310
|
owned.delete(pid);
|
|
3278
3311
|
if (owned.size === 0)
|
|
3279
|
-
this.
|
|
3280
|
-
return
|
|
3312
|
+
this.runnerDescendants.delete(message.runnerEpoch);
|
|
3313
|
+
return runnerResponse(message, "accepted", { pid });
|
|
3281
3314
|
}
|
|
3282
|
-
if (active.
|
|
3283
|
-
throw new Error("
|
|
3315
|
+
if (active.runnerPid === undefined || processParentPid(pid) !== active.runnerPid) {
|
|
3316
|
+
throw new Error("Runner process is not a direct child of the active runner");
|
|
3284
3317
|
}
|
|
3285
|
-
for (const [
|
|
3286
|
-
if (
|
|
3287
|
-
throw new Error("
|
|
3318
|
+
for (const [runnerEpoch, pids] of this.runnerDescendants) {
|
|
3319
|
+
if (runnerEpoch !== message.runnerEpoch && pids.has(pid)) {
|
|
3320
|
+
throw new Error("Runner process is already registered to another runner epoch");
|
|
3288
3321
|
}
|
|
3289
3322
|
}
|
|
3290
3323
|
this.registry.register(pid);
|
|
3291
3324
|
const descendants = owned ?? new Set();
|
|
3292
3325
|
descendants.add(pid);
|
|
3293
|
-
this.
|
|
3294
|
-
return
|
|
3326
|
+
this.runnerDescendants.set(message.runnerEpoch, descendants);
|
|
3327
|
+
return runnerResponse(message, "accepted", { pid });
|
|
3295
3328
|
}
|
|
3296
3329
|
catch (error) {
|
|
3297
|
-
return
|
|
3330
|
+
return runnerResponse(message, "rejected", undefined, errorMessage(error));
|
|
3298
3331
|
}
|
|
3299
3332
|
}
|
|
3300
|
-
|
|
3301
|
-
const descendants = this.
|
|
3333
|
+
reapRunnerDescendants(runnerEpoch) {
|
|
3334
|
+
const descendants = this.runnerDescendants.get(runnerEpoch);
|
|
3302
3335
|
if (descendants === undefined)
|
|
3303
3336
|
return;
|
|
3304
|
-
this.
|
|
3337
|
+
this.runnerDescendants.delete(runnerEpoch);
|
|
3305
3338
|
for (const pid of descendants)
|
|
3306
3339
|
this.registry.kill(pid);
|
|
3307
3340
|
}
|
|
3308
|
-
|
|
3309
|
-
const candidate = this.
|
|
3310
|
-
return this.
|
|
3341
|
+
acceptRunnerInteraction(message, payload) {
|
|
3342
|
+
const candidate = this.requireRunnerInteraction(message, payload, true);
|
|
3343
|
+
return this.serverState.finishInteractionValidation({
|
|
3311
3344
|
requestId: candidate.requestId,
|
|
3312
3345
|
submissionId: candidate.submissionId,
|
|
3313
3346
|
accepted: true,
|
|
3314
3347
|
receipt: { status: "accepted" },
|
|
3315
3348
|
});
|
|
3316
3349
|
}
|
|
3317
|
-
|
|
3318
|
-
const candidate = this.
|
|
3350
|
+
rejectRunnerInteraction(active, message, payload) {
|
|
3351
|
+
const candidate = this.requireRunnerInteraction(message, payload, false);
|
|
3319
3352
|
return this.settleRejectedInteraction(active, candidate, requireString(payload.error, "interaction validation error"));
|
|
3320
3353
|
}
|
|
3321
|
-
|
|
3322
|
-
const candidate = this.
|
|
3323
|
-
(acceptSettled ? this.
|
|
3354
|
+
requireRunnerInteraction(message, payload, acceptSettled) {
|
|
3355
|
+
const candidate = this.serverState.validatingInteraction(message.runId) ??
|
|
3356
|
+
(acceptSettled ? this.serverState.acceptedInteraction(message.runId) : undefined);
|
|
3324
3357
|
if (candidate === undefined ||
|
|
3325
3358
|
candidate.requestId !== requireString(payload.requestId, "interaction requestId") ||
|
|
3326
3359
|
candidate.submissionId !== requireString(payload.submissionId, "interaction submissionId") ||
|
|
3327
3360
|
candidate.attemptId !== message.attemptId ||
|
|
3328
3361
|
candidate.attemptId !== requireString(payload.attemptId, "interaction attemptId")) {
|
|
3329
|
-
throw new Error("Interactive submission does not match the
|
|
3362
|
+
throw new Error("Interactive submission does not match the runner candidate");
|
|
3330
3363
|
}
|
|
3331
3364
|
return candidate;
|
|
3332
3365
|
}
|
|
3333
3366
|
settleRejectedInteraction(active, candidate, error) {
|
|
3334
3367
|
const result = this.state.transaction(() => {
|
|
3335
|
-
const submission = this.
|
|
3368
|
+
const submission = this.serverState.finishInteractionValidation({
|
|
3336
3369
|
requestId: candidate.requestId,
|
|
3337
3370
|
submissionId: candidate.submissionId,
|
|
3338
3371
|
accepted: false,
|
|
@@ -3359,7 +3392,7 @@ export class WorkflowHost {
|
|
|
3359
3392
|
active.control = "handoff";
|
|
3360
3393
|
return result;
|
|
3361
3394
|
}
|
|
3362
|
-
|
|
3395
|
+
enqueueRunnerNotification(active, message, payload) {
|
|
3363
3396
|
if (active.record.originSessionId === null) {
|
|
3364
3397
|
throw new Error("Workflow notification has no origin Pi session");
|
|
3365
3398
|
}
|
|
@@ -3381,7 +3414,7 @@ export class WorkflowHost {
|
|
|
3381
3414
|
const content = requireString(request.content, "notification.content");
|
|
3382
3415
|
const notificationId = `notification-${message.runId}-${attemptId}-${notificationIndex}`;
|
|
3383
3416
|
const workflowMessageId = workflowMessageIdFor("notification", notificationId, "initial");
|
|
3384
|
-
const record = this.
|
|
3417
|
+
const record = this.serverState.workflowMessages.create({
|
|
3385
3418
|
workflowMessageId,
|
|
3386
3419
|
runId: message.runId,
|
|
3387
3420
|
targetSessionId: active.record.originSessionId,
|
|
@@ -3401,7 +3434,7 @@ export class WorkflowHost {
|
|
|
3401
3434
|
targetSessionId: record.targetSessionId,
|
|
3402
3435
|
};
|
|
3403
3436
|
}
|
|
3404
|
-
|
|
3437
|
+
requestRunnerPresentation(active, message, payload) {
|
|
3405
3438
|
if (active.record.originSessionId === null) {
|
|
3406
3439
|
throw new Error("Workflow presentation has no origin Pi session");
|
|
3407
3440
|
}
|
|
@@ -3434,7 +3467,7 @@ export class WorkflowHost {
|
|
|
3434
3467
|
const attemptId = requireString(payload.attemptId, "attemptId");
|
|
3435
3468
|
const requestId = `interaction-${message.runId}-${attemptId}`;
|
|
3436
3469
|
return this.state.transaction(() => {
|
|
3437
|
-
const request = this.
|
|
3470
|
+
const request = this.serverState.createInteractiveRequest({
|
|
3438
3471
|
requestId,
|
|
3439
3472
|
runId: message.runId,
|
|
3440
3473
|
attemptId,
|
|
@@ -3460,9 +3493,9 @@ export class WorkflowHost {
|
|
|
3460
3493
|
});
|
|
3461
3494
|
}
|
|
3462
3495
|
async prepareInteractionResume(runId) {
|
|
3463
|
-
const accepted = this.
|
|
3464
|
-
const candidate = accepted ?? this.
|
|
3465
|
-
const timedOut = this.
|
|
3496
|
+
const accepted = this.serverState.acceptedInteraction(runId);
|
|
3497
|
+
const candidate = accepted ?? this.serverState.validatingInteraction(runId);
|
|
3498
|
+
const timedOut = this.serverState.timedOutInteraction(runId);
|
|
3466
3499
|
if (candidate === undefined && timedOut === undefined)
|
|
3467
3500
|
return;
|
|
3468
3501
|
const loaded = this.runStore.readRun(runId);
|
|
@@ -3495,7 +3528,7 @@ export class WorkflowHost {
|
|
|
3495
3528
|
if (active === undefined)
|
|
3496
3529
|
return;
|
|
3497
3530
|
if (context.event.type === "node_finished" && context.event.attemptId !== undefined) {
|
|
3498
|
-
this.
|
|
3531
|
+
this.serverState.consumeAcceptedInteraction(context.runId, context.event.attemptId, context.now);
|
|
3499
3532
|
}
|
|
3500
3533
|
if (context.state.status === "running") {
|
|
3501
3534
|
if (context.event.type === "run_started" || context.event.type === "run_resumed") {
|
|
@@ -3506,10 +3539,10 @@ export class WorkflowHost {
|
|
|
3506
3539
|
}
|
|
3507
3540
|
return;
|
|
3508
3541
|
}
|
|
3509
|
-
this.
|
|
3542
|
+
this.completeResourceManagerWorkflow(context.runId, context.state);
|
|
3510
3543
|
if (context.state.status !== "waiting") {
|
|
3511
|
-
this.
|
|
3512
|
-
this.
|
|
3544
|
+
this.serverState.workflowMessages.settleOpenTurnsForRun(context.runId, "lost", context.now);
|
|
3545
|
+
this.serverState.workflowMessages.cancelPendingForRun(context.runId, context.now, context.state.status === "completed" ? ["step", "decision"] : undefined);
|
|
3513
3546
|
}
|
|
3514
3547
|
const queueStatus = context.state.status === "completed"
|
|
3515
3548
|
? "done"
|
|
@@ -3588,7 +3621,7 @@ export class WorkflowHost {
|
|
|
3588
3621
|
"Workflow result:",
|
|
3589
3622
|
canonicalJson({ ...terminalFacts, terminalFingerprint }),
|
|
3590
3623
|
].join("\n");
|
|
3591
|
-
this.
|
|
3624
|
+
this.serverState.workflowMessages.create({
|
|
3592
3625
|
workflowMessageId,
|
|
3593
3626
|
runId,
|
|
3594
3627
|
targetSessionId: queue.originSessionId,
|
|
@@ -3630,7 +3663,8 @@ export class WorkflowHost {
|
|
|
3630
3663
|
}
|
|
3631
3664
|
terminalWorkflowMessageMissing(runId) {
|
|
3632
3665
|
return (this.terminalWorkflowMessageRequired(runId) &&
|
|
3633
|
-
this.
|
|
3666
|
+
this.serverState.workflowMessages.latestForSource("terminal", `terminal:${runId}`) ===
|
|
3667
|
+
undefined);
|
|
3634
3668
|
}
|
|
3635
3669
|
discoverMissingTerminalWorkflowMessages() {
|
|
3636
3670
|
for (const run of this.queue.listWorkflowRuns({
|
|
@@ -3689,9 +3723,9 @@ export class WorkflowHost {
|
|
|
3689
3723
|
};
|
|
3690
3724
|
});
|
|
3691
3725
|
}
|
|
3692
|
-
|
|
3726
|
+
completeResourceManagerWorkflow(runId, state) {
|
|
3693
3727
|
const row = this.state.connection
|
|
3694
|
-
.prepare(`SELECT w.request_id AS requestId, c.controller_name AS
|
|
3728
|
+
.prepare(`SELECT w.request_id AS requestId, c.controller_name AS resourceManager,
|
|
3695
3729
|
c.resource_key AS resourceKey, p.canonical_path AS projectPath
|
|
3696
3730
|
FROM controller_workflows w
|
|
3697
3731
|
JOIN controller_resources c ON c.controller_resource_id = w.controller_resource_id
|
|
@@ -3699,12 +3733,12 @@ export class WorkflowHost {
|
|
|
3699
3733
|
WHERE w.run_id = ? LIMIT 1`)
|
|
3700
3734
|
.get(runId);
|
|
3701
3735
|
if (typeof row?.requestId !== "string" ||
|
|
3702
|
-
typeof row.
|
|
3736
|
+
typeof row.resourceManager !== "string" ||
|
|
3703
3737
|
typeof row.resourceKey !== "string" ||
|
|
3704
3738
|
typeof row.projectPath !== "string") {
|
|
3705
3739
|
return;
|
|
3706
3740
|
}
|
|
3707
|
-
const store = new
|
|
3741
|
+
const store = new SqliteResourceManagerStore(this.databasePath, {
|
|
3708
3742
|
state: this.state,
|
|
3709
3743
|
projectPath: row.projectPath,
|
|
3710
3744
|
});
|
|
@@ -3718,7 +3752,7 @@ export class WorkflowHost {
|
|
|
3718
3752
|
runId,
|
|
3719
3753
|
...(state.error === undefined ? {} : { error: state.error }),
|
|
3720
3754
|
});
|
|
3721
|
-
store.enqueue({
|
|
3755
|
+
store.enqueue({ resourceManager: row.resourceManager, key: row.resourceKey });
|
|
3722
3756
|
}
|
|
3723
3757
|
commitActivePause(active) {
|
|
3724
3758
|
const now = Date.now();
|
|
@@ -3734,7 +3768,7 @@ export class WorkflowHost {
|
|
|
3734
3768
|
}
|
|
3735
3769
|
});
|
|
3736
3770
|
}
|
|
3737
|
-
async
|
|
3771
|
+
async recoverRunnerExit(active, outcome) {
|
|
3738
3772
|
if (!this.queue.verifyWorkflowRunClaim({
|
|
3739
3773
|
runId: active.record.runId,
|
|
3740
3774
|
claimToken: active.claimToken,
|
|
@@ -3753,9 +3787,9 @@ export class WorkflowHost {
|
|
|
3753
3787
|
this.log(`run ${active.record.runId} parked for ambiguous effect recovery`);
|
|
3754
3788
|
return;
|
|
3755
3789
|
}
|
|
3756
|
-
const validating = this.
|
|
3790
|
+
const validating = this.serverState.validatingInteraction(active.record.runId);
|
|
3757
3791
|
if (validating !== undefined) {
|
|
3758
|
-
this.settleRejectedInteraction(active, validating, `Workflow
|
|
3792
|
+
this.settleRejectedInteraction(active, validating, `Workflow runner ${outcome} before it completed interaction validation`);
|
|
3759
3793
|
this.log(`run ${active.record.runId} rejected an interrupted interaction validation`);
|
|
3760
3794
|
return;
|
|
3761
3795
|
}
|
|
@@ -3769,7 +3803,7 @@ export class WorkflowHost {
|
|
|
3769
3803
|
this.tryEnsureTerminalWorkflowMessage(active.record.runId, Date.now(), {
|
|
3770
3804
|
status: "failed",
|
|
3771
3805
|
error: active.workflowLoadFailure,
|
|
3772
|
-
statusDetail: "The supervised
|
|
3806
|
+
statusDetail: "The supervised runner could not load the saved workflow source.",
|
|
3773
3807
|
});
|
|
3774
3808
|
this.log(`run ${active.record.runId} failed because its workflow source could not load`);
|
|
3775
3809
|
}
|
|
@@ -3777,23 +3811,23 @@ export class WorkflowHost {
|
|
|
3777
3811
|
}
|
|
3778
3812
|
const currentProgressRevision = runRevision(this.state, active.record.runId);
|
|
3779
3813
|
if (currentProgressRevision <= active.launchProgressRevision) {
|
|
3780
|
-
const detail = `Workflow
|
|
3814
|
+
const detail = `Workflow runner ${outcome} before it committed workflow progress`;
|
|
3781
3815
|
this.blockedRuns.add(active.record.runId);
|
|
3782
|
-
this.queue.
|
|
3816
|
+
this.queue.parkWorkflowRunForRunnerNoProgress({
|
|
3783
3817
|
runId: active.record.runId,
|
|
3784
3818
|
claimToken: active.claimToken,
|
|
3785
3819
|
detail,
|
|
3786
3820
|
});
|
|
3787
|
-
this.log(`run ${active.record.runId} parked after a
|
|
3821
|
+
this.log(`run ${active.record.runId} parked after a runner made no progress`);
|
|
3788
3822
|
return;
|
|
3789
3823
|
}
|
|
3790
3824
|
this.queue.parkWorkflowRun({ runId: active.record.runId, claimToken: active.claimToken });
|
|
3791
|
-
this.log(`run ${active.record.runId} parked after
|
|
3825
|
+
this.log(`run ${active.record.runId} parked after runner ${outcome}`);
|
|
3792
3826
|
}
|
|
3793
3827
|
releaseLock() {
|
|
3794
3828
|
try {
|
|
3795
3829
|
const current = JSON.parse(fs.readFileSync(this.lockPath, "utf8"));
|
|
3796
|
-
if (isLockRecord(current) && current.
|
|
3830
|
+
if (isLockRecord(current) && current.serverId === this.serverId)
|
|
3797
3831
|
fs.rmSync(this.lockPath);
|
|
3798
3832
|
}
|
|
3799
3833
|
catch {
|
|
@@ -3871,21 +3905,21 @@ function channelEventPayload(message) {
|
|
|
3871
3905
|
const { expectedRevision: _expectedRevision, sequence: _sequence, ...payload } = message;
|
|
3872
3906
|
return payload;
|
|
3873
3907
|
}
|
|
3874
|
-
function
|
|
3908
|
+
function acquireServerLock(lockPath, record) {
|
|
3875
3909
|
try {
|
|
3876
3910
|
const existing = JSON.parse(fs.readFileSync(lockPath, "utf8"));
|
|
3877
3911
|
if (isLockRecord(existing) && matchesProcessIdentity(existing)) {
|
|
3878
|
-
throw new Error(`A workflow
|
|
3912
|
+
throw new Error(`A workflow server is already running with PID ${existing.pid}`);
|
|
3879
3913
|
}
|
|
3880
3914
|
fs.rmSync(lockPath, { force: true });
|
|
3881
3915
|
}
|
|
3882
3916
|
catch (error) {
|
|
3883
|
-
if (error instanceof Error && error.message.startsWith("A workflow
|
|
3917
|
+
if (error instanceof Error && error.message.startsWith("A workflow server is already"))
|
|
3884
3918
|
throw error;
|
|
3885
3919
|
}
|
|
3886
3920
|
fs.writeFileSync(lockPath, `${JSON.stringify({ schema: "pi-workflows.host-lock.v1", ...record })}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
3887
3921
|
}
|
|
3888
|
-
function
|
|
3922
|
+
function runnerRunCommand(record, resumeInteractionAttemptId) {
|
|
3889
3923
|
if (record.initialized) {
|
|
3890
3924
|
return {
|
|
3891
3925
|
kind: "resume",
|
|
@@ -3914,7 +3948,7 @@ function workerRunCommand(record, resumeInteractionAttemptId) {
|
|
|
3914
3948
|
}
|
|
3915
3949
|
return { kind: "start", input };
|
|
3916
3950
|
}
|
|
3917
|
-
function
|
|
3951
|
+
function runnerResponse(message, outcome, result, error, revision) {
|
|
3918
3952
|
return {
|
|
3919
3953
|
schema: "pi-workflows.worker-response.v1",
|
|
3920
3954
|
messageId: message.messageId,
|
|
@@ -3924,7 +3958,7 @@ function workerResponse(message, outcome, result, error, revision) {
|
|
|
3924
3958
|
...(error === undefined ? {} : { error }),
|
|
3925
3959
|
};
|
|
3926
3960
|
}
|
|
3927
|
-
function
|
|
3961
|
+
function resourceRunnerResponse(message, outcome, result, error) {
|
|
3928
3962
|
return {
|
|
3929
3963
|
schema: "pi-workflows.controller-worker-response.v1",
|
|
3930
3964
|
messageId: message.messageId,
|
|
@@ -3933,7 +3967,7 @@ function controllerWorkerResponse(message, outcome, result, error) {
|
|
|
3933
3967
|
...(error === undefined ? {} : { error }),
|
|
3934
3968
|
};
|
|
3935
3969
|
}
|
|
3936
|
-
function
|
|
3970
|
+
function resourceManagerWorkflowResult(record) {
|
|
3937
3971
|
switch (record.status) {
|
|
3938
3972
|
case "queued":
|
|
3939
3973
|
case "starting":
|
|
@@ -3968,12 +4002,12 @@ function hasUncertainEffect(state, runId) {
|
|
|
3968
4002
|
.get(runId);
|
|
3969
4003
|
return row !== undefined;
|
|
3970
4004
|
}
|
|
3971
|
-
function
|
|
3972
|
-
if (
|
|
3973
|
-
!/\.
|
|
4005
|
+
function resourceManagerPathAllowed(projectPath, resourceManagerName, resourceManagerPath) {
|
|
4006
|
+
if (resourceManagerFileStem(resourceManagerPath) !== resourceManagerName ||
|
|
4007
|
+
!/\.resource-manager\.(?:ts|js|mts|mjs)$/u.test(resourceManagerPath)) {
|
|
3974
4008
|
return false;
|
|
3975
4009
|
}
|
|
3976
|
-
return
|
|
4010
|
+
return resourceManagerSearchDirs({ cwd: projectPath }).some(({ dir }) => path.dirname(resourceManagerPath) === path.resolve(dir));
|
|
3977
4011
|
}
|
|
3978
4012
|
function waitForSocketDrain(socket) {
|
|
3979
4013
|
if (socket.destroyed)
|
|
@@ -4003,7 +4037,7 @@ function waitForSocketDrain(socket) {
|
|
|
4003
4037
|
onClose();
|
|
4004
4038
|
});
|
|
4005
4039
|
}
|
|
4006
|
-
function
|
|
4040
|
+
function serverDelay(ms) {
|
|
4007
4041
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
4008
4042
|
}
|
|
4009
4043
|
function runPageReceipt(view, kind, cursor) {
|
|
@@ -4223,6 +4257,6 @@ function isLockRecord(value) {
|
|
|
4223
4257
|
value.schema === "pi-workflows.host-lock.v1" &&
|
|
4224
4258
|
typeof value.pid === "number" &&
|
|
4225
4259
|
typeof value.startIdentity === "string" &&
|
|
4226
|
-
typeof value.
|
|
4260
|
+
typeof value.serverId === "string");
|
|
4227
4261
|
}
|
|
4228
|
-
//# sourceMappingURL=
|
|
4262
|
+
//# sourceMappingURL=server.js.map
|