@osolmaz/pi-workflows 0.16.2 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -41
- package/dist/builtins/monitor.workflow.d.ts +1 -1
- package/dist/builtins/monitor.workflow.js +8 -8
- package/dist/builtins/pi-agent-group.js +3 -3
- package/dist/builtins/pi-agent-group.js.map +1 -1
- package/dist/channels/adapter-entry.js +14 -14
- package/dist/channels/adapter-entry.js.map +1 -1
- package/dist/client/client.d.ts +6 -6
- package/dist/client/client.js +27 -27
- package/dist/client/client.js.map +1 -1
- package/dist/client/protocol.d.ts +4 -1
- package/dist/client/protocol.js +17 -11
- package/dist/client/protocol.js.map +1 -1
- package/dist/client/resolver.d.ts +4 -4
- package/dist/client/view.d.ts +1 -1
- package/dist/extension/index.d.ts +1 -1
- package/dist/extension/index.js +32 -32
- package/dist/extension/index.js.map +1 -1
- package/dist/extension/recorder.d.ts +1 -1
- package/dist/extension/recorder.js +1 -1
- package/dist/extension/recorder.js.map +1 -1
- package/dist/extension/remote-recorder-store.d.ts +1 -1
- package/dist/extension/remote-recorder-store.js +2 -2
- package/dist/extension/remote-recorder-store.js.map +1 -1
- package/dist/extension/{controller-command.d.ts → resource-manager-command.d.ts} +6 -6
- package/dist/extension/{controller-command.js → resource-manager-command.js} +7 -7
- package/dist/extension/resource-manager-command.js.map +1 -0
- package/dist/extension/session-delivery.d.ts +1 -1
- package/dist/extension/session-delivery.js +2 -2
- package/dist/extension/session-delivery.js.map +1 -1
- package/dist/extension/session-view.d.ts +1 -1
- package/dist/extension/session-view.js +1 -1
- package/dist/extension/session-view.js.map +1 -1
- package/dist/extension/workflow-message-coordinator.d.ts +1 -1
- package/dist/extension/workflow-message-coordinator.js +6 -6
- package/dist/extension/workflow-message-coordinator.js.map +1 -1
- package/dist/resource-managers/conditions.d.ts +6 -0
- package/dist/{controllers → resource-managers}/conditions.js +3 -3
- package/dist/resource-managers/conditions.js.map +1 -0
- package/dist/resource-managers/definition.d.ts +6 -0
- package/dist/resource-managers/definition.js +45 -0
- package/dist/resource-managers/definition.js.map +1 -0
- package/dist/resource-managers/effects.d.ts +15 -0
- package/dist/{controllers → resource-managers}/effects.js +2 -2
- package/dist/resource-managers/effects.js.map +1 -0
- package/dist/resource-managers/errors.d.ts +12 -0
- package/dist/resource-managers/errors.js +25 -0
- package/dist/resource-managers/errors.js.map +1 -0
- package/dist/resource-managers/index.d.ts +12 -0
- package/dist/resource-managers/index.js +12 -0
- package/dist/resource-managers/index.js.map +1 -0
- package/dist/resource-managers/json.js.map +1 -0
- package/dist/resource-managers/loader.d.ts +23 -0
- package/dist/resource-managers/loader.js +75 -0
- package/dist/resource-managers/loader.js.map +1 -0
- package/dist/resource-managers/results.d.ts +5 -0
- package/dist/resource-managers/results.js.map +1 -0
- package/dist/resource-managers/runtime.d.ts +59 -0
- package/dist/{controllers/manager.js → resource-managers/runtime.js} +57 -57
- package/dist/resource-managers/runtime.js.map +1 -0
- package/dist/{controllers → resource-managers}/sqlite.d.ts +44 -44
- package/dist/{controllers → resource-managers}/sqlite.js +139 -130
- package/dist/resource-managers/sqlite.js.map +1 -0
- package/dist/{controllers → resource-managers}/store.d.ts +33 -33
- package/dist/{controllers → resource-managers}/store.js.map +1 -1
- package/dist/{controllers → resource-managers}/types.d.ts +41 -41
- package/dist/{controllers → resource-managers}/types.js.map +1 -1
- package/dist/resource-managers/workflows.d.ts +34 -0
- package/dist/{controllers → resource-managers}/workflows.js +15 -15
- package/dist/resource-managers/workflows.js.map +1 -0
- package/dist/server/channel-effects.js.map +1 -0
- package/dist/{host → server}/channel-supervisor.d.ts +6 -6
- package/dist/{host → server}/channel-supervisor.js +3 -3
- package/dist/server/channel-supervisor.js.map +1 -0
- package/dist/{host/child-worker-supervisor.d.ts → server/child-runner-supervisor.d.ts} +15 -13
- package/dist/{host/child-worker-supervisor.js → server/child-runner-supervisor.js} +6 -5
- package/dist/server/child-runner-supervisor.js.map +1 -0
- package/dist/{host → server}/processes.d.ts +3 -3
- package/dist/{host → server}/processes.js +3 -3
- package/dist/server/processes.js.map +1 -0
- package/dist/{host → server}/resolver-entry.d.ts +6 -6
- package/dist/{host → server}/resolver-entry.js +14 -14
- package/dist/server/resolver-entry.js.map +1 -0
- package/dist/server/resource-runner-entry.d.ts +2 -0
- package/dist/{host/controller-worker-entry.js → server/resource-runner-entry.js} +33 -33
- package/dist/server/resource-runner-entry.js.map +1 -0
- package/dist/server/resource-runner-protocol.d.ts +36 -0
- package/dist/server/resource-runner-protocol.js +49 -0
- package/dist/server/resource-runner-protocol.js.map +1 -0
- package/dist/server/resource-runner-supervisor.d.ts +21 -0
- package/dist/{host/worker-supervisor.js → server/resource-runner-supervisor.js} +17 -17
- package/dist/server/resource-runner-supervisor.js.map +1 -0
- package/dist/{host → server}/rpc-bridge.d.ts +3 -3
- package/dist/{host → server}/rpc-bridge.js +3 -3
- package/dist/server/rpc-bridge.js.map +1 -0
- package/dist/{host → server}/rpc-executor.d.ts +1 -1
- package/dist/{host → server}/rpc-executor.js +3 -3
- package/dist/server/rpc-executor.js.map +1 -0
- package/dist/{host/host-entry.js → server/server-entry.js} +6 -6
- package/dist/server/server-entry.js.map +1 -0
- package/dist/{host/runner.d.ts → server/server.d.ts} +39 -38
- package/dist/{host/runner.js → server/server.js} +408 -374
- package/dist/server/server.js.map +1 -0
- package/dist/{host → server}/state.d.ts +25 -25
- package/dist/{host → server}/state.js +49 -49
- package/dist/server/state.js.map +1 -0
- package/dist/{host → server}/view.d.ts +7 -7
- package/dist/{host → server}/view.js +13 -13
- package/dist/server/view.js.map +1 -0
- package/dist/server/workflow-runner-content.d.ts +6 -0
- package/dist/server/workflow-runner-content.js +96 -0
- package/dist/server/workflow-runner-content.js.map +1 -0
- package/dist/{host/worker-entry.d.ts → server/workflow-runner-entry.d.ts} +3 -3
- package/dist/{host/worker-entry.js → server/workflow-runner-entry.js} +68 -35
- package/dist/server/workflow-runner-entry.js.map +1 -0
- package/dist/server/workflow-runner-protocol.d.ts +69 -0
- package/dist/server/workflow-runner-protocol.js +176 -0
- package/dist/server/workflow-runner-protocol.js.map +1 -0
- package/dist/{host/worker-store.d.ts → server/workflow-runner-store.d.ts} +11 -11
- package/dist/{host/worker-store.js → server/workflow-runner-store.js} +8 -8
- package/dist/server/workflow-runner-store.js.map +1 -0
- package/dist/server/workflow-runner-supervisor.d.ts +22 -0
- package/dist/server/workflow-runner-supervisor.js +56 -0
- package/dist/server/workflow-runner-supervisor.js.map +1 -0
- package/dist/state/prune.d.ts +1 -1
- package/dist/state/prune.js +17 -10
- package/dist/state/prune.js.map +1 -1
- package/dist/state/schema.js +2 -2
- package/dist/state/workflow-messages.d.ts +1 -1
- package/dist/state/workflow-messages.js +1 -1
- package/dist/state/workflow-messages.js.map +1 -1
- package/dist/viewer/backup.js +1 -1
- package/dist/viewer/backup.js.map +1 -1
- package/dist/viewer/cli.d.ts +2 -2
- package/dist/viewer/cli.js +46 -32
- package/dist/viewer/cli.js.map +1 -1
- package/dist/viewer/tui.d.ts +1 -1
- package/dist/viewer/tui.js +1 -1
- package/dist/viewer/tui.js.map +1 -1
- package/dist/workflows/command-batch.js +2 -2
- package/dist/workflows/engine.js +14 -15
- package/dist/workflows/engine.js.map +1 -1
- package/dist/workflows/store.d.ts +12 -11
- package/dist/workflows/store.js +24 -18
- package/dist/workflows/store.js.map +1 -1
- package/dist/workflows/types.d.ts +2 -2
- package/docs/2026-08-20-durable-workflow-launch-plan.md +12 -12
- package/docs/2026-08-25-workflow-follow-ups.md +7 -7
- package/docs/2026-08-25-workflow-settings.md +3 -3
- package/docs/2026-08-30-out-of-process-workflow-host-plan.md +3 -3
- package/docs/2026-09-01-restore-session-delivery-controls-plan.md +5 -5
- package/docs/2026-09-01-unified-workflow-client-plan.md +6 -6
- package/docs/2026-09-02-installed-live-e2e-plan.md +3 -3
- package/docs/2026-09-02-unify-workflow-messages-plan.md +8 -8
- package/docs/2026-09-04-workflow-run-state-plan.md +6 -6
- package/docs/DEFERRED_TURNS.md +7 -7
- package/docs/DESIGN_PHILOSOPHY.md +1 -1
- package/docs/HUMAN_DECISIONS.md +11 -11
- package/docs/HUMAN_DECISION_PRESENTATIONS.md +4 -4
- package/docs/MONITOR.md +1 -1
- package/docs/RESOURCE_MANAGERS.md +221 -0
- package/docs/SQLITE_STATE.md +35 -33
- package/docs/WORKFLOW_COMPOSITION.md +1 -1
- package/docs/{WORKFLOW_HOST.md → WORKFLOW_SERVER.md} +125 -119
- package/docs/WORKFLOW_STEP_MESSAGES.md +18 -18
- package/docs/WORKFLOW_UPDATES.md +16 -16
- package/docs/development.md +15 -14
- package/docs/live-replay-protocol.md +20 -20
- package/docs/plans/2026-08-04-controller-runtime-plan.md +25 -25
- package/docs/plans/2026-08-05-always-on-workflows-plan.md +8 -8
- package/docs/plans/2026-08-10-agent-managed-monitor-workflows-plan.md +2 -2
- package/docs/plans/2026-08-13-session-addressed-workflow-notifications-plan.md +1 -1
- package/docs/plans/2026-08-16-workflow-updates-plan.md +5 -5
- package/docs/plans/2026-08-19-human-decision-gates-plan.md +1 -1
- package/docs/plans/2026-08-20-bounded-command-batches-plan.md +1 -1
- package/docs/plans/2026-08-21-deferred-turn-intents-plan.md +3 -3
- package/docs/plans/2026-08-21-sanity-check-plan.md +2 -2
- package/docs/plans/2026-08-23-assistant-agent-completion-plan.md +5 -5
- package/docs/plans/2026-08-23-sqlite-state-plan.md +25 -25
- package/docs/plans/2026-08-25-live-workflow-settings-plan.md +4 -4
- package/docs/plans/2026-08-27-workflow-terminal-restart-plan.md +3 -3
- package/docs/plans/2026-09-04-workflow-runner-resume-state-plan.md +318 -0
- package/docs/plans/piw-viewer-experience-implementation-plan.md +1 -1
- package/docs/tui-viewer.md +9 -9
- package/docs/workflows.md +72 -66
- package/examples/{controllers/pull-request.controller.ts → resource-managers/pull-request.resource-manager.ts} +12 -12
- package/herdr-plugin.toml +1 -1
- package/package.json +5 -5
- package/protocol/client.v1.schema.json +7 -7
- package/protocol/fixtures/client-v1.json +1 -1
- package/src/builtins/monitor.workflow.ts +9 -9
- package/src/builtins/pi-agent-group.ts +3 -3
- package/src/channels/adapter-entry.ts +14 -14
- package/src/client/client.ts +33 -32
- package/src/client/protocol.ts +16 -11
- package/src/client/resolver.ts +4 -4
- package/src/client/view.ts +1 -1
- package/src/extension/index.ts +44 -34
- package/src/extension/recorder.ts +1 -1
- package/src/extension/remote-recorder-store.ts +2 -2
- package/src/extension/resource-manager-command.ts +45 -0
- package/src/extension/session-delivery.ts +2 -2
- package/src/extension/session-view.ts +1 -1
- package/src/extension/workflow-message-coordinator.ts +6 -6
- package/src/{controllers → resource-managers}/conditions.ts +23 -23
- package/src/resource-managers/definition.ts +71 -0
- package/src/{controllers → resource-managers}/effects.ts +9 -9
- package/src/resource-managers/errors.ts +27 -0
- package/src/resource-managers/index.ts +88 -0
- package/src/resource-managers/loader.ts +111 -0
- package/src/{controllers → resource-managers}/results.ts +4 -4
- package/src/{controllers/manager.ts → resource-managers/runtime.ts} +95 -92
- package/src/{controllers → resource-managers}/sqlite.ts +198 -186
- package/src/{controllers → resource-managers}/store.ts +46 -38
- package/src/{controllers → resource-managers}/types.ts +50 -41
- package/src/{controllers → resource-managers}/workflows.ts +43 -41
- package/src/{host → server}/channel-supervisor.ts +8 -8
- package/src/{host/child-worker-supervisor.ts → server/child-runner-supervisor.ts} +22 -18
- package/src/{host → server}/processes.ts +3 -3
- package/src/{host → server}/resolver-entry.ts +28 -25
- package/src/{host/controller-worker-entry.ts → server/resource-runner-entry.ts} +67 -65
- package/src/server/resource-runner-protocol.ts +103 -0
- package/src/server/resource-runner-supervisor.ts +76 -0
- package/src/{host → server}/rpc-bridge.ts +3 -3
- package/src/{host → server}/rpc-executor.ts +4 -4
- package/src/{host/host-entry.ts → server/server-entry.ts} +5 -5
- package/src/{host/runner.ts → server/server.ts} +526 -461
- package/src/{host → server}/state.ts +78 -67
- package/src/{host → server}/view.ts +15 -15
- package/src/server/workflow-runner-content.ts +119 -0
- package/src/{host/worker-entry.ts → server/workflow-runner-entry.ts} +98 -57
- package/src/server/workflow-runner-protocol.ts +241 -0
- package/src/{host/worker-store.ts → server/workflow-runner-store.ts} +18 -20
- package/src/server/workflow-runner-supervisor.ts +79 -0
- package/src/state/prune.ts +27 -9
- package/src/state/schema.ts +2 -2
- package/src/state/workflow-messages.ts +1 -1
- package/src/viewer/backup.ts +1 -1
- package/src/viewer/cli.ts +50 -36
- package/src/viewer/tui.ts +1 -1
- package/src/workflows/command-batch.ts +2 -2
- package/src/workflows/engine.ts +14 -15
- package/src/workflows/store.ts +31 -28
- package/src/workflows/types.ts +2 -2
- package/dist/controllers/conditions.d.ts +0 -6
- package/dist/controllers/conditions.js.map +0 -1
- package/dist/controllers/definition.d.ts +0 -6
- package/dist/controllers/definition.js +0 -45
- package/dist/controllers/definition.js.map +0 -1
- package/dist/controllers/effects.d.ts +0 -15
- package/dist/controllers/effects.js.map +0 -1
- package/dist/controllers/errors.d.ts +0 -12
- package/dist/controllers/errors.js +0 -25
- package/dist/controllers/errors.js.map +0 -1
- package/dist/controllers/index.d.ts +0 -12
- package/dist/controllers/index.js +0 -12
- package/dist/controllers/index.js.map +0 -1
- package/dist/controllers/json.js.map +0 -1
- package/dist/controllers/loader.d.ts +0 -23
- package/dist/controllers/loader.js +0 -74
- package/dist/controllers/loader.js.map +0 -1
- package/dist/controllers/manager.d.ts +0 -59
- package/dist/controllers/manager.js.map +0 -1
- package/dist/controllers/results.d.ts +0 -5
- package/dist/controllers/results.js.map +0 -1
- package/dist/controllers/sqlite.js.map +0 -1
- package/dist/controllers/workflows.d.ts +0 -34
- package/dist/controllers/workflows.js.map +0 -1
- package/dist/extension/controller-command.js.map +0 -1
- package/dist/host/channel-effects.js.map +0 -1
- package/dist/host/channel-supervisor.js.map +0 -1
- package/dist/host/child-worker-supervisor.js.map +0 -1
- package/dist/host/controller-worker-entry.d.ts +0 -2
- package/dist/host/controller-worker-entry.js.map +0 -1
- package/dist/host/controller-worker-protocol.d.ts +0 -36
- package/dist/host/controller-worker-protocol.js +0 -49
- package/dist/host/controller-worker-protocol.js.map +0 -1
- package/dist/host/controller-worker-supervisor.d.ts +0 -21
- package/dist/host/controller-worker-supervisor.js +0 -54
- package/dist/host/controller-worker-supervisor.js.map +0 -1
- package/dist/host/host-entry.js.map +0 -1
- package/dist/host/processes.js.map +0 -1
- package/dist/host/resolver-entry.js.map +0 -1
- package/dist/host/rpc-bridge.js.map +0 -1
- package/dist/host/rpc-executor.js.map +0 -1
- package/dist/host/runner.js.map +0 -1
- package/dist/host/state.js.map +0 -1
- package/dist/host/view.js.map +0 -1
- package/dist/host/worker-entry.js.map +0 -1
- package/dist/host/worker-protocol.d.ts +0 -46
- package/dist/host/worker-protocol.js +0 -122
- package/dist/host/worker-protocol.js.map +0 -1
- package/dist/host/worker-store.js.map +0 -1
- package/dist/host/worker-supervisor.d.ts +0 -22
- package/dist/host/worker-supervisor.js.map +0 -1
- package/docs/CONTROLLERS.md +0 -217
- package/src/controllers/definition.ts +0 -65
- package/src/controllers/errors.ts +0 -27
- package/src/controllers/index.ts +0 -88
- package/src/controllers/loader.ts +0 -104
- package/src/extension/controller-command.ts +0 -45
- package/src/host/controller-worker-protocol.ts +0 -104
- package/src/host/controller-worker-supervisor.ts +0 -79
- package/src/host/worker-protocol.ts +0 -176
- package/src/host/worker-supervisor.ts +0 -74
- /package/dist/{controllers → resource-managers}/json.d.ts +0 -0
- /package/dist/{controllers → resource-managers}/json.js +0 -0
- /package/dist/{controllers → resource-managers}/results.js +0 -0
- /package/dist/{controllers → resource-managers}/store.js +0 -0
- /package/dist/{controllers → resource-managers}/types.js +0 -0
- /package/dist/{host → server}/channel-effects.d.ts +0 -0
- /package/dist/{host → server}/channel-effects.js +0 -0
- /package/dist/{host/host-entry.d.ts → server/server-entry.d.ts} +0 -0
- /package/src/{controllers → resource-managers}/json.ts +0 -0
- /package/src/{host → server}/channel-effects.ts +0 -0
|
@@ -34,28 +34,31 @@ import {
|
|
|
34
34
|
type WorkflowTurnReport,
|
|
35
35
|
type WorkflowTurnReportReceipt,
|
|
36
36
|
} from "../client/view.js";
|
|
37
|
-
import { applyStatusPatch } from "../
|
|
38
|
-
import {
|
|
37
|
+
import { applyStatusPatch } from "../resource-managers/conditions.js";
|
|
38
|
+
import { ManagedResourceConflictError } from "../resource-managers/errors.js";
|
|
39
39
|
import {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
} from "../
|
|
44
|
-
import {
|
|
40
|
+
resourceManagerFileStem,
|
|
41
|
+
resourceManagerSearchDirs,
|
|
42
|
+
discoverResourceManagers,
|
|
43
|
+
} from "../resource-managers/loader.js";
|
|
44
|
+
import {
|
|
45
|
+
SqliteResourceManagerStore,
|
|
46
|
+
type WorkflowRunQueueRecord,
|
|
47
|
+
} from "../resource-managers/sqlite.js";
|
|
45
48
|
import type {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
ResourceManagerQueueClaim,
|
|
50
|
+
ManagedResource,
|
|
51
|
+
ResourceManagerSettingsChangeRequest,
|
|
52
|
+
ResourceManagerSettingsChangeResult,
|
|
50
53
|
ReconcileResult,
|
|
51
|
-
} from "../
|
|
54
|
+
} from "../resource-managers/types.js";
|
|
52
55
|
import {
|
|
53
|
-
|
|
54
|
-
type
|
|
55
|
-
type
|
|
56
|
+
ResourceManagerWorkflowCoordinator,
|
|
57
|
+
type ResourceManagerWorkflowControlRequest,
|
|
58
|
+
type ResourceManagerWorkflowScheduler,
|
|
56
59
|
type WorkflowSchedulerRequest,
|
|
57
60
|
type WorkflowSchedulerResult,
|
|
58
|
-
} from "../
|
|
61
|
+
} from "../resource-managers/workflows.js";
|
|
59
62
|
import { StateDatabase, workflowStatePath } from "../state/database.js";
|
|
60
63
|
import { canonicalJson, type JsonValue } from "../state/json.js";
|
|
61
64
|
import { resourceIdFor } from "../state/mutation.js";
|
|
@@ -99,55 +102,60 @@ import {
|
|
|
99
102
|
type ChannelEffectRecord,
|
|
100
103
|
} from "./channel-effects.js";
|
|
101
104
|
import { ChannelAdapterSupervisor } from "./channel-supervisor.js";
|
|
102
|
-
import type {
|
|
103
|
-
ControllerWorkerLaunchEnvelope,
|
|
104
|
-
ControllerWorkerMessage,
|
|
105
|
-
ControllerWorkerResponse,
|
|
106
|
-
} from "./controller-worker-protocol.js";
|
|
107
|
-
import { ControllerWorkerSupervisor } from "./controller-worker-supervisor.js";
|
|
108
105
|
import {
|
|
109
|
-
|
|
106
|
+
ServerProcessRegistry,
|
|
110
107
|
matchesProcessIdentity,
|
|
111
108
|
processParentPid,
|
|
112
109
|
processStartIdentity,
|
|
113
110
|
} from "./processes.js";
|
|
111
|
+
import type {
|
|
112
|
+
ResourceRunnerLaunchEnvelope,
|
|
113
|
+
ResourceRunnerMessage,
|
|
114
|
+
ResourceRunnerResponse,
|
|
115
|
+
} from "./resource-runner-protocol.js";
|
|
116
|
+
import { ResourceRunnerSupervisor } from "./resource-runner-supervisor.js";
|
|
114
117
|
import {
|
|
115
|
-
|
|
116
|
-
type
|
|
118
|
+
ServerStateStore,
|
|
119
|
+
type ServerClaim,
|
|
117
120
|
type InteractiveRequestRecord,
|
|
118
121
|
type InteractiveSubmissionRecord,
|
|
119
|
-
type
|
|
122
|
+
type WorkflowRunnerLaunchEnvelope,
|
|
120
123
|
} from "./state.js";
|
|
121
|
-
import {
|
|
122
|
-
import
|
|
123
|
-
import {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
124
|
+
import { ServerViewStore, WORKFLOW_PAGE_KINDS, type WorkflowPageKind } from "./view.js";
|
|
125
|
+
import { boundRunnerResponse, readRunnerContentChunk } from "./workflow-runner-content.js";
|
|
126
|
+
import type {
|
|
127
|
+
WorkflowRunnerMessage,
|
|
128
|
+
WorkflowRunnerResponse,
|
|
129
|
+
WorkflowRunnerCommand,
|
|
130
|
+
} from "./workflow-runner-protocol.js";
|
|
131
|
+
import { WorkflowRunnerSupervisor } from "./workflow-runner-supervisor.js";
|
|
132
|
+
|
|
133
|
+
const SERVER_LEASE_MS = 30_000;
|
|
134
|
+
const SERVER_RENEW_MS = 10_000;
|
|
127
135
|
const PACKAGE_VERSION = runtimePackageVersion();
|
|
128
136
|
const CLAIM_POLL_MS = 2_000;
|
|
129
137
|
const TERMINAL_MESSAGE_RECONCILE_MS = 1_000;
|
|
130
138
|
const RUN_CLAIM_LEASE_MS = 30_000;
|
|
131
|
-
const
|
|
132
|
-
const
|
|
133
|
-
const
|
|
134
|
-
const
|
|
139
|
+
const RESOURCE_MANAGER_CLAIM_LEASE_MS = 120_000;
|
|
140
|
+
const RESOURCE_MANAGER_RENEW_MS = 30_000;
|
|
141
|
+
const MAX_RESOURCE_RUNNERS = 4;
|
|
142
|
+
const DEFAULT_RESOURCE_MANAGER_TIMEOUT_MS = 60_000;
|
|
135
143
|
|
|
136
|
-
export type
|
|
137
|
-
/** Retained for callers; the
|
|
144
|
+
export type WorkflowServerOptions = {
|
|
145
|
+
/** Retained for callers; the server itself is global to the state database. */
|
|
138
146
|
cwd?: string;
|
|
139
147
|
runnerId?: string;
|
|
140
148
|
databasePath?: string;
|
|
141
|
-
registry?:
|
|
149
|
+
registry?: ServerProcessRegistry;
|
|
142
150
|
piArgs?: string[];
|
|
143
151
|
env?: Record<string, string>;
|
|
144
152
|
claimPollMs?: number;
|
|
145
|
-
|
|
146
|
-
|
|
153
|
+
serverLeaseMs?: number;
|
|
154
|
+
serverRenewMs?: number;
|
|
147
155
|
runClaimLeaseMs?: number;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
156
|
+
runnerEntryPath?: string;
|
|
157
|
+
runnerStartupTimeoutMs?: number;
|
|
158
|
+
resourceRunnerEntryPath?: string;
|
|
151
159
|
channelAdapterEntryPath?: string;
|
|
152
160
|
onLog?: (message: string) => void;
|
|
153
161
|
};
|
|
@@ -156,13 +164,14 @@ type ActiveRun = {
|
|
|
156
164
|
record: WorkflowRunQueueRecord;
|
|
157
165
|
claimToken: string;
|
|
158
166
|
generation: number;
|
|
159
|
-
supervisor:
|
|
167
|
+
supervisor: WorkflowRunnerSupervisor;
|
|
160
168
|
launchProgressRevision: number;
|
|
161
|
-
|
|
169
|
+
runnerPid?: number;
|
|
162
170
|
exiting: boolean;
|
|
163
171
|
workflowLoadFailure?: string;
|
|
164
172
|
control?: "cancel" | "pause" | "handoff";
|
|
165
173
|
claimLost?: boolean;
|
|
174
|
+
contentDigests: Set<string>;
|
|
166
175
|
};
|
|
167
176
|
|
|
168
177
|
type ClientSubscription = {
|
|
@@ -199,38 +208,38 @@ type ActiveChannel = {
|
|
|
199
208
|
stopping: boolean;
|
|
200
209
|
};
|
|
201
210
|
|
|
202
|
-
type
|
|
211
|
+
type ActiveResourceManager = {
|
|
203
212
|
key: string;
|
|
204
213
|
projectPath: string;
|
|
205
|
-
store:
|
|
206
|
-
claim:
|
|
207
|
-
resource:
|
|
214
|
+
store: SqliteResourceManagerStore;
|
|
215
|
+
claim: ResourceManagerQueueClaim;
|
|
216
|
+
resource: ManagedResource;
|
|
208
217
|
reconcileId: string;
|
|
209
218
|
startedAt: number;
|
|
210
|
-
supervisor:
|
|
219
|
+
supervisor: ResourceRunnerSupervisor;
|
|
211
220
|
renewTimer: ReturnType<typeof setInterval>;
|
|
212
221
|
settled: boolean;
|
|
213
222
|
};
|
|
214
223
|
|
|
215
|
-
/** Global package-owned
|
|
216
|
-
export class
|
|
217
|
-
private readonly options:
|
|
218
|
-
private readonly
|
|
224
|
+
/** Global package-owned server. It writes state and supervises code-only runners. */
|
|
225
|
+
export class WorkflowServer {
|
|
226
|
+
private readonly options: WorkflowServerOptions;
|
|
227
|
+
private readonly serverId: string;
|
|
219
228
|
private readonly databasePath: string;
|
|
220
229
|
private readonly stateDirectory: string;
|
|
221
230
|
private readonly socketPath: string;
|
|
222
231
|
private readonly lockPath: string;
|
|
223
232
|
private readonly state: StateDatabase;
|
|
224
|
-
private readonly
|
|
225
|
-
private readonly queue:
|
|
233
|
+
private readonly serverState: ServerStateStore;
|
|
234
|
+
private readonly queue: SqliteResourceManagerStore;
|
|
226
235
|
private readonly decisions: HumanDecisionStore;
|
|
227
236
|
private readonly channelEffects: ChannelEffectStore;
|
|
228
237
|
private readonly runStore: WorkflowRunStore;
|
|
229
|
-
private readonly views:
|
|
230
|
-
private readonly registry:
|
|
238
|
+
private readonly views: ServerViewStore;
|
|
239
|
+
private readonly registry: ServerProcessRegistry;
|
|
231
240
|
private readonly activeRuns = new Map<string, ActiveRun>();
|
|
232
|
-
private readonly
|
|
233
|
-
private readonly
|
|
241
|
+
private readonly runnerDescendants = new Map<string, Set<number>>();
|
|
242
|
+
private readonly activeResourceManagers = new Map<string, ActiveResourceManager>();
|
|
234
243
|
private readonly activeChannels = new Map<string, ActiveChannel>();
|
|
235
244
|
private readonly controlClaims = new Map<string, string>();
|
|
236
245
|
private readonly activationTasks = new Map<string, Promise<void>>();
|
|
@@ -244,29 +253,32 @@ export class WorkflowHost {
|
|
|
244
253
|
private readonly sockets = new Set<Socket>();
|
|
245
254
|
private readonly connections = new Map<Socket, ClientConnection>();
|
|
246
255
|
private server: net.Server | null = null;
|
|
247
|
-
private claim:
|
|
256
|
+
private claim: ServerClaim | null = null;
|
|
248
257
|
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
|
249
258
|
private pollTimer: ReturnType<typeof setInterval> | null = null;
|
|
250
259
|
private viewTimer: ReturnType<typeof setInterval> | null = null;
|
|
251
260
|
private stopping = false;
|
|
252
261
|
private started = false;
|
|
253
|
-
private
|
|
262
|
+
private resourceManagerPollActive = false;
|
|
254
263
|
private decisionTimeoutActive = false;
|
|
255
264
|
private decisionChannelConfig: DecisionChannelConfig | null = null;
|
|
256
265
|
private decisionChannelError: string | null = null;
|
|
257
266
|
private channelReloading = false;
|
|
258
267
|
private nextTerminalMessageReconciliationAt = 0;
|
|
259
268
|
|
|
260
|
-
constructor(options:
|
|
269
|
+
constructor(options: WorkflowServerOptions = {}) {
|
|
261
270
|
this.options = options;
|
|
262
|
-
this.
|
|
271
|
+
this.serverId = options.runnerId ?? `host-${randomUUID()}`;
|
|
263
272
|
this.databasePath = path.resolve(options.databasePath ?? workflowStatePath());
|
|
264
273
|
this.stateDirectory = path.join(path.dirname(this.databasePath), "host");
|
|
265
274
|
this.socketPath = clientSocketPath(this.databasePath);
|
|
266
275
|
this.lockPath = path.join(this.stateDirectory, "host.lock.json");
|
|
267
276
|
this.state = new StateDatabase({ filePath: this.databasePath });
|
|
268
|
-
this.
|
|
269
|
-
this.queue = new
|
|
277
|
+
this.serverState = new ServerStateStore(this.databasePath, { state: this.state });
|
|
278
|
+
this.queue = new SqliteResourceManagerStore(this.databasePath, {
|
|
279
|
+
state: this.state,
|
|
280
|
+
global: true,
|
|
281
|
+
});
|
|
270
282
|
this.decisions = new HumanDecisionStore(this.databasePath, {
|
|
271
283
|
state: this.state,
|
|
272
284
|
authorityProvider: (runId) => {
|
|
@@ -282,12 +294,12 @@ export class WorkflowHost {
|
|
|
282
294
|
return token === undefined ? undefined : this.queue.workflowRunAuthority(runId, token);
|
|
283
295
|
},
|
|
284
296
|
snapshotLifecycle: (context) => this.applyLifecycleProjection(context),
|
|
285
|
-
|
|
297
|
+
allowServerSessionRecording: true,
|
|
286
298
|
});
|
|
287
|
-
this.views = new
|
|
299
|
+
this.views = new ServerViewStore(
|
|
288
300
|
this.state,
|
|
289
301
|
this.queue,
|
|
290
|
-
this.
|
|
302
|
+
this.serverState,
|
|
291
303
|
this.runStore,
|
|
292
304
|
(runId) => this.activeRuns.has(runId),
|
|
293
305
|
(targetSessionId) => {
|
|
@@ -295,7 +307,7 @@ export class WorkflowHost {
|
|
|
295
307
|
return coordinator?.branchReported === true && coordinator.modelTurnActive;
|
|
296
308
|
},
|
|
297
309
|
);
|
|
298
|
-
this.registry = options.registry ?? new
|
|
310
|
+
this.registry = options.registry ?? new ServerProcessRegistry(this.stateDirectory);
|
|
299
311
|
}
|
|
300
312
|
|
|
301
313
|
get endpoint(): string {
|
|
@@ -312,23 +324,23 @@ export class WorkflowHost {
|
|
|
312
324
|
fs.chmodSync(this.stateDirectory, 0o700);
|
|
313
325
|
const startIdentity = processStartIdentity(process.pid);
|
|
314
326
|
if (startIdentity === undefined) {
|
|
315
|
-
throw new Error("Cannot attest the workflow
|
|
327
|
+
throw new Error("Cannot attest the workflow server process start identity");
|
|
316
328
|
}
|
|
317
|
-
|
|
329
|
+
acquireServerLock(this.lockPath, { pid: process.pid, startIdentity, serverId: this.serverId });
|
|
318
330
|
try {
|
|
319
331
|
const reaped = this.registry.reapOrphans();
|
|
320
332
|
if (reaped.length > 0) this.log(`reaped ${reaped.length} exact orphan process(es)`);
|
|
321
|
-
const
|
|
322
|
-
this.claim = this.
|
|
323
|
-
|
|
333
|
+
const previousServer = this.serverState.serverStatus();
|
|
334
|
+
this.claim = this.serverState.acquireServer({
|
|
335
|
+
serverId: this.serverId,
|
|
324
336
|
pid: process.pid,
|
|
325
337
|
processStartIdentity: startIdentity,
|
|
326
|
-
leaseMs: this.options.
|
|
338
|
+
leaseMs: this.options.serverLeaseMs ?? SERVER_LEASE_MS,
|
|
327
339
|
});
|
|
328
|
-
this.
|
|
340
|
+
this.recoverPreviousServer(previousServer.serverId, this.claim.epoch);
|
|
329
341
|
const previousHeartbeatAt =
|
|
330
|
-
|
|
331
|
-
this.
|
|
342
|
+
previousServer.heartbeatAt === null ? undefined : Date.parse(previousServer.heartbeatAt);
|
|
343
|
+
this.serverState.recoverInactiveModelTurns(previousHeartbeatAt);
|
|
332
344
|
this.discoverMissingTerminalWorkflowMessages();
|
|
333
345
|
this.reconcilePendingTerminalWorkflowMessages(Date.now(), true);
|
|
334
346
|
await this.listen();
|
|
@@ -338,7 +350,7 @@ export class WorkflowHost {
|
|
|
338
350
|
this.expireTimedOutInteraction();
|
|
339
351
|
void this.expireTimedOutDecision();
|
|
340
352
|
void this.claimOne();
|
|
341
|
-
void this.
|
|
353
|
+
void this.claimResourceManagerOne();
|
|
342
354
|
void this.reloadDecisionChannels().catch((error) => {
|
|
343
355
|
this.log(`decision channel startup failed: ${errorMessage(error)}`);
|
|
344
356
|
});
|
|
@@ -348,7 +360,7 @@ export class WorkflowHost {
|
|
|
348
360
|
await this.closeServer(server);
|
|
349
361
|
if (this.claim !== null) {
|
|
350
362
|
try {
|
|
351
|
-
this.
|
|
363
|
+
this.serverState.releaseServer(this.claim);
|
|
352
364
|
} catch {
|
|
353
365
|
// Preserve the startup error when cleanup cannot release an already-lost claim.
|
|
354
366
|
}
|
|
@@ -386,13 +398,13 @@ export class WorkflowHost {
|
|
|
386
398
|
}),
|
|
387
399
|
);
|
|
388
400
|
await Promise.allSettled(
|
|
389
|
-
[...this.
|
|
401
|
+
[...this.activeResourceManagers.values()].map(async (active) => {
|
|
390
402
|
clearInterval(active.renewTimer);
|
|
391
403
|
await active.supervisor.stop("orphaned");
|
|
392
404
|
if (!active.settled) {
|
|
393
405
|
active.store.requeueClaim(
|
|
394
406
|
active.claim,
|
|
395
|
-
{ availableAt: new Date().toISOString(), error: "Workflow
|
|
407
|
+
{ availableAt: new Date().toISOString(), error: "Workflow server stopped" },
|
|
396
408
|
new Date().toISOString(),
|
|
397
409
|
);
|
|
398
410
|
}
|
|
@@ -408,13 +420,13 @@ export class WorkflowHost {
|
|
|
408
420
|
await Promise.allSettled(this.activationTasks.values());
|
|
409
421
|
await Promise.allSettled(this.maintenanceCommands.values());
|
|
410
422
|
this.registry.killAll();
|
|
411
|
-
if (this.claim !== null) this.
|
|
423
|
+
if (this.claim !== null) this.serverState.releaseServer(this.claim);
|
|
412
424
|
this.claim = null;
|
|
413
425
|
if (process.platform !== "win32") fs.rmSync(this.socketPath, { force: true });
|
|
414
426
|
this.releaseLock();
|
|
415
427
|
this.runStore.close();
|
|
416
428
|
this.queue.close();
|
|
417
|
-
this.
|
|
429
|
+
this.serverState.close();
|
|
418
430
|
this.state.close();
|
|
419
431
|
this.started = false;
|
|
420
432
|
}
|
|
@@ -436,8 +448,8 @@ export class WorkflowHost {
|
|
|
436
448
|
this.sockets.clear();
|
|
437
449
|
}
|
|
438
450
|
|
|
439
|
-
private
|
|
440
|
-
if (
|
|
451
|
+
private recoverPreviousServer(previousServerId: string | null, serverEpoch: number): void {
|
|
452
|
+
if (previousServerId === null || previousServerId === this.serverId) return;
|
|
441
453
|
const now = Date.now();
|
|
442
454
|
this.state.transaction(() => {
|
|
443
455
|
this.state.connection
|
|
@@ -445,13 +457,13 @@ export class WorkflowHost {
|
|
|
445
457
|
`UPDATE leases SET expires_at = ?
|
|
446
458
|
WHERE owner_id = ? AND owner_type IN ('host', 'controller') AND expires_at > ?`,
|
|
447
459
|
)
|
|
448
|
-
.run(now,
|
|
460
|
+
.run(now, previousServerId, now);
|
|
449
461
|
this.state.connection
|
|
450
462
|
.prepare(
|
|
451
463
|
`UPDATE run_workers SET status = 'orphaned', finished_at = ?
|
|
452
464
|
WHERE host_epoch < ? AND status IN ('starting', 'ready', 'running')`,
|
|
453
465
|
)
|
|
454
|
-
.run(now,
|
|
466
|
+
.run(now, serverEpoch);
|
|
455
467
|
});
|
|
456
468
|
}
|
|
457
469
|
|
|
@@ -459,9 +471,9 @@ export class WorkflowHost {
|
|
|
459
471
|
this.heartbeatTimer = setInterval(() => {
|
|
460
472
|
try {
|
|
461
473
|
if (this.claim === null) return;
|
|
462
|
-
this.claim = this.
|
|
474
|
+
this.claim = this.serverState.renewServer(
|
|
463
475
|
this.claim,
|
|
464
|
-
this.options.
|
|
476
|
+
this.options.serverLeaseMs ?? SERVER_LEASE_MS,
|
|
465
477
|
);
|
|
466
478
|
for (const active of this.activeRuns.values()) {
|
|
467
479
|
if (active.control !== undefined || active.exiting) continue;
|
|
@@ -477,16 +489,16 @@ export class WorkflowHost {
|
|
|
477
489
|
}
|
|
478
490
|
}
|
|
479
491
|
} catch (error) {
|
|
480
|
-
this.log(`
|
|
492
|
+
this.log(`server claim lost: ${errorMessage(error)}`);
|
|
481
493
|
void this.stop();
|
|
482
494
|
}
|
|
483
|
-
}, this.options.
|
|
495
|
+
}, this.options.serverRenewMs ?? SERVER_RENEW_MS);
|
|
484
496
|
this.heartbeatTimer.unref?.();
|
|
485
497
|
this.pollTimer = setInterval(() => {
|
|
486
498
|
this.expireTimedOutInteraction();
|
|
487
499
|
void this.expireTimedOutDecision();
|
|
488
500
|
void this.claimOne();
|
|
489
|
-
void this.
|
|
501
|
+
void this.claimResourceManagerOne();
|
|
490
502
|
this.reconcilePendingTerminalWorkflowMessages();
|
|
491
503
|
}, this.options.claimPollMs ?? CLAIM_POLL_MS);
|
|
492
504
|
this.pollTimer.unref?.();
|
|
@@ -580,7 +592,7 @@ export class WorkflowHost {
|
|
|
580
592
|
let changed = false;
|
|
581
593
|
for (const [sessionId, coordinator] of this.sessionCoordinators) {
|
|
582
594
|
if (connectionId !== undefined && coordinator.connectionId !== connectionId) continue;
|
|
583
|
-
this.
|
|
595
|
+
this.serverState.markSessionModelTurnsInactive(sessionId);
|
|
584
596
|
this.sessionCoordinators.delete(sessionId);
|
|
585
597
|
changed = true;
|
|
586
598
|
}
|
|
@@ -646,7 +658,7 @@ export class WorkflowHost {
|
|
|
646
658
|
if (payload.coordinator === true) {
|
|
647
659
|
const previous = this.sessionCoordinators.get(sessionId);
|
|
648
660
|
if (previous !== undefined && previous.connectionId !== connection.id) {
|
|
649
|
-
this.
|
|
661
|
+
this.serverState.markSessionModelTurnsInactive(sessionId);
|
|
650
662
|
}
|
|
651
663
|
const coordinator = {
|
|
652
664
|
connectionId: connection.id,
|
|
@@ -758,11 +770,16 @@ export class WorkflowHost {
|
|
|
758
770
|
payload.backupPath === undefined
|
|
759
771
|
? undefined
|
|
760
772
|
: requireAbsolutePath(payload.backupPath, "backupPath");
|
|
761
|
-
const report = await pruneState(
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
773
|
+
const report = await pruneState(
|
|
774
|
+
this.state,
|
|
775
|
+
this.databasePath,
|
|
776
|
+
{
|
|
777
|
+
before,
|
|
778
|
+
apply,
|
|
779
|
+
...(backupPath === undefined ? {} : { backupPath }),
|
|
780
|
+
},
|
|
781
|
+
() => this.activeRunnerContentHashes(),
|
|
782
|
+
);
|
|
766
783
|
return toJsonValue(report);
|
|
767
784
|
});
|
|
768
785
|
default:
|
|
@@ -777,7 +794,7 @@ export class WorkflowHost {
|
|
|
777
794
|
request: ClientRequest,
|
|
778
795
|
operation: () => Promise<JsonValue>,
|
|
779
796
|
): Promise<ClientResponse> {
|
|
780
|
-
const adopted = this.
|
|
797
|
+
const adopted = this.serverState.adoptCommand(request);
|
|
781
798
|
if (adopted !== undefined) return adopted;
|
|
782
799
|
const claim = this.claim;
|
|
783
800
|
if (claim === null || this.stopping) {
|
|
@@ -785,7 +802,7 @@ export class WorkflowHost {
|
|
|
785
802
|
request.requestId,
|
|
786
803
|
"unavailable",
|
|
787
804
|
undefined,
|
|
788
|
-
"Workflow
|
|
805
|
+
"Workflow server is stopping",
|
|
789
806
|
);
|
|
790
807
|
}
|
|
791
808
|
const key = canonicalJson([request.clientId, request.idempotencyKey]);
|
|
@@ -793,7 +810,7 @@ export class WorkflowHost {
|
|
|
793
810
|
if (active !== undefined) {
|
|
794
811
|
await active;
|
|
795
812
|
return (
|
|
796
|
-
this.
|
|
813
|
+
this.serverState.adoptCommand(request) ??
|
|
797
814
|
clientResponse(
|
|
798
815
|
request.requestId,
|
|
799
816
|
"unavailable",
|
|
@@ -810,7 +827,7 @@ export class WorkflowHost {
|
|
|
810
827
|
} catch (error) {
|
|
811
828
|
result = { outcome: "rejected", error: errorMessage(error) };
|
|
812
829
|
}
|
|
813
|
-
return this.
|
|
830
|
+
return this.serverState.executeCommand(request, claim.epoch, () => result);
|
|
814
831
|
})();
|
|
815
832
|
this.maintenanceCommands.set(key, execution);
|
|
816
833
|
try {
|
|
@@ -903,16 +920,20 @@ export class WorkflowHost {
|
|
|
903
920
|
report.targetSessionId,
|
|
904
921
|
report.coordinatorEpoch,
|
|
905
922
|
);
|
|
906
|
-
const messages = this.
|
|
923
|
+
const messages = this.serverState.workflowMessages.listSession(report.targetSessionId);
|
|
907
924
|
const allowed = new Set(messages.map((message) => message.workflowMessageId));
|
|
908
925
|
const reconciledRunIds = new Set<string>();
|
|
909
926
|
this.state.transaction(() => {
|
|
910
927
|
if (coordinator.needsTimerResume) {
|
|
911
|
-
this.
|
|
928
|
+
this.serverState.resumeSessionModelTurns(report.targetSessionId);
|
|
912
929
|
}
|
|
913
|
-
this.
|
|
930
|
+
this.serverState.workflowMessages.adoptBranch(
|
|
931
|
+
report.targetSessionId,
|
|
932
|
+
report.entries,
|
|
933
|
+
allowed,
|
|
934
|
+
);
|
|
914
935
|
if (report.isIdle && !report.hasPendingMessages) {
|
|
915
|
-
for (const turn of this.
|
|
936
|
+
for (const turn of this.serverState.workflowMessages.openTurnsForSession(
|
|
916
937
|
report.targetSessionId,
|
|
917
938
|
)) {
|
|
918
939
|
reconciledRunIds.add(turn.runId);
|
|
@@ -928,8 +949,10 @@ export class WorkflowHost {
|
|
|
928
949
|
});
|
|
929
950
|
}
|
|
930
951
|
const branchIds = new Set(report.entries.map((entry) => entry.workflowMessageId));
|
|
931
|
-
const refreshed = this.
|
|
932
|
-
for (const interaction of this.
|
|
952
|
+
const refreshed = this.serverState.workflowMessages.listSession(report.targetSessionId);
|
|
953
|
+
for (const interaction of this.serverState.listPendingInteractions(
|
|
954
|
+
report.targetSessionId,
|
|
955
|
+
)) {
|
|
933
956
|
const sourceMessages = refreshed.filter(
|
|
934
957
|
(message) => message.sourceId === interaction.requestId,
|
|
935
958
|
);
|
|
@@ -937,8 +960,8 @@ export class WorkflowHost {
|
|
|
937
960
|
sourceMessages.some((message) => message.status === "sent") &&
|
|
938
961
|
!sourceMessages.some((message) => branchIds.has(message.workflowMessageId))
|
|
939
962
|
) {
|
|
940
|
-
this.
|
|
941
|
-
this.
|
|
963
|
+
this.serverState.workflowMessages.cancelPendingForSource(interaction.requestId);
|
|
964
|
+
this.serverState.ensureInteractionMessage(
|
|
942
965
|
interaction,
|
|
943
966
|
interaction.kind === "decision" ? "initial" : "resumed",
|
|
944
967
|
);
|
|
@@ -970,7 +993,7 @@ export class WorkflowHost {
|
|
|
970
993
|
}
|
|
971
994
|
let outcome: "accepted" | "adopted" = "accepted";
|
|
972
995
|
const receipt = this.state.transaction((): WorkflowTurnReportReceipt => {
|
|
973
|
-
const existing = this.
|
|
996
|
+
const existing = this.serverState.workflowMessages.getTurn(report.workflowTurnId);
|
|
974
997
|
if (report.state === "ended") {
|
|
975
998
|
if (existing === undefined) {
|
|
976
999
|
outcome = "adopted";
|
|
@@ -994,7 +1017,7 @@ export class WorkflowHost {
|
|
|
994
1017
|
return workflowTurnReceipt("settled", turn);
|
|
995
1018
|
}
|
|
996
1019
|
if (existing !== undefined) {
|
|
997
|
-
const turn = this.
|
|
1020
|
+
const turn = this.serverState.workflowMessages.startTurn(report);
|
|
998
1021
|
outcome = "adopted";
|
|
999
1022
|
return workflowTurnReceipt(turn.state === "started" ? "active" : "settled", turn);
|
|
1000
1023
|
}
|
|
@@ -1010,7 +1033,7 @@ export class WorkflowHost {
|
|
|
1010
1033
|
outcome = "adopted";
|
|
1011
1034
|
return workflowTurnReceipt("absent", null);
|
|
1012
1035
|
}
|
|
1013
|
-
const message = this.
|
|
1036
|
+
const message = this.serverState.workflowMessages.require(report.workflowMessageId);
|
|
1014
1037
|
const queuedRun = this.queue.getWorkflowRun(report.runId);
|
|
1015
1038
|
if (
|
|
1016
1039
|
message.kind === "step" &&
|
|
@@ -1021,14 +1044,14 @@ export class WorkflowHost {
|
|
|
1021
1044
|
}
|
|
1022
1045
|
if (message.kind === "step") {
|
|
1023
1046
|
const now = Date.now();
|
|
1024
|
-
this.
|
|
1025
|
-
this.
|
|
1047
|
+
this.serverState.beginInteractionModelTurn(message.sourceId, now);
|
|
1048
|
+
this.serverState.workflowMessages.cancelPendingForSource(message.sourceId, "step", now);
|
|
1026
1049
|
return workflowTurnReceipt(
|
|
1027
1050
|
"active",
|
|
1028
|
-
this.
|
|
1051
|
+
this.serverState.workflowMessages.startTurn({ ...report, now }),
|
|
1029
1052
|
);
|
|
1030
1053
|
}
|
|
1031
|
-
return workflowTurnReceipt("active", this.
|
|
1054
|
+
return workflowTurnReceipt("active", this.serverState.workflowMessages.startTurn(report));
|
|
1032
1055
|
});
|
|
1033
1056
|
coordinator.modelTurnActive = receipt.ownership === "active";
|
|
1034
1057
|
this.views.noteOriginActivityChange();
|
|
@@ -1038,24 +1061,24 @@ export class WorkflowHost {
|
|
|
1038
1061
|
}
|
|
1039
1062
|
|
|
1040
1063
|
private applyWorkflowTurnEnd(report: Extract<WorkflowTurnReport, { state: "ended" }>) {
|
|
1041
|
-
const current = this.
|
|
1064
|
+
const current = this.serverState.workflowMessages.requireTurn(report.workflowTurnId);
|
|
1042
1065
|
if (current.state === "ended") {
|
|
1043
|
-
return this.
|
|
1066
|
+
return this.serverState.workflowMessages.endTurn(report);
|
|
1044
1067
|
}
|
|
1045
|
-
const turn = this.
|
|
1046
|
-
const message = this.
|
|
1068
|
+
const turn = this.serverState.workflowMessages.endTurn(report);
|
|
1069
|
+
const message = this.serverState.workflowMessages.require(report.workflowMessageId);
|
|
1047
1070
|
if (message.kind !== "step") return turn;
|
|
1048
|
-
const interaction = this.
|
|
1071
|
+
const interaction = this.serverState.getInteraction(message.sourceId);
|
|
1049
1072
|
if (interaction === undefined || interaction.status !== "pending") return turn;
|
|
1050
1073
|
if (report.stopReason === "aborted") {
|
|
1051
1074
|
if (!this.queue.pauseParkedWorkflowRun({ runId: report.runId })) {
|
|
1052
1075
|
throw new Error("Interrupted workflow turn could not pause its exact parked run");
|
|
1053
1076
|
}
|
|
1054
|
-
this.
|
|
1077
|
+
this.serverState.workflowMessages.cancelPendingForSource(interaction.requestId, "step");
|
|
1055
1078
|
return turn;
|
|
1056
1079
|
}
|
|
1057
1080
|
if (this.queue.isWorkflowRunPaused(report.runId)) return turn;
|
|
1058
|
-
if (this.
|
|
1081
|
+
if (this.serverState.validatingInteraction(report.runId) !== undefined) return turn;
|
|
1059
1082
|
const changed = this.state.connection
|
|
1060
1083
|
.prepare(
|
|
1061
1084
|
`UPDATE interactive_requests
|
|
@@ -1065,13 +1088,13 @@ export class WorkflowHost {
|
|
|
1065
1088
|
)
|
|
1066
1089
|
.run(Date.now(), interaction.requestId);
|
|
1067
1090
|
if (changed.changes !== 1) return turn;
|
|
1068
|
-
const updated = this.
|
|
1091
|
+
const updated = this.serverState.getInteraction(interaction.requestId);
|
|
1069
1092
|
if (updated === undefined) throw new Error("Workflow interaction disappeared after turn end");
|
|
1070
1093
|
if (updated.unproductiveTurnEnds <= 2) {
|
|
1071
|
-
this.
|
|
1094
|
+
this.serverState.ensureInteractionMessage(updated, "reminder");
|
|
1072
1095
|
return turn;
|
|
1073
1096
|
}
|
|
1074
|
-
this.
|
|
1097
|
+
this.serverState.workflowMessages.cancelPendingForSource(updated.requestId, "step");
|
|
1075
1098
|
this.state.connection
|
|
1076
1099
|
.prepare(
|
|
1077
1100
|
`UPDATE interactive_requests SET status = 'cancelled', revision = revision + 1, updated_at = ?
|
|
@@ -1160,10 +1183,10 @@ export class WorkflowHost {
|
|
|
1160
1183
|
request.requestId,
|
|
1161
1184
|
"unavailable",
|
|
1162
1185
|
undefined,
|
|
1163
|
-
"Workflow
|
|
1186
|
+
"Workflow server stopped while validating the submission",
|
|
1164
1187
|
);
|
|
1165
1188
|
}
|
|
1166
|
-
const submission = this.
|
|
1189
|
+
const submission = this.serverState.interactionSubmission(requestId, submissionId);
|
|
1167
1190
|
if (submission?.outcome === "accepted" || submission?.outcome === "adopted") {
|
|
1168
1191
|
return clientResponse(
|
|
1169
1192
|
request.requestId,
|
|
@@ -1179,7 +1202,7 @@ export class WorkflowHost {
|
|
|
1179
1202
|
: "Workflow step output failed validation";
|
|
1180
1203
|
return clientResponse(request.requestId, "rejected", receipt, detail);
|
|
1181
1204
|
}
|
|
1182
|
-
await
|
|
1205
|
+
await serverDelay(25);
|
|
1183
1206
|
}
|
|
1184
1207
|
}
|
|
1185
1208
|
|
|
@@ -1190,13 +1213,13 @@ export class WorkflowHost {
|
|
|
1190
1213
|
type: "response",
|
|
1191
1214
|
requestId: request.requestId,
|
|
1192
1215
|
outcome: "unavailable",
|
|
1193
|
-
error: "Workflow
|
|
1216
|
+
error: "Workflow server is stopping",
|
|
1194
1217
|
};
|
|
1195
1218
|
}
|
|
1196
1219
|
const afterCommit: Array<() => void> = [];
|
|
1197
1220
|
let response: ClientResponse;
|
|
1198
1221
|
try {
|
|
1199
|
-
response = this.
|
|
1222
|
+
response = this.serverState.executeCommand(request, this.claim.epoch, () =>
|
|
1200
1223
|
this.executeOperation(request, afterCommit, connection),
|
|
1201
1224
|
);
|
|
1202
1225
|
} catch (error) {
|
|
@@ -1218,9 +1241,9 @@ export class WorkflowHost {
|
|
|
1218
1241
|
connection?: ClientConnection,
|
|
1219
1242
|
): Omit<ClientResponse, "schema" | "type" | "requestId"> {
|
|
1220
1243
|
switch (request.operation) {
|
|
1221
|
-
case "
|
|
1244
|
+
case "server.status":
|
|
1222
1245
|
return { outcome: "accepted", receipt: this.statusReceipt() };
|
|
1223
|
-
case "
|
|
1246
|
+
case "server.stop":
|
|
1224
1247
|
afterCommit.push(() => void this.stop());
|
|
1225
1248
|
return { outcome: "accepted", receipt: { stopping: true } };
|
|
1226
1249
|
case "run.list":
|
|
@@ -1281,7 +1304,7 @@ export class WorkflowHost {
|
|
|
1281
1304
|
status: "parked",
|
|
1282
1305
|
paused: true,
|
|
1283
1306
|
alreadyPaused: true,
|
|
1284
|
-
detail: "The supervised
|
|
1307
|
+
detail: "The supervised runner is already stopping at the pause boundary.",
|
|
1285
1308
|
},
|
|
1286
1309
|
};
|
|
1287
1310
|
}
|
|
@@ -1313,7 +1336,7 @@ export class WorkflowHost {
|
|
|
1313
1336
|
status: "parked",
|
|
1314
1337
|
paused: true,
|
|
1315
1338
|
alreadyPaused: false,
|
|
1316
|
-
detail: "The run is durably paused; the supervised
|
|
1339
|
+
detail: "The run is durably paused; the supervised runner is stopping.",
|
|
1317
1340
|
},
|
|
1318
1341
|
};
|
|
1319
1342
|
}
|
|
@@ -1335,12 +1358,12 @@ export class WorkflowHost {
|
|
|
1335
1358
|
const runId = requireRunId(request);
|
|
1336
1359
|
const resumedInteraction = this.state.transaction(() => {
|
|
1337
1360
|
const now = Date.now();
|
|
1338
|
-
const pausedAt = this.
|
|
1361
|
+
const pausedAt = this.serverState.interactionPauseStartedAt(runId);
|
|
1339
1362
|
if (!this.queue.resumePausedInteraction({ runId, now: new Date(now).toISOString() })) {
|
|
1340
1363
|
return false;
|
|
1341
1364
|
}
|
|
1342
|
-
this.
|
|
1343
|
-
this.
|
|
1365
|
+
this.serverState.resumeInteractionModelTurn(runId, pausedAt, now);
|
|
1366
|
+
this.serverState.resumePendingInteraction(runId, now);
|
|
1344
1367
|
return true;
|
|
1345
1368
|
});
|
|
1346
1369
|
if (resumedInteraction) {
|
|
@@ -1372,7 +1395,7 @@ export class WorkflowHost {
|
|
|
1372
1395
|
const token = randomUUID();
|
|
1373
1396
|
const claimed = this.queue.claimWorkflowRun({
|
|
1374
1397
|
runId,
|
|
1375
|
-
runnerId: this.
|
|
1398
|
+
runnerId: this.serverId,
|
|
1376
1399
|
claimToken: token,
|
|
1377
1400
|
leaseMs: this.runClaimLeaseMs,
|
|
1378
1401
|
});
|
|
@@ -1412,12 +1435,12 @@ export class WorkflowHost {
|
|
|
1412
1435
|
return this.startRun(request, afterCommit);
|
|
1413
1436
|
case "checkpoint.answer":
|
|
1414
1437
|
return this.answerCheckpoint(request, afterCommit);
|
|
1415
|
-
case "
|
|
1416
|
-
case "
|
|
1417
|
-
case "
|
|
1418
|
-
case "
|
|
1419
|
-
case "
|
|
1420
|
-
return this.
|
|
1438
|
+
case "resourceManager.list":
|
|
1439
|
+
case "resourceManager.get":
|
|
1440
|
+
case "resourceManager.apply":
|
|
1441
|
+
case "resourceManager.reconcile":
|
|
1442
|
+
case "resourceManager.delete":
|
|
1443
|
+
return this.executeResourceManagerOperation(request);
|
|
1421
1444
|
case "interaction.update":
|
|
1422
1445
|
return this.publishInteractionUpdate(request);
|
|
1423
1446
|
case "interaction.submit":
|
|
@@ -1471,7 +1494,7 @@ export class WorkflowHost {
|
|
|
1471
1494
|
}
|
|
1472
1495
|
const payload = requireRecord(request.payload, "run.restart payload");
|
|
1473
1496
|
const workflowMessageId = requireString(payload.workflowMessageId, "workflowMessageId");
|
|
1474
|
-
const terminal = this.
|
|
1497
|
+
const terminal = this.serverState.workflowMessages.latestForSource(
|
|
1475
1498
|
"terminal",
|
|
1476
1499
|
`terminal:${sourceRunId}`,
|
|
1477
1500
|
);
|
|
@@ -1487,7 +1510,7 @@ export class WorkflowHost {
|
|
|
1487
1510
|
if (sessionView.run?.runId !== sourceRunId) {
|
|
1488
1511
|
return { outcome: "rejected", error: "Workflow terminal result is no longer current" };
|
|
1489
1512
|
}
|
|
1490
|
-
const turn = this.
|
|
1513
|
+
const turn = this.serverState.workflowMessages.latestTurnForMessage(workflowMessageId);
|
|
1491
1514
|
if (turn === undefined) {
|
|
1492
1515
|
return { outcome: "rejected", error: "Workflow restart requires a terminal model turn" };
|
|
1493
1516
|
}
|
|
@@ -1547,7 +1570,7 @@ export class WorkflowHost {
|
|
|
1547
1570
|
}
|
|
1548
1571
|
const runId = `restart-${createHash("sha256").update(workflowMessageId).digest("hex").slice(0, 40)}`;
|
|
1549
1572
|
const claimToken = randomUUID();
|
|
1550
|
-
const scoped = new
|
|
1573
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
1551
1574
|
state: this.state,
|
|
1552
1575
|
projectPath,
|
|
1553
1576
|
});
|
|
@@ -1560,7 +1583,7 @@ export class WorkflowHost {
|
|
|
1560
1583
|
definitionSnapshot: this.state.readJson(definition.definitionHash),
|
|
1561
1584
|
input: source.input,
|
|
1562
1585
|
launchOptions: source.launchOptions,
|
|
1563
|
-
runnerId: this.
|
|
1586
|
+
runnerId: this.serverId,
|
|
1564
1587
|
claimToken,
|
|
1565
1588
|
leaseMs: this.runClaimLeaseMs,
|
|
1566
1589
|
originSessionId: session.targetSessionId,
|
|
@@ -1769,58 +1792,72 @@ export class WorkflowHost {
|
|
|
1769
1792
|
};
|
|
1770
1793
|
}
|
|
1771
1794
|
|
|
1772
|
-
private
|
|
1795
|
+
private executeResourceManagerOperation(
|
|
1773
1796
|
request: ClientRequest,
|
|
1774
1797
|
): Omit<ClientResponse, "schema" | "type" | "requestId"> {
|
|
1775
|
-
const payload = requireRecord(request.payload, "
|
|
1798
|
+
const payload = requireRecord(request.payload, "resource manager payload");
|
|
1776
1799
|
const projectPath = path.resolve(requireString(payload.projectPath, "projectPath"));
|
|
1777
1800
|
if (payload.projectPath !== projectPath) {
|
|
1778
1801
|
return {
|
|
1779
1802
|
outcome: "rejected",
|
|
1780
|
-
error: "
|
|
1803
|
+
error: "ResourceManager projectPath must be absolute and normalized",
|
|
1781
1804
|
};
|
|
1782
1805
|
}
|
|
1783
|
-
const store = new
|
|
1806
|
+
const store = new SqliteResourceManagerStore(this.databasePath, {
|
|
1784
1807
|
state: this.state,
|
|
1785
1808
|
projectPath,
|
|
1786
1809
|
});
|
|
1787
|
-
if (request.operation === "
|
|
1810
|
+
if (request.operation === "resourceManager.list") {
|
|
1788
1811
|
return { outcome: "accepted", receipt: store.listResources() as unknown as JsonValue };
|
|
1789
1812
|
}
|
|
1790
|
-
const
|
|
1813
|
+
const resourceManager = requireString(payload.resourceManager, "resourceManager");
|
|
1791
1814
|
const key = requireString(payload.key, "key");
|
|
1792
|
-
const ref = {
|
|
1793
|
-
if (request.operation === "
|
|
1815
|
+
const ref = { resourceManager, key };
|
|
1816
|
+
if (request.operation === "resourceManager.get") {
|
|
1794
1817
|
const resource = store.getResource(ref);
|
|
1795
1818
|
return resource === undefined
|
|
1796
|
-
? { outcome: "notFound", error: `
|
|
1819
|
+
? { outcome: "notFound", error: `Managed resource not found: ${resourceManager}/${key}` }
|
|
1797
1820
|
: { outcome: "accepted", receipt: resource as unknown as JsonValue };
|
|
1798
1821
|
}
|
|
1799
|
-
if (request.operation === "
|
|
1822
|
+
if (request.operation === "resourceManager.apply") {
|
|
1800
1823
|
if (!Object.hasOwn(payload, "spec") || !Object.hasOwn(payload, "initialStatus")) {
|
|
1801
|
-
return {
|
|
1824
|
+
return {
|
|
1825
|
+
outcome: "rejected",
|
|
1826
|
+
error: "ResourceManager apply requires spec and initialStatus",
|
|
1827
|
+
};
|
|
1802
1828
|
}
|
|
1803
1829
|
const spec = payload.spec as JsonValue;
|
|
1804
1830
|
const initialStatus = payload.initialStatus as JsonValue;
|
|
1805
|
-
const
|
|
1831
|
+
const resourceManagerPath = path.resolve(
|
|
1832
|
+
requireString(payload.resourceManagerPath, "resourceManagerPath"),
|
|
1833
|
+
);
|
|
1806
1834
|
const sourceHash = requireString(payload.sourceHash, "sourceHash");
|
|
1807
|
-
if (!
|
|
1808
|
-
return {
|
|
1835
|
+
if (!resourceManagerPathAllowed(projectPath, resourceManager, resourceManagerPath)) {
|
|
1836
|
+
return {
|
|
1837
|
+
outcome: "rejected",
|
|
1838
|
+
error: "ResourceManager source does not match discovery rules",
|
|
1839
|
+
};
|
|
1809
1840
|
}
|
|
1810
1841
|
const observedHash = createHash("sha256")
|
|
1811
|
-
.update(fs.readFileSync(
|
|
1842
|
+
.update(fs.readFileSync(resourceManagerPath))
|
|
1812
1843
|
.digest("hex");
|
|
1813
1844
|
if (observedHash !== sourceHash) {
|
|
1814
|
-
return {
|
|
1845
|
+
return {
|
|
1846
|
+
outcome: "conflict",
|
|
1847
|
+
error: "ResourceManager source changed before apply committed",
|
|
1848
|
+
};
|
|
1815
1849
|
}
|
|
1816
|
-
const resource = store.putResource({
|
|
1850
|
+
const resource = store.putResource({ resourceManager, key, spec, initialStatus });
|
|
1817
1851
|
return { outcome: "accepted", receipt: resource as unknown as JsonValue };
|
|
1818
1852
|
}
|
|
1819
1853
|
const existing = store.getResource(ref);
|
|
1820
1854
|
if (existing === undefined) {
|
|
1821
|
-
return {
|
|
1855
|
+
return {
|
|
1856
|
+
outcome: "notFound",
|
|
1857
|
+
error: `Managed resource not found: ${resourceManager}/${key}`,
|
|
1858
|
+
};
|
|
1822
1859
|
}
|
|
1823
|
-
if (request.operation === "
|
|
1860
|
+
if (request.operation === "resourceManager.reconcile") {
|
|
1824
1861
|
store.enqueue(ref);
|
|
1825
1862
|
return { outcome: "accepted", receipt: existing as unknown as JsonValue };
|
|
1826
1863
|
}
|
|
@@ -1830,7 +1867,7 @@ export class WorkflowHost {
|
|
|
1830
1867
|
}
|
|
1831
1868
|
|
|
1832
1869
|
private statusReceipt(): JsonValue {
|
|
1833
|
-
const
|
|
1870
|
+
const serverStatus = this.serverState.serverStatus();
|
|
1834
1871
|
const count = (sql: string, ...params: unknown[]): number => {
|
|
1835
1872
|
const row = this.state.connection.prepare(sql).get(...params) as
|
|
1836
1873
|
| { count?: unknown }
|
|
@@ -1839,13 +1876,13 @@ export class WorkflowHost {
|
|
|
1839
1876
|
};
|
|
1840
1877
|
const now = Date.now();
|
|
1841
1878
|
return {
|
|
1842
|
-
state:
|
|
1843
|
-
epoch:
|
|
1879
|
+
state: serverStatus.live ? "running" : "stale",
|
|
1880
|
+
epoch: serverStatus.epoch,
|
|
1844
1881
|
socketAvailable: this.server?.listening === true,
|
|
1845
|
-
startedAt:
|
|
1846
|
-
heartbeatAt:
|
|
1847
|
-
expiresAt:
|
|
1848
|
-
|
|
1882
|
+
startedAt: serverStatus.startedAt,
|
|
1883
|
+
heartbeatAt: serverStatus.heartbeatAt,
|
|
1884
|
+
expiresAt: serverStatus.expiresAt,
|
|
1885
|
+
activeRunners: this.activeRuns.size,
|
|
1849
1886
|
connectedClients: this.sockets.size,
|
|
1850
1887
|
queuedRuns: count("SELECT COUNT(*) AS count FROM run_queue WHERE status = 'queued'"),
|
|
1851
1888
|
runningRuns: count(
|
|
@@ -1865,8 +1902,8 @@ export class WorkflowHost {
|
|
|
1865
1902
|
"SELECT COUNT(*) AS count FROM interactive_requests WHERE status = 'pending'",
|
|
1866
1903
|
),
|
|
1867
1904
|
ambiguousEffects: count("SELECT COUNT(*) AS count FROM effects WHERE status = 'ambiguous'"),
|
|
1868
|
-
|
|
1869
|
-
|
|
1905
|
+
pendingResourceManagers: count("SELECT COUNT(*) AS count FROM controller_queue"),
|
|
1906
|
+
activeResourceRunners: this.activeResourceManagers.size,
|
|
1870
1907
|
lifecycleContradictions: count(
|
|
1871
1908
|
`SELECT COUNT(*) AS count
|
|
1872
1909
|
FROM runs r JOIN run_queue q ON q.run_id = r.run_id
|
|
@@ -1882,6 +1919,14 @@ export class WorkflowHost {
|
|
|
1882
1919
|
};
|
|
1883
1920
|
}
|
|
1884
1921
|
|
|
1922
|
+
private activeRunnerContentHashes(): Buffer[] {
|
|
1923
|
+
const digests = new Set<string>();
|
|
1924
|
+
for (const active of this.activeRuns.values()) {
|
|
1925
|
+
for (const digest of active.contentDigests) digests.add(digest);
|
|
1926
|
+
}
|
|
1927
|
+
return [...digests].map((digest) => Buffer.from(digest, "hex"));
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1885
1930
|
private stateStatusReceipt(): JsonValue {
|
|
1886
1931
|
const count = (sql: string, ...params: unknown[]): number => {
|
|
1887
1932
|
const row = this.state.connection.prepare(sql).get(...params) as
|
|
@@ -1895,7 +1940,7 @@ export class WorkflowHost {
|
|
|
1895
1940
|
counts: {
|
|
1896
1941
|
resources: count("SELECT COUNT(*) AS count FROM resources"),
|
|
1897
1942
|
runs: count("SELECT COUNT(*) AS count FROM runs"),
|
|
1898
|
-
|
|
1943
|
+
resourceManagers: count("SELECT COUNT(*) AS count FROM controller_resources"),
|
|
1899
1944
|
decisions: count("SELECT COUNT(*) AS count FROM human_decisions"),
|
|
1900
1945
|
settingsScopes: count("SELECT COUNT(*) AS count FROM workflow_settings"),
|
|
1901
1946
|
pendingInteractions: count(
|
|
@@ -1985,7 +2030,7 @@ export class WorkflowHost {
|
|
|
1985
2030
|
): Omit<ClientResponse, "schema" | "type" | "requestId"> {
|
|
1986
2031
|
const payload = requireRecord(command.payload, "decision answer payload");
|
|
1987
2032
|
const requestId = requireString(payload.requestId, "requestId");
|
|
1988
|
-
const interaction = this.
|
|
2033
|
+
const interaction = this.serverState.getInteraction(requestId);
|
|
1989
2034
|
if (interaction === undefined || interaction.kind !== "decision") {
|
|
1990
2035
|
return { outcome: "notFound", error: `Decision request not found: ${requestId}` };
|
|
1991
2036
|
}
|
|
@@ -2030,7 +2075,7 @@ export class WorkflowHost {
|
|
|
2030
2075
|
if (accepted.status === "conflict") {
|
|
2031
2076
|
return { outcome: "conflict", error: "Another human decision answer already won" };
|
|
2032
2077
|
}
|
|
2033
|
-
this.
|
|
2078
|
+
this.serverState.submitInteraction({
|
|
2034
2079
|
requestId: options.interaction.requestId,
|
|
2035
2080
|
submissionId: options.submissionId,
|
|
2036
2081
|
idempotencyKey: options.idempotencyKey,
|
|
@@ -2100,7 +2145,7 @@ export class WorkflowHost {
|
|
|
2100
2145
|
afterCommit: Array<() => void>,
|
|
2101
2146
|
) {
|
|
2102
2147
|
const token = randomUUID();
|
|
2103
|
-
const scoped = new
|
|
2148
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
2104
2149
|
state: this.state,
|
|
2105
2150
|
projectPath: options.projectPath,
|
|
2106
2151
|
});
|
|
@@ -2113,7 +2158,7 @@ export class WorkflowHost {
|
|
|
2113
2158
|
definitionSnapshot: options.definitionSnapshot,
|
|
2114
2159
|
input: options.input,
|
|
2115
2160
|
launchOptions: options.launchOptions,
|
|
2116
|
-
runnerId: this.
|
|
2161
|
+
runnerId: this.serverId,
|
|
2117
2162
|
claimToken: token,
|
|
2118
2163
|
leaseMs: this.runClaimLeaseMs,
|
|
2119
2164
|
originSessionId: options.originSessionId,
|
|
@@ -2140,7 +2185,7 @@ export class WorkflowHost {
|
|
|
2140
2185
|
const originSessionId = requireString(payload.originSessionId, "originSessionId");
|
|
2141
2186
|
const executionMode = payload.executionMode === "headless" ? "headless" : "interactive";
|
|
2142
2187
|
const token = randomUUID();
|
|
2143
|
-
const scoped = new
|
|
2188
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
2144
2189
|
state: this.state,
|
|
2145
2190
|
projectPath,
|
|
2146
2191
|
});
|
|
@@ -2153,7 +2198,7 @@ export class WorkflowHost {
|
|
|
2153
2198
|
definitionSnapshot: payload.definitionSnapshot,
|
|
2154
2199
|
input: payload.input,
|
|
2155
2200
|
launchOptions: payload.launchOptions ?? {},
|
|
2156
|
-
runnerId: this.
|
|
2201
|
+
runnerId: this.serverId,
|
|
2157
2202
|
claimToken: token,
|
|
2158
2203
|
leaseMs: this.runClaimLeaseMs,
|
|
2159
2204
|
originSessionId,
|
|
@@ -2181,7 +2226,7 @@ export class WorkflowHost {
|
|
|
2181
2226
|
const runId = requireRunId(request);
|
|
2182
2227
|
const payload = requireRecord(request.payload, "interaction update payload");
|
|
2183
2228
|
const requestId = requireString(payload.requestId, "requestId");
|
|
2184
|
-
const interaction = this.
|
|
2229
|
+
const interaction = this.serverState.getInteraction(requestId);
|
|
2185
2230
|
if (interaction === undefined || interaction.runId !== runId) {
|
|
2186
2231
|
return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
|
|
2187
2232
|
}
|
|
@@ -2210,7 +2255,7 @@ export class WorkflowHost {
|
|
|
2210
2255
|
const token = randomUUID();
|
|
2211
2256
|
const claimed = this.queue.claimWorkflowRunForControl({
|
|
2212
2257
|
runId,
|
|
2213
|
-
runnerId: this.
|
|
2258
|
+
runnerId: this.serverId,
|
|
2214
2259
|
claimToken: token,
|
|
2215
2260
|
leaseMs: this.runClaimLeaseMs,
|
|
2216
2261
|
});
|
|
@@ -2249,7 +2294,7 @@ export class WorkflowHost {
|
|
|
2249
2294
|
): Omit<ClientResponse, "schema" | "type" | "requestId"> {
|
|
2250
2295
|
const payload = requireRecord(request.payload, "interaction payload");
|
|
2251
2296
|
const requestId = requireString(payload.requestId, "requestId");
|
|
2252
|
-
const current = this.
|
|
2297
|
+
const current = this.serverState.getInteraction(requestId);
|
|
2253
2298
|
if (current === undefined || current.runId !== requireRunId(request)) {
|
|
2254
2299
|
return { outcome: "notFound", error: `Interactive request not found: ${requestId}` };
|
|
2255
2300
|
}
|
|
@@ -2267,7 +2312,7 @@ export class WorkflowHost {
|
|
|
2267
2312
|
return { outcome: "conflict", error: "Interactive request attempt is stale" };
|
|
2268
2313
|
}
|
|
2269
2314
|
const submissionId = requireString(payload.submissionId, "submissionId");
|
|
2270
|
-
const submission = this.
|
|
2315
|
+
const submission = this.serverState.beginInteractionValidation({
|
|
2271
2316
|
requestId,
|
|
2272
2317
|
submissionId,
|
|
2273
2318
|
idempotencyKey: request.idempotencyKey,
|
|
@@ -2290,7 +2335,7 @@ export class WorkflowHost {
|
|
|
2290
2335
|
const token = randomUUID();
|
|
2291
2336
|
const claimed = this.queue.claimWorkflowRunForInteractionValidation({
|
|
2292
2337
|
runId: interaction.runId,
|
|
2293
|
-
runnerId: this.
|
|
2338
|
+
runnerId: this.serverId,
|
|
2294
2339
|
claimToken: token,
|
|
2295
2340
|
leaseMs: this.runClaimLeaseMs,
|
|
2296
2341
|
});
|
|
@@ -2334,7 +2379,7 @@ export class WorkflowHost {
|
|
|
2334
2379
|
...(sourceResourceId === undefined ? {} : { sourceResourceId }),
|
|
2335
2380
|
...(stableMessageIds === undefined ? {} : { stableMessageIds }),
|
|
2336
2381
|
error: errorCode,
|
|
2337
|
-
actorId: this.
|
|
2382
|
+
actorId: this.serverId,
|
|
2338
2383
|
});
|
|
2339
2384
|
for (const effect of effects) {
|
|
2340
2385
|
this.recordChannelEffectResult(effect, "unknown", errorCode);
|
|
@@ -2349,13 +2394,13 @@ export class WorkflowHost {
|
|
|
2349
2394
|
if (action !== "confirm" && action !== "retry") {
|
|
2350
2395
|
throw new Error("Channel recovery action must be confirm or retry");
|
|
2351
2396
|
}
|
|
2352
|
-
if (this.claim === null) throw new Error("Workflow
|
|
2397
|
+
if (this.claim === null) throw new Error("Workflow server claim is unavailable");
|
|
2353
2398
|
const effect = this.state.transaction(() => {
|
|
2354
2399
|
const recovered = this.channelEffects.recover({
|
|
2355
2400
|
stableMessageId: messageId,
|
|
2356
2401
|
action,
|
|
2357
2402
|
actorId,
|
|
2358
|
-
ownerId: this.
|
|
2403
|
+
ownerId: this.serverId,
|
|
2359
2404
|
leaseGeneration: this.claim?.epoch ?? 0,
|
|
2360
2405
|
});
|
|
2361
2406
|
const attemptId = channelEffectAttemptId(recovered.effectId, recovered.attemptNumber);
|
|
@@ -2622,9 +2667,9 @@ export class WorkflowHost {
|
|
|
2622
2667
|
!config.allowedUserIds.includes(message.actorId) ||
|
|
2623
2668
|
!config.allowedChatIds.includes(message.chatId)
|
|
2624
2669
|
) {
|
|
2625
|
-
throw new Error("Telegram answer source is not authorized by the
|
|
2670
|
+
throw new Error("Telegram answer source is not authorized by the server profile");
|
|
2626
2671
|
}
|
|
2627
|
-
const interaction = this.
|
|
2672
|
+
const interaction = this.serverState.listPendingDecisionInteractions().find((candidate) => {
|
|
2628
2673
|
const request = candidate.contract as unknown as HumanDecisionRequest;
|
|
2629
2674
|
return request.decisionId === message.decisionId;
|
|
2630
2675
|
});
|
|
@@ -2661,7 +2706,7 @@ export class WorkflowHost {
|
|
|
2661
2706
|
}
|
|
2662
2707
|
|
|
2663
2708
|
private async nextChannelCommand(active: ActiveChannel): Promise<ChannelAdapterCommand> {
|
|
2664
|
-
const interactions = this.
|
|
2709
|
+
const interactions = this.serverState.listPendingDecisionInteractions();
|
|
2665
2710
|
for (const interaction of interactions) {
|
|
2666
2711
|
const request = interaction.contract as unknown as HumanDecisionRequest;
|
|
2667
2712
|
if (
|
|
@@ -2751,7 +2796,7 @@ export class WorkflowHost {
|
|
|
2751
2796
|
request: HumanDecisionChannelRequest,
|
|
2752
2797
|
purpose: "delivery" | "settlement",
|
|
2753
2798
|
): ChannelEffectRecord {
|
|
2754
|
-
if (this.claim === null) throw new Error("Workflow
|
|
2799
|
+
if (this.claim === null) throw new Error("Workflow server claim is unavailable");
|
|
2755
2800
|
return this.state.transaction(() => {
|
|
2756
2801
|
const effect = this.channelEffects.ensureApplying({
|
|
2757
2802
|
channelResourceId: active.resourceId,
|
|
@@ -2760,7 +2805,7 @@ export class WorkflowHost {
|
|
|
2760
2805
|
decisionId: request.decisionId,
|
|
2761
2806
|
purpose,
|
|
2762
2807
|
request,
|
|
2763
|
-
ownerId: this.
|
|
2808
|
+
ownerId: this.serverId,
|
|
2764
2809
|
leaseGeneration: this.claim?.epoch ?? 0,
|
|
2765
2810
|
maxAutomaticAttempts: 3,
|
|
2766
2811
|
});
|
|
@@ -2950,16 +2995,16 @@ export class WorkflowHost {
|
|
|
2950
2995
|
return row.revision;
|
|
2951
2996
|
}
|
|
2952
2997
|
|
|
2953
|
-
private async
|
|
2998
|
+
private async claimResourceManagerOne(): Promise<void> {
|
|
2954
2999
|
if (
|
|
2955
3000
|
this.stopping ||
|
|
2956
3001
|
this.claim === null ||
|
|
2957
|
-
this.
|
|
2958
|
-
this.
|
|
3002
|
+
this.resourceManagerPollActive ||
|
|
3003
|
+
this.activeResourceManagers.size >= MAX_RESOURCE_RUNNERS
|
|
2959
3004
|
) {
|
|
2960
3005
|
return;
|
|
2961
3006
|
}
|
|
2962
|
-
this.
|
|
3007
|
+
this.resourceManagerPollActive = true;
|
|
2963
3008
|
try {
|
|
2964
3009
|
const rows = this.state.connection
|
|
2965
3010
|
.prepare(
|
|
@@ -2974,87 +3019,90 @@ export class WorkflowHost {
|
|
|
2974
3019
|
.all(Date.now(), Date.now()) as Array<{ projectPath?: unknown }>;
|
|
2975
3020
|
for (const row of rows) {
|
|
2976
3021
|
if (typeof row.projectPath !== "string") continue;
|
|
2977
|
-
const discovered = await
|
|
3022
|
+
const discovered = await discoverResourceManagers({ cwd: row.projectPath });
|
|
2978
3023
|
if (discovered.length === 0) continue;
|
|
2979
3024
|
const byName = new Map(discovered.map((item) => [item.name, item.path]));
|
|
2980
|
-
const store = new
|
|
3025
|
+
const store = new SqliteResourceManagerStore(this.databasePath, {
|
|
2981
3026
|
state: this.state,
|
|
2982
3027
|
projectPath: row.projectPath,
|
|
2983
3028
|
});
|
|
2984
3029
|
const claim = store.claimNext({
|
|
2985
|
-
|
|
2986
|
-
ownerId: this.
|
|
2987
|
-
leaseMs:
|
|
2988
|
-
exclude: [...this.
|
|
3030
|
+
resourceManagers: [...byName.keys()],
|
|
3031
|
+
ownerId: this.serverId,
|
|
3032
|
+
leaseMs: RESOURCE_MANAGER_CLAIM_LEASE_MS,
|
|
3033
|
+
exclude: [...this.activeResourceManagers.values()]
|
|
2989
3034
|
.filter((active) => active.projectPath === row.projectPath)
|
|
2990
3035
|
.map((active) => ({
|
|
2991
|
-
|
|
3036
|
+
resourceManager: active.claim.resourceManager,
|
|
2992
3037
|
key: active.claim.key,
|
|
2993
3038
|
})),
|
|
2994
3039
|
});
|
|
2995
3040
|
if (claim === undefined) continue;
|
|
2996
|
-
const definitionPath = byName.get(claim.
|
|
2997
|
-
const resource = store.getResource({
|
|
3041
|
+
const definitionPath = byName.get(claim.resourceManager);
|
|
3042
|
+
const resource = store.getResource({
|
|
3043
|
+
resourceManager: claim.resourceManager,
|
|
3044
|
+
key: claim.key,
|
|
3045
|
+
});
|
|
2998
3046
|
if (definitionPath === undefined || resource === undefined) {
|
|
2999
3047
|
store.requeueClaim(claim, {
|
|
3000
3048
|
availableAt: new Date().toISOString(),
|
|
3001
|
-
error: "
|
|
3049
|
+
error: "ResourceManager source is unavailable",
|
|
3002
3050
|
});
|
|
3003
3051
|
continue;
|
|
3004
3052
|
}
|
|
3005
|
-
void this.
|
|
3053
|
+
void this.activateResourceManager(row.projectPath, definitionPath, store, claim, resource);
|
|
3006
3054
|
break;
|
|
3007
3055
|
}
|
|
3008
3056
|
} catch (error) {
|
|
3009
|
-
this.log(`
|
|
3057
|
+
this.log(`resource manager scheduling failed: ${errorMessage(error)}`);
|
|
3010
3058
|
} finally {
|
|
3011
|
-
this.
|
|
3059
|
+
this.resourceManagerPollActive = false;
|
|
3012
3060
|
}
|
|
3013
3061
|
}
|
|
3014
3062
|
|
|
3015
|
-
private async
|
|
3063
|
+
private async activateResourceManager(
|
|
3016
3064
|
projectPath: string,
|
|
3017
3065
|
definitionPath: string,
|
|
3018
|
-
store:
|
|
3019
|
-
claim:
|
|
3020
|
-
resource:
|
|
3066
|
+
store: SqliteResourceManagerStore,
|
|
3067
|
+
claim: ResourceManagerQueueClaim,
|
|
3068
|
+
resource: ManagedResource,
|
|
3021
3069
|
): Promise<void> {
|
|
3022
3070
|
if (this.claim === null || this.stopping) {
|
|
3023
3071
|
store.requeueClaim(claim, { availableAt: new Date().toISOString() });
|
|
3024
3072
|
return;
|
|
3025
3073
|
}
|
|
3026
|
-
const
|
|
3027
|
-
const key = `${projectPath}\u0000${claim.
|
|
3074
|
+
const runnerEpoch = randomUUID();
|
|
3075
|
+
const key = `${projectPath}\u0000${claim.resourceManager}\u0000${claim.key}`;
|
|
3028
3076
|
const reconcileId = randomUUID();
|
|
3029
|
-
const envelope:
|
|
3077
|
+
const envelope: ResourceRunnerLaunchEnvelope = {
|
|
3030
3078
|
schema: "pi-workflows.controller-worker-launch.v1",
|
|
3031
|
-
|
|
3032
|
-
|
|
3079
|
+
runnerEpoch,
|
|
3080
|
+
serverEpoch: this.claim.epoch,
|
|
3033
3081
|
generation: claim.generation,
|
|
3034
3082
|
projectPath,
|
|
3035
3083
|
definitionPath,
|
|
3036
|
-
|
|
3084
|
+
resourceManagerName: claim.resourceManager,
|
|
3037
3085
|
resource,
|
|
3038
|
-
timeoutMs:
|
|
3086
|
+
timeoutMs: DEFAULT_RESOURCE_MANAGER_TIMEOUT_MS,
|
|
3039
3087
|
};
|
|
3040
|
-
let active:
|
|
3041
|
-
const supervisor = new
|
|
3088
|
+
let active: ActiveResourceManager;
|
|
3089
|
+
const supervisor = new ResourceRunnerSupervisor(envelope, {
|
|
3042
3090
|
registry: this.registry,
|
|
3043
|
-
onMessage: async (message) => await this.
|
|
3091
|
+
onMessage: async (message) => await this.handleResourceRunnerMessage(active, message),
|
|
3044
3092
|
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
3045
|
-
...(this.options.
|
|
3093
|
+
...(this.options.resourceRunnerEntryPath === undefined
|
|
3046
3094
|
? {}
|
|
3047
|
-
: {
|
|
3048
|
-
...(this.options.
|
|
3095
|
+
: { runnerEntryPath: this.options.resourceRunnerEntryPath }),
|
|
3096
|
+
...(this.options.runnerStartupTimeoutMs === undefined
|
|
3049
3097
|
? {}
|
|
3050
|
-
: { startupTimeoutMs: this.options.
|
|
3051
|
-
onDiagnostic: (message) => this.log(`
|
|
3098
|
+
: { startupTimeoutMs: this.options.runnerStartupTimeoutMs }),
|
|
3099
|
+
onDiagnostic: (message) => this.log(`resource runner ${runnerEpoch}: ${message}`),
|
|
3052
3100
|
});
|
|
3053
3101
|
const renewTimer = setInterval(() => {
|
|
3054
|
-
if (!store.renewClaim(claim,
|
|
3102
|
+
if (!store.renewClaim(claim, RESOURCE_MANAGER_CLAIM_LEASE_MS)) {
|
|
3055
3103
|
void supervisor.stop("claimLost");
|
|
3056
3104
|
}
|
|
3057
|
-
},
|
|
3105
|
+
}, RESOURCE_MANAGER_RENEW_MS);
|
|
3058
3106
|
renewTimer.unref?.();
|
|
3059
3107
|
active = {
|
|
3060
3108
|
key,
|
|
@@ -3068,9 +3116,9 @@ export class WorkflowHost {
|
|
|
3068
3116
|
renewTimer,
|
|
3069
3117
|
settled: false,
|
|
3070
3118
|
};
|
|
3071
|
-
this.
|
|
3119
|
+
this.activeResourceManagers.set(key, active);
|
|
3072
3120
|
store.recordEvent({
|
|
3073
|
-
|
|
3121
|
+
resourceManager: claim.resourceManager,
|
|
3074
3122
|
key: claim.key,
|
|
3075
3123
|
claim,
|
|
3076
3124
|
type: "reconcile_started",
|
|
@@ -3080,46 +3128,46 @@ export class WorkflowHost {
|
|
|
3080
3128
|
await supervisor.start();
|
|
3081
3129
|
const result = await supervisor.wait();
|
|
3082
3130
|
if (result.outcome !== "exited" && !active.settled) {
|
|
3083
|
-
this.
|
|
3131
|
+
this.finishResourceManagerFailure(
|
|
3084
3132
|
active,
|
|
3085
|
-
result.diagnostic ?? `
|
|
3133
|
+
result.diagnostic ?? `Resource runner ${result.outcome}`,
|
|
3086
3134
|
);
|
|
3087
3135
|
}
|
|
3088
3136
|
} catch (error) {
|
|
3089
3137
|
await supervisor.stop("crashed");
|
|
3090
|
-
if (!active.settled) this.
|
|
3138
|
+
if (!active.settled) this.finishResourceManagerFailure(active, errorMessage(error));
|
|
3091
3139
|
} finally {
|
|
3092
3140
|
clearInterval(renewTimer);
|
|
3093
|
-
this.
|
|
3094
|
-
if (!this.stopping) setImmediate(() => void this.
|
|
3141
|
+
this.activeResourceManagers.delete(key);
|
|
3142
|
+
if (!this.stopping) setImmediate(() => void this.claimResourceManagerOne());
|
|
3095
3143
|
}
|
|
3096
3144
|
}
|
|
3097
3145
|
|
|
3098
|
-
private async
|
|
3099
|
-
active:
|
|
3100
|
-
message:
|
|
3101
|
-
): Promise<
|
|
3102
|
-
const current = this.
|
|
3146
|
+
private async handleResourceRunnerMessage(
|
|
3147
|
+
active: ActiveResourceManager,
|
|
3148
|
+
message: ResourceRunnerMessage,
|
|
3149
|
+
): Promise<ResourceRunnerResponse> {
|
|
3150
|
+
const current = this.activeResourceManagers.get(active.key);
|
|
3103
3151
|
if (
|
|
3104
3152
|
current !== active ||
|
|
3105
|
-
message.
|
|
3153
|
+
message.runnerEpoch !== active.supervisor.envelope.runnerEpoch ||
|
|
3106
3154
|
message.generation !== active.claim.generation
|
|
3107
3155
|
) {
|
|
3108
|
-
return
|
|
3156
|
+
return resourceRunnerResponse(message, "claimLost", undefined, "Resource runner is stale");
|
|
3157
|
+
}
|
|
3158
|
+
if (!active.store.renewClaim(active.claim, RESOURCE_MANAGER_CLAIM_LEASE_MS)) {
|
|
3159
|
+
return resourceRunnerResponse(
|
|
3109
3160
|
message,
|
|
3110
3161
|
"claimLost",
|
|
3111
3162
|
undefined,
|
|
3112
|
-
"
|
|
3163
|
+
"ResourceManager claim expired",
|
|
3113
3164
|
);
|
|
3114
3165
|
}
|
|
3115
|
-
if (!active.store.renewClaim(active.claim, CONTROLLER_CLAIM_LEASE_MS)) {
|
|
3116
|
-
return controllerWorkerResponse(message, "claimLost", undefined, "Controller claim expired");
|
|
3117
|
-
}
|
|
3118
3166
|
try {
|
|
3119
|
-
const payload = requireRecord(message.payload, "
|
|
3167
|
+
const payload = requireRecord(message.payload, "resource runner payload");
|
|
3120
3168
|
switch (message.operation) {
|
|
3121
|
-
case "
|
|
3122
|
-
return
|
|
3169
|
+
case "runner.ready":
|
|
3170
|
+
return resourceRunnerResponse(message, "accepted", {});
|
|
3123
3171
|
case "effect.reserve": {
|
|
3124
3172
|
const reservation = active.store.reserveEffect({
|
|
3125
3173
|
key: requireString(payload.key, "key"),
|
|
@@ -3129,12 +3177,12 @@ export class WorkflowHost {
|
|
|
3129
3177
|
kind: requireString(payload.kind, "kind"),
|
|
3130
3178
|
requestFingerprint: requireString(payload.requestFingerprint, "requestFingerprint"),
|
|
3131
3179
|
});
|
|
3132
|
-
return
|
|
3180
|
+
return resourceRunnerResponse(message, "accepted", reservation as unknown as JsonValue);
|
|
3133
3181
|
}
|
|
3134
3182
|
case "effect.settle": {
|
|
3135
3183
|
const state = requireString(payload.state, "state");
|
|
3136
3184
|
if (!["applied", "rejected", "indeterminate"].includes(state)) {
|
|
3137
|
-
throw new Error("
|
|
3185
|
+
throw new Error("ResourceManager effect state is invalid");
|
|
3138
3186
|
}
|
|
3139
3187
|
const record = active.store.updateEffect({
|
|
3140
3188
|
resourceUid: active.resource.metadata.uid,
|
|
@@ -3147,21 +3195,21 @@ export class WorkflowHost {
|
|
|
3147
3195
|
...(typeof payload.error === "string" ? { error: payload.error } : {}),
|
|
3148
3196
|
});
|
|
3149
3197
|
active.store.recordEvent({
|
|
3150
|
-
|
|
3198
|
+
resourceManager: active.claim.resourceManager,
|
|
3151
3199
|
key: active.claim.key,
|
|
3152
3200
|
claim: active.claim,
|
|
3153
3201
|
type: `effect_${state}`,
|
|
3154
3202
|
payload: { effectKey: record.key },
|
|
3155
3203
|
});
|
|
3156
|
-
return
|
|
3204
|
+
return resourceRunnerResponse(message, "accepted", record as unknown as JsonValue);
|
|
3157
3205
|
}
|
|
3158
3206
|
case "workflow.ensure":
|
|
3159
3207
|
case "workflow.changeSettings":
|
|
3160
3208
|
case "workflow.queueFollowUp":
|
|
3161
3209
|
case "workflow.removeFollowUp": {
|
|
3162
|
-
const workflows = new
|
|
3210
|
+
const workflows = new ResourceManagerWorkflowCoordinator(
|
|
3163
3211
|
active.store,
|
|
3164
|
-
this.
|
|
3212
|
+
this.resourceManagerWorkflowScheduler(active),
|
|
3165
3213
|
).forResource(active.resource, active.claim, new AbortController().signal);
|
|
3166
3214
|
const result =
|
|
3167
3215
|
message.operation === "workflow.ensure"
|
|
@@ -3171,39 +3219,41 @@ export class WorkflowHost {
|
|
|
3171
3219
|
: message.operation === "workflow.queueFollowUp"
|
|
3172
3220
|
? await workflows.queueFollowUp(payload as never)
|
|
3173
3221
|
: await workflows.removeFollowUp(payload as never);
|
|
3174
|
-
return
|
|
3222
|
+
return resourceRunnerResponse(message, "accepted", result as unknown as JsonValue);
|
|
3175
3223
|
}
|
|
3176
|
-
case "
|
|
3177
|
-
this.
|
|
3224
|
+
case "runner.finished":
|
|
3225
|
+
this.finishResourceManagerSuccess(
|
|
3178
3226
|
active,
|
|
3179
3227
|
message.payload as unknown as ReconcileResult<unknown>,
|
|
3180
3228
|
);
|
|
3181
|
-
return
|
|
3182
|
-
case "
|
|
3183
|
-
this.
|
|
3184
|
-
return
|
|
3229
|
+
return resourceRunnerResponse(message, "accepted", {});
|
|
3230
|
+
case "runner.failed":
|
|
3231
|
+
this.finishResourceManagerFailure(active, requireString(payload.error, "error"));
|
|
3232
|
+
return resourceRunnerResponse(message, "accepted", {});
|
|
3185
3233
|
}
|
|
3186
3234
|
} catch (error) {
|
|
3187
|
-
return
|
|
3235
|
+
return resourceRunnerResponse(message, "rejected", undefined, errorMessage(error));
|
|
3188
3236
|
}
|
|
3189
3237
|
}
|
|
3190
3238
|
|
|
3191
|
-
private
|
|
3239
|
+
private resourceManagerWorkflowScheduler(
|
|
3240
|
+
active: ActiveResourceManager,
|
|
3241
|
+
): ResourceManagerWorkflowScheduler {
|
|
3192
3242
|
return {
|
|
3193
3243
|
ensure: async (request, _signal, _onComplete) =>
|
|
3194
|
-
await this.
|
|
3244
|
+
await this.ensureResourceManagerWorkflow(active, request),
|
|
3195
3245
|
changeSettings: async (request) =>
|
|
3196
|
-
await this.
|
|
3246
|
+
await this.changeResourceManagerWorkflowSettings(active, request),
|
|
3197
3247
|
queueFollowUp: async (request) => {
|
|
3198
3248
|
const targetSessionId = this.runStore.originSessionId(request.runId);
|
|
3199
3249
|
if (targetSessionId === undefined) {
|
|
3200
|
-
throw new Error("
|
|
3250
|
+
throw new Error("ResourceManager follow-up requires an origin Pi session");
|
|
3201
3251
|
}
|
|
3202
3252
|
const result = this.runStore.queueFollowUp({
|
|
3203
3253
|
runId: request.runId,
|
|
3204
3254
|
requestId: request.actorRequestKey,
|
|
3205
3255
|
targetSessionId,
|
|
3206
|
-
actor: { type: "controller", id: request.
|
|
3256
|
+
actor: { type: "controller", id: request.managedResourceUid },
|
|
3207
3257
|
source: "controller-request",
|
|
3208
3258
|
prompt: request.prompt,
|
|
3209
3259
|
});
|
|
@@ -3219,7 +3269,7 @@ export class WorkflowHost {
|
|
|
3219
3269
|
const result = this.runStore.removeFollowUp({
|
|
3220
3270
|
runId: request.runId,
|
|
3221
3271
|
followUpId: request.followUpId,
|
|
3222
|
-
actor: { type: "controller", id: request.
|
|
3272
|
+
actor: { type: "controller", id: request.managedResourceUid },
|
|
3223
3273
|
source: "controller-request",
|
|
3224
3274
|
});
|
|
3225
3275
|
return {
|
|
@@ -3233,16 +3283,16 @@ export class WorkflowHost {
|
|
|
3233
3283
|
};
|
|
3234
3284
|
}
|
|
3235
3285
|
|
|
3236
|
-
private async
|
|
3237
|
-
_active:
|
|
3238
|
-
request:
|
|
3239
|
-
): Promise<
|
|
3286
|
+
private async changeResourceManagerWorkflowSettings(
|
|
3287
|
+
_active: ActiveResourceManager,
|
|
3288
|
+
request: ResourceManagerWorkflowControlRequest<ResourceManagerSettingsChangeRequest>,
|
|
3289
|
+
): Promise<ResourceManagerSettingsChangeResult> {
|
|
3240
3290
|
const run = this.queue.getWorkflowRun(request.runId);
|
|
3241
3291
|
if (run === undefined) throw new Error(`Workflow run not found: ${request.runId}`);
|
|
3242
3292
|
const scopes = this.runStore.listSettingsScopes(request.runId);
|
|
3243
3293
|
const scopeId = request.scopeId ?? (scopes.length === 1 ? scopes[0]?.scopeId : undefined);
|
|
3244
3294
|
if (scopeId === undefined) {
|
|
3245
|
-
throw new Error("
|
|
3295
|
+
throw new Error("ResourceManager settings changes require scopeId when several scopes exist");
|
|
3246
3296
|
}
|
|
3247
3297
|
const scope = this.runStore.getSettingsScope(scopeId);
|
|
3248
3298
|
if (scope === undefined || scope.activeRunId !== request.runId) {
|
|
@@ -3270,7 +3320,7 @@ export class WorkflowHost {
|
|
|
3270
3320
|
mountPath: scope.mountPath,
|
|
3271
3321
|
current: scope.settings,
|
|
3272
3322
|
patch: request.patch as unknown as JsonValue,
|
|
3273
|
-
actorId: request.
|
|
3323
|
+
actorId: request.managedResourceUid,
|
|
3274
3324
|
});
|
|
3275
3325
|
if (proposal.definitionDigest !== run.definitionDigest || !Array.isArray(proposal.paths)) {
|
|
3276
3326
|
throw new Error("Workflow settings proposal does not match the durable source");
|
|
@@ -3291,7 +3341,7 @@ export class WorkflowHost {
|
|
|
3291
3341
|
scopeId,
|
|
3292
3342
|
requestId: request.actorRequestKey,
|
|
3293
3343
|
expectedChangeNumber: request.expectedChangeNumber ?? scope.changeNumber,
|
|
3294
|
-
actor: { type: "controller", id: request.
|
|
3344
|
+
actor: { type: "controller", id: request.managedResourceUid },
|
|
3295
3345
|
source: "controller-request",
|
|
3296
3346
|
patch: proposal.patch,
|
|
3297
3347
|
});
|
|
@@ -3303,12 +3353,12 @@ export class WorkflowHost {
|
|
|
3303
3353
|
};
|
|
3304
3354
|
}
|
|
3305
3355
|
|
|
3306
|
-
private async
|
|
3307
|
-
active:
|
|
3356
|
+
private async ensureResourceManagerWorkflow(
|
|
3357
|
+
active: ActiveResourceManager,
|
|
3308
3358
|
request: WorkflowSchedulerRequest,
|
|
3309
3359
|
): Promise<WorkflowSchedulerResult> {
|
|
3310
3360
|
const existing = this.queue.getWorkflowRun(request.runId);
|
|
3311
|
-
if (existing !== undefined) return
|
|
3361
|
+
if (existing !== undefined) return resourceManagerWorkflowResult(existing);
|
|
3312
3362
|
const resolver = new WorkflowClient({
|
|
3313
3363
|
databasePath: this.databasePath,
|
|
3314
3364
|
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
@@ -3318,7 +3368,7 @@ export class WorkflowHost {
|
|
|
3318
3368
|
workflowRef: request.workflow,
|
|
3319
3369
|
});
|
|
3320
3370
|
const token = randomUUID();
|
|
3321
|
-
const scoped = new
|
|
3371
|
+
const scoped = new SqliteResourceManagerStore(this.databasePath, {
|
|
3322
3372
|
state: this.state,
|
|
3323
3373
|
projectPath: active.projectPath,
|
|
3324
3374
|
});
|
|
@@ -3331,26 +3381,26 @@ export class WorkflowHost {
|
|
|
3331
3381
|
definitionSnapshot: resolved.definitionSnapshot,
|
|
3332
3382
|
input: request.input,
|
|
3333
3383
|
launchOptions: {},
|
|
3334
|
-
runnerId: this.
|
|
3384
|
+
runnerId: this.serverId,
|
|
3335
3385
|
claimToken: token,
|
|
3336
3386
|
leaseMs: this.runClaimLeaseMs,
|
|
3337
|
-
originSessionId: `
|
|
3387
|
+
originSessionId: `resource-manager-${active.resource.metadata.uid}`,
|
|
3338
3388
|
executionMode: "headless",
|
|
3339
3389
|
});
|
|
3340
3390
|
if (prepared.state === "claimed") {
|
|
3341
3391
|
this.markPendingStart(request.runId, token);
|
|
3342
3392
|
setImmediate(() => void this.activateRun(prepared.run, token));
|
|
3343
3393
|
}
|
|
3344
|
-
return
|
|
3394
|
+
return resourceManagerWorkflowResult(prepared.run);
|
|
3345
3395
|
}
|
|
3346
3396
|
|
|
3347
|
-
private
|
|
3348
|
-
active:
|
|
3397
|
+
private finishResourceManagerSuccess(
|
|
3398
|
+
active: ActiveResourceManager,
|
|
3349
3399
|
result: ReconcileResult<unknown>,
|
|
3350
3400
|
): void {
|
|
3351
3401
|
if (active.settled) return;
|
|
3352
3402
|
const now = new Date();
|
|
3353
|
-
const ref = {
|
|
3403
|
+
const ref = { resourceManager: active.claim.resourceManager, key: active.claim.key };
|
|
3354
3404
|
const status = applyStatusPatch(
|
|
3355
3405
|
active.resource.status,
|
|
3356
3406
|
result.status,
|
|
@@ -3405,7 +3455,7 @@ export class WorkflowHost {
|
|
|
3405
3455
|
}
|
|
3406
3456
|
active.settled = true;
|
|
3407
3457
|
} catch (error) {
|
|
3408
|
-
if (error instanceof
|
|
3458
|
+
if (error instanceof ManagedResourceConflictError) {
|
|
3409
3459
|
active.store.recordEvent({
|
|
3410
3460
|
...ref,
|
|
3411
3461
|
claim: active.claim,
|
|
@@ -3424,13 +3474,13 @@ export class WorkflowHost {
|
|
|
3424
3474
|
}
|
|
3425
3475
|
}
|
|
3426
3476
|
|
|
3427
|
-
private
|
|
3477
|
+
private finishResourceManagerFailure(active: ActiveResourceManager, failure: string): void {
|
|
3428
3478
|
if (active.settled) return;
|
|
3429
3479
|
const now = new Date();
|
|
3430
3480
|
const delay = Math.min(60_000, 1_000 * 2 ** Math.min(active.claim.consecutiveErrors, 6));
|
|
3431
3481
|
const error = failure.slice(0, 8_192);
|
|
3432
3482
|
active.store.recordEvent({
|
|
3433
|
-
|
|
3483
|
+
resourceManager: active.claim.resourceManager,
|
|
3434
3484
|
key: active.claim.key,
|
|
3435
3485
|
claim: active.claim,
|
|
3436
3486
|
type: "reconcile_failed",
|
|
@@ -3458,7 +3508,7 @@ export class WorkflowHost {
|
|
|
3458
3508
|
const candidate = requests
|
|
3459
3509
|
.map((request) => ({
|
|
3460
3510
|
request,
|
|
3461
|
-
interaction: this.
|
|
3511
|
+
interaction: this.serverState.getInteraction(request.decisionId),
|
|
3462
3512
|
}))
|
|
3463
3513
|
.find(
|
|
3464
3514
|
({ request, interaction }) =>
|
|
@@ -3479,7 +3529,7 @@ export class WorkflowHost {
|
|
|
3479
3529
|
const claimToken = randomUUID();
|
|
3480
3530
|
const claimed = this.queue.claimWorkflowRunForControl({
|
|
3481
3531
|
runId: request.runId,
|
|
3482
|
-
runnerId: this.
|
|
3532
|
+
runnerId: this.serverId,
|
|
3483
3533
|
claimToken,
|
|
3484
3534
|
leaseMs: this.runClaimLeaseMs,
|
|
3485
3535
|
});
|
|
@@ -3491,7 +3541,7 @@ export class WorkflowHost {
|
|
|
3491
3541
|
this.state.transaction(() => {
|
|
3492
3542
|
const accepted = this.decisions.resolveTimeoutSync(request, new Date());
|
|
3493
3543
|
const submissionId = `timeout-${request.decisionId}`;
|
|
3494
|
-
this.
|
|
3544
|
+
this.serverState.submitInteraction({
|
|
3495
3545
|
requestId: interaction.requestId,
|
|
3496
3546
|
submissionId,
|
|
3497
3547
|
idempotencyKey: submissionId,
|
|
@@ -3528,7 +3578,7 @@ export class WorkflowHost {
|
|
|
3528
3578
|
.filter(([, coordinator]) => coordinator.branchReported && coordinator.modelTurnActive)
|
|
3529
3579
|
.map(([sessionId]) => sessionId),
|
|
3530
3580
|
);
|
|
3531
|
-
const expired = this.
|
|
3581
|
+
const expired = this.serverState
|
|
3532
3582
|
.expiredInteractionRuns()
|
|
3533
3583
|
.find(
|
|
3534
3584
|
(candidate) =>
|
|
@@ -3541,7 +3591,7 @@ export class WorkflowHost {
|
|
|
3541
3591
|
const claimToken = randomUUID();
|
|
3542
3592
|
const claimed = this.queue.claimWorkflowRunForControl({
|
|
3543
3593
|
runId,
|
|
3544
|
-
runnerId: this.
|
|
3594
|
+
runnerId: this.serverId,
|
|
3545
3595
|
claimToken,
|
|
3546
3596
|
leaseMs: this.runClaimLeaseMs,
|
|
3547
3597
|
});
|
|
@@ -3583,14 +3633,14 @@ export class WorkflowHost {
|
|
|
3583
3633
|
...this.pendingStarts,
|
|
3584
3634
|
...this.blockedRuns,
|
|
3585
3635
|
]);
|
|
3586
|
-
const validatingRunId = this.
|
|
3636
|
+
const validatingRunId = this.serverState
|
|
3587
3637
|
.validatingInteractionRunIds()
|
|
3588
3638
|
.find((runId) => !excluded.has(runId));
|
|
3589
3639
|
if (validatingRunId !== undefined) {
|
|
3590
3640
|
const validationToken = randomUUID();
|
|
3591
3641
|
const validationRun = this.queue.claimWorkflowRunForInteractionValidation({
|
|
3592
3642
|
runId: validatingRunId,
|
|
3593
|
-
runnerId: this.
|
|
3643
|
+
runnerId: this.serverId,
|
|
3594
3644
|
claimToken: validationToken,
|
|
3595
3645
|
leaseMs: this.runClaimLeaseMs,
|
|
3596
3646
|
});
|
|
@@ -3602,7 +3652,7 @@ export class WorkflowHost {
|
|
|
3602
3652
|
}
|
|
3603
3653
|
const token = randomUUID();
|
|
3604
3654
|
const claimed = this.queue.claimNextWorkflowRun({
|
|
3605
|
-
runnerId: this.
|
|
3655
|
+
runnerId: this.serverId,
|
|
3606
3656
|
claimToken: token,
|
|
3607
3657
|
leaseMs: this.runClaimLeaseMs,
|
|
3608
3658
|
excludeRunIds: [...excluded],
|
|
@@ -3653,38 +3703,38 @@ export class WorkflowHost {
|
|
|
3653
3703
|
this.queue.parkWorkflowRun({ runId, claimToken });
|
|
3654
3704
|
return;
|
|
3655
3705
|
}
|
|
3656
|
-
const envelope:
|
|
3706
|
+
const envelope: WorkflowRunnerLaunchEnvelope = {
|
|
3657
3707
|
schema: "pi-workflows.worker-launch.v1",
|
|
3658
3708
|
runId,
|
|
3659
3709
|
generation,
|
|
3660
|
-
|
|
3710
|
+
runnerEpoch: randomUUID(),
|
|
3661
3711
|
projectPath,
|
|
3662
3712
|
workflowSource: record.workflowSource as JsonValue,
|
|
3663
3713
|
definitionDigest: record.definitionDigest,
|
|
3664
3714
|
inputHash: `sha256:${createHash("sha256").update(canonicalJson(record.input)).digest("hex")}`,
|
|
3665
3715
|
protocolVersion: 1,
|
|
3666
3716
|
};
|
|
3667
|
-
this.
|
|
3717
|
+
this.serverState.recordRunnerStart(envelope, this.claim.epoch);
|
|
3668
3718
|
let active: ActiveRun;
|
|
3669
|
-
const supervisor = new
|
|
3719
|
+
const supervisor = new WorkflowRunnerSupervisor(envelope, {
|
|
3670
3720
|
registry: this.registry,
|
|
3671
|
-
onMessage: async (message) => await this.
|
|
3721
|
+
onMessage: async (message) => await this.handleRunnerMessage(message),
|
|
3672
3722
|
...(this.options.env === undefined ? {} : { env: this.options.env }),
|
|
3673
|
-
...(this.options.
|
|
3723
|
+
...(this.options.runnerEntryPath === undefined
|
|
3674
3724
|
? {}
|
|
3675
|
-
: {
|
|
3676
|
-
...(this.options.
|
|
3725
|
+
: { runnerEntryPath: this.options.runnerEntryPath }),
|
|
3726
|
+
...(this.options.runnerStartupTimeoutMs === undefined
|
|
3677
3727
|
? {}
|
|
3678
|
-
: { startupTimeoutMs: this.options.
|
|
3728
|
+
: { startupTimeoutMs: this.options.runnerStartupTimeoutMs }),
|
|
3679
3729
|
onSpawn: (identity) => {
|
|
3680
|
-
active.
|
|
3681
|
-
this.
|
|
3682
|
-
envelope.
|
|
3730
|
+
active.runnerPid = identity.pid;
|
|
3731
|
+
this.serverState.attachRunnerProcess(
|
|
3732
|
+
envelope.runnerEpoch,
|
|
3683
3733
|
identity.pid,
|
|
3684
3734
|
identity.startIdentity,
|
|
3685
3735
|
);
|
|
3686
3736
|
},
|
|
3687
|
-
onDiagnostic: (message) => this.log(`
|
|
3737
|
+
onDiagnostic: (message) => this.log(`runner ${envelope.runnerEpoch}: ${message}`),
|
|
3688
3738
|
});
|
|
3689
3739
|
active = {
|
|
3690
3740
|
record,
|
|
@@ -3693,6 +3743,7 @@ export class WorkflowHost {
|
|
|
3693
3743
|
supervisor,
|
|
3694
3744
|
launchProgressRevision: runRevision(this.state, runId),
|
|
3695
3745
|
exiting: false,
|
|
3746
|
+
contentDigests: new Set(),
|
|
3696
3747
|
};
|
|
3697
3748
|
this.activeRuns.set(runId, active);
|
|
3698
3749
|
this.clearPendingStart(runId, claimToken);
|
|
@@ -3703,8 +3754,8 @@ export class WorkflowHost {
|
|
|
3703
3754
|
if (!this.queue.parkWorkflowRunForAmbiguousEffect({ runId, claimToken })) {
|
|
3704
3755
|
throw new Error("Run claim was lost during ambiguous-effect recovery");
|
|
3705
3756
|
}
|
|
3706
|
-
this.
|
|
3707
|
-
|
|
3757
|
+
this.serverState.finishRunner({
|
|
3758
|
+
runnerEpoch: envelope.runnerEpoch,
|
|
3708
3759
|
outcome: "orphaned",
|
|
3709
3760
|
diagnostic: "effect outcome is ambiguous",
|
|
3710
3761
|
});
|
|
@@ -3712,8 +3763,8 @@ export class WorkflowHost {
|
|
|
3712
3763
|
}
|
|
3713
3764
|
await supervisor.start();
|
|
3714
3765
|
const result = await supervisor.wait();
|
|
3715
|
-
this.
|
|
3716
|
-
|
|
3766
|
+
this.serverState.finishRunner({
|
|
3767
|
+
runnerEpoch: envelope.runnerEpoch,
|
|
3717
3768
|
outcome: result.outcome,
|
|
3718
3769
|
exitCode: result.exitCode,
|
|
3719
3770
|
signal: result.signal,
|
|
@@ -3722,30 +3773,30 @@ export class WorkflowHost {
|
|
|
3722
3773
|
if (
|
|
3723
3774
|
active.control === undefined &&
|
|
3724
3775
|
(result.outcome !== "exited" ||
|
|
3725
|
-
this.
|
|
3776
|
+
this.serverState.validatingInteraction(active.record.runId) !== undefined)
|
|
3726
3777
|
) {
|
|
3727
|
-
await this.
|
|
3778
|
+
await this.recoverRunnerExit(active, result.outcome);
|
|
3728
3779
|
}
|
|
3729
3780
|
} catch (error) {
|
|
3730
|
-
this.log(`
|
|
3781
|
+
this.log(`runner ${envelope.runnerEpoch} failed: ${errorMessage(error)}`);
|
|
3731
3782
|
await supervisor.stop("crashed");
|
|
3732
3783
|
const result = await supervisor.wait();
|
|
3733
3784
|
try {
|
|
3734
|
-
this.
|
|
3735
|
-
|
|
3785
|
+
this.serverState.finishRunner({
|
|
3786
|
+
runnerEpoch: envelope.runnerEpoch,
|
|
3736
3787
|
outcome: "crashed",
|
|
3737
3788
|
exitCode: result.exitCode,
|
|
3738
3789
|
signal: result.signal,
|
|
3739
3790
|
diagnostic: result.diagnostic ?? errorMessage(error),
|
|
3740
3791
|
});
|
|
3741
3792
|
} catch {
|
|
3742
|
-
// A completed
|
|
3793
|
+
// A completed runner record wins.
|
|
3743
3794
|
}
|
|
3744
3795
|
if (active.control === undefined) {
|
|
3745
|
-
await this.
|
|
3796
|
+
await this.recoverRunnerExit(active, "crashed");
|
|
3746
3797
|
}
|
|
3747
3798
|
} finally {
|
|
3748
|
-
this.
|
|
3799
|
+
this.reapRunnerDescendants(envelope.runnerEpoch);
|
|
3749
3800
|
this.activeRuns.delete(runId);
|
|
3750
3801
|
}
|
|
3751
3802
|
}
|
|
@@ -3755,7 +3806,7 @@ export class WorkflowHost {
|
|
|
3755
3806
|
const token = randomUUID();
|
|
3756
3807
|
const claimed = this.queue.claimWorkflowRunForInteractionValidation({
|
|
3757
3808
|
runId,
|
|
3758
|
-
runnerId: this.
|
|
3809
|
+
runnerId: this.serverId,
|
|
3759
3810
|
claimToken: token,
|
|
3760
3811
|
leaseMs: this.runClaimLeaseMs,
|
|
3761
3812
|
});
|
|
@@ -3767,50 +3818,55 @@ export class WorkflowHost {
|
|
|
3767
3818
|
await this.activateRun(claimed, token);
|
|
3768
3819
|
}
|
|
3769
3820
|
|
|
3770
|
-
private async
|
|
3821
|
+
private async handleRunnerMessage(
|
|
3822
|
+
message: WorkflowRunnerMessage,
|
|
3823
|
+
): Promise<WorkflowRunnerResponse> {
|
|
3771
3824
|
const active = this.activeRuns.get(message.runId);
|
|
3772
3825
|
if (
|
|
3773
3826
|
active === undefined ||
|
|
3774
3827
|
active.generation !== message.generation ||
|
|
3775
|
-
active.supervisor.envelope.
|
|
3828
|
+
active.supervisor.envelope.runnerEpoch !== message.runnerEpoch
|
|
3776
3829
|
) {
|
|
3777
|
-
return
|
|
3830
|
+
return runnerResponse(message, "claimLost", undefined, "Runner epoch is stale");
|
|
3778
3831
|
}
|
|
3779
|
-
const stored = this.
|
|
3780
|
-
if (stored !== undefined)
|
|
3781
|
-
|
|
3782
|
-
|
|
3832
|
+
const stored = this.serverState.readRunnerMessage(message);
|
|
3833
|
+
if (stored !== undefined) {
|
|
3834
|
+
return boundRunnerResponse(this.state, active.contentDigests, stored);
|
|
3835
|
+
}
|
|
3836
|
+
const response = await this.handleFreshRunnerMessage(active, message);
|
|
3837
|
+
const recorded = this.serverState.recordRunnerMessage(message, response);
|
|
3838
|
+
return boundRunnerResponse(this.state, active.contentDigests, recorded);
|
|
3783
3839
|
}
|
|
3784
3840
|
|
|
3785
|
-
private async
|
|
3841
|
+
private async handleFreshRunnerMessage(
|
|
3786
3842
|
active: ActiveRun,
|
|
3787
|
-
message:
|
|
3788
|
-
): Promise<
|
|
3789
|
-
if (message.operation === "
|
|
3843
|
+
message: WorkflowRunnerMessage,
|
|
3844
|
+
): Promise<WorkflowRunnerResponse> {
|
|
3845
|
+
if (message.operation === "runner.ready") {
|
|
3790
3846
|
if (
|
|
3791
3847
|
!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
|
|
3792
3848
|
) {
|
|
3793
3849
|
active.claimLost = true;
|
|
3794
3850
|
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
3795
|
-
return
|
|
3851
|
+
return runnerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
3796
3852
|
}
|
|
3797
|
-
this.
|
|
3853
|
+
this.serverState.markRunnerReady(message.runnerEpoch);
|
|
3798
3854
|
this.queue.markWorkflowRunRunning({ runId: message.runId, claimToken: active.claimToken });
|
|
3799
3855
|
const revision = this.runStore.synchronizeRevision(message.runId);
|
|
3800
3856
|
active.launchProgressRevision = revision;
|
|
3801
3857
|
const current = this.queue.getWorkflowRun(message.runId);
|
|
3802
3858
|
const candidateInteraction =
|
|
3803
|
-
this.
|
|
3804
|
-
this.
|
|
3805
|
-
const timedOutInteraction = this.
|
|
3859
|
+
this.serverState.acceptedInteraction(message.runId) ??
|
|
3860
|
+
this.serverState.validatingInteraction(message.runId);
|
|
3861
|
+
const timedOutInteraction = this.serverState.timedOutInteraction(message.runId);
|
|
3806
3862
|
const resumeInteractionAttemptId =
|
|
3807
3863
|
candidateInteraction?.attemptId ?? timedOutInteraction?.attemptId;
|
|
3808
3864
|
const record = current ?? active.record;
|
|
3809
|
-
return
|
|
3865
|
+
return runnerResponse(
|
|
3810
3866
|
message,
|
|
3811
3867
|
"accepted",
|
|
3812
3868
|
{
|
|
3813
|
-
command:
|
|
3869
|
+
command: runnerRunCommand(record, resumeInteractionAttemptId),
|
|
3814
3870
|
originSessionId: record.originSessionId,
|
|
3815
3871
|
stateDirectory: this.stateDirectory,
|
|
3816
3872
|
...(candidateInteraction === undefined ? {} : { candidateInteraction }),
|
|
@@ -3820,8 +3876,8 @@ export class WorkflowHost {
|
|
|
3820
3876
|
revision,
|
|
3821
3877
|
);
|
|
3822
3878
|
}
|
|
3823
|
-
if (message.operation === "
|
|
3824
|
-
const payload = requireRecord(message.payload, "
|
|
3879
|
+
if (message.operation === "runner.exiting") {
|
|
3880
|
+
const payload = requireRecord(message.payload, "runner exit payload");
|
|
3825
3881
|
if (payload.status === "workflowLoadFailed") {
|
|
3826
3882
|
active.workflowLoadFailure = requireString(payload.error, "workflow load error").slice(
|
|
3827
3883
|
0,
|
|
@@ -3829,14 +3885,14 @@ export class WorkflowHost {
|
|
|
3829
3885
|
);
|
|
3830
3886
|
}
|
|
3831
3887
|
active.exiting = true;
|
|
3832
|
-
return
|
|
3888
|
+
return runnerResponse(message, "accepted", {});
|
|
3833
3889
|
}
|
|
3834
3890
|
if (message.operation === "process.register" || message.operation === "process.unregister") {
|
|
3835
|
-
return this.
|
|
3891
|
+
return this.handleRunnerProcessOperation(active, message);
|
|
3836
3892
|
}
|
|
3837
3893
|
const currentRevision = runRevision(this.state, message.runId);
|
|
3838
3894
|
if (message.expectedRevision !== currentRevision) {
|
|
3839
|
-
return
|
|
3895
|
+
return runnerResponse(
|
|
3840
3896
|
message,
|
|
3841
3897
|
"rejected",
|
|
3842
3898
|
undefined,
|
|
@@ -3849,10 +3905,10 @@ export class WorkflowHost {
|
|
|
3849
3905
|
) {
|
|
3850
3906
|
active.claimLost = true;
|
|
3851
3907
|
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
3852
|
-
return
|
|
3908
|
+
return runnerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
3853
3909
|
}
|
|
3854
3910
|
try {
|
|
3855
|
-
const payload = requireRecord(message.payload, "
|
|
3911
|
+
const payload = requireRecord(message.payload, "runner payload");
|
|
3856
3912
|
let result: unknown;
|
|
3857
3913
|
switch (message.operation) {
|
|
3858
3914
|
case "store.initializeRun":
|
|
@@ -3867,11 +3923,8 @@ export class WorkflowHost {
|
|
|
3867
3923
|
await this.prepareInteractionResume(message.runId);
|
|
3868
3924
|
result = await this.runStore.prepareRunResume(message.runId);
|
|
3869
3925
|
break;
|
|
3870
|
-
case "store.
|
|
3871
|
-
result = this.runStore.
|
|
3872
|
-
requireString(payload.runId, "runId"),
|
|
3873
|
-
payload.options as never,
|
|
3874
|
-
);
|
|
3926
|
+
case "store.readRunState":
|
|
3927
|
+
result = this.runStore.readRunState(requireString(payload.runId, "runId"));
|
|
3875
3928
|
break;
|
|
3876
3929
|
case "store.writeSnapshot": {
|
|
3877
3930
|
const state = payload.state as WorkflowRunState;
|
|
@@ -3914,7 +3967,7 @@ export class WorkflowHost {
|
|
|
3914
3967
|
const decision = payload.request as HumanDecisionRequest;
|
|
3915
3968
|
result = await this.runStore.createHumanDecisionRequest(decision);
|
|
3916
3969
|
if (active.record.originSessionId !== null) {
|
|
3917
|
-
this.
|
|
3970
|
+
this.serverState.createInteractiveRequest({
|
|
3918
3971
|
requestId: decision.decisionId,
|
|
3919
3972
|
runId: decision.runId,
|
|
3920
3973
|
attemptId: decision.attemptId,
|
|
@@ -3944,6 +3997,14 @@ export class WorkflowHost {
|
|
|
3944
3997
|
throw new Error("Run claim was lost during ambiguous-effect recovery");
|
|
3945
3998
|
}
|
|
3946
3999
|
break;
|
|
4000
|
+
case "content.read":
|
|
4001
|
+
result = readRunnerContentChunk(
|
|
4002
|
+
this.state,
|
|
4003
|
+
active.contentDigests,
|
|
4004
|
+
requireString(payload.sha256, "runner content digest"),
|
|
4005
|
+
requireNonNegativeInteger(payload.offset, "runner content offset"),
|
|
4006
|
+
);
|
|
4007
|
+
break;
|
|
3947
4008
|
case "store.settleEffect": {
|
|
3948
4009
|
const options = payload.options as {
|
|
3949
4010
|
outcome?: unknown;
|
|
@@ -3962,16 +4023,16 @@ export class WorkflowHost {
|
|
|
3962
4023
|
break;
|
|
3963
4024
|
}
|
|
3964
4025
|
case "interaction.accept":
|
|
3965
|
-
result = this.
|
|
4026
|
+
result = this.acceptRunnerInteraction(message, payload);
|
|
3966
4027
|
break;
|
|
3967
4028
|
case "interaction.reject":
|
|
3968
|
-
result = this.
|
|
4029
|
+
result = this.rejectRunnerInteraction(active, message, payload);
|
|
3969
4030
|
break;
|
|
3970
4031
|
case "notification.request":
|
|
3971
|
-
result = this.
|
|
4032
|
+
result = this.enqueueRunnerNotification(active, message, payload);
|
|
3972
4033
|
break;
|
|
3973
4034
|
case "presentation.request":
|
|
3974
|
-
result = this.
|
|
4035
|
+
result = this.requestRunnerPresentation(active, message, payload);
|
|
3975
4036
|
break;
|
|
3976
4037
|
case "interaction.request": {
|
|
3977
4038
|
result = this.parkForInteraction(active, message, payload);
|
|
@@ -3985,9 +4046,9 @@ export class WorkflowHost {
|
|
|
3985
4046
|
break;
|
|
3986
4047
|
}
|
|
3987
4048
|
default:
|
|
3988
|
-
return
|
|
4049
|
+
return runnerResponse(message, "rejected", undefined, "Unknown runner operation");
|
|
3989
4050
|
}
|
|
3990
|
-
return
|
|
4051
|
+
return runnerResponse(
|
|
3991
4052
|
message,
|
|
3992
4053
|
"accepted",
|
|
3993
4054
|
result === undefined ? null : (result as JsonValue),
|
|
@@ -3995,64 +4056,67 @@ export class WorkflowHost {
|
|
|
3995
4056
|
runRevision(this.state, message.runId),
|
|
3996
4057
|
);
|
|
3997
4058
|
} catch (error) {
|
|
3998
|
-
return
|
|
4059
|
+
return runnerResponse(message, "rejected", undefined, errorMessage(error));
|
|
3999
4060
|
}
|
|
4000
4061
|
}
|
|
4001
4062
|
|
|
4002
|
-
private
|
|
4063
|
+
private handleRunnerProcessOperation(
|
|
4064
|
+
active: ActiveRun,
|
|
4065
|
+
message: WorkflowRunnerMessage,
|
|
4066
|
+
): WorkflowRunnerResponse {
|
|
4003
4067
|
if (
|
|
4004
4068
|
message.operation === "process.register" &&
|
|
4005
4069
|
!this.queue.verifyWorkflowRunClaim({ runId: message.runId, claimToken: active.claimToken })
|
|
4006
4070
|
) {
|
|
4007
4071
|
active.claimLost = true;
|
|
4008
4072
|
setImmediate(() => void active.supervisor.stop("claimLost"));
|
|
4009
|
-
return
|
|
4073
|
+
return runnerResponse(message, "claimLost", undefined, "Run claim expired");
|
|
4010
4074
|
}
|
|
4011
4075
|
try {
|
|
4012
|
-
const payload = requireRecord(message.payload, "
|
|
4076
|
+
const payload = requireRecord(message.payload, "runner process payload");
|
|
4013
4077
|
const pid = requireNonNegativeInteger(payload.pid, "pid");
|
|
4014
|
-
if (pid === 0) throw new Error("
|
|
4015
|
-
const owned = this.
|
|
4078
|
+
if (pid === 0) throw new Error("Runner process PID must be positive");
|
|
4079
|
+
const owned = this.runnerDescendants.get(message.runnerEpoch);
|
|
4016
4080
|
if (message.operation === "process.unregister") {
|
|
4017
4081
|
if (owned?.has(pid) !== true) {
|
|
4018
|
-
throw new Error("
|
|
4082
|
+
throw new Error("Runner process is not registered to this runner epoch");
|
|
4019
4083
|
}
|
|
4020
4084
|
this.registry.unregister(pid);
|
|
4021
4085
|
owned.delete(pid);
|
|
4022
|
-
if (owned.size === 0) this.
|
|
4023
|
-
return
|
|
4086
|
+
if (owned.size === 0) this.runnerDescendants.delete(message.runnerEpoch);
|
|
4087
|
+
return runnerResponse(message, "accepted", { pid });
|
|
4024
4088
|
}
|
|
4025
|
-
if (active.
|
|
4026
|
-
throw new Error("
|
|
4089
|
+
if (active.runnerPid === undefined || processParentPid(pid) !== active.runnerPid) {
|
|
4090
|
+
throw new Error("Runner process is not a direct child of the active runner");
|
|
4027
4091
|
}
|
|
4028
|
-
for (const [
|
|
4029
|
-
if (
|
|
4030
|
-
throw new Error("
|
|
4092
|
+
for (const [runnerEpoch, pids] of this.runnerDescendants) {
|
|
4093
|
+
if (runnerEpoch !== message.runnerEpoch && pids.has(pid)) {
|
|
4094
|
+
throw new Error("Runner process is already registered to another runner epoch");
|
|
4031
4095
|
}
|
|
4032
4096
|
}
|
|
4033
4097
|
this.registry.register(pid);
|
|
4034
4098
|
const descendants = owned ?? new Set<number>();
|
|
4035
4099
|
descendants.add(pid);
|
|
4036
|
-
this.
|
|
4037
|
-
return
|
|
4100
|
+
this.runnerDescendants.set(message.runnerEpoch, descendants);
|
|
4101
|
+
return runnerResponse(message, "accepted", { pid });
|
|
4038
4102
|
} catch (error) {
|
|
4039
|
-
return
|
|
4103
|
+
return runnerResponse(message, "rejected", undefined, errorMessage(error));
|
|
4040
4104
|
}
|
|
4041
4105
|
}
|
|
4042
4106
|
|
|
4043
|
-
private
|
|
4044
|
-
const descendants = this.
|
|
4107
|
+
private reapRunnerDescendants(runnerEpoch: string): void {
|
|
4108
|
+
const descendants = this.runnerDescendants.get(runnerEpoch);
|
|
4045
4109
|
if (descendants === undefined) return;
|
|
4046
|
-
this.
|
|
4110
|
+
this.runnerDescendants.delete(runnerEpoch);
|
|
4047
4111
|
for (const pid of descendants) this.registry.kill(pid);
|
|
4048
4112
|
}
|
|
4049
4113
|
|
|
4050
|
-
private
|
|
4051
|
-
message:
|
|
4114
|
+
private acceptRunnerInteraction(
|
|
4115
|
+
message: WorkflowRunnerMessage,
|
|
4052
4116
|
payload: Record<string, unknown>,
|
|
4053
4117
|
): InteractiveSubmissionRecord {
|
|
4054
|
-
const candidate = this.
|
|
4055
|
-
return this.
|
|
4118
|
+
const candidate = this.requireRunnerInteraction(message, payload, true);
|
|
4119
|
+
return this.serverState.finishInteractionValidation({
|
|
4056
4120
|
requestId: candidate.requestId,
|
|
4057
4121
|
submissionId: candidate.submissionId,
|
|
4058
4122
|
accepted: true,
|
|
@@ -4060,12 +4124,12 @@ export class WorkflowHost {
|
|
|
4060
4124
|
});
|
|
4061
4125
|
}
|
|
4062
4126
|
|
|
4063
|
-
private
|
|
4127
|
+
private rejectRunnerInteraction(
|
|
4064
4128
|
active: ActiveRun,
|
|
4065
|
-
message:
|
|
4129
|
+
message: WorkflowRunnerMessage,
|
|
4066
4130
|
payload: Record<string, unknown>,
|
|
4067
4131
|
): InteractiveSubmissionRecord {
|
|
4068
|
-
const candidate = this.
|
|
4132
|
+
const candidate = this.requireRunnerInteraction(message, payload, false);
|
|
4069
4133
|
return this.settleRejectedInteraction(
|
|
4070
4134
|
active,
|
|
4071
4135
|
candidate,
|
|
@@ -4073,14 +4137,14 @@ export class WorkflowHost {
|
|
|
4073
4137
|
);
|
|
4074
4138
|
}
|
|
4075
4139
|
|
|
4076
|
-
private
|
|
4077
|
-
message:
|
|
4140
|
+
private requireRunnerInteraction(
|
|
4141
|
+
message: WorkflowRunnerMessage,
|
|
4078
4142
|
payload: Record<string, unknown>,
|
|
4079
4143
|
acceptSettled: boolean,
|
|
4080
4144
|
) {
|
|
4081
4145
|
const candidate =
|
|
4082
|
-
this.
|
|
4083
|
-
(acceptSettled ? this.
|
|
4146
|
+
this.serverState.validatingInteraction(message.runId) ??
|
|
4147
|
+
(acceptSettled ? this.serverState.acceptedInteraction(message.runId) : undefined);
|
|
4084
4148
|
if (
|
|
4085
4149
|
candidate === undefined ||
|
|
4086
4150
|
candidate.requestId !== requireString(payload.requestId, "interaction requestId") ||
|
|
@@ -4088,7 +4152,7 @@ export class WorkflowHost {
|
|
|
4088
4152
|
candidate.attemptId !== message.attemptId ||
|
|
4089
4153
|
candidate.attemptId !== requireString(payload.attemptId, "interaction attemptId")
|
|
4090
4154
|
) {
|
|
4091
|
-
throw new Error("Interactive submission does not match the
|
|
4155
|
+
throw new Error("Interactive submission does not match the runner candidate");
|
|
4092
4156
|
}
|
|
4093
4157
|
return candidate;
|
|
4094
4158
|
}
|
|
@@ -4103,7 +4167,7 @@ export class WorkflowHost {
|
|
|
4103
4167
|
error: string,
|
|
4104
4168
|
): InteractiveSubmissionRecord {
|
|
4105
4169
|
const result = this.state.transaction(() => {
|
|
4106
|
-
const submission = this.
|
|
4170
|
+
const submission = this.serverState.finishInteractionValidation({
|
|
4107
4171
|
requestId: candidate.requestId,
|
|
4108
4172
|
submissionId: candidate.submissionId,
|
|
4109
4173
|
accepted: false,
|
|
@@ -4142,9 +4206,9 @@ export class WorkflowHost {
|
|
|
4142
4206
|
return result;
|
|
4143
4207
|
}
|
|
4144
4208
|
|
|
4145
|
-
private
|
|
4209
|
+
private enqueueRunnerNotification(
|
|
4146
4210
|
active: ActiveRun,
|
|
4147
|
-
message:
|
|
4211
|
+
message: WorkflowRunnerMessage,
|
|
4148
4212
|
payload: Record<string, unknown>,
|
|
4149
4213
|
): JsonValue {
|
|
4150
4214
|
if (active.record.originSessionId === null) {
|
|
@@ -4172,7 +4236,7 @@ export class WorkflowHost {
|
|
|
4172
4236
|
const content = requireString(request.content, "notification.content");
|
|
4173
4237
|
const notificationId = `notification-${message.runId}-${attemptId}-${notificationIndex}`;
|
|
4174
4238
|
const workflowMessageId = workflowMessageIdFor("notification", notificationId, "initial");
|
|
4175
|
-
const record = this.
|
|
4239
|
+
const record = this.serverState.workflowMessages.create({
|
|
4176
4240
|
workflowMessageId,
|
|
4177
4241
|
runId: message.runId,
|
|
4178
4242
|
targetSessionId: active.record.originSessionId,
|
|
@@ -4193,9 +4257,9 @@ export class WorkflowHost {
|
|
|
4193
4257
|
};
|
|
4194
4258
|
}
|
|
4195
4259
|
|
|
4196
|
-
private
|
|
4260
|
+
private requestRunnerPresentation(
|
|
4197
4261
|
active: ActiveRun,
|
|
4198
|
-
message:
|
|
4262
|
+
message: WorkflowRunnerMessage,
|
|
4199
4263
|
payload: Record<string, unknown>,
|
|
4200
4264
|
): JsonValue {
|
|
4201
4265
|
if (active.record.originSessionId === null) {
|
|
@@ -4227,7 +4291,7 @@ export class WorkflowHost {
|
|
|
4227
4291
|
|
|
4228
4292
|
private parkForInteraction(
|
|
4229
4293
|
active: ActiveRun,
|
|
4230
|
-
message:
|
|
4294
|
+
message: WorkflowRunnerMessage,
|
|
4231
4295
|
payload: Record<string, unknown>,
|
|
4232
4296
|
): JsonValue {
|
|
4233
4297
|
if (active.record.originSessionId === null) {
|
|
@@ -4236,7 +4300,7 @@ export class WorkflowHost {
|
|
|
4236
4300
|
const attemptId = requireString(payload.attemptId, "attemptId");
|
|
4237
4301
|
const requestId = `interaction-${message.runId}-${attemptId}`;
|
|
4238
4302
|
return this.state.transaction(() => {
|
|
4239
|
-
const request = this.
|
|
4303
|
+
const request = this.serverState.createInteractiveRequest({
|
|
4240
4304
|
requestId,
|
|
4241
4305
|
runId: message.runId,
|
|
4242
4306
|
attemptId,
|
|
@@ -4272,9 +4336,9 @@ export class WorkflowHost {
|
|
|
4272
4336
|
}
|
|
4273
4337
|
|
|
4274
4338
|
private async prepareInteractionResume(runId: string): Promise<void> {
|
|
4275
|
-
const accepted = this.
|
|
4276
|
-
const candidate = accepted ?? this.
|
|
4277
|
-
const timedOut = this.
|
|
4339
|
+
const accepted = this.serverState.acceptedInteraction(runId);
|
|
4340
|
+
const candidate = accepted ?? this.serverState.validatingInteraction(runId);
|
|
4341
|
+
const timedOut = this.serverState.timedOutInteraction(runId);
|
|
4278
4342
|
if (candidate === undefined && timedOut === undefined) return;
|
|
4279
4343
|
const loaded = this.runStore.readRun(runId);
|
|
4280
4344
|
if (loaded === null) return;
|
|
@@ -4310,7 +4374,7 @@ export class WorkflowHost {
|
|
|
4310
4374
|
const active = this.activeRuns.get(context.runId);
|
|
4311
4375
|
if (active === undefined) return;
|
|
4312
4376
|
if (context.event.type === "node_finished" && context.event.attemptId !== undefined) {
|
|
4313
|
-
this.
|
|
4377
|
+
this.serverState.consumeAcceptedInteraction(
|
|
4314
4378
|
context.runId,
|
|
4315
4379
|
context.event.attemptId,
|
|
4316
4380
|
context.now,
|
|
@@ -4327,10 +4391,10 @@ export class WorkflowHost {
|
|
|
4327
4391
|
}
|
|
4328
4392
|
return;
|
|
4329
4393
|
}
|
|
4330
|
-
this.
|
|
4394
|
+
this.completeResourceManagerWorkflow(context.runId, context.state);
|
|
4331
4395
|
if (context.state.status !== "waiting") {
|
|
4332
|
-
this.
|
|
4333
|
-
this.
|
|
4396
|
+
this.serverState.workflowMessages.settleOpenTurnsForRun(context.runId, "lost", context.now);
|
|
4397
|
+
this.serverState.workflowMessages.cancelPendingForRun(
|
|
4334
4398
|
context.runId,
|
|
4335
4399
|
context.now,
|
|
4336
4400
|
context.state.status === "completed" ? ["step", "decision"] : undefined,
|
|
@@ -4435,7 +4499,7 @@ export class WorkflowHost {
|
|
|
4435
4499
|
"Workflow result:",
|
|
4436
4500
|
canonicalJson({ ...terminalFacts, terminalFingerprint }),
|
|
4437
4501
|
].join("\n");
|
|
4438
|
-
this.
|
|
4502
|
+
this.serverState.workflowMessages.create({
|
|
4439
4503
|
workflowMessageId,
|
|
4440
4504
|
runId,
|
|
4441
4505
|
targetSessionId: queue.originSessionId,
|
|
@@ -4488,7 +4552,8 @@ export class WorkflowHost {
|
|
|
4488
4552
|
private terminalWorkflowMessageMissing(runId: string): boolean {
|
|
4489
4553
|
return (
|
|
4490
4554
|
this.terminalWorkflowMessageRequired(runId) &&
|
|
4491
|
-
this.
|
|
4555
|
+
this.serverState.workflowMessages.latestForSource("terminal", `terminal:${runId}`) ===
|
|
4556
|
+
undefined
|
|
4492
4557
|
);
|
|
4493
4558
|
}
|
|
4494
4559
|
|
|
@@ -4562,10 +4627,10 @@ export class WorkflowHost {
|
|
|
4562
4627
|
});
|
|
4563
4628
|
}
|
|
4564
4629
|
|
|
4565
|
-
private
|
|
4630
|
+
private completeResourceManagerWorkflow(runId: string, state: WorkflowRunState): void {
|
|
4566
4631
|
const row = this.state.connection
|
|
4567
4632
|
.prepare(
|
|
4568
|
-
`SELECT w.request_id AS requestId, c.controller_name AS
|
|
4633
|
+
`SELECT w.request_id AS requestId, c.controller_name AS resourceManager,
|
|
4569
4634
|
c.resource_key AS resourceKey, p.canonical_path AS projectPath
|
|
4570
4635
|
FROM controller_workflows w
|
|
4571
4636
|
JOIN controller_resources c ON c.controller_resource_id = w.controller_resource_id
|
|
@@ -4575,20 +4640,20 @@ export class WorkflowHost {
|
|
|
4575
4640
|
.get(runId) as
|
|
4576
4641
|
| {
|
|
4577
4642
|
requestId?: unknown;
|
|
4578
|
-
|
|
4643
|
+
resourceManager?: unknown;
|
|
4579
4644
|
resourceKey?: unknown;
|
|
4580
4645
|
projectPath?: unknown;
|
|
4581
4646
|
}
|
|
4582
4647
|
| undefined;
|
|
4583
4648
|
if (
|
|
4584
4649
|
typeof row?.requestId !== "string" ||
|
|
4585
|
-
typeof row.
|
|
4650
|
+
typeof row.resourceManager !== "string" ||
|
|
4586
4651
|
typeof row.resourceKey !== "string" ||
|
|
4587
4652
|
typeof row.projectPath !== "string"
|
|
4588
4653
|
) {
|
|
4589
4654
|
return;
|
|
4590
4655
|
}
|
|
4591
|
-
const store = new
|
|
4656
|
+
const store = new SqliteResourceManagerStore(this.databasePath, {
|
|
4592
4657
|
state: this.state,
|
|
4593
4658
|
projectPath: row.projectPath,
|
|
4594
4659
|
});
|
|
@@ -4603,7 +4668,7 @@ export class WorkflowHost {
|
|
|
4603
4668
|
runId,
|
|
4604
4669
|
...(state.error === undefined ? {} : { error: state.error }),
|
|
4605
4670
|
});
|
|
4606
|
-
store.enqueue({
|
|
4671
|
+
store.enqueue({ resourceManager: row.resourceManager, key: row.resourceKey });
|
|
4607
4672
|
}
|
|
4608
4673
|
|
|
4609
4674
|
private commitActivePause(active: ActiveRun): void {
|
|
@@ -4623,7 +4688,7 @@ export class WorkflowHost {
|
|
|
4623
4688
|
});
|
|
4624
4689
|
}
|
|
4625
4690
|
|
|
4626
|
-
private async
|
|
4691
|
+
private async recoverRunnerExit(active: ActiveRun, outcome: string): Promise<void> {
|
|
4627
4692
|
if (
|
|
4628
4693
|
!this.queue.verifyWorkflowRunClaim({
|
|
4629
4694
|
runId: active.record.runId,
|
|
@@ -4646,12 +4711,12 @@ export class WorkflowHost {
|
|
|
4646
4711
|
this.log(`run ${active.record.runId} parked for ambiguous effect recovery`);
|
|
4647
4712
|
return;
|
|
4648
4713
|
}
|
|
4649
|
-
const validating = this.
|
|
4714
|
+
const validating = this.serverState.validatingInteraction(active.record.runId);
|
|
4650
4715
|
if (validating !== undefined) {
|
|
4651
4716
|
this.settleRejectedInteraction(
|
|
4652
4717
|
active,
|
|
4653
4718
|
validating,
|
|
4654
|
-
`Workflow
|
|
4719
|
+
`Workflow runner ${outcome} before it completed interaction validation`,
|
|
4655
4720
|
);
|
|
4656
4721
|
this.log(`run ${active.record.runId} rejected an interrupted interaction validation`);
|
|
4657
4722
|
return;
|
|
@@ -4668,7 +4733,7 @@ export class WorkflowHost {
|
|
|
4668
4733
|
this.tryEnsureTerminalWorkflowMessage(active.record.runId, Date.now(), {
|
|
4669
4734
|
status: "failed",
|
|
4670
4735
|
error: active.workflowLoadFailure,
|
|
4671
|
-
statusDetail: "The supervised
|
|
4736
|
+
statusDetail: "The supervised runner could not load the saved workflow source.",
|
|
4672
4737
|
});
|
|
4673
4738
|
this.log(`run ${active.record.runId} failed because its workflow source could not load`);
|
|
4674
4739
|
}
|
|
@@ -4676,24 +4741,24 @@ export class WorkflowHost {
|
|
|
4676
4741
|
}
|
|
4677
4742
|
const currentProgressRevision = runRevision(this.state, active.record.runId);
|
|
4678
4743
|
if (currentProgressRevision <= active.launchProgressRevision) {
|
|
4679
|
-
const detail = `Workflow
|
|
4744
|
+
const detail = `Workflow runner ${outcome} before it committed workflow progress`;
|
|
4680
4745
|
this.blockedRuns.add(active.record.runId);
|
|
4681
|
-
this.queue.
|
|
4746
|
+
this.queue.parkWorkflowRunForRunnerNoProgress({
|
|
4682
4747
|
runId: active.record.runId,
|
|
4683
4748
|
claimToken: active.claimToken,
|
|
4684
4749
|
detail,
|
|
4685
4750
|
});
|
|
4686
|
-
this.log(`run ${active.record.runId} parked after a
|
|
4751
|
+
this.log(`run ${active.record.runId} parked after a runner made no progress`);
|
|
4687
4752
|
return;
|
|
4688
4753
|
}
|
|
4689
4754
|
this.queue.parkWorkflowRun({ runId: active.record.runId, claimToken: active.claimToken });
|
|
4690
|
-
this.log(`run ${active.record.runId} parked after
|
|
4755
|
+
this.log(`run ${active.record.runId} parked after runner ${outcome}`);
|
|
4691
4756
|
}
|
|
4692
4757
|
|
|
4693
4758
|
private releaseLock(): void {
|
|
4694
4759
|
try {
|
|
4695
4760
|
const current = JSON.parse(fs.readFileSync(this.lockPath, "utf8")) as unknown;
|
|
4696
|
-
if (isLockRecord(current) && current.
|
|
4761
|
+
if (isLockRecord(current) && current.serverId === this.serverId) fs.rmSync(this.lockPath);
|
|
4697
4762
|
} catch {
|
|
4698
4763
|
// The lock is already gone.
|
|
4699
4764
|
}
|
|
@@ -4795,18 +4860,18 @@ function channelEventPayload(message: ChannelAdapterMessage): JsonValue {
|
|
|
4795
4860
|
return payload as unknown as JsonValue;
|
|
4796
4861
|
}
|
|
4797
4862
|
|
|
4798
|
-
function
|
|
4863
|
+
function acquireServerLock(
|
|
4799
4864
|
lockPath: string,
|
|
4800
|
-
record: { pid: number; startIdentity: string;
|
|
4865
|
+
record: { pid: number; startIdentity: string; serverId: string },
|
|
4801
4866
|
): void {
|
|
4802
4867
|
try {
|
|
4803
4868
|
const existing = JSON.parse(fs.readFileSync(lockPath, "utf8")) as unknown;
|
|
4804
4869
|
if (isLockRecord(existing) && matchesProcessIdentity(existing)) {
|
|
4805
|
-
throw new Error(`A workflow
|
|
4870
|
+
throw new Error(`A workflow server is already running with PID ${existing.pid}`);
|
|
4806
4871
|
}
|
|
4807
4872
|
fs.rmSync(lockPath, { force: true });
|
|
4808
4873
|
} catch (error) {
|
|
4809
|
-
if (error instanceof Error && error.message.startsWith("A workflow
|
|
4874
|
+
if (error instanceof Error && error.message.startsWith("A workflow server is already"))
|
|
4810
4875
|
throw error;
|
|
4811
4876
|
}
|
|
4812
4877
|
fs.writeFileSync(
|
|
@@ -4816,10 +4881,10 @@ function acquireHostLock(
|
|
|
4816
4881
|
);
|
|
4817
4882
|
}
|
|
4818
4883
|
|
|
4819
|
-
function
|
|
4884
|
+
function runnerRunCommand(
|
|
4820
4885
|
record: WorkflowRunQueueRecord,
|
|
4821
4886
|
resumeInteractionAttemptId: string | undefined,
|
|
4822
|
-
):
|
|
4887
|
+
): WorkflowRunnerCommand {
|
|
4823
4888
|
if (record.initialized) {
|
|
4824
4889
|
return {
|
|
4825
4890
|
kind: "resume",
|
|
@@ -4848,13 +4913,13 @@ function workerRunCommand(
|
|
|
4848
4913
|
return { kind: "start", input };
|
|
4849
4914
|
}
|
|
4850
4915
|
|
|
4851
|
-
function
|
|
4852
|
-
message:
|
|
4853
|
-
outcome:
|
|
4916
|
+
function runnerResponse(
|
|
4917
|
+
message: WorkflowRunnerMessage,
|
|
4918
|
+
outcome: WorkflowRunnerResponse["outcome"],
|
|
4854
4919
|
result?: JsonValue,
|
|
4855
4920
|
error?: string,
|
|
4856
4921
|
revision?: number,
|
|
4857
|
-
):
|
|
4922
|
+
): WorkflowRunnerResponse {
|
|
4858
4923
|
return {
|
|
4859
4924
|
schema: "pi-workflows.worker-response.v1",
|
|
4860
4925
|
messageId: message.messageId,
|
|
@@ -4865,12 +4930,12 @@ function workerResponse(
|
|
|
4865
4930
|
};
|
|
4866
4931
|
}
|
|
4867
4932
|
|
|
4868
|
-
function
|
|
4869
|
-
message:
|
|
4870
|
-
outcome:
|
|
4933
|
+
function resourceRunnerResponse(
|
|
4934
|
+
message: ResourceRunnerMessage,
|
|
4935
|
+
outcome: ResourceRunnerResponse["outcome"],
|
|
4871
4936
|
result?: JsonValue,
|
|
4872
4937
|
error?: string,
|
|
4873
|
-
):
|
|
4938
|
+
): ResourceRunnerResponse {
|
|
4874
4939
|
return {
|
|
4875
4940
|
schema: "pi-workflows.controller-worker-response.v1",
|
|
4876
4941
|
messageId: message.messageId,
|
|
@@ -4880,7 +4945,7 @@ function controllerWorkerResponse(
|
|
|
4880
4945
|
};
|
|
4881
4946
|
}
|
|
4882
4947
|
|
|
4883
|
-
function
|
|
4948
|
+
function resourceManagerWorkflowResult(record: WorkflowRunQueueRecord): WorkflowSchedulerResult {
|
|
4884
4949
|
switch (record.status) {
|
|
4885
4950
|
case "queued":
|
|
4886
4951
|
case "starting":
|
|
@@ -4921,19 +4986,19 @@ function hasUncertainEffect(state: StateDatabase, runId: string): boolean {
|
|
|
4921
4986
|
return row !== undefined;
|
|
4922
4987
|
}
|
|
4923
4988
|
|
|
4924
|
-
function
|
|
4989
|
+
function resourceManagerPathAllowed(
|
|
4925
4990
|
projectPath: string,
|
|
4926
|
-
|
|
4927
|
-
|
|
4991
|
+
resourceManagerName: string,
|
|
4992
|
+
resourceManagerPath: string,
|
|
4928
4993
|
): boolean {
|
|
4929
4994
|
if (
|
|
4930
|
-
|
|
4931
|
-
!/\.
|
|
4995
|
+
resourceManagerFileStem(resourceManagerPath) !== resourceManagerName ||
|
|
4996
|
+
!/\.resource-manager\.(?:ts|js|mts|mjs)$/u.test(resourceManagerPath)
|
|
4932
4997
|
) {
|
|
4933
4998
|
return false;
|
|
4934
4999
|
}
|
|
4935
|
-
return
|
|
4936
|
-
({ dir }) => path.dirname(
|
|
5000
|
+
return resourceManagerSearchDirs({ cwd: projectPath }).some(
|
|
5001
|
+
({ dir }) => path.dirname(resourceManagerPath) === path.resolve(dir),
|
|
4937
5002
|
);
|
|
4938
5003
|
}
|
|
4939
5004
|
|
|
@@ -4964,7 +5029,7 @@ function waitForSocketDrain(socket: Socket): Promise<void> {
|
|
|
4964
5029
|
});
|
|
4965
5030
|
}
|
|
4966
5031
|
|
|
4967
|
-
function
|
|
5032
|
+
function serverDelay(ms: number): Promise<void> {
|
|
4968
5033
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
4969
5034
|
}
|
|
4970
5035
|
|
|
@@ -5210,13 +5275,13 @@ function runtimePackageVersion(): string {
|
|
|
5210
5275
|
|
|
5211
5276
|
function isLockRecord(
|
|
5212
5277
|
value: unknown,
|
|
5213
|
-
): value is { schema: string; pid: number; startIdentity: string;
|
|
5278
|
+
): value is { schema: string; pid: number; startIdentity: string; serverId: string } {
|
|
5214
5279
|
return (
|
|
5215
5280
|
typeof value === "object" &&
|
|
5216
5281
|
value !== null &&
|
|
5217
5282
|
(value as { schema?: unknown }).schema === "pi-workflows.host-lock.v1" &&
|
|
5218
5283
|
typeof (value as { pid?: unknown }).pid === "number" &&
|
|
5219
5284
|
typeof (value as { startIdentity?: unknown }).startIdentity === "string" &&
|
|
5220
|
-
typeof (value as {
|
|
5285
|
+
typeof (value as { serverId?: unknown }).serverId === "string"
|
|
5221
5286
|
);
|
|
5222
5287
|
}
|