@provos/ironcurtain 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -14
- package/dist/auth/oauth-provider.js +1 -4
- package/dist/auth/oauth-provider.js.map +1 -1
- package/dist/cli-help.d.ts +12 -0
- package/dist/cli-help.js +21 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.js +46 -6
- package/dist/cli.js.map +1 -1
- package/dist/config/agent-model-guard.d.ts +14 -0
- package/dist/config/agent-model-guard.js +24 -0
- package/dist/config/agent-model-guard.js.map +1 -0
- package/dist/config/config-command.js +277 -7
- package/dist/config/config-command.js.map +1 -1
- package/dist/config/first-start.js +62 -7
- package/dist/config/first-start.js.map +1 -1
- package/dist/config/generated/compiled-policy.json +528 -161
- package/dist/config/generated/test-scenarios.json +4810 -170
- package/dist/config/generated/tool-annotations.json +34 -47
- package/dist/config/index.d.ts +58 -0
- package/dist/config/index.js +97 -15
- package/dist/config/index.js.map +1 -1
- package/dist/config/mcp-servers.json +2 -2
- package/dist/config/model-provider.d.ts +9 -0
- package/dist/config/model-provider.js +13 -0
- package/dist/config/model-provider.js.map +1 -1
- package/dist/config/paths.d.ts +217 -0
- package/dist/config/paths.js +363 -14
- package/dist/config/paths.js.map +1 -1
- package/dist/config/types.d.ts +7 -1
- package/dist/config/user-config.d.ts +101 -6
- package/dist/config/user-config.js +145 -31
- package/dist/config/user-config.js.map +1 -1
- package/dist/config/validate-policy-dir.d.ts +10 -0
- package/dist/config/validate-policy-dir.js +59 -0
- package/dist/config/validate-policy-dir.js.map +1 -0
- package/dist/cron/job-commands.js +22 -2
- package/dist/cron/job-commands.js.map +1 -1
- package/dist/cron/types.d.ts +16 -2
- package/dist/cron/types.js +1 -3
- package/dist/cron/types.js.map +1 -1
- package/dist/daemon/daemon-command.js +30 -7
- package/dist/daemon/daemon-command.js.map +1 -1
- package/dist/daemon/ironcurtain-daemon.d.ts +62 -1
- package/dist/daemon/ironcurtain-daemon.js +231 -8
- package/dist/daemon/ironcurtain-daemon.js.map +1 -1
- package/dist/daemon-client/daemon-client.d.ts +123 -0
- package/dist/daemon-client/daemon-client.js +324 -0
- package/dist/daemon-client/daemon-client.js.map +1 -0
- package/dist/docker/adapters/claude-code.js +194 -20
- package/dist/docker/adapters/claude-code.js.map +1 -1
- package/dist/docker/adapters/codex.d.ts +9 -0
- package/dist/docker/adapters/codex.js +297 -0
- package/dist/docker/adapters/codex.js.map +1 -0
- package/dist/docker/adapters/goose.d.ts +12 -11
- package/dist/docker/adapters/goose.js +80 -27
- package/dist/docker/adapters/goose.js.map +1 -1
- package/dist/docker/agent-adapter.d.ts +183 -5
- package/dist/docker/agent-adapter.js +16 -0
- package/dist/docker/agent-adapter.js.map +1 -1
- package/dist/docker/agent-registry.js +5 -0
- package/dist/docker/agent-registry.js.map +1 -1
- package/dist/docker/apple-container-manager.d.ts +49 -0
- package/dist/docker/apple-container-manager.js +464 -0
- package/dist/docker/apple-container-manager.js.map +1 -0
- package/dist/docker/code-mode-proxy.d.ts +38 -0
- package/dist/docker/code-mode-proxy.js +6 -1
- package/dist/docker/code-mode-proxy.js.map +1 -1
- package/dist/docker/container-lifecycle.d.ts +2 -2
- package/dist/docker/container-lifecycle.js.map +1 -1
- package/dist/docker/container-runtime.d.ts +29 -0
- package/dist/docker/container-runtime.js +76 -0
- package/dist/docker/container-runtime.js.map +1 -0
- package/dist/docker/docker-agent-session.d.ts +113 -62
- package/dist/docker/docker-agent-session.js +271 -294
- package/dist/docker/docker-agent-session.js.map +1 -1
- package/dist/docker/docker-infrastructure.d.ts +428 -26
- package/dist/docker/docker-infrastructure.js +1005 -70
- package/dist/docker/docker-infrastructure.js.map +1 -1
- package/dist/docker/docker-manager.d.ts +58 -3
- package/dist/docker/docker-manager.js +378 -33
- package/dist/docker/docker-manager.js.map +1 -1
- package/dist/docker/docker-probe.d.ts +45 -0
- package/dist/docker/docker-probe.js +86 -0
- package/dist/docker/docker-probe.js.map +1 -0
- package/dist/docker/docker-progress-sink.d.ts +43 -0
- package/dist/docker/docker-progress-sink.js +299 -0
- package/dist/docker/docker-progress-sink.js.map +1 -0
- package/dist/docker/mitm-proxy.d.ts +135 -1
- package/dist/docker/mitm-proxy.js +611 -25
- package/dist/docker/mitm-proxy.js.map +1 -1
- package/dist/docker/network-topology.d.ts +81 -0
- package/dist/docker/network-topology.js +160 -0
- package/dist/docker/network-topology.js.map +1 -0
- package/dist/docker/oauth-credentials.d.ts +59 -3
- package/dist/docker/oauth-credentials.js +181 -14
- package/dist/docker/oauth-credentials.js.map +1 -1
- package/dist/docker/oauth-token-manager.js +2 -2
- package/dist/docker/oauth-token-manager.js.map +1 -1
- package/dist/docker/orientation.d.ts +1 -1
- package/dist/docker/package-types.d.ts +1 -1
- package/dist/docker/provider-config.d.ts +78 -3
- package/dist/docker/provider-config.js +295 -31
- package/dist/docker/provider-config.js.map +1 -1
- package/dist/docker/provision-lock.d.ts +46 -0
- package/dist/docker/provision-lock.js +152 -0
- package/dist/docker/provision-lock.js.map +1 -0
- package/dist/docker/proxy-tools.d.ts +0 -2
- package/dist/docker/proxy-tools.js +2 -7
- package/dist/docker/proxy-tools.js.map +1 -1
- package/dist/docker/pty-session.d.ts +64 -1
- package/dist/docker/pty-session.js +306 -104
- package/dist/docker/pty-session.js.map +1 -1
- package/dist/docker/pty-types.d.ts +0 -2
- package/dist/docker/pty-types.js +0 -2
- package/dist/docker/pty-types.js.map +1 -1
- package/dist/docker/registry-proxy.d.ts +46 -0
- package/dist/docker/registry-proxy.js +255 -1
- package/dist/docker/registry-proxy.js.map +1 -1
- package/dist/docker/resource-limits.d.ts +134 -0
- package/dist/docker/resource-limits.js +203 -0
- package/dist/docker/resource-limits.js.map +1 -0
- package/dist/docker/spawn-with-idle-timeout.d.ts +39 -0
- package/dist/docker/spawn-with-idle-timeout.js +152 -0
- package/dist/docker/spawn-with-idle-timeout.js.map +1 -0
- package/dist/docker/sse-extractor.d.ts +51 -0
- package/dist/docker/sse-extractor.js +303 -0
- package/dist/docker/sse-extractor.js.map +1 -0
- package/dist/docker/token-stream-bus.d.ts +83 -0
- package/dist/docker/token-stream-bus.js +98 -0
- package/dist/docker/token-stream-bus.js.map +1 -0
- package/dist/docker/token-stream-types.d.ts +71 -0
- package/dist/docker/token-stream-types.js +8 -0
- package/dist/docker/token-stream-types.js.map +1 -0
- package/dist/docker/trajectory-capture.d.ts +79 -0
- package/dist/docker/trajectory-capture.js +521 -0
- package/dist/docker/trajectory-capture.js.map +1 -0
- package/dist/docker/trajectory-reassembler.d.ts +286 -0
- package/dist/docker/trajectory-reassembler.js +990 -0
- package/dist/docker/trajectory-reassembler.js.map +1 -0
- package/dist/docker/trajectory-tap.d.ts +100 -0
- package/dist/docker/trajectory-tap.js +410 -0
- package/dist/docker/trajectory-tap.js.map +1 -0
- package/dist/docker/trajectory-types.d.ts +187 -0
- package/dist/docker/trajectory-types.js +43 -0
- package/dist/docker/trajectory-types.js.map +1 -0
- package/dist/docker/types.d.ts +105 -12
- package/dist/doctor/checks.d.ts +171 -0
- package/dist/doctor/checks.js +598 -0
- package/dist/doctor/checks.js.map +1 -0
- package/dist/doctor/doctor-command.d.ts +29 -0
- package/dist/doctor/doctor-command.js +190 -0
- package/dist/doctor/doctor-command.js.map +1 -0
- package/dist/doctor/mcp-liveness.d.ts +28 -0
- package/dist/doctor/mcp-liveness.js +92 -0
- package/dist/doctor/mcp-liveness.js.map +1 -0
- package/dist/doctor/oauth-checks.d.ts +23 -0
- package/dist/doctor/oauth-checks.js +157 -0
- package/dist/doctor/oauth-checks.js.map +1 -0
- package/dist/doctor/output.d.ts +26 -0
- package/dist/doctor/output.js +82 -0
- package/dist/doctor/output.js.map +1 -0
- package/dist/escalation/listener-command.js +0 -3
- package/dist/escalation/listener-command.js.map +1 -1
- package/dist/event-bus/typed-event-bus.d.ts +26 -0
- package/dist/event-bus/typed-event-bus.js +35 -0
- package/dist/event-bus/typed-event-bus.js.map +1 -0
- package/dist/index.js +54 -21
- package/dist/index.js.map +1 -1
- package/dist/logger.js +31 -2
- package/dist/logger.js.map +1 -1
- package/dist/memory/auto-save.d.ts +18 -4
- package/dist/memory/auto-save.js +46 -6
- package/dist/memory/auto-save.js.map +1 -1
- package/dist/memory/memory-policy.d.ts +41 -0
- package/dist/memory/memory-policy.js +36 -0
- package/dist/memory/memory-policy.js.map +1 -0
- package/dist/mux/mux-app.d.ts +2 -0
- package/dist/mux/mux-app.js +1 -0
- package/dist/mux/mux-app.js.map +1 -1
- package/dist/mux/mux-command.d.ts +10 -1
- package/dist/mux/mux-command.js +74 -40
- package/dist/mux/mux-command.js.map +1 -1
- package/dist/mux/mux-renderer.js +2 -12
- package/dist/mux/mux-renderer.js.map +1 -1
- package/dist/mux/pty-bridge.d.ts +2 -0
- package/dist/mux/pty-bridge.js +3 -0
- package/dist/mux/pty-bridge.js.map +1 -1
- package/dist/mux/session-scanner.js +1 -2
- package/dist/mux/session-scanner.js.map +1 -1
- package/dist/{pipeline → observability}/llm-logger.d.ts +8 -3
- package/dist/{pipeline → observability}/llm-logger.js +8 -3
- package/dist/observability/llm-logger.js.map +1 -0
- package/dist/observe/observe-command.d.ts +12 -0
- package/dist/observe/observe-command.js +285 -0
- package/dist/observe/observe-command.js.map +1 -0
- package/dist/observe/observe-renderer.d.ts +24 -0
- package/dist/observe/observe-renderer.js +87 -0
- package/dist/observe/observe-renderer.js.map +1 -0
- package/dist/observe/observe-tui-rain.d.ts +42 -0
- package/dist/observe/observe-tui-rain.js +531 -0
- package/dist/observe/observe-tui-rain.js.map +1 -0
- package/dist/observe/observe-tui-text-panel.d.ts +46 -0
- package/dist/observe/observe-tui-text-panel.js +525 -0
- package/dist/observe/observe-tui-text-panel.js.map +1 -0
- package/dist/observe/observe-tui-types.d.ts +270 -0
- package/dist/observe/observe-tui-types.js +200 -0
- package/dist/observe/observe-tui-types.js.map +1 -0
- package/dist/observe/observe-tui-word-scorer.d.ts +143 -0
- package/dist/observe/observe-tui-word-scorer.js +572 -0
- package/dist/observe/observe-tui-word-scorer.js.map +1 -0
- package/dist/observe/observe-tui.d.ts +59 -0
- package/dist/observe/observe-tui.js +592 -0
- package/dist/observe/observe-tui.js.map +1 -0
- package/dist/persona/broad-policy-validator.d.ts +95 -0
- package/dist/persona/broad-policy-validator.js +171 -0
- package/dist/persona/broad-policy-validator.js.map +1 -0
- package/dist/persona/compile-persona-policy.d.ts +54 -2
- package/dist/persona/compile-persona-policy.js +29 -4
- package/dist/persona/compile-persona-policy.js.map +1 -1
- package/dist/persona/event-bus-progress-reporter.d.ts +57 -0
- package/dist/persona/event-bus-progress-reporter.js +66 -0
- package/dist/persona/event-bus-progress-reporter.js.map +1 -0
- package/dist/persona/memory-gate.d.ts +19 -0
- package/dist/persona/memory-gate.js +33 -0
- package/dist/persona/memory-gate.js.map +1 -0
- package/dist/persona/persona-command.js +77 -72
- package/dist/persona/persona-command.js.map +1 -1
- package/dist/persona/persona-compile-orchestrator.d.ts +101 -0
- package/dist/persona/persona-compile-orchestrator.js +504 -0
- package/dist/persona/persona-compile-orchestrator.js.map +1 -0
- package/dist/persona/persona-service.d.ts +137 -0
- package/dist/persona/persona-service.js +263 -0
- package/dist/persona/persona-service.js.map +1 -0
- package/dist/persona/policy-mutation-audit.d.ts +109 -0
- package/dist/persona/policy-mutation-audit.js +179 -0
- package/dist/persona/policy-mutation-audit.js.map +1 -0
- package/dist/persona/resolve.d.ts +22 -0
- package/dist/persona/resolve.js +45 -2
- package/dist/persona/resolve.js.map +1 -1
- package/dist/persona/rule-delta.d.ts +30 -0
- package/dist/persona/rule-delta.js +82 -0
- package/dist/persona/rule-delta.js.map +1 -0
- package/dist/persona/types.d.ts +28 -0
- package/dist/pipeline/annotate.js +10 -2
- package/dist/pipeline/annotate.js.map +1 -1
- package/dist/pipeline/compile.js +14 -2
- package/dist/pipeline/compile.js.map +1 -1
- package/dist/pipeline/constitution-compiler.d.ts +25 -14
- package/dist/pipeline/constitution-compiler.js +107 -38
- package/dist/pipeline/constitution-compiler.js.map +1 -1
- package/dist/pipeline/dynamic-list-types.d.ts +3 -6
- package/dist/pipeline/dynamic-list-types.js +3 -16
- package/dist/pipeline/dynamic-list-types.js.map +1 -1
- package/dist/pipeline/pipeline-runner.d.ts +63 -4
- package/dist/pipeline/pipeline-runner.js +242 -94
- package/dist/pipeline/pipeline-runner.js.map +1 -1
- package/dist/pipeline/pipeline-shared.d.ts +33 -4
- package/dist/pipeline/pipeline-shared.js +79 -19
- package/dist/pipeline/pipeline-shared.js.map +1 -1
- package/dist/pipeline/policy-verifier.js +16 -3
- package/dist/pipeline/policy-verifier.js.map +1 -1
- package/dist/pipeline/proxy-mcp-connections.js +17 -2
- package/dist/pipeline/proxy-mcp-connections.js.map +1 -1
- package/dist/pipeline/refresh-lists.js +3 -5
- package/dist/pipeline/refresh-lists.js.map +1 -1
- package/dist/pipeline/scenario-generator.d.ts +4 -8
- package/dist/pipeline/scenario-generator.js +13 -46
- package/dist/pipeline/scenario-generator.js.map +1 -1
- package/dist/sandbox/index.d.ts +29 -0
- package/dist/sandbox/index.js +351 -119
- package/dist/sandbox/index.js.map +1 -1
- package/dist/sandbox/ironcurtain-protocol.d.ts +52 -0
- package/dist/sandbox/ironcurtain-protocol.js +185 -0
- package/dist/sandbox/ironcurtain-protocol.js.map +1 -0
- package/dist/session/agent-session.d.ts +6 -0
- package/dist/session/agent-session.js +16 -5
- package/dist/session/agent-session.js.map +1 -1
- package/dist/session/cli-transport.js +0 -6
- package/dist/session/cli-transport.js.map +1 -1
- package/dist/session/index.d.ts +35 -14
- package/dist/session/index.js +333 -130
- package/dist/session/index.js.map +1 -1
- package/dist/session/preflight.d.ts +21 -13
- package/dist/session/preflight.js +227 -88
- package/dist/session/preflight.js.map +1 -1
- package/dist/session/prompts.d.ts +2 -5
- package/dist/session/prompts.js.map +1 -1
- package/dist/session/session-manager.d.ts +61 -6
- package/dist/session/session-manager.js +72 -10
- package/dist/session/session-manager.js.map +1 -1
- package/dist/session/session-metadata.d.ts +21 -5
- package/dist/session/session-metadata.js +34 -12
- package/dist/session/session-metadata.js.map +1 -1
- package/dist/session/types.d.ts +320 -5
- package/dist/session/types.js +57 -1
- package/dist/session/types.js.map +1 -1
- package/dist/signal/bot-command.js +6 -1
- package/dist/signal/bot-command.js.map +1 -1
- package/dist/signal/setup-signal.js +13 -21
- package/dist/signal/setup-signal.js.map +1 -1
- package/dist/signal/signal-bot-daemon.d.ts +16 -1
- package/dist/signal/signal-bot-daemon.js +39 -27
- package/dist/signal/signal-bot-daemon.js.map +1 -1
- package/dist/signal/signal-container.d.ts +3 -3
- package/dist/signal/signal-container.js +9 -3
- package/dist/signal/signal-container.js.map +1 -1
- package/dist/skills/discovery.d.ts +77 -0
- package/dist/skills/discovery.js +206 -0
- package/dist/skills/discovery.js.map +1 -0
- package/dist/skills/index.d.ts +4 -0
- package/dist/skills/index.js +3 -0
- package/dist/skills/index.js.map +1 -0
- package/dist/skills/staging.d.ts +28 -0
- package/dist/skills/staging.js +68 -0
- package/dist/skills/staging.js.map +1 -0
- package/dist/skills/types.d.ts +46 -0
- package/dist/skills/types.js +2 -0
- package/dist/skills/types.js.map +1 -0
- package/dist/trusted-process/approval-whitelist.d.ts +0 -2
- package/dist/trusted-process/approval-whitelist.js +17 -17
- package/dist/trusted-process/approval-whitelist.js.map +1 -1
- package/dist/trusted-process/async-mutex.d.ts +32 -0
- package/dist/trusted-process/async-mutex.js +50 -0
- package/dist/trusted-process/async-mutex.js.map +1 -0
- package/dist/trusted-process/audit-log.d.ts +38 -1
- package/dist/trusted-process/audit-log.js +82 -13
- package/dist/trusted-process/audit-log.js.map +1 -1
- package/dist/trusted-process/call-circuit-breaker.d.ts +6 -0
- package/dist/trusted-process/call-circuit-breaker.js +23 -3
- package/dist/trusted-process/call-circuit-breaker.js.map +1 -1
- package/dist/trusted-process/container-command.d.ts +35 -0
- package/dist/trusted-process/container-command.js +49 -0
- package/dist/trusted-process/container-command.js.map +1 -0
- package/dist/trusted-process/control-server.d.ts +101 -0
- package/dist/trusted-process/control-server.js +282 -0
- package/dist/trusted-process/control-server.js.map +1 -0
- package/dist/trusted-process/domain-utils.d.ts +6 -6
- package/dist/trusted-process/domain-utils.js +7 -23
- package/dist/trusted-process/domain-utils.js.map +1 -1
- package/dist/trusted-process/error-prefixes.d.ts +16 -0
- package/dist/trusted-process/error-prefixes.js +17 -0
- package/dist/trusted-process/error-prefixes.js.map +1 -0
- package/dist/trusted-process/index.d.ts +40 -23
- package/dist/trusted-process/index.js +131 -225
- package/dist/trusted-process/index.js.map +1 -1
- package/dist/trusted-process/list-matcher.d.ts +17 -0
- package/dist/trusted-process/list-matcher.js +31 -0
- package/dist/trusted-process/list-matcher.js.map +1 -0
- package/dist/trusted-process/mcp-client-manager.d.ts +35 -6
- package/dist/trusted-process/mcp-client-manager.js +40 -45
- package/dist/trusted-process/mcp-client-manager.js.map +1 -1
- package/dist/trusted-process/mcp-proxy-server.d.ts +31 -120
- package/dist/trusted-process/mcp-proxy-server.js +201 -660
- package/dist/trusted-process/mcp-proxy-server.js.map +1 -1
- package/dist/trusted-process/policy-engine.d.ts +16 -4
- package/dist/trusted-process/policy-engine.js +103 -47
- package/dist/trusted-process/policy-engine.js.map +1 -1
- package/dist/trusted-process/policy-roots.d.ts +6 -0
- package/dist/trusted-process/policy-roots.js +20 -0
- package/dist/trusted-process/policy-roots.js.map +1 -1
- package/dist/trusted-process/sandbox-integration.d.ts +12 -0
- package/dist/trusted-process/sandbox-integration.js +18 -0
- package/dist/trusted-process/sandbox-integration.js.map +1 -1
- package/dist/trusted-process/tcp-server-transport.d.ts +12 -1
- package/dist/trusted-process/tcp-server-transport.js +9 -1
- package/dist/trusted-process/tcp-server-transport.js.map +1 -1
- package/dist/trusted-process/tool-call-coordinator.d.ts +283 -0
- package/dist/trusted-process/tool-call-coordinator.js +592 -0
- package/dist/trusted-process/tool-call-coordinator.js.map +1 -0
- package/dist/trusted-process/tool-call-pipeline.d.ts +213 -0
- package/dist/trusted-process/tool-call-pipeline.js +832 -0
- package/dist/trusted-process/tool-call-pipeline.js.map +1 -0
- package/dist/trusted-process/tool-description-hints.d.ts +1 -1
- package/dist/types/argument-roles.d.ts +1 -1
- package/dist/types/argument-roles.js +14 -1
- package/dist/types/argument-roles.js.map +1 -1
- package/dist/types/audit.d.ts +9 -0
- package/dist/{session → types}/errors.d.ts +8 -2
- package/dist/{session → types}/errors.js +8 -2
- package/dist/types/errors.js.map +1 -0
- package/dist/types/reserved-servers.d.ts +11 -0
- package/dist/types/reserved-servers.js +12 -0
- package/dist/types/reserved-servers.js.map +1 -0
- package/dist/types/server-listing.d.ts +14 -0
- package/dist/types/server-listing.js +2 -0
- package/dist/types/server-listing.js.map +1 -0
- package/dist/types/slug.d.ts +2 -0
- package/dist/types/slug.js +2 -0
- package/dist/types/slug.js.map +1 -1
- package/dist/types/url-normalize.d.ts +16 -0
- package/dist/types/url-normalize.js +33 -0
- package/dist/types/url-normalize.js.map +1 -0
- package/dist/util/atomic-write.d.ts +31 -0
- package/dist/util/atomic-write.js +43 -0
- package/dist/util/atomic-write.js.map +1 -0
- package/dist/utils/error-message.d.ts +8 -0
- package/dist/utils/error-message.js +11 -0
- package/dist/utils/error-message.js.map +1 -0
- package/dist/utils/exec-error.d.ts +19 -0
- package/dist/utils/exec-error.js +16 -0
- package/dist/utils/exec-error.js.map +1 -0
- package/dist/utils/is-plain-object.d.ts +7 -0
- package/dist/utils/is-plain-object.js +12 -0
- package/dist/utils/is-plain-object.js.map +1 -0
- package/dist/utils/preflight-checks.d.ts +16 -0
- package/dist/utils/preflight-checks.js +178 -0
- package/dist/utils/preflight-checks.js.map +1 -0
- package/dist/web-ui/__tests__/json-rpc-dispatch.test.d.ts +7 -0
- package/dist/web-ui/__tests__/json-rpc-dispatch.test.js +725 -0
- package/dist/web-ui/__tests__/json-rpc-dispatch.test.js.map +1 -0
- package/dist/web-ui/conn-registry.d.ts +14 -0
- package/dist/web-ui/conn-registry.js +25 -0
- package/dist/web-ui/conn-registry.js.map +1 -0
- package/dist/web-ui/dispatch/escalation-dispatch.d.ts +7 -0
- package/dist/web-ui/dispatch/escalation-dispatch.js +52 -0
- package/dist/web-ui/dispatch/escalation-dispatch.js.map +1 -0
- package/dist/web-ui/dispatch/job-dispatch.d.ts +8 -0
- package/dist/web-ui/dispatch/job-dispatch.js +83 -0
- package/dist/web-ui/dispatch/job-dispatch.js.map +1 -0
- package/dist/web-ui/dispatch/persona-dispatch.d.ts +50 -0
- package/dist/web-ui/dispatch/persona-dispatch.js +269 -0
- package/dist/web-ui/dispatch/persona-dispatch.js.map +1 -0
- package/dist/web-ui/dispatch/session-dispatch.d.ts +9 -0
- package/dist/web-ui/dispatch/session-dispatch.js +219 -0
- package/dist/web-ui/dispatch/session-dispatch.js.map +1 -0
- package/dist/web-ui/dispatch/token-stream-dispatch.d.ts +15 -0
- package/dist/web-ui/dispatch/token-stream-dispatch.js +55 -0
- package/dist/web-ui/dispatch/token-stream-dispatch.js.map +1 -0
- package/dist/web-ui/dispatch/types.d.ts +54 -0
- package/dist/web-ui/dispatch/types.js +72 -0
- package/dist/web-ui/dispatch/types.js.map +1 -0
- package/dist/web-ui/dispatch/workflow-dispatch.d.ts +103 -0
- package/dist/web-ui/dispatch/workflow-dispatch.js +1036 -0
- package/dist/web-ui/dispatch/workflow-dispatch.js.map +1 -0
- package/dist/web-ui/json-rpc-dispatch.d.ts +12 -0
- package/dist/web-ui/json-rpc-dispatch.js +34 -0
- package/dist/web-ui/json-rpc-dispatch.js.map +1 -0
- package/dist/web-ui/state-graph.d.ts +13 -0
- package/dist/web-ui/state-graph.js +82 -0
- package/dist/web-ui/state-graph.js.map +1 -0
- package/dist/web-ui/token-stream-bridge.d.ts +76 -0
- package/dist/web-ui/token-stream-bridge.js +265 -0
- package/dist/web-ui/token-stream-bridge.js.map +1 -0
- package/dist/web-ui/web-event-bus.d.ts +176 -0
- package/dist/web-ui/web-event-bus.js +19 -0
- package/dist/web-ui/web-event-bus.js.map +1 -0
- package/dist/web-ui/web-session-transport.d.ts +31 -0
- package/dist/web-ui/web-session-transport.js +109 -0
- package/dist/web-ui/web-session-transport.js.map +1 -0
- package/dist/web-ui/web-ui-server.d.ts +90 -0
- package/dist/web-ui/web-ui-server.js +477 -0
- package/dist/web-ui/web-ui-server.js.map +1 -0
- package/dist/web-ui/web-ui-types.d.ts +418 -0
- package/dist/web-ui/web-ui-types.js +32 -0
- package/dist/web-ui/web-ui-types.js.map +1 -0
- package/dist/web-ui/ws-utils.d.ts +5 -0
- package/dist/web-ui/ws-utils.js +12 -0
- package/dist/web-ui/ws-utils.js.map +1 -0
- package/dist/web-ui-static/assets/index-BD5U6ILh.css +1 -0
- package/dist/web-ui-static/assets/index-FMsHWEfK.js +79 -0
- package/dist/web-ui-static/assets/orbitron-latin-700-normal-4jsRXGGJ.woff2 +0 -0
- package/dist/web-ui-static/assets/orbitron-latin-700-normal-DmYOh-W9.woff +0 -0
- package/dist/web-ui-static/index.html +14 -0
- package/dist/workflow/artifacts.d.ts +59 -0
- package/dist/workflow/artifacts.js +188 -0
- package/dist/workflow/artifacts.js.map +1 -0
- package/dist/workflow/checkpoint-selection.d.ts +34 -0
- package/dist/workflow/checkpoint-selection.js +48 -0
- package/dist/workflow/checkpoint-selection.js.map +1 -0
- package/dist/workflow/checkpoint.d.ts +35 -0
- package/dist/workflow/checkpoint.js +84 -0
- package/dist/workflow/checkpoint.js.map +1 -0
- package/dist/workflow/cli-shared.d.ts +62 -0
- package/dist/workflow/cli-shared.js +117 -0
- package/dist/workflow/cli-shared.js.map +1 -0
- package/dist/workflow/cli-support.d.ts +92 -0
- package/dist/workflow/cli-support.js +421 -0
- package/dist/workflow/cli-support.js.map +1 -0
- package/dist/workflow/container-snapshots.d.ts +29 -0
- package/dist/workflow/container-snapshots.js +137 -0
- package/dist/workflow/container-snapshots.js.map +1 -0
- package/dist/workflow/daemon-gate-commands.d.ts +21 -0
- package/dist/workflow/daemon-gate-commands.js +599 -0
- package/dist/workflow/daemon-gate-commands.js.map +1 -0
- package/dist/workflow/definition-loader.d.ts +32 -0
- package/dist/workflow/definition-loader.js +56 -0
- package/dist/workflow/definition-loader.js.map +1 -0
- package/dist/workflow/discovery.d.ts +107 -0
- package/dist/workflow/discovery.js +263 -0
- package/dist/workflow/discovery.js.map +1 -0
- package/dist/workflow/errors.d.ts +66 -0
- package/dist/workflow/errors.js +49 -0
- package/dist/workflow/errors.js.map +1 -0
- package/dist/workflow/guards.d.ts +8 -0
- package/dist/workflow/guards.js +38 -0
- package/dist/workflow/guards.js.map +1 -0
- package/dist/workflow/lane-template.d.ts +89 -0
- package/dist/workflow/lane-template.js +141 -0
- package/dist/workflow/lane-template.js.map +1 -0
- package/dist/workflow/lint-integration.d.ts +64 -0
- package/dist/workflow/lint-integration.js +56 -0
- package/dist/workflow/lint-integration.js.map +1 -0
- package/dist/workflow/lint.d.ts +72 -0
- package/dist/workflow/lint.js +593 -0
- package/dist/workflow/lint.js.map +1 -0
- package/dist/workflow/machine-builder.d.ts +178 -0
- package/dist/workflow/machine-builder.js +678 -0
- package/dist/workflow/machine-builder.js.map +1 -0
- package/dist/workflow/message-log.d.ts +99 -0
- package/dist/workflow/message-log.js +40 -0
- package/dist/workflow/message-log.js.map +1 -0
- package/dist/workflow/orchestrator.d.ts +954 -0
- package/dist/workflow/orchestrator.js +3594 -0
- package/dist/workflow/orchestrator.js.map +1 -0
- package/dist/workflow/prompt-builder.d.ts +39 -0
- package/dist/workflow/prompt-builder.js +243 -0
- package/dist/workflow/prompt-builder.js.map +1 -0
- package/dist/workflow/run-state-command.d.ts +9 -0
- package/dist/workflow/run-state-command.js +414 -0
- package/dist/workflow/run-state-command.js.map +1 -0
- package/dist/workflow/run-state-debug-capture.d.ts +40 -0
- package/dist/workflow/run-state-debug-capture.js +101 -0
- package/dist/workflow/run-state-debug-capture.js.map +1 -0
- package/dist/workflow/status-parser.d.ts +68 -0
- package/dist/workflow/status-parser.js +274 -0
- package/dist/workflow/status-parser.js.map +1 -0
- package/dist/workflow/types.d.ts +598 -0
- package/dist/workflow/types.js +85 -0
- package/dist/workflow/types.js.map +1 -0
- package/dist/workflow/validate.d.ts +80 -0
- package/dist/workflow/validate.js +788 -0
- package/dist/workflow/validate.js.map +1 -0
- package/dist/workflow/workflow-command.d.ts +9 -0
- package/dist/workflow/workflow-command.js +670 -0
- package/dist/workflow/workflow-command.js.map +1 -0
- package/dist/workflow/workflow-discovery.d.ts +27 -0
- package/dist/workflow/workflow-discovery.js +121 -0
- package/dist/workflow/workflow-discovery.js.map +1 -0
- package/dist/workflow/workflow-manager.d.ts +156 -0
- package/dist/workflow/workflow-manager.js +359 -0
- package/dist/workflow/workflow-manager.js.map +1 -0
- package/dist/workflow/workflows/design-and-code/README.md +50 -0
- package/dist/workflow/workflows/design-and-code/workflow.yaml +169 -0
- package/dist/workflow/workflows/deterministic-eval-smoke/scripts/format_report.js +29 -0
- package/dist/workflow/workflows/deterministic-eval-smoke/scripts/package.json +5 -0
- package/dist/workflow/workflows/deterministic-eval-smoke/scripts/requirements.txt +1 -0
- package/dist/workflow/workflows/deterministic-eval-smoke/scripts/run_eval.py +41 -0
- package/dist/workflow/workflows/deterministic-eval-smoke/workflow.yaml +57 -0
- package/dist/workflow/workflows/deterministic-verdict-smoke/scripts/classify.py +18 -0
- package/dist/workflow/workflows/deterministic-verdict-smoke/workflow.yaml +51 -0
- package/dist/workflow/workflows/evolve/scripts/LICENSE +201 -0
- package/dist/workflow/workflows/evolve/scripts/README.md +314 -0
- package/dist/workflow/workflows/evolve/scripts/evolve-brief +16 -0
- package/dist/workflow/workflows/evolve/scripts/evolve-cognition +16 -0
- package/dist/workflow/workflows/evolve/scripts/evolve-db +16 -0
- package/dist/workflow/workflows/evolve/scripts/evolve-eval +16 -0
- package/dist/workflow/workflows/evolve/scripts/evolve-files +16 -0
- package/dist/workflow/workflows/evolve/scripts/evolve-summary +16 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/__init__.py +14 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/algorithms/__init__.py +17 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/algorithms/base.py +21 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/algorithms/factory.py +90 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/algorithms/greedy.py +17 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/algorithms/island.py +316 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/algorithms/random.py +18 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/algorithms/ucb1.py +55 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/best_snapshot.py +62 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/cli.py +752 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/cognition.py +127 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/database.py +218 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/diff.py +57 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/embedding.py +84 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/file_lock.py +45 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/run_state.py +343 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/sampling_config.py +173 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/structures.py +93 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_core/vector_index.py +191 -0
- package/dist/workflow/workflows/evolve/scripts/evolve_result.py +1503 -0
- package/dist/workflow/workflows/evolve/scripts/requirements.txt +2 -0
- package/dist/workflow/workflows/evolve/workflow.yaml +646 -0
- package/dist/workflow/workflows/test-email-summary/skills/email-formatting/SKILL.md +55 -0
- package/dist/workflow/workflows/test-email-summary/skills/gmail-query-syntax/SKILL.md +63 -0
- package/dist/workflow/workflows/test-email-summary/skills/json-schema-emails/SKILL.md +41 -0
- package/dist/workflow/workflows/test-email-summary/skills/json-schema-emails/schema.json +34 -0
- package/dist/workflow/workflows/test-email-summary/skills/summary-style-guide/SKILL.md +51 -0
- package/dist/workflow/workflows/test-email-summary/workflow.yaml +90 -0
- package/dist/workflow/workflows/vuln-discovery/skills/harness-design-fuzzing/SKILL.md +206 -0
- package/dist/workflow/workflows/vuln-discovery/skills/memory-safety-c-cpp/SKILL.md +167 -0
- package/dist/workflow/workflows/vuln-discovery/skills/vulnerability-triage/SKILL.md +189 -0
- package/dist/workflow/workflows/vuln-discovery/workflow.yaml +925 -0
- package/dist/workflow/worktree.d.ts +43 -0
- package/dist/workflow/worktree.js +106 -0
- package/dist/workflow/worktree.js.map +1 -0
- package/docker/Dockerfile.base +31 -0
- package/docker/Dockerfile.base.arm64 +27 -0
- package/docker/Dockerfile.claude-code +3 -1
- package/docker/Dockerfile.codex +17 -0
- package/docker/Dockerfile.goose +3 -1
- package/docker/entrypoint-claude-code.sh +7 -0
- package/docker/entrypoint-codex.sh +42 -0
- package/docker/entrypoint-goose.sh +6 -0
- package/docker/entrypoint-uid-remap.sh +93 -0
- package/package.json +27 -11
- package/packages/web-ui/README.md +113 -0
- package/packages/web-ui/docs/workflow-state-machine.png +0 -0
- package/src/config/generated/compiled-policy.json +528 -161
- package/src/config/generated/test-scenarios.json +4810 -170
- package/src/config/generated/tool-annotations.json +34 -47
- package/src/config/mcp-servers.json +2 -2
- package/src/workflow/workflows/design-and-code/README.md +50 -0
- package/src/workflow/workflows/design-and-code/workflow.yaml +169 -0
- package/src/workflow/workflows/deterministic-eval-smoke/scripts/format_report.js +29 -0
- package/src/workflow/workflows/deterministic-eval-smoke/scripts/package.json +5 -0
- package/src/workflow/workflows/deterministic-eval-smoke/scripts/requirements.txt +1 -0
- package/src/workflow/workflows/deterministic-eval-smoke/scripts/run_eval.py +41 -0
- package/src/workflow/workflows/deterministic-eval-smoke/workflow.yaml +57 -0
- package/src/workflow/workflows/deterministic-verdict-smoke/scripts/classify.py +18 -0
- package/src/workflow/workflows/deterministic-verdict-smoke/workflow.yaml +51 -0
- package/src/workflow/workflows/evolve/scripts/LICENSE +201 -0
- package/src/workflow/workflows/evolve/scripts/README.md +314 -0
- package/src/workflow/workflows/evolve/scripts/evolve-brief +16 -0
- package/src/workflow/workflows/evolve/scripts/evolve-cognition +16 -0
- package/src/workflow/workflows/evolve/scripts/evolve-db +16 -0
- package/src/workflow/workflows/evolve/scripts/evolve-eval +16 -0
- package/src/workflow/workflows/evolve/scripts/evolve-files +16 -0
- package/src/workflow/workflows/evolve/scripts/evolve-summary +16 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/__init__.py +14 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/algorithms/__init__.py +17 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/algorithms/base.py +21 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/algorithms/factory.py +90 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/algorithms/greedy.py +17 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/algorithms/island.py +316 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/algorithms/random.py +18 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/algorithms/ucb1.py +55 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/best_snapshot.py +62 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/cli.py +752 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/cognition.py +127 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/database.py +218 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/diff.py +57 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/embedding.py +84 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/file_lock.py +45 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/run_state.py +343 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/sampling_config.py +173 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/structures.py +93 -0
- package/src/workflow/workflows/evolve/scripts/evolve_core/vector_index.py +191 -0
- package/src/workflow/workflows/evolve/scripts/evolve_result.py +1503 -0
- package/src/workflow/workflows/evolve/scripts/requirements.txt +2 -0
- package/src/workflow/workflows/evolve/workflow.yaml +646 -0
- package/src/workflow/workflows/test-email-summary/skills/email-formatting/SKILL.md +55 -0
- package/src/workflow/workflows/test-email-summary/skills/gmail-query-syntax/SKILL.md +63 -0
- package/src/workflow/workflows/test-email-summary/skills/json-schema-emails/SKILL.md +41 -0
- package/src/workflow/workflows/test-email-summary/skills/json-schema-emails/schema.json +34 -0
- package/src/workflow/workflows/test-email-summary/skills/summary-style-guide/SKILL.md +51 -0
- package/src/workflow/workflows/test-email-summary/workflow.yaml +90 -0
- package/src/workflow/workflows/vuln-discovery/skills/harness-design-fuzzing/SKILL.md +206 -0
- package/src/workflow/workflows/vuln-discovery/skills/memory-safety-c-cpp/SKILL.md +167 -0
- package/src/workflow/workflows/vuln-discovery/skills/vulnerability-triage/SKILL.md +189 -0
- package/src/workflow/workflows/vuln-discovery/workflow.yaml +925 -0
- package/dist/pipeline/llm-logger.js.map +0 -1
- package/dist/session/errors.js.map +0 -1
|
@@ -0,0 +1,3594 @@
|
|
|
1
|
+
import { readFileSync, existsSync, mkdirSync, writeFileSync, appendFileSync, readdirSync, statSync, unlinkSync, cpSync, } from 'node:fs';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { isWithinDirectory } from '../types/argument-roles.js';
|
|
4
|
+
import { errorMessage } from '../utils/error-message.js';
|
|
5
|
+
import { MessageLog } from './message-log.js';
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import { execFile as execFileCb } from 'node:child_process';
|
|
8
|
+
import { promisify } from 'node:util';
|
|
9
|
+
import * as http from 'node:http';
|
|
10
|
+
import { createActor, fromPromise } from 'xstate';
|
|
11
|
+
import { createWorkflowId, WORKFLOW_ARTIFACT_DIR, GLOBAL_PERSONA, DEFAULT_CONTAINER_SCOPE, DETERMINISTIC_RESULT_ERROR_VERDICT, resolveWorkflowSkillsOptions, } from './types.js';
|
|
12
|
+
import { getBundleAuditLogPath, getBundleBundleDir, getBundleControlSocketPath, getBundleRuntimeRoot, getBundleStatesDir, getInvocationDir, nextStateSlug, } from '../config/paths.js';
|
|
13
|
+
import { POLICY_LOAD_PATH } from '../trusted-process/control-server.js';
|
|
14
|
+
import { loadConfig, loadPersonaPolicyArtifacts } from '../config/index.js';
|
|
15
|
+
import { getBundleCapturesDir } from '../config/paths.js';
|
|
16
|
+
import { validatePolicyDir } from '../config/validate-policy-dir.js';
|
|
17
|
+
import { getTokenStreamBus } from '../docker/token-stream-bus.js';
|
|
18
|
+
import { getPersonaDefinitionPath, resolvePersona } from '../persona/resolve.js';
|
|
19
|
+
import { isMemoryEnabledForPersonaName } from '../persona/memory-gate.js';
|
|
20
|
+
import { extractRequiredServers } from '../trusted-process/policy-roots.js';
|
|
21
|
+
import { createPersonaName } from '../persona/types.js';
|
|
22
|
+
import { MEMORY_SERVER_NAME } from '../memory/memory-annotations.js';
|
|
23
|
+
import { createAgentConversationId, createBundleId } from '../session/types.js';
|
|
24
|
+
import { CONTAINER_WORKSPACE_DIR, describeTransientFailureKind, } from '../docker/agent-adapter.js';
|
|
25
|
+
import { buildWorkflowExecCommand, ensureSecureBundleDir, } from '../docker/docker-infrastructure.js';
|
|
26
|
+
import { buildWorkflowMachine, } from './machine-builder.js';
|
|
27
|
+
import { parseAgentStatus, AgentStatusParseError, buildStatusBlockReprompt, getValidVerdicts, buildInvalidVerdictReprompt, } from './status-parser.js';
|
|
28
|
+
import { buildAgentCommand, buildArtifactReprompt, buildStatusInstructions } from './prompt-builder.js';
|
|
29
|
+
import { DEFAULT_EVOLVE_LANE_DIR, DEFAULT_EVOLVE_LANE_RELATIVE_DIR, evolveResultScriptIndex, evolveStepName, resolveFanOutWorkers, } from './lane-template.js';
|
|
30
|
+
import { collectFilesRecursive, hasAnyFiles, snapshotArtifacts } from './artifacts.js';
|
|
31
|
+
import { isSafeWorkspaceRelativePath, parseArtifactRef, validateDefinition, validateWorkflowSkillReferences, } from './validate.js';
|
|
32
|
+
import { parseDefinitionFile, getWorkflowPackageDir } from './discovery.js';
|
|
33
|
+
import { resolveSkillsForSession } from '../skills/discovery.js';
|
|
34
|
+
import { discoverWorkflowRuns } from './workflow-discovery.js';
|
|
35
|
+
import { isCheckpointResumable } from './checkpoint.js';
|
|
36
|
+
import { AgentInvocationError, WorkflowQuotaExhaustedError, WorkflowTransientFailureError, isWorkflowQuotaExhaustedError, isWorkflowTransientFailureError, } from './errors.js';
|
|
37
|
+
import { commitContainerSnapshot, removeContainerSnapshotImages } from './container-snapshots.js';
|
|
38
|
+
const execFileAsync = promisify(execFileCb);
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// Transition message truncation
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
const MAX_TRANSITION_MESSAGE_BYTES = 4096;
|
|
43
|
+
const TRANSITION_TRUNCATION_NOTICE = '\n\n[... truncated]';
|
|
44
|
+
/**
|
|
45
|
+
* Default timeout for a `loadPolicy` RPC to the workflow coordinator.
|
|
46
|
+
* Generous because the coordinator's mutex may be held briefly by an
|
|
47
|
+
* in-flight tool call; tight because "control socket unreachable" must
|
|
48
|
+
* surface fast rather than wedge the workflow.
|
|
49
|
+
*/
|
|
50
|
+
const LOAD_POLICY_RPC_TIMEOUT_MS = 10_000;
|
|
51
|
+
function truncateForTransition(text) {
|
|
52
|
+
if (!text)
|
|
53
|
+
return undefined;
|
|
54
|
+
if (Buffer.byteLength(text, 'utf-8') <= MAX_TRANSITION_MESSAGE_BYTES) {
|
|
55
|
+
return text;
|
|
56
|
+
}
|
|
57
|
+
const noticeBudget = MAX_TRANSITION_MESSAGE_BYTES - Buffer.byteLength(TRANSITION_TRUNCATION_NOTICE, 'utf-8');
|
|
58
|
+
let truncated = text;
|
|
59
|
+
while (Buffer.byteLength(truncated, 'utf-8') > noticeBudget) {
|
|
60
|
+
truncated = truncated.slice(0, Math.floor(truncated.length * 0.9));
|
|
61
|
+
}
|
|
62
|
+
return truncated + TRANSITION_TRUNCATION_NOTICE;
|
|
63
|
+
}
|
|
64
|
+
function toErrorMessage(err) {
|
|
65
|
+
return err instanceof Error ? err.message : String(err);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Narrows a finished child round actor's `snapshot.value` to the closed
|
|
69
|
+
* {@link RoundChildStatus} union. The child machine always terminates in one
|
|
70
|
+
* of its three synthetic terminals (`recorded`/`blocked`/`errored`), so this
|
|
71
|
+
* normally just confirms the name. An unrecognized terminal — which should be
|
|
72
|
+
* unreachable in a validated definition — is treated as `errored`, the
|
|
73
|
+
* conservative fail verdict, rather than silently passing as `recorded`.
|
|
74
|
+
*/
|
|
75
|
+
function toRoundChildStatus(value, index) {
|
|
76
|
+
if (value === 'recorded' || value === 'blocked' || value === 'errored') {
|
|
77
|
+
return value;
|
|
78
|
+
}
|
|
79
|
+
writeStderr(`[workflow] fan-out child ${index} ended in unexpected terminal "${String(value)}"; treating as errored`);
|
|
80
|
+
return 'errored';
|
|
81
|
+
}
|
|
82
|
+
// Parses the fan-out step name `step_<batch>(_lane_<k>)?`: group 1 = batch index,
|
|
83
|
+
// group 2 = lane id (undefined for legacy non-lane `step_<batch>` nodes). The
|
|
84
|
+
// TS twin of Python STEP_NAME_RE in scripts/evolve_result.py; the writer is
|
|
85
|
+
// {@link evolveStepName} in lane-template.ts. All three move together.
|
|
86
|
+
const EVOLVE_LANE_STEP_RE = /^step_(\d+)(?:_lane_(\d+))?$/;
|
|
87
|
+
function tryParseAgentStatus(responseText) {
|
|
88
|
+
try {
|
|
89
|
+
const output = parseAgentStatus(responseText);
|
|
90
|
+
return output ? { kind: 'ok', output } : { kind: 'missing' };
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
if (err instanceof AgentStatusParseError) {
|
|
94
|
+
return { kind: 'malformed', error: err };
|
|
95
|
+
}
|
|
96
|
+
throw err;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Writes directly to process.stderr, bypassing any console hijacking
|
|
101
|
+
* (e.g., logger.setup() redirecting console.error to a file).
|
|
102
|
+
* Used for critical error messages that MUST reach the terminal.
|
|
103
|
+
*/
|
|
104
|
+
function writeStderr(message) {
|
|
105
|
+
process.stderr.write(`${message}\n`);
|
|
106
|
+
}
|
|
107
|
+
/** Per-run subdirectory name for the staged copy of the workflow's bundled `skills/` tree. */
|
|
108
|
+
const STAGED_WORKFLOW_SKILLS_SUBDIR = 'workflow-skills';
|
|
109
|
+
/** Per-run subdirectory name for the staged copy of workflow helper scripts. */
|
|
110
|
+
const STAGED_WORKFLOW_SCRIPTS_SUBDIR = 'workflow-scripts';
|
|
111
|
+
/**
|
|
112
|
+
* Workspace-relative segments of the evolve engine's per-step code directory:
|
|
113
|
+
* `<workspace>/.evolve_runs/main/steps/<step_name>/code`.
|
|
114
|
+
*
|
|
115
|
+
* This layout is OWNED by the Python bridge (`evolve_result.py` — the engine
|
|
116
|
+
* writes each candidate's source under `steps/<step_name>/code`, and
|
|
117
|
+
* `evolve-eval` reads it from there). The TS side only READS it host-side to
|
|
118
|
+
* content-hash fan-out candidates ({@link computeFanOutCandidateDuplicateStats}),
|
|
119
|
+
* so this const duplicates a path the Python defines. Keep the two in sync: if
|
|
120
|
+
* the engine's on-disk step layout changes, update both here and in the bridge.
|
|
121
|
+
*/
|
|
122
|
+
const EVOLVE_STEP_CODE_PATH_SEGMENTS = ['.evolve_runs', 'main', 'steps'];
|
|
123
|
+
const EVOLVE_STEP_CODE_FILENAME = 'code';
|
|
124
|
+
/**
|
|
125
|
+
* Stages the workflow package's `skills/` tree into the run directory
|
|
126
|
+
* so the workflow becomes self-contained at start time. After this call
|
|
127
|
+
* the per-run path (`<runMetaDir>/workflow-skills/`) is the canonical
|
|
128
|
+
* source of workflow-bundled skills for both initial-set staging and
|
|
129
|
+
* per-state restage; the original package path under
|
|
130
|
+
* `getWorkflowPackageDir(definitionPath)` is no longer consulted at
|
|
131
|
+
* runtime.
|
|
132
|
+
*
|
|
133
|
+
* Returns the absolute path to the staged copy when source `skills/`
|
|
134
|
+
* exists and was copied, `undefined` when the source has no `skills/`
|
|
135
|
+
* directory (the workflow simply ships none — not an error). Throws
|
|
136
|
+
* only on copy I/O failure, since a half-copied tree would be worse
|
|
137
|
+
* than a missing one.
|
|
138
|
+
*
|
|
139
|
+
* Why copy at start (not symlink): the package directory may be
|
|
140
|
+
* deleted, moved, or rebuilt between start and resume — symlink chases
|
|
141
|
+
* lead to "skills silently empty on resume" without diagnostic. A
|
|
142
|
+
* shallow-recursive copy decouples the run from its source-of-record
|
|
143
|
+
* location.
|
|
144
|
+
*/
|
|
145
|
+
export function stageWorkflowSkillsAtStart(packageDir, runMetaDir) {
|
|
146
|
+
return stageWorkflowSubdir(packageDir, runMetaDir, 'skills', STAGED_WORKFLOW_SKILLS_SUBDIR);
|
|
147
|
+
}
|
|
148
|
+
export function stageWorkflowScriptsAtStart(packageDir, runMetaDir) {
|
|
149
|
+
return stageWorkflowSubdir(packageDir, runMetaDir, 'scripts', STAGED_WORKFLOW_SCRIPTS_SUBDIR);
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Shallow-recursive copy of a workflow package subdir (`skills` / `scripts`) into
|
|
153
|
+
* the run dir. Returns the staged path, or `undefined` when the source subdir is
|
|
154
|
+
* absent (the workflow ships none — not an error). Throws only on copy I/O
|
|
155
|
+
* failure (a half-copied tree is worse than a missing one). See
|
|
156
|
+
* `stageWorkflowSkillsAtStart` above for why we copy rather than symlink.
|
|
157
|
+
*/
|
|
158
|
+
function stageWorkflowSubdir(packageDir, runMetaDir, sourceSubdir, stagedSubdir) {
|
|
159
|
+
const source = resolve(packageDir, sourceSubdir);
|
|
160
|
+
if (!existsSync(source))
|
|
161
|
+
return undefined;
|
|
162
|
+
const stagedDir = resolve(runMetaDir, stagedSubdir);
|
|
163
|
+
cpSync(source, stagedDir, { recursive: true });
|
|
164
|
+
return stagedDir;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Resolves the workflow-skills directory to use for a resumed workflow.
|
|
168
|
+
*
|
|
169
|
+
* Priority order, from most-trusted to most-degraded:
|
|
170
|
+
* 1. The checkpointed staged path, if it still exists. The orchestrator
|
|
171
|
+
* writes this at start under the run dir; we own the lifecycle so
|
|
172
|
+
* it should normally still be there.
|
|
173
|
+
* 2. A fresh stage from the original package dir, when the staged
|
|
174
|
+
* copy is gone but the source is reachable. Recovers from manual
|
|
175
|
+
* pruning of the run dir without losing skills on resume.
|
|
176
|
+
* 3. `undefined`, with a stderr warning. Both the staged copy and
|
|
177
|
+
* the source are gone; degrade gracefully but make the loss
|
|
178
|
+
* visible (the original bug — silent empty discovery — is what
|
|
179
|
+
* this whole resolver exists to prevent).
|
|
180
|
+
*
|
|
181
|
+
* Pre-feature checkpoints have no `workflowSkillsDir` field; they
|
|
182
|
+
* trigger the case-(2) fresh stage transparently.
|
|
183
|
+
*/
|
|
184
|
+
function resolveWorkflowSkillsDirOnResume(opts) {
|
|
185
|
+
const { workflowId, checkpointedStagedDir, packageDir, runMetaDir } = opts;
|
|
186
|
+
if (checkpointedStagedDir !== undefined && existsSync(checkpointedStagedDir)) {
|
|
187
|
+
return checkpointedStagedDir;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
const restaged = stageWorkflowSkillsAtStart(packageDir, runMetaDir);
|
|
191
|
+
if (restaged !== undefined) {
|
|
192
|
+
writeStderr(`[workflow] Resume ${workflowId}: workflow-skills staged copy missing, re-staged from ${packageDir}.`);
|
|
193
|
+
return restaged;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch (err) {
|
|
197
|
+
writeStderr(`[workflow] Resume ${workflowId}: failed to re-stage workflow skills from ${packageDir}: ${toErrorMessage(err)}`);
|
|
198
|
+
return undefined;
|
|
199
|
+
}
|
|
200
|
+
// Both paths gone. Without this warning the resumed run silently
|
|
201
|
+
// loses every workflow-bundled skill — the exact failure mode this
|
|
202
|
+
// function was added to prevent.
|
|
203
|
+
writeStderr(`[workflow] Resume ${workflowId}: no workflow-skills available (staged copy and ${packageDir}/skills both missing). Workflow-bundled skills will not be available in this run.`);
|
|
204
|
+
return undefined;
|
|
205
|
+
}
|
|
206
|
+
function resolveWorkflowScriptsDirOnResume(opts) {
|
|
207
|
+
const { workflowId, checkpointedStagedDir, packageDir, runMetaDir } = opts;
|
|
208
|
+
if (checkpointedStagedDir !== undefined && existsSync(checkpointedStagedDir)) {
|
|
209
|
+
return checkpointedStagedDir;
|
|
210
|
+
}
|
|
211
|
+
try {
|
|
212
|
+
const restaged = stageWorkflowScriptsAtStart(packageDir, runMetaDir);
|
|
213
|
+
if (restaged !== undefined) {
|
|
214
|
+
writeStderr(`[workflow] Resume ${workflowId}: workflow-scripts staged copy missing, re-staged from ${packageDir}.`);
|
|
215
|
+
return restaged;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
catch (err) {
|
|
219
|
+
writeStderr(`[workflow] Resume ${workflowId}: failed to re-stage workflow scripts from ${packageDir}: ${toErrorMessage(err)}`);
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
222
|
+
// Both paths gone. Only warn if scripts were previously staged
|
|
223
|
+
// (`checkpointedStagedDir` set): a workflow that never shipped scripts must not
|
|
224
|
+
// warn about "missing" scripts on every resume. `stageWorkflowScriptsAtStart`
|
|
225
|
+
// already proved `scripts/` is absent here, so no source re-stat is needed.
|
|
226
|
+
if (checkpointedStagedDir === undefined) {
|
|
227
|
+
return undefined;
|
|
228
|
+
}
|
|
229
|
+
writeStderr(`[workflow] Resume ${workflowId}: no workflow-scripts available (staged copy and ${packageDir}/scripts both missing). Container deterministic helpers will not be available in this run.`);
|
|
230
|
+
return undefined;
|
|
231
|
+
}
|
|
232
|
+
// ---------------------------------------------------------------------------
|
|
233
|
+
// Orchestrator
|
|
234
|
+
// ---------------------------------------------------------------------------
|
|
235
|
+
export class WorkflowOrchestrator {
|
|
236
|
+
deps;
|
|
237
|
+
workflows = new Map();
|
|
238
|
+
lifecycleCallbacks = [];
|
|
239
|
+
constructor(deps) {
|
|
240
|
+
this.deps = deps;
|
|
241
|
+
}
|
|
242
|
+
/** Build a partial log entry with shared fields for the given workflow instance. */
|
|
243
|
+
logBase(instance) {
|
|
244
|
+
return {
|
|
245
|
+
ts: new Date().toISOString(),
|
|
246
|
+
workflowId: instance.id,
|
|
247
|
+
state: instance.currentState,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
// -----------------------------------------------------------------------
|
|
251
|
+
// Pre-flight validation
|
|
252
|
+
// -----------------------------------------------------------------------
|
|
253
|
+
/**
|
|
254
|
+
* Validates that all non-"global" personas referenced in the workflow
|
|
255
|
+
* definition actually exist on disk. Fails fast with a clear error
|
|
256
|
+
* listing all missing personas.
|
|
257
|
+
*
|
|
258
|
+
* Only checks for persona.json existence -- does NOT verify compiled
|
|
259
|
+
* policy (that happens at session creation time via resolvePersona).
|
|
260
|
+
*/
|
|
261
|
+
validatePersonas(definition) {
|
|
262
|
+
const missing = [];
|
|
263
|
+
for (const [stateId, state] of Object.entries(definition.states)) {
|
|
264
|
+
if (state.type !== 'agent')
|
|
265
|
+
continue;
|
|
266
|
+
const persona = state.persona;
|
|
267
|
+
if (persona === GLOBAL_PERSONA)
|
|
268
|
+
continue;
|
|
269
|
+
const defPath = getPersonaDefinitionPath(createPersonaName(persona));
|
|
270
|
+
if (!existsSync(defPath)) {
|
|
271
|
+
missing.push(`"${persona}" (used by state "${stateId}")`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (missing.length > 0) {
|
|
275
|
+
throw new Error(`Workflow references personas that do not exist:\n` +
|
|
276
|
+
` ${missing.join('\n ')}\n` +
|
|
277
|
+
`Create them with: ironcurtain persona create <name>`);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
// -----------------------------------------------------------------------
|
|
281
|
+
// Workflow-scoped Docker infrastructure lifecycle
|
|
282
|
+
// -----------------------------------------------------------------------
|
|
283
|
+
/**
|
|
284
|
+
* Determines whether a workflow should run in shared-container mode.
|
|
285
|
+
* Requires both an explicit opt-in via `settings.sharedContainer` and
|
|
286
|
+
* a Docker-backed agent. Builtin workflows ignore the flag entirely.
|
|
287
|
+
*/
|
|
288
|
+
shouldUseSharedContainer(definition) {
|
|
289
|
+
const settings = definition.settings ?? {};
|
|
290
|
+
if (settings.sharedContainer !== true)
|
|
291
|
+
return false;
|
|
292
|
+
if (settings.mode === 'builtin')
|
|
293
|
+
return false;
|
|
294
|
+
return true;
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Returns the bundle for `scope`, minting a new one on first use.
|
|
298
|
+
*
|
|
299
|
+
* Each unique `containerScope` used across the workflow's states gets
|
|
300
|
+
* exactly one bundle: the first `executeAgentState` call under a given
|
|
301
|
+
* scope creates the bundle, attaches its coordinator control server,
|
|
302
|
+
* and inserts the result into `instance.bundlesByScope`. Later states
|
|
303
|
+
* with the same scope borrow the existing entry (no factory call).
|
|
304
|
+
*
|
|
305
|
+
* Replaces the pre-Step-6 eager workflow-start mint. Only invoked when
|
|
306
|
+
* `shouldUseSharedContainer(definition)` returns true; callers must
|
|
307
|
+
* gate on that check.
|
|
308
|
+
*
|
|
309
|
+
* On control-server attach failure, the just-created bundle is torn
|
|
310
|
+
* down before the error propagates so we do not leak Docker resources
|
|
311
|
+
* on partial initialization. The map is **not** populated in that case
|
|
312
|
+
* — the scope entry remains absent so a later state can retry.
|
|
313
|
+
*
|
|
314
|
+
* Concurrency: relies on serial XState invocations — there is no
|
|
315
|
+
* parallel agent-state fan-out today. If that ever lands, add an
|
|
316
|
+
* in-flight-promise guard to prevent concurrent lazy-mint races for
|
|
317
|
+
* the same scope.
|
|
318
|
+
*
|
|
319
|
+
* Abort race: if `destroyWorkflowInfrastructure` begins while a mint
|
|
320
|
+
* is suspended at either `await factory(...)` or `await startServer(...)`,
|
|
321
|
+
* the mint completes but must not publish into `bundlesByScope` —
|
|
322
|
+
* destroy already snapshot-cleared the map and would miss the late
|
|
323
|
+
* insertion. `instance.aborted` is the barrier, consulted at both
|
|
324
|
+
* `await` points; on observed abort we tear down the just-built
|
|
325
|
+
* bundle and throw without updating the map.
|
|
326
|
+
*/
|
|
327
|
+
async ensureBundleForScope(instance, scope) {
|
|
328
|
+
if (instance.aborted) {
|
|
329
|
+
throw new Error(`Workflow ${instance.id} is aborting; cannot mint bundle for scope "${scope}"`);
|
|
330
|
+
}
|
|
331
|
+
const existing = instance.bundlesByScope.get(scope);
|
|
332
|
+
if (existing)
|
|
333
|
+
return existing;
|
|
334
|
+
const settings = instance.definition.settings ?? {};
|
|
335
|
+
const agentId = (settings.dockerAgent ?? 'claude-code');
|
|
336
|
+
// Mint a fresh UUID per scope: each bundle needs its own on-disk
|
|
337
|
+
// directory tree, coordinator control socket, and Docker container
|
|
338
|
+
// name. Unlike single-session CLI (`bundleId === sessionId`),
|
|
339
|
+
// workflow scopes produce genuine standalone `BundleId`s.
|
|
340
|
+
const bundleId = createBundleId();
|
|
341
|
+
const controlSocketPath = getBundleControlSocketPath(bundleId);
|
|
342
|
+
// Ensure the per-bundle runtime root exists BEFORE the coordinator
|
|
343
|
+
// tries to bind its UDS there. Routes through `ensureSecureBundleDir`
|
|
344
|
+
// so the same symlink-rejection + 0o700-enforcement hardening that
|
|
345
|
+
// guards `sockets/` and `host/` (in `prepareDockerInfrastructure`)
|
|
346
|
+
// also guards the root the coordinator's `ctrl.sock` binds into.
|
|
347
|
+
ensureSecureBundleDir(getBundleRuntimeRoot(bundleId));
|
|
348
|
+
const requiredServers = this.getRequiredServersForScope(instance, scope);
|
|
349
|
+
// Persona-less initial set; per-state restaging fills in persona skills later.
|
|
350
|
+
// Reads the cached per-run staged path (set at start / resume) so
|
|
351
|
+
// the bundle never depends on the original package directory still
|
|
352
|
+
// being on disk.
|
|
353
|
+
const resolvedSkills = resolveSkillsForSession({ workflowSkillsDir: instance.workflowSkillsDir });
|
|
354
|
+
const factory = this.deps.createWorkflowInfrastructure ?? (await this.loadDefaultInfrastructureFactory());
|
|
355
|
+
const baseImageOverride = instance.containerSnapshots?.[scope]?.image;
|
|
356
|
+
const infra = await factory({
|
|
357
|
+
workflowId: instance.id,
|
|
358
|
+
bundleId,
|
|
359
|
+
agentId,
|
|
360
|
+
controlSocketPath,
|
|
361
|
+
workspacePath: instance.workspacePath,
|
|
362
|
+
scope,
|
|
363
|
+
requiredServers,
|
|
364
|
+
resolvedSkills,
|
|
365
|
+
workflowScriptsDir: instance.workflowScriptsDir,
|
|
366
|
+
baseImageOverride,
|
|
367
|
+
});
|
|
368
|
+
// Abort may have landed while `factory()` was suspended. Publishing
|
|
369
|
+
// into `bundlesByScope` now would leak past the destroy's leak
|
|
370
|
+
// assertion; destroy the orphan inline and throw. ESLint cannot see
|
|
371
|
+
// that `instance.aborted` is mutated by the concurrent
|
|
372
|
+
// `destroyWorkflowInfrastructure` call between the two reads.
|
|
373
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
374
|
+
if (instance.aborted) {
|
|
375
|
+
const destroy = this.deps.destroyWorkflowInfrastructure ?? (await this.loadDefaultInfrastructureTeardown());
|
|
376
|
+
await destroy(infra).catch((teardownErr) => {
|
|
377
|
+
writeStderr(`[workflow] destroyDockerInfrastructure during abort recovery for ${instance.id} (bundle ${bundleId}): ${toErrorMessage(teardownErr)}`);
|
|
378
|
+
});
|
|
379
|
+
throw new Error(`Workflow ${instance.id} was aborted during bundle mint for scope "${scope}"`);
|
|
380
|
+
}
|
|
381
|
+
// Attach the control server. On failure, tear down the bundle so
|
|
382
|
+
// we don't leak containers/proxies; the error propagates so the
|
|
383
|
+
// caller (executeAgentState) can fail the state invoke.
|
|
384
|
+
try {
|
|
385
|
+
const startServer = this.deps.startWorkflowControlServer ?? defaultStartWorkflowControlServer;
|
|
386
|
+
await startServer({ infra, socketPath: controlSocketPath });
|
|
387
|
+
}
|
|
388
|
+
catch (err) {
|
|
389
|
+
const destroy = this.deps.destroyWorkflowInfrastructure ?? (await this.loadDefaultInfrastructureTeardown());
|
|
390
|
+
await destroy(infra).catch((teardownErr) => {
|
|
391
|
+
writeStderr(`[workflow] destroyDockerInfrastructure during control-server recovery for ${instance.id}: ${toErrorMessage(teardownErr)}`);
|
|
392
|
+
});
|
|
393
|
+
throw err;
|
|
394
|
+
}
|
|
395
|
+
// Abort check #2: abort may have landed while `startServer()` was
|
|
396
|
+
// suspended. Publishing into `bundlesByScope` now would leak past
|
|
397
|
+
// the destroy's leak assertion; tear down the just-built bundle
|
|
398
|
+
// and throw. ESLint cannot see that `instance.aborted` is mutated
|
|
399
|
+
// by the concurrent `destroyWorkflowInfrastructure` call between
|
|
400
|
+
// the two reads.
|
|
401
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
402
|
+
if (instance.aborted) {
|
|
403
|
+
const destroy = this.deps.destroyWorkflowInfrastructure ?? (await this.loadDefaultInfrastructureTeardown());
|
|
404
|
+
await destroy(infra).catch((teardownErr) => {
|
|
405
|
+
writeStderr(`[workflow] destroyDockerInfrastructure during abort recovery (post-startServer) for ${instance.id} (bundle ${bundleId}): ${toErrorMessage(teardownErr)}`);
|
|
406
|
+
});
|
|
407
|
+
throw new Error(`Workflow ${instance.id} was aborted during control-server attach for scope "${scope}"`);
|
|
408
|
+
}
|
|
409
|
+
instance.bundlesByScope.set(scope, infra);
|
|
410
|
+
instance.mintedServersByBundle.set(bundleId, requiredServers);
|
|
411
|
+
return infra;
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Returns the cached policyDir for `persona`, canonicalizing on first
|
|
415
|
+
* lookup so required-server derivation and the coordinator's
|
|
416
|
+
* `loadPolicy` RPC operate on the same realpath-resolved file (any
|
|
417
|
+
* symlink under the persona dir is collapsed once, here, instead of
|
|
418
|
+
* re-resolved by every reader).
|
|
419
|
+
*/
|
|
420
|
+
getPolicyDir(instance, persona) {
|
|
421
|
+
const cached = instance.policyDirByPersona.get(persona);
|
|
422
|
+
if (cached !== undefined)
|
|
423
|
+
return cached;
|
|
424
|
+
const dir = validatePolicyDir(resolvePersonaPolicyDir(persona));
|
|
425
|
+
instance.policyDirByPersona.set(persona, dir);
|
|
426
|
+
return dir;
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Computes the union of MCP server names required by every agent state
|
|
430
|
+
* sharing `scope`. Threaded into the workflow infrastructure factory so
|
|
431
|
+
* the bundle only spawns proxies the policies actually reference.
|
|
432
|
+
*/
|
|
433
|
+
getRequiredServersForScope(instance, scope) {
|
|
434
|
+
const union = new Set();
|
|
435
|
+
const seenPersonas = new Set();
|
|
436
|
+
for (const stateConfig of Object.values(instance.definition.states)) {
|
|
437
|
+
if (stateConfig.type !== 'agent')
|
|
438
|
+
continue;
|
|
439
|
+
const stateScope = stateConfig.containerScope ?? DEFAULT_CONTAINER_SCOPE;
|
|
440
|
+
if (stateScope !== scope)
|
|
441
|
+
continue;
|
|
442
|
+
if (seenPersonas.has(stateConfig.persona))
|
|
443
|
+
continue;
|
|
444
|
+
seenPersonas.add(stateConfig.persona);
|
|
445
|
+
const { compiledPolicy } = loadPersonaPolicyArtifacts(this.getPolicyDir(instance, stateConfig.persona));
|
|
446
|
+
for (const server of extractRequiredServers(compiledPolicy)) {
|
|
447
|
+
union.add(server);
|
|
448
|
+
}
|
|
449
|
+
// Memory is bolt-on (not in compiled policy). `Set.add` is
|
|
450
|
+
// idempotent, so adding here per-persona preserves any-wants-it.
|
|
451
|
+
if (isMemoryEnabledForPersonaName(stateConfig.persona, this.deps.userConfig)) {
|
|
452
|
+
union.add(MEMORY_SERVER_NAME);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return union;
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* Reloads the coordinator's policy for the given persona on the given
|
|
459
|
+
* bundle. Called once per agent state invocation (including re-entries)
|
|
460
|
+
* in shared-container mode. The coordinator stamps `persona` onto every
|
|
461
|
+
* subsequent audit entry, so consumers can reconstruct per-persona /
|
|
462
|
+
* per-re-entry slices from the bundle's `audit.jsonl` file.
|
|
463
|
+
*
|
|
464
|
+
* Each scope owns its own bundle, coordinator, and control socket, so
|
|
465
|
+
* the bundle is passed explicitly — cycle the scope the state is
|
|
466
|
+
* about to enter, not some other scope's bundle.
|
|
467
|
+
*
|
|
468
|
+
* On failure (control socket unreachable, coordinator reports a load
|
|
469
|
+
* error) this throws — the workflow must not proceed under the
|
|
470
|
+
* previous persona's policy.
|
|
471
|
+
*/
|
|
472
|
+
async cyclePolicy(instance, persona, bundle) {
|
|
473
|
+
// Skip the RPC when the bundle's coordinator already has this
|
|
474
|
+
// persona loaded. Consecutive states on the same scope+persona
|
|
475
|
+
// would otherwise re-send an identical `loadPolicy` — a no-op on
|
|
476
|
+
// the coordinator that still costs a UDS round-trip and a policy
|
|
477
|
+
// engine rebuild.
|
|
478
|
+
if (instance.currentPersonaByBundle.get(bundle.bundleId) === persona)
|
|
479
|
+
return;
|
|
480
|
+
const policyDir = this.getPolicyDir(instance, persona);
|
|
481
|
+
// Guard against mid-workflow recompiles that add a server: the bundle
|
|
482
|
+
// was minted with a fixed required-server set, so a new policy that
|
|
483
|
+
// expands the set would have the coordinator allow tools no relay was
|
|
484
|
+
// spawned for. Fail fast with a clear error instead.
|
|
485
|
+
const minted = instance.mintedServersByBundle.get(bundle.bundleId);
|
|
486
|
+
if (minted) {
|
|
487
|
+
const { compiledPolicy } = loadPersonaPolicyArtifacts(policyDir);
|
|
488
|
+
const newServers = extractRequiredServers(compiledPolicy);
|
|
489
|
+
const added = [];
|
|
490
|
+
for (const s of newServers)
|
|
491
|
+
if (!minted.has(s))
|
|
492
|
+
added.push(s);
|
|
493
|
+
if (added.length > 0) {
|
|
494
|
+
throw new Error(`Persona "${persona}" requires server(s) [${added.join(', ')}] that were not spawned for this scope ` +
|
|
495
|
+
`(bundle minted with [${[...minted].sort().join(', ')}]). ` +
|
|
496
|
+
`A persona recompile that expands the server set requires restarting the workflow.`);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
// Target this bundle's coordinator. Under bifurcated workflows every
|
|
500
|
+
// bundle has its own socket, so the bundle argument fully determines
|
|
501
|
+
// the route.
|
|
502
|
+
const socketPath = getBundleControlSocketPath(bundle.bundleId);
|
|
503
|
+
// Invalidate the cache entry BEFORE the RPC. If `loadPolicy` fails
|
|
504
|
+
// after the coordinator already applied the new policy (e.g., the
|
|
505
|
+
// RPC times out during the response), leaving the cache pointed at
|
|
506
|
+
// the previous persona would silently skip the next cycle and run
|
|
507
|
+
// a subsequent state under the wrong policy. Clear-then-rewrite
|
|
508
|
+
// ensures the cache is never stale relative to the coordinator's
|
|
509
|
+
// actual state — at worst we re-send an identical `loadPolicy`.
|
|
510
|
+
instance.currentPersonaByBundle.delete(bundle.bundleId);
|
|
511
|
+
const loadPolicy = this.deps.loadPolicyRpc ?? defaultLoadPolicyRpc;
|
|
512
|
+
await loadPolicy({
|
|
513
|
+
socketPath,
|
|
514
|
+
persona,
|
|
515
|
+
policyDir,
|
|
516
|
+
timeoutMs: LOAD_POLICY_RPC_TIMEOUT_MS,
|
|
517
|
+
});
|
|
518
|
+
instance.currentPersonaByBundle.set(bundle.bundleId, persona);
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Tears down every bundle this workflow owns. Walks
|
|
522
|
+
* `instance.bundlesByScope.values()` in parallel via `Promise.allSettled`
|
|
523
|
+
* so failures in one bundle's teardown do not block the others. The
|
|
524
|
+
* map is drained before the awaits so a second concurrent caller (e.g.
|
|
525
|
+
* `shutdownAll` racing the fire-and-forget destroy in
|
|
526
|
+
* `handleWorkflowComplete`) sees an empty map and returns without
|
|
527
|
+
* re-entering `destroy`.
|
|
528
|
+
*
|
|
529
|
+
* Callers in recovery paths rely on **expected** failure modes being
|
|
530
|
+
* swallowed so abort/shutdown flows can complete:
|
|
531
|
+
* - Failures in `destroy(infra)` are logged to stderr and swallowed.
|
|
532
|
+
* - Failures in socket unlinks are logged and swallowed.
|
|
533
|
+
*
|
|
534
|
+
* **One exception — intentional throw on internal invariant violation.**
|
|
535
|
+
* After teardown the helper asserts `bundlesByScope.size === 0`. A
|
|
536
|
+
* non-empty map at that point is a bug: something added an entry back
|
|
537
|
+
* while teardown was in flight. The invariant is enforced by the
|
|
538
|
+
* abort-guard in `ensureBundleForScope` (post-`factory` flag check);
|
|
539
|
+
* reaching this throw means the guard has a hole and we want it to
|
|
540
|
+
* surface loudly rather than silently strand a bundle. Callers are
|
|
541
|
+
* `abort()`, `shutdownAll()`, and the fire-and-forget `.catch()` in
|
|
542
|
+
* `handleWorkflowComplete` — in the expected invariant-holds case
|
|
543
|
+
* none of them see an exception. A thrown leak assertion in
|
|
544
|
+
* `abort()` / `shutdownAll()` will propagate to their own caller (the
|
|
545
|
+
* test or CLI layer), which is the intended signal.
|
|
546
|
+
*/
|
|
547
|
+
destroyWorkflowInfrastructure(instance) {
|
|
548
|
+
// Set BEFORE any short-circuit: an in-flight `ensureBundleForScope`
|
|
549
|
+
// that resumes from its `await factory(...)` must observe this flag
|
|
550
|
+
// and tear down its own orphan rather than publishing into the map.
|
|
551
|
+
instance.aborted = true;
|
|
552
|
+
// Join semantics: the first caller starts the teardown; concurrent
|
|
553
|
+
// and later callers await the SAME promise. This is what lets the
|
|
554
|
+
// CLI's shutdownAll() block until the fire-and-forget destroy from
|
|
555
|
+
// handleWorkflowComplete has actually finished, instead of bailing
|
|
556
|
+
// on the already-cleared map and letting process.exit() kill the
|
|
557
|
+
// teardown mid-flight (leaking the container, its network, and the
|
|
558
|
+
// MCP relay subprocesses).
|
|
559
|
+
instance.teardownPromise ??= this.runBundleTeardown(instance);
|
|
560
|
+
return instance.teardownPromise;
|
|
561
|
+
}
|
|
562
|
+
/** Body of the bundle teardown; only ever started once per instance. */
|
|
563
|
+
async runBundleTeardown(instance) {
|
|
564
|
+
if (instance.bundlesByScope.size === 0)
|
|
565
|
+
return;
|
|
566
|
+
// Snapshot and clear BEFORE the awaits so a concurrent caller sees
|
|
567
|
+
// an empty map and bails out. See JSDoc for the rationale.
|
|
568
|
+
const bundles = [...instance.bundlesByScope.values()];
|
|
569
|
+
instance.bundlesByScope.clear();
|
|
570
|
+
const destroy = this.deps.destroyWorkflowInfrastructure ?? (await this.loadDefaultInfrastructureTeardown());
|
|
571
|
+
await Promise.allSettled(bundles.map(async (infra) => {
|
|
572
|
+
try {
|
|
573
|
+
await destroy(infra);
|
|
574
|
+
}
|
|
575
|
+
catch (err) {
|
|
576
|
+
writeStderr(`[workflow] destroyDockerInfrastructure failed for ${instance.id} (bundle ${infra.bundleId}): ${toErrorMessage(err)}`);
|
|
577
|
+
}
|
|
578
|
+
// Best-effort: unlink the coordinator control socket. Swallow
|
|
579
|
+
// ENOENT because the socket may never have been bound (e.g.,
|
|
580
|
+
// when destroy runs after a failure before control-server attach).
|
|
581
|
+
try {
|
|
582
|
+
unlinkSync(getBundleControlSocketPath(infra.bundleId));
|
|
583
|
+
}
|
|
584
|
+
catch (err) {
|
|
585
|
+
const code = err.code;
|
|
586
|
+
if (code !== 'ENOENT') {
|
|
587
|
+
writeStderr(`[workflow] Failed to unlink control socket for ${instance.id} (bundle ${infra.bundleId}): ${toErrorMessage(err)}`);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
}));
|
|
591
|
+
// Leak assertion: after snapshot-and-clear + parallel destroy, the
|
|
592
|
+
// map MUST be empty. A non-empty map means a destroy path silently
|
|
593
|
+
// added an entry back after teardown began — a real bug we surface
|
|
594
|
+
// as a synchronous error so it cannot be swallowed.
|
|
595
|
+
if (instance.bundlesByScope.size !== 0) {
|
|
596
|
+
const leaked = [...instance.bundlesByScope.keys()];
|
|
597
|
+
throw new Error(`Leaked workflow bundle scopes after teardown: ${JSON.stringify(leaked)}`);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
shouldSnapshotOnStop(instance) {
|
|
601
|
+
if (!this.shouldUseSharedContainer(instance.definition))
|
|
602
|
+
return false;
|
|
603
|
+
if (instance.definition.settings?.snapshotOnStop !== true)
|
|
604
|
+
return false;
|
|
605
|
+
return this.deps.userConfig.snapshot.enabled;
|
|
606
|
+
}
|
|
607
|
+
async snapshotResumableScopes(instance) {
|
|
608
|
+
if (!this.shouldSnapshotOnStop(instance))
|
|
609
|
+
return undefined;
|
|
610
|
+
const entries = [...instance.bundlesByScope.entries()];
|
|
611
|
+
if (entries.length === 0)
|
|
612
|
+
return undefined;
|
|
613
|
+
const snapshots = {};
|
|
614
|
+
for (const [scope, infra] of entries) {
|
|
615
|
+
// Container snapshots require Docker's commit/image APIs, which the Apple
|
|
616
|
+
// `container` runtime does not provide. Skip cleanly (the workflow stops
|
|
617
|
+
// without a resumable container snapshot) rather than attempting a commit
|
|
618
|
+
// that the runtime would reject.
|
|
619
|
+
if (infra.runtimeKind !== 'docker') {
|
|
620
|
+
writeStderr(`[workflow] Container snapshots are not supported on the "${infra.runtimeKind}" runtime; ` +
|
|
621
|
+
`stopping ${instance.id} scope "${scope}" without a snapshot.`);
|
|
622
|
+
continue;
|
|
623
|
+
}
|
|
624
|
+
try {
|
|
625
|
+
snapshots[scope] = await commitContainerSnapshot({
|
|
626
|
+
docker: infra.docker,
|
|
627
|
+
workflowId: instance.id,
|
|
628
|
+
scope,
|
|
629
|
+
containerId: infra.containerId,
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
catch (err) {
|
|
633
|
+
writeStderr(`[workflow] Failed to snapshot container for ${instance.id} scope "${scope}": ${toErrorMessage(err)}`);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
return Object.keys(snapshots).length > 0 ? snapshots : undefined;
|
|
637
|
+
}
|
|
638
|
+
async dockerForSnapshotCleanup(instance) {
|
|
639
|
+
if (instance.bundlesByScope.size > 0) {
|
|
640
|
+
return [...instance.bundlesByScope.values()][0].docker;
|
|
641
|
+
}
|
|
642
|
+
const { createDockerManager } = await import('../docker/docker-manager.js');
|
|
643
|
+
return createDockerManager();
|
|
644
|
+
}
|
|
645
|
+
async removeSnapshotImagesAfterTeardown(docker, snapshots) {
|
|
646
|
+
if (!snapshots || Object.keys(snapshots).length === 0)
|
|
647
|
+
return;
|
|
648
|
+
await removeContainerSnapshotImages(docker, snapshots).catch((err) => {
|
|
649
|
+
writeStderr(`[workflow] Failed to remove container snapshot image(s): ${toErrorMessage(err)}`);
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
653
|
+
* Builds and persists the terminal checkpoint, preserving the prior on-disk
|
|
654
|
+
* machineState/context (which point at the last non-terminal save point) so
|
|
655
|
+
* resume re-enters that state instead of the terminal snapshot. Shared by the
|
|
656
|
+
* abort() and handleWorkflowComplete() stop paths.
|
|
657
|
+
*/
|
|
658
|
+
saveTerminalCheckpoint(instance, finalStatus, containerSnapshots, existing) {
|
|
659
|
+
try {
|
|
660
|
+
const terminalCheckpoint = this.buildCheckpoint(instance, instance.actor.getSnapshot(), finalStatus, containerSnapshots);
|
|
661
|
+
const checkpoint = existing
|
|
662
|
+
? { ...terminalCheckpoint, machineState: existing.machineState, context: existing.context }
|
|
663
|
+
: terminalCheckpoint;
|
|
664
|
+
this.deps.checkpointStore.save(instance.id, checkpoint);
|
|
665
|
+
}
|
|
666
|
+
catch (err) {
|
|
667
|
+
writeStderr(`[workflow] Failed to save terminal checkpoint for ${instance.id}: ${toErrorMessage(err)}`);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
snapshotsSupersededBy(previous, next) {
|
|
671
|
+
if (!previous)
|
|
672
|
+
return undefined;
|
|
673
|
+
const nextImages = new Set(Object.values(next ?? {}).map((snapshot) => snapshot.image));
|
|
674
|
+
const superseded = {};
|
|
675
|
+
for (const [scope, snapshot] of Object.entries(previous)) {
|
|
676
|
+
if (!nextImages.has(snapshot.image)) {
|
|
677
|
+
superseded[scope] = snapshot;
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
return Object.keys(superseded).length > 0 ? superseded : undefined;
|
|
681
|
+
}
|
|
682
|
+
/**
|
|
683
|
+
* Lazy-loads the default `createDockerInfrastructure` wrapper. The real
|
|
684
|
+
* helper requires Docker dependencies and the session config, so we
|
|
685
|
+
* construct them here rather than eagerly at orchestrator construction
|
|
686
|
+
* time (which would pay the import cost for every workflow run,
|
|
687
|
+
* including builtin ones that don't need Docker).
|
|
688
|
+
*/
|
|
689
|
+
async loadDefaultInfrastructureFactory() {
|
|
690
|
+
const { createDockerInfrastructure } = await import('../docker/docker-infrastructure.js');
|
|
691
|
+
const { loadConfig, applyAllowedDirectoryToMcpArgs } = await import('../config/index.js');
|
|
692
|
+
const { filterMcpServersByPolicy } = await import('../persona/resolve.js');
|
|
693
|
+
// The raw `--capture-traces` override is threaded through unresolved;
|
|
694
|
+
// the infrastructure factory is the single place that resolves it
|
|
695
|
+
// against `userConfig.capture?.enabled`. The same raw override is
|
|
696
|
+
// applied to every scope's bundle.
|
|
697
|
+
const captureTracesOverride = this.deps.captureTracesOverride;
|
|
698
|
+
return async (input) => {
|
|
699
|
+
const config = loadConfig();
|
|
700
|
+
// Bundle dir holds the bundle's shared Docker artifacts
|
|
701
|
+
// (orientation, sockets, claude-state, escalations, system
|
|
702
|
+
// prompt). It lives under the bundle's per-bundleId subtree so
|
|
703
|
+
// bifurcated workflows can host multiple bundles side by side
|
|
704
|
+
// without key collisions.
|
|
705
|
+
// Audit entries go to the per-bundle audit file (one file per
|
|
706
|
+
// bundle, with per-entry persona tagging).
|
|
707
|
+
const bundleDir = getBundleBundleDir(input.workflowId, input.bundleId);
|
|
708
|
+
const bundleEscalationDir = resolve(bundleDir, 'escalations');
|
|
709
|
+
// All sessions in a workflow share the workflow's workspace dir
|
|
710
|
+
// as their allowed directory. Do NOT use a session-scoped sandbox
|
|
711
|
+
// here: the orchestrator's artifact checks look inside the
|
|
712
|
+
// workspace, so artifacts the agent writes must land in the same
|
|
713
|
+
// tree.
|
|
714
|
+
mkdirSync(bundleDir, { recursive: true });
|
|
715
|
+
mkdirSync(bundleEscalationDir, { recursive: true });
|
|
716
|
+
mkdirSync(input.workspacePath, { recursive: true });
|
|
717
|
+
// Rewrite the allowed directory and audit path onto the loaded
|
|
718
|
+
// config. `config.allowedDirectory` drives the filesystem MCP
|
|
719
|
+
// server's --allowed-directory; `config.auditLogPath` is what the
|
|
720
|
+
// ToolCallCoordinator (via Sandbox) reads when creating the
|
|
721
|
+
// AuditLog. Without the audit rewrite, audit entries fall through
|
|
722
|
+
// to the default `./audit.jsonl` against process.cwd().
|
|
723
|
+
config.allowedDirectory = input.workspacePath;
|
|
724
|
+
config.auditLogPath = getBundleAuditLogPath(input.workflowId, input.bundleId);
|
|
725
|
+
config.mcpServers = filterMcpServersByPolicy(config.mcpServers, input.requiredServers);
|
|
726
|
+
applyAllowedDirectoryToMcpArgs(config.mcpServers, input.workspacePath);
|
|
727
|
+
return createDockerInfrastructure(config, { kind: 'docker', agent: input.agentId }, bundleDir, input.workspacePath, bundleEscalationDir, input.bundleId,
|
|
728
|
+
// Threading the workflowId + scope here drives the
|
|
729
|
+
// `ironcurtain.workflow` and `ironcurtain.scope` Docker labels
|
|
730
|
+
// emitted by `createSessionContainers()` via `buildBundleLabels`.
|
|
731
|
+
input.workflowId, input.scope, input.resolvedSkills, {
|
|
732
|
+
override: captureTracesOverride,
|
|
733
|
+
capturesDir: getBundleCapturesDir(input.workflowId, input.bundleId),
|
|
734
|
+
recordedAgentName: input.agentId,
|
|
735
|
+
workflowRunId: input.workflowId,
|
|
736
|
+
}, input.workflowScriptsDir, input.baseImageOverride ? { baseImageOverride: input.baseImageOverride } : undefined);
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
/** Lazy-loads the default `destroyDockerInfrastructure` helper. */
|
|
740
|
+
async loadDefaultInfrastructureTeardown() {
|
|
741
|
+
const { destroyDockerInfrastructure } = await import('../docker/docker-infrastructure.js');
|
|
742
|
+
return destroyDockerInfrastructure;
|
|
743
|
+
}
|
|
744
|
+
// -----------------------------------------------------------------------
|
|
745
|
+
// Token-stream accumulation (workflow-scoped totalTokens counter)
|
|
746
|
+
// -----------------------------------------------------------------------
|
|
747
|
+
//
|
|
748
|
+
// The token-stream bus is global (see `getTokenStreamBus()` in
|
|
749
|
+
// `src/docker/token-stream-bus.ts`), so the orchestrator subscribes with
|
|
750
|
+
// `subscribeAll()` and filters inside the listener by the workflow's own
|
|
751
|
+
// session-ID set. Per-workflow rather than per-agent: `ctx.totalTokens`
|
|
752
|
+
// is a workflow-level cumulative sum, so one long-lived subscription is
|
|
753
|
+
// cheaper (one bus listener, one unsubscribe on completion) than
|
|
754
|
+
// subscribing and unsubscribing on every `agent_started` /
|
|
755
|
+
// `agent_session_ended` pair.
|
|
756
|
+
/**
|
|
757
|
+
* Subscribes the workflow to the global token-stream bus. Accumulates
|
|
758
|
+
* `message_end.outputTokens` into `instance.tokens.outputTokens` for any
|
|
759
|
+
* event whose session ID is in `instance.tokens.sessionIds`. The set is managed
|
|
760
|
+
* by `executeAgentState` (added before `agent_started`, removed in the
|
|
761
|
+
* `finally` block after `agent_session_ended`).
|
|
762
|
+
*
|
|
763
|
+
* Installed synchronously so the subscription is live before any agent
|
|
764
|
+
* state runs — no window where bus events go unobserved.
|
|
765
|
+
*/
|
|
766
|
+
setupTokenSubscription(instance) {
|
|
767
|
+
// Defensive: re-entry by resume-after-crash should not double-subscribe.
|
|
768
|
+
if (instance.tokens.unsubscribe)
|
|
769
|
+
return;
|
|
770
|
+
instance.tokens.unsubscribe = getTokenStreamBus().subscribeAll((sessionId, event) => {
|
|
771
|
+
if (event.kind !== 'message_end')
|
|
772
|
+
return;
|
|
773
|
+
if (!instance.tokens.sessionIds.has(sessionId))
|
|
774
|
+
return;
|
|
775
|
+
instance.tokens.outputTokens += event.outputTokens;
|
|
776
|
+
});
|
|
777
|
+
}
|
|
778
|
+
/**
|
|
779
|
+
* Tears down the token-stream bus subscription. Idempotent: safe to
|
|
780
|
+
* call from both normal completion and abort paths.
|
|
781
|
+
*/
|
|
782
|
+
teardownTokenSubscription(instance) {
|
|
783
|
+
if (instance.tokens.unsubscribe) {
|
|
784
|
+
try {
|
|
785
|
+
instance.tokens.unsubscribe();
|
|
786
|
+
}
|
|
787
|
+
catch {
|
|
788
|
+
// Unsubscribe is best-effort; errors here should never prevent
|
|
789
|
+
// workflow teardown from completing.
|
|
790
|
+
}
|
|
791
|
+
instance.tokens.unsubscribe = undefined;
|
|
792
|
+
}
|
|
793
|
+
instance.tokens.sessionIds.clear();
|
|
794
|
+
}
|
|
795
|
+
// -----------------------------------------------------------------------
|
|
796
|
+
// WorkflowController implementation
|
|
797
|
+
// -----------------------------------------------------------------------
|
|
798
|
+
// Keeps the `async` signature even though the body no longer awaits:
|
|
799
|
+
// (a) preserves the `WorkflowController.start()` contract (Promise<WorkflowId>),
|
|
800
|
+
// (b) leaves a hook for future async bootstrap work without another
|
|
801
|
+
// interface change.
|
|
802
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
803
|
+
async start(definitionPath, taskDescription, workspacePath) {
|
|
804
|
+
const raw = parseDefinitionFile(definitionPath);
|
|
805
|
+
const definition = validateDefinition(raw);
|
|
806
|
+
this.validatePersonas(definition);
|
|
807
|
+
validateWorkflowSkillReferences(definition, getWorkflowPackageDir(definitionPath));
|
|
808
|
+
const workflowId = createWorkflowId();
|
|
809
|
+
const resolvedWorkspace = workspacePath ?? resolve(this.deps.baseDir, workflowId, 'workspace');
|
|
810
|
+
// Reject if any active workflow already uses this workspace path
|
|
811
|
+
for (const instance of this.workflows.values()) {
|
|
812
|
+
if (instance.workspacePath === resolvedWorkspace && !instance.finalStatus) {
|
|
813
|
+
throw new Error(`Workspace ${resolvedWorkspace} is already in use by workflow ${instance.id}`);
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
mkdirSync(resolvedWorkspace, { recursive: true });
|
|
817
|
+
const artifactDir = resolve(resolvedWorkspace, WORKFLOW_ARTIFACT_DIR);
|
|
818
|
+
mkdirSync(artifactDir, { recursive: true });
|
|
819
|
+
const taskDir = resolve(artifactDir, 'task');
|
|
820
|
+
mkdirSync(taskDir, { recursive: true });
|
|
821
|
+
writeFileSync(resolve(taskDir, 'description.md'), taskDescription);
|
|
822
|
+
ensureWorkflowGitignored(resolvedWorkspace);
|
|
823
|
+
// Copy definition to baseDir for resume portability
|
|
824
|
+
const metaDir = resolve(this.deps.baseDir, workflowId);
|
|
825
|
+
mkdirSync(metaDir, { recursive: true });
|
|
826
|
+
writeFileSync(resolve(metaDir, 'definition.json'), JSON.stringify(definition, null, 2));
|
|
827
|
+
// Stage the workflow package's `skills/` tree into the run dir so
|
|
828
|
+
// resume is independent of whether the original package path still
|
|
829
|
+
// exists. Cached on the instance and persisted in every checkpoint
|
|
830
|
+
// so all subsequent reads point at this stable per-run copy.
|
|
831
|
+
const workflowSkillsDir = stageWorkflowSkillsAtStart(getWorkflowPackageDir(definitionPath), metaDir);
|
|
832
|
+
const workflowScriptsDir = stageWorkflowScriptsAtStart(getWorkflowPackageDir(definitionPath), metaDir);
|
|
833
|
+
const { machine, roundMachinesByState, gateStateNames, terminalStateNames } = buildWorkflowMachine(definition, taskDescription);
|
|
834
|
+
const providedMachine = this.provideActors(machine, workflowId, definition, roundMachinesByState);
|
|
835
|
+
const actor = createActor(providedMachine);
|
|
836
|
+
const tab = this.deps.createWorkflowTab(definition.name, workflowId);
|
|
837
|
+
const messageLog = new MessageLog(resolve(this.deps.baseDir, workflowId, 'messages.jsonl'));
|
|
838
|
+
const instance = {
|
|
839
|
+
id: workflowId,
|
|
840
|
+
definition,
|
|
841
|
+
definitionPath,
|
|
842
|
+
workflowSkillsDir,
|
|
843
|
+
workflowScriptsDir,
|
|
844
|
+
containerSnapshots: undefined,
|
|
845
|
+
actor,
|
|
846
|
+
roundMachinesByState,
|
|
847
|
+
gateStateNames,
|
|
848
|
+
terminalStateNames,
|
|
849
|
+
activeSessions: new Set(),
|
|
850
|
+
artifactDir,
|
|
851
|
+
workspacePath: resolvedWorkspace,
|
|
852
|
+
tab,
|
|
853
|
+
transitionHistory: [],
|
|
854
|
+
currentState: definition.initial,
|
|
855
|
+
stateEnteredAt: Date.now(),
|
|
856
|
+
messageLog,
|
|
857
|
+
bundlesByScope: new Map(),
|
|
858
|
+
policyDirByPersona: new Map(),
|
|
859
|
+
currentPersonaByBundle: new Map(),
|
|
860
|
+
mintedServersByBundle: new Map(),
|
|
861
|
+
aborted: false,
|
|
862
|
+
tokens: {
|
|
863
|
+
outputTokens: 0,
|
|
864
|
+
sessionIds: new Set(),
|
|
865
|
+
},
|
|
866
|
+
};
|
|
867
|
+
// Under `sharedContainer: true`, bundles are minted lazily by
|
|
868
|
+
// `ensureBundleForScope` on the first `executeAgentState` call for
|
|
869
|
+
// each scope. There is no eager workflow-start mint — the first
|
|
870
|
+
// state pays the cost of spinning up the bundle it needs. This also
|
|
871
|
+
// removes a latent "Docker not available" failure mode at workflow
|
|
872
|
+
// registration time: the workflow enters the actor's first state
|
|
873
|
+
// before any Docker resource is touched.
|
|
874
|
+
this.workflows.set(workflowId, instance);
|
|
875
|
+
this.setupTokenSubscription(instance);
|
|
876
|
+
this.emitLifecycleEvent({ kind: 'started', workflowId, name: definition.name, taskDescription });
|
|
877
|
+
this.subscribeToActor(instance);
|
|
878
|
+
actor.start();
|
|
879
|
+
return workflowId;
|
|
880
|
+
}
|
|
881
|
+
async resume(workflowId) {
|
|
882
|
+
const checkpoint = await Promise.resolve(this.deps.checkpointStore.load(workflowId));
|
|
883
|
+
if (!checkpoint) {
|
|
884
|
+
throw new Error(`No checkpoint found for workflow ${workflowId}`);
|
|
885
|
+
}
|
|
886
|
+
const definitionCopyPath = resolve(this.deps.baseDir, workflowId, 'definition.json');
|
|
887
|
+
const definitionPath = existsSync(definitionCopyPath) ? definitionCopyPath : checkpoint.definitionPath;
|
|
888
|
+
// Runtime copies are always JSON; original user files may be YAML
|
|
889
|
+
const raw = parseDefinitionFile(definitionPath);
|
|
890
|
+
const definition = validateDefinition(raw);
|
|
891
|
+
const { machine, roundMachinesByState, gateStateNames, terminalStateNames } = buildWorkflowMachine(definition, checkpoint.context.taskDescription);
|
|
892
|
+
const providedMachine = this.provideActors(machine, workflowId, definition, roundMachinesByState);
|
|
893
|
+
const restoredSnapshot = providedMachine.resolveState({
|
|
894
|
+
value: checkpoint.machineState,
|
|
895
|
+
context: checkpoint.context,
|
|
896
|
+
});
|
|
897
|
+
const actor = createActor(providedMachine, { snapshot: restoredSnapshot });
|
|
898
|
+
const workspacePath = checkpoint.workspacePath ?? resolve(this.deps.baseDir, workflowId, 'workspace');
|
|
899
|
+
const artifactDir = resolve(workspacePath, WORKFLOW_ARTIFACT_DIR);
|
|
900
|
+
// Ensure artifact dir exists (backward compat with pre-workspace checkpoints)
|
|
901
|
+
mkdirSync(artifactDir, { recursive: true });
|
|
902
|
+
const tab = this.deps.createWorkflowTab(definition.name, workflowId);
|
|
903
|
+
// Append to existing log file (resume must not overwrite)
|
|
904
|
+
const messageLog = new MessageLog(resolve(this.deps.baseDir, workflowId, 'messages.jsonl'));
|
|
905
|
+
// Resolve workflow-skills with the run-dir staged copy as primary,
|
|
906
|
+
// a fresh re-stage from the original package dir as recovery, and
|
|
907
|
+
// a logged `undefined` as the last-resort degradation. See
|
|
908
|
+
// `resolveWorkflowSkillsDirOnResume` for the full priority order.
|
|
909
|
+
//
|
|
910
|
+
// Recovery uses `checkpoint.definitionPath` (the user's original
|
|
911
|
+
// package), not `definitionPath` — the latter resolves to the
|
|
912
|
+
// run-dir copy of `definition.json` whose `getWorkflowPackageDir`
|
|
913
|
+
// is the run dir itself, which never contains a sibling `skills/`.
|
|
914
|
+
const runMetaDir = resolve(this.deps.baseDir, workflowId);
|
|
915
|
+
const workflowSkillsDir = resolveWorkflowSkillsDirOnResume({
|
|
916
|
+
workflowId,
|
|
917
|
+
checkpointedStagedDir: checkpoint.workflowSkillsDir,
|
|
918
|
+
packageDir: getWorkflowPackageDir(checkpoint.definitionPath),
|
|
919
|
+
runMetaDir,
|
|
920
|
+
});
|
|
921
|
+
const workflowScriptsDir = resolveWorkflowScriptsDirOnResume({
|
|
922
|
+
workflowId,
|
|
923
|
+
checkpointedStagedDir: checkpoint.workflowScriptsDir,
|
|
924
|
+
packageDir: getWorkflowPackageDir(checkpoint.definitionPath),
|
|
925
|
+
runMetaDir,
|
|
926
|
+
});
|
|
927
|
+
const instance = {
|
|
928
|
+
id: workflowId,
|
|
929
|
+
definition,
|
|
930
|
+
definitionPath: checkpoint.definitionPath,
|
|
931
|
+
workflowSkillsDir,
|
|
932
|
+
workflowScriptsDir,
|
|
933
|
+
containerSnapshots: checkpoint.containerSnapshots,
|
|
934
|
+
actor,
|
|
935
|
+
roundMachinesByState,
|
|
936
|
+
gateStateNames,
|
|
937
|
+
terminalStateNames,
|
|
938
|
+
activeSessions: new Set(),
|
|
939
|
+
artifactDir,
|
|
940
|
+
workspacePath,
|
|
941
|
+
tab,
|
|
942
|
+
transitionHistory: [...checkpoint.transitionHistory],
|
|
943
|
+
currentState: String(checkpoint.machineState),
|
|
944
|
+
stateEnteredAt: Date.now(),
|
|
945
|
+
messageLog,
|
|
946
|
+
bundlesByScope: new Map(),
|
|
947
|
+
policyDirByPersona: new Map(),
|
|
948
|
+
currentPersonaByBundle: new Map(),
|
|
949
|
+
mintedServersByBundle: new Map(),
|
|
950
|
+
aborted: false,
|
|
951
|
+
tokens: {
|
|
952
|
+
// Resume picks up the checkpointed totalTokens as the accumulator's
|
|
953
|
+
// starting point so post-resume message_end events keep adding to
|
|
954
|
+
// the running total instead of resetting it.
|
|
955
|
+
outputTokens: checkpoint.context.totalTokens,
|
|
956
|
+
sessionIds: new Set(),
|
|
957
|
+
},
|
|
958
|
+
};
|
|
959
|
+
// Under the lazy-mint model there is nothing to create at resume
|
|
960
|
+
// time: `bundlesByScope` starts empty and the first state to execute
|
|
961
|
+
// mints its bundle. When the checkpoint carries snapshot digests,
|
|
962
|
+
// the per-scope mint path uses them as immutable image overrides
|
|
963
|
+
// after an imageExists guard; missing images degrade to fresh.
|
|
964
|
+
if (this.shouldUseSharedContainer(definition)) {
|
|
965
|
+
const restoredScopes = Object.keys(checkpoint.containerSnapshots ?? {});
|
|
966
|
+
const snapshotNote = restoredScopes.length > 0
|
|
967
|
+
? ` Snapshot restore available for scope(s): ${restoredScopes.join(', ')}.`
|
|
968
|
+
: ' No container snapshots recorded; scopes will start fresh.';
|
|
969
|
+
writeStderr(`[workflow] Resuming ${workflowId} in shared-container mode: bundles will be re-created lazily per scope.${snapshotNote}`);
|
|
970
|
+
}
|
|
971
|
+
this.workflows.set(workflowId, instance);
|
|
972
|
+
this.setupTokenSubscription(instance);
|
|
973
|
+
this.emitLifecycleEvent({
|
|
974
|
+
kind: 'started',
|
|
975
|
+
workflowId,
|
|
976
|
+
name: definition.name,
|
|
977
|
+
taskDescription: checkpoint.context.taskDescription,
|
|
978
|
+
});
|
|
979
|
+
this.subscribeToActor(instance);
|
|
980
|
+
actor.start();
|
|
981
|
+
// XState v5's resolveState() restores *to* a state but does not *enter* it.
|
|
982
|
+
// Invoke services (agent/deterministic) only start on state entry via transition.
|
|
983
|
+
// For invoke states, we must manually execute the service and feed the result
|
|
984
|
+
// back to the actor as an xstate.done.actor / xstate.error.actor event.
|
|
985
|
+
const restoredState = String(checkpoint.machineState);
|
|
986
|
+
const stateDef = definition.states[restoredState];
|
|
987
|
+
if (stateDef.type === 'agent' || stateDef.type === 'deterministic') {
|
|
988
|
+
this.replayInvokeForRestoredState(workflowId, restoredState, stateDef, definition);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
/**
|
|
992
|
+
* Manually executes the invoke service for a state that was restored from
|
|
993
|
+
* a checkpoint. XState does not re-trigger invocations when an actor is
|
|
994
|
+
* started from a persisted snapshot, so we run the service ourselves and
|
|
995
|
+
* send the result event to the actor.
|
|
996
|
+
*/
|
|
997
|
+
replayInvokeForRestoredState(workflowId, stateId, stateDef, definition) {
|
|
998
|
+
const instance = this.workflows.get(workflowId);
|
|
999
|
+
if (!instance)
|
|
1000
|
+
return;
|
|
1001
|
+
const snapshot = instance.actor.getSnapshot();
|
|
1002
|
+
const context = snapshot.context;
|
|
1003
|
+
// Build the service promise based on state type
|
|
1004
|
+
let servicePromise;
|
|
1005
|
+
if (stateDef.type === 'agent') {
|
|
1006
|
+
servicePromise =
|
|
1007
|
+
stateDef.fanOut !== undefined
|
|
1008
|
+
? this.runFanOutSegment(workflowId, { stateId, stateConfig: stateDef, context }, definition, instance.roundMachinesByState)
|
|
1009
|
+
: this.executeAgentState(workflowId, { stateId, stateConfig: stateDef, context }, definition);
|
|
1010
|
+
}
|
|
1011
|
+
else if (stateDef.fanOut !== undefined) {
|
|
1012
|
+
servicePromise = this.runFanOutSegment(workflowId, { stateId, stateConfig: stateDef, context }, definition, instance.roundMachinesByState);
|
|
1013
|
+
}
|
|
1014
|
+
else {
|
|
1015
|
+
servicePromise = this.executeDeterministicState(workflowId, {
|
|
1016
|
+
stateId,
|
|
1017
|
+
commands: stateDef.run,
|
|
1018
|
+
context,
|
|
1019
|
+
container: stateDef.container ?? false,
|
|
1020
|
+
containerScope: stateDef.containerScope,
|
|
1021
|
+
timeoutMs: stateDef.timeoutMs,
|
|
1022
|
+
resultFile: stateDef.resultFile,
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
// Feed the result back to the actor as an XState internal invoke event.
|
|
1026
|
+
// These event types don't exist in our WorkflowEvent union, so we cast
|
|
1027
|
+
// through unknown to satisfy TypeScript while matching XState's internal
|
|
1028
|
+
// event format that onDone/onError handlers expect.
|
|
1029
|
+
servicePromise
|
|
1030
|
+
.then((output) => {
|
|
1031
|
+
instance.actor.send({ type: `xstate.done.actor.${stateId}`, output });
|
|
1032
|
+
})
|
|
1033
|
+
.catch((err) => {
|
|
1034
|
+
instance.actor.send({ type: `xstate.error.actor.${stateId}`, error: err });
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
listResumable() {
|
|
1038
|
+
const runs = discoverWorkflowRuns(this.deps.baseDir);
|
|
1039
|
+
const activeIds = new Set(this.listActive());
|
|
1040
|
+
const resumable = [];
|
|
1041
|
+
for (const run of runs) {
|
|
1042
|
+
if (!run.hasCheckpoint || activeIds.has(run.workflowId))
|
|
1043
|
+
continue;
|
|
1044
|
+
// FileCheckpointStore.load() re-throws on JSON.parse failure (it only
|
|
1045
|
+
// swallows ENOENT). A single corrupt checkpoint.json would otherwise
|
|
1046
|
+
// poison the whole list and break `workflow inspect` discovery + the
|
|
1047
|
+
// web UI past-runs panel. Skip and log instead.
|
|
1048
|
+
let cp;
|
|
1049
|
+
try {
|
|
1050
|
+
cp = this.deps.checkpointStore.load(run.workflowId);
|
|
1051
|
+
}
|
|
1052
|
+
catch (err) {
|
|
1053
|
+
writeStderr(`[workflow] Failed to load checkpoint for ${run.workflowId}: ${toErrorMessage(err)}; skipping in resumable list`);
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
1056
|
+
if (cp !== undefined && isCheckpointResumable(cp)) {
|
|
1057
|
+
resumable.push(run.workflowId);
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
return resumable;
|
|
1061
|
+
}
|
|
1062
|
+
getStatus(id) {
|
|
1063
|
+
const instance = this.workflows.get(id);
|
|
1064
|
+
if (!instance)
|
|
1065
|
+
return undefined;
|
|
1066
|
+
if (instance.finalStatus)
|
|
1067
|
+
return instance.finalStatus;
|
|
1068
|
+
if (instance.activeGateId) {
|
|
1069
|
+
const gateName = instance.currentState;
|
|
1070
|
+
const stateDef = instance.definition.states[gateName];
|
|
1071
|
+
if (stateDef.type === 'human_gate') {
|
|
1072
|
+
return {
|
|
1073
|
+
phase: 'waiting_human',
|
|
1074
|
+
gate: this.buildGateRequest(id, gateName, stateDef),
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
return {
|
|
1079
|
+
phase: 'running',
|
|
1080
|
+
currentState: instance.currentState,
|
|
1081
|
+
activeAgents: [],
|
|
1082
|
+
};
|
|
1083
|
+
}
|
|
1084
|
+
getDetail(id) {
|
|
1085
|
+
const instance = this.workflows.get(id);
|
|
1086
|
+
if (!instance)
|
|
1087
|
+
return undefined;
|
|
1088
|
+
const snapshot = instance.actor.getSnapshot();
|
|
1089
|
+
const ctx = snapshot.context;
|
|
1090
|
+
return {
|
|
1091
|
+
definition: instance.definition,
|
|
1092
|
+
transitionHistory: [...instance.transitionHistory],
|
|
1093
|
+
workspacePath: instance.workspacePath,
|
|
1094
|
+
context: {
|
|
1095
|
+
taskDescription: ctx.taskDescription,
|
|
1096
|
+
round: ctx.round,
|
|
1097
|
+
maxRounds: ctx.maxRounds,
|
|
1098
|
+
totalTokens: ctx.totalTokens,
|
|
1099
|
+
visitCounts: { ...ctx.visitCounts },
|
|
1100
|
+
},
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
listActive() {
|
|
1104
|
+
return [...this.workflows.keys()].filter((id) => {
|
|
1105
|
+
const instance = this.workflows.get(id);
|
|
1106
|
+
return instance && !instance.finalStatus;
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
resolveGate(id, event) {
|
|
1110
|
+
// FORCE_REVISION and REPLAN loop back to an earlier state with the
|
|
1111
|
+
// feedback injected into the next agent's prompt (via context.humanPrompt).
|
|
1112
|
+
// Without feedback the agent has no signal for what to change and the
|
|
1113
|
+
// re-entry prompt ("Revise it to address the human feedback above")
|
|
1114
|
+
// references content that does not exist. Require non-empty feedback at
|
|
1115
|
+
// the source so every entry point (CLI, web UI, programmatic) fails fast.
|
|
1116
|
+
if (event.type === 'FORCE_REVISION' || event.type === 'REPLAN') {
|
|
1117
|
+
if (!event.prompt || event.prompt.trim().length === 0) {
|
|
1118
|
+
throw new Error(`Feedback is required for ${event.type} events`);
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
const instance = this.workflows.get(id);
|
|
1122
|
+
if (!instance)
|
|
1123
|
+
return;
|
|
1124
|
+
const gateId = instance.activeGateId;
|
|
1125
|
+
if (!gateId) {
|
|
1126
|
+
// No active gate -- prevent double-resolution
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
// Clear before sending to prevent concurrent double-resolution
|
|
1130
|
+
instance.activeGateId = undefined;
|
|
1131
|
+
this.deps.dismissGate(instance.id, gateId);
|
|
1132
|
+
instance.messageLog.append({
|
|
1133
|
+
...this.logBase(instance),
|
|
1134
|
+
type: 'gate_resolved',
|
|
1135
|
+
event: event.type,
|
|
1136
|
+
prompt: event.prompt ?? null,
|
|
1137
|
+
});
|
|
1138
|
+
const xstateEventName = `HUMAN_${event.type}`;
|
|
1139
|
+
instance.actor.send({
|
|
1140
|
+
type: xstateEventName,
|
|
1141
|
+
prompt: event.prompt,
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
1144
|
+
async abort(id) {
|
|
1145
|
+
const instance = this.workflows.get(id);
|
|
1146
|
+
if (!instance)
|
|
1147
|
+
return;
|
|
1148
|
+
if (instance.finalStatus?.phase === 'completed' ||
|
|
1149
|
+
instance.finalStatus?.phase === 'aborted' ||
|
|
1150
|
+
instance.finalStatus?.phase === 'failed') {
|
|
1151
|
+
// A terminal transition is already in flight (e.g. a natural completion
|
|
1152
|
+
// racing this abort). Wait for its teardown instead of starting a second
|
|
1153
|
+
// one. `teardownPromise` is only set by the fire-and-forget completion
|
|
1154
|
+
// path; awaiting `undefined` is a no-op on the abort-vs-abort race.
|
|
1155
|
+
await instance.teardownPromise;
|
|
1156
|
+
return;
|
|
1157
|
+
}
|
|
1158
|
+
// Claim the terminal synchronously, BEFORE the first await below, so a
|
|
1159
|
+
// natural completion firing during session close sees `finalStatus` set
|
|
1160
|
+
// and bails at its own guard. This is the single-flight gate that
|
|
1161
|
+
// guarantees exactly-once snapshot + teardown across abort/complete.
|
|
1162
|
+
instance.finalStatus = {
|
|
1163
|
+
phase: 'aborted',
|
|
1164
|
+
reason: 'Workflow aborted by user',
|
|
1165
|
+
};
|
|
1166
|
+
const closePromises = [];
|
|
1167
|
+
for (const session of instance.activeSessions) {
|
|
1168
|
+
closePromises.push(session.close().catch(() => { }));
|
|
1169
|
+
}
|
|
1170
|
+
await Promise.allSettled(closePromises);
|
|
1171
|
+
instance.activeSessions.clear();
|
|
1172
|
+
instance.actor.stop();
|
|
1173
|
+
let previousCheckpoint;
|
|
1174
|
+
try {
|
|
1175
|
+
previousCheckpoint = this.deps.checkpointStore.load(id);
|
|
1176
|
+
}
|
|
1177
|
+
catch (err) {
|
|
1178
|
+
writeStderr(`[workflow] Failed to load existing checkpoint during abort for ${id}: ${toErrorMessage(err)}`);
|
|
1179
|
+
}
|
|
1180
|
+
const containerSnapshots = await this.snapshotResumableScopes(instance);
|
|
1181
|
+
this.saveTerminalCheckpoint(instance, instance.finalStatus, containerSnapshots, previousCheckpoint);
|
|
1182
|
+
// Release the token-stream bus subscription before the async infra
|
|
1183
|
+
// teardown so no late events land against a finalized workflow.
|
|
1184
|
+
this.teardownTokenSubscription(instance);
|
|
1185
|
+
// Tear down workflow-scoped Docker infrastructure after all sessions
|
|
1186
|
+
// are closed. Error-tolerant (see destroyWorkflowInfrastructure).
|
|
1187
|
+
const snapshotsToRemove = this.snapshotsSupersededBy(previousCheckpoint?.containerSnapshots, containerSnapshots);
|
|
1188
|
+
const cleanupDocker = snapshotsToRemove ? await this.dockerForSnapshotCleanup(instance) : undefined;
|
|
1189
|
+
await this.destroyWorkflowInfrastructure(instance);
|
|
1190
|
+
if (cleanupDocker) {
|
|
1191
|
+
await this.removeSnapshotImagesAfterTeardown(cleanupDocker, snapshotsToRemove);
|
|
1192
|
+
}
|
|
1193
|
+
// Intentionally leave the checkpoint in place: a user-triggered abort
|
|
1194
|
+
// should remain resumable via `workflow resume`. The checkpoint is only
|
|
1195
|
+
// removed on successful completion (see handleWorkflowComplete).
|
|
1196
|
+
instance.tab.write('[aborted]');
|
|
1197
|
+
instance.tab.close();
|
|
1198
|
+
this.emitLifecycleEvent({
|
|
1199
|
+
kind: 'failed',
|
|
1200
|
+
workflowId: id,
|
|
1201
|
+
error: 'Workflow aborted by user',
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
onEvent(callback) {
|
|
1205
|
+
this.lifecycleCallbacks.push(callback);
|
|
1206
|
+
}
|
|
1207
|
+
async shutdownAll() {
|
|
1208
|
+
const ids = [...this.workflows.keys()];
|
|
1209
|
+
// Abort runs full teardown (including destroyWorkflowInfrastructure) for
|
|
1210
|
+
// workflows not yet in a terminal state. For already-terminal workflows,
|
|
1211
|
+
// abort() early-returns — so we follow up with an explicit destroy pass
|
|
1212
|
+
// to defend against any case where a terminal transition skipped infra
|
|
1213
|
+
// teardown (e.g., tests forcing finalStatus directly). The destroy call
|
|
1214
|
+
// is idempotent when instance.bundlesByScope is empty.
|
|
1215
|
+
await Promise.allSettled(ids.map((id) => this.abort(id)));
|
|
1216
|
+
await Promise.allSettled(ids.map((id) => {
|
|
1217
|
+
const instance = this.workflows.get(id);
|
|
1218
|
+
return instance ? this.destroyWorkflowInfrastructure(instance) : Promise.resolve();
|
|
1219
|
+
}));
|
|
1220
|
+
// Drain any fire-and-forget teardown started by `handleWorkflowComplete`
|
|
1221
|
+
// on a normal terminal. Because `destroyWorkflowInfrastructure`
|
|
1222
|
+
// snapshot-and-clears `bundlesByScope` up front, the explicit pass above
|
|
1223
|
+
// no-ops for an already-completed workflow whose teardown is still in
|
|
1224
|
+
// flight — so we must await the stored promise to guarantee the Docker
|
|
1225
|
+
// network/container removal has actually finished before the caller (the
|
|
1226
|
+
// CLI) calls process.exit().
|
|
1227
|
+
await Promise.allSettled(ids.map((id) => this.workflows.get(id)?.teardownPromise ?? Promise.resolve()));
|
|
1228
|
+
}
|
|
1229
|
+
// -----------------------------------------------------------------------
|
|
1230
|
+
// Actor setup helpers
|
|
1231
|
+
// -----------------------------------------------------------------------
|
|
1232
|
+
/**
|
|
1233
|
+
* Injects concrete service implementations into the machine: `agentService`
|
|
1234
|
+
* and `deterministicService` for executable states, plus `fanOutService` —
|
|
1235
|
+
* the invoke body of a `workers` fan-out state, which runs
|
|
1236
|
+
* {@link runFanOutSegment} and is re-provided onto each child round machine
|
|
1237
|
+
* (the children reuse the same agent/deterministic actors, lane-scoped via
|
|
1238
|
+
* input).
|
|
1239
|
+
*/
|
|
1240
|
+
provideActors(machine, workflowId, definition, roundMachinesByState) {
|
|
1241
|
+
return machine.provide({
|
|
1242
|
+
actors: {
|
|
1243
|
+
agentService: fromPromise(async ({ input }) => {
|
|
1244
|
+
try {
|
|
1245
|
+
return await this.executeAgentState(workflowId, input, definition);
|
|
1246
|
+
}
|
|
1247
|
+
catch (err) {
|
|
1248
|
+
writeStderr(`[workflow] agentService invoke rejected for "${input.stateId}": ${toErrorMessage(err)}`);
|
|
1249
|
+
throw err;
|
|
1250
|
+
}
|
|
1251
|
+
}),
|
|
1252
|
+
deterministicService: fromPromise(async ({ input }) => {
|
|
1253
|
+
try {
|
|
1254
|
+
return await this.executeDeterministicState(workflowId, input);
|
|
1255
|
+
}
|
|
1256
|
+
catch (err) {
|
|
1257
|
+
writeStderr(`[workflow] deterministicService invoke rejected for "${input.stateId}": ${toErrorMessage(err)}`);
|
|
1258
|
+
throw err;
|
|
1259
|
+
}
|
|
1260
|
+
}),
|
|
1261
|
+
fanOutService: fromPromise(async ({ input }) => {
|
|
1262
|
+
try {
|
|
1263
|
+
return await this.runFanOutSegment(workflowId, input, definition, roundMachinesByState);
|
|
1264
|
+
}
|
|
1265
|
+
catch (err) {
|
|
1266
|
+
writeStderr(`[workflow] fanOutService invoke rejected for "${input.stateId}": ${toErrorMessage(err)}`);
|
|
1267
|
+
throw err;
|
|
1268
|
+
}
|
|
1269
|
+
}),
|
|
1270
|
+
},
|
|
1271
|
+
});
|
|
1272
|
+
}
|
|
1273
|
+
/** Subscribes to actor snapshot changes for lifecycle events, gates, and checkpointing. */
|
|
1274
|
+
subscribeToActor(instance) {
|
|
1275
|
+
const { actor, gateStateNames, definition, id: workflowId } = instance;
|
|
1276
|
+
actor.subscribe((rawSnapshot) => {
|
|
1277
|
+
const snapshot = rawSnapshot;
|
|
1278
|
+
const stateValue = String(snapshot.value);
|
|
1279
|
+
const previousState = instance.currentState;
|
|
1280
|
+
const now = Date.now();
|
|
1281
|
+
// Record transition and checkpoint only on actual state changes
|
|
1282
|
+
if (stateValue !== previousState) {
|
|
1283
|
+
const duration = instance.stateEnteredAt ? now - instance.stateEnteredAt : 0;
|
|
1284
|
+
const stateType = definition.states[previousState].type;
|
|
1285
|
+
let agentMessage;
|
|
1286
|
+
if (stateType === 'human_gate') {
|
|
1287
|
+
agentMessage = truncateForTransition(snapshot.context.humanPrompt ?? snapshot.context.previousAgentOutput);
|
|
1288
|
+
}
|
|
1289
|
+
else if (stateType === 'agent') {
|
|
1290
|
+
agentMessage = truncateForTransition(snapshot.context.previousAgentOutput);
|
|
1291
|
+
}
|
|
1292
|
+
// deterministic and terminal: no agentMessage (leave undefined)
|
|
1293
|
+
instance.transitionHistory.push({
|
|
1294
|
+
from: previousState,
|
|
1295
|
+
to: stateValue,
|
|
1296
|
+
event: 'transition',
|
|
1297
|
+
timestamp: new Date(now).toISOString(),
|
|
1298
|
+
duration_ms: duration,
|
|
1299
|
+
agentMessage,
|
|
1300
|
+
});
|
|
1301
|
+
instance.stateEnteredAt = now;
|
|
1302
|
+
// Skip checkpointing transitions into terminal states. On successful
|
|
1303
|
+
// completion `handleWorkflowComplete` removes the checkpoint anyway;
|
|
1304
|
+
// on abort we want the surviving checkpoint to point at the last
|
|
1305
|
+
// non-terminal state so `resume()` can restart the run meaningfully.
|
|
1306
|
+
if (!this.isTerminalStateValue(definition, stateValue)) {
|
|
1307
|
+
this.saveCheckpoint(instance, snapshot);
|
|
1308
|
+
}
|
|
1309
|
+
instance.messageLog.append({
|
|
1310
|
+
...this.logBase(instance),
|
|
1311
|
+
type: 'state_transition',
|
|
1312
|
+
from: previousState,
|
|
1313
|
+
event: stateValue,
|
|
1314
|
+
});
|
|
1315
|
+
}
|
|
1316
|
+
instance.currentState = stateValue;
|
|
1317
|
+
instance.tab.write(`[state] ${stateValue}`);
|
|
1318
|
+
// Surface errors from invoke failures (storeError action sets lastError)
|
|
1319
|
+
const ctx = snapshot.context;
|
|
1320
|
+
if (ctx.lastError && ctx.lastError !== instance.lastSurfacedError) {
|
|
1321
|
+
instance.lastSurfacedError = ctx.lastError;
|
|
1322
|
+
writeStderr(`[workflow] Error in context after state "${stateValue}": ${ctx.lastError}`);
|
|
1323
|
+
instance.tab.write(`[error] Agent invoke failed: ${ctx.lastError}`);
|
|
1324
|
+
this.emitLifecycleEvent({
|
|
1325
|
+
kind: 'failed',
|
|
1326
|
+
workflowId,
|
|
1327
|
+
error: `Agent "${previousState}" failed: ${ctx.lastError}`,
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1330
|
+
else if (!ctx.lastError && instance.lastSurfacedError) {
|
|
1331
|
+
instance.lastSurfacedError = undefined;
|
|
1332
|
+
}
|
|
1333
|
+
for (const gateName of gateStateNames) {
|
|
1334
|
+
if (snapshot.matches(gateName)) {
|
|
1335
|
+
const stateDef = definition.states[gateName];
|
|
1336
|
+
if (stateDef.type === 'human_gate') {
|
|
1337
|
+
this.handleGateEntry(workflowId, gateName, stateDef);
|
|
1338
|
+
}
|
|
1339
|
+
break;
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
this.emitLifecycleEvent({
|
|
1343
|
+
kind: 'state_entered',
|
|
1344
|
+
workflowId,
|
|
1345
|
+
state: stateValue,
|
|
1346
|
+
});
|
|
1347
|
+
// Check for terminal states
|
|
1348
|
+
if (snapshot.status === 'done') {
|
|
1349
|
+
this.handleWorkflowComplete(workflowId, snapshot.context).catch((err) => {
|
|
1350
|
+
writeStderr(`[workflow] terminal handling failed for ${workflowId}: ${toErrorMessage(err)}`);
|
|
1351
|
+
});
|
|
1352
|
+
}
|
|
1353
|
+
});
|
|
1354
|
+
}
|
|
1355
|
+
// -----------------------------------------------------------------------
|
|
1356
|
+
// Checkpointing
|
|
1357
|
+
// -----------------------------------------------------------------------
|
|
1358
|
+
/**
|
|
1359
|
+
* Returns true if `stateValue` names a terminal state in the workflow
|
|
1360
|
+
* definition. Used to skip checkpointing on the final transition so the
|
|
1361
|
+
* last on-disk checkpoint points at the last non-terminal state (important
|
|
1362
|
+
* for resume-after-abort, which would otherwise reload an `aborted`
|
|
1363
|
+
* snapshot and immediately re-terminate).
|
|
1364
|
+
*/
|
|
1365
|
+
isTerminalStateValue(definition, stateValue) {
|
|
1366
|
+
return definition.states[stateValue].type === 'terminal';
|
|
1367
|
+
}
|
|
1368
|
+
// `waiting_human` would smuggle a ReadonlyMap (gate.presentedArtifacts) into
|
|
1369
|
+
// JSON.stringify, which silently emits `{}`. handleWorkflowComplete only
|
|
1370
|
+
// assigns `completed` or `aborted`, so the cycle is safe today.
|
|
1371
|
+
buildCheckpoint(instance, snapshot, finalStatus, containerSnapshots) {
|
|
1372
|
+
return {
|
|
1373
|
+
machineState: snapshot.value,
|
|
1374
|
+
context: snapshot.context,
|
|
1375
|
+
timestamp: new Date().toISOString(),
|
|
1376
|
+
transitionHistory: [...instance.transitionHistory],
|
|
1377
|
+
definitionPath: instance.definitionPath,
|
|
1378
|
+
workspacePath: instance.workspacePath,
|
|
1379
|
+
// Persisted so resume can read the run-dir staged copy directly
|
|
1380
|
+
// instead of recomputing from the (possibly moved/deleted)
|
|
1381
|
+
// package path. Skipped when the workflow shipped no skills, to
|
|
1382
|
+
// keep the checkpoint shape tight for the common case.
|
|
1383
|
+
...(instance.workflowSkillsDir !== undefined ? { workflowSkillsDir: instance.workflowSkillsDir } : {}),
|
|
1384
|
+
...(instance.workflowScriptsDir !== undefined ? { workflowScriptsDir: instance.workflowScriptsDir } : {}),
|
|
1385
|
+
...(containerSnapshots !== undefined && Object.keys(containerSnapshots).length > 0 ? { containerSnapshots } : {}),
|
|
1386
|
+
...(finalStatus !== undefined ? { finalStatus } : {}),
|
|
1387
|
+
};
|
|
1388
|
+
}
|
|
1389
|
+
saveCheckpoint(instance, snapshot) {
|
|
1390
|
+
const checkpoint = this.buildCheckpoint(instance, snapshot);
|
|
1391
|
+
try {
|
|
1392
|
+
this.deps.checkpointStore.save(instance.id, checkpoint);
|
|
1393
|
+
}
|
|
1394
|
+
catch (err) {
|
|
1395
|
+
writeStderr(`[workflow] Failed to save checkpoint for ${instance.id}: ${toErrorMessage(err)}`);
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
// -----------------------------------------------------------------------
|
|
1399
|
+
// Agent state execution
|
|
1400
|
+
// -----------------------------------------------------------------------
|
|
1401
|
+
// TODO(refactor): this method is ~230 lines and carries three
|
|
1402
|
+
// invocation-scoped closures (`logReceived`, `logAgentRetry`,
|
|
1403
|
+
// `sendAgentTurn`) that all capture the same bundle of state —
|
|
1404
|
+
// `session`, `instance`, `stateConfig`, `stateId`, `messageLog`, and
|
|
1405
|
+
// `this`. The closure pattern is idiomatic here only because every
|
|
1406
|
+
// alternative (methods, helpers) would propagate that bundle through
|
|
1407
|
+
// 5–6 parameters per site.
|
|
1408
|
+
//
|
|
1409
|
+
// The right fix is to extract the whole turn-handling pipeline
|
|
1410
|
+
// (send/parse/retry/reprompt + the three log helpers) into a
|
|
1411
|
+
// dedicated class — e.g. `AgentTurnRunner` — constructed once per
|
|
1412
|
+
// invocation with the shared state injected. That shrinks this
|
|
1413
|
+
// method from ~230 lines to ~80, and makes turn handling
|
|
1414
|
+
// unit-testable in isolation (today it is reachable only through
|
|
1415
|
+
// end-to-end workflow tests). Pair the extraction with a review of
|
|
1416
|
+
// `AgentInvocationError` wrapping so the conversation-id recovery
|
|
1417
|
+
// semantics survive the move. Worth doing as its own PR; do NOT
|
|
1418
|
+
// fold it into a feature change.
|
|
1419
|
+
async executeAgentState(workflowId, input, definition) {
|
|
1420
|
+
const { stateId, stateConfig, context } = input;
|
|
1421
|
+
const instance = this.workflows.get(workflowId);
|
|
1422
|
+
if (!instance)
|
|
1423
|
+
throw new Error(`Workflow ${workflowId} not found`);
|
|
1424
|
+
const settings = definition.settings ?? {};
|
|
1425
|
+
// Version artifact directories before re-entering a state
|
|
1426
|
+
const visitCount = context.visitCounts[stateId] ?? 0;
|
|
1427
|
+
if (visitCount > 1) {
|
|
1428
|
+
const unversioned = new Set(settings.unversionedArtifacts ?? []);
|
|
1429
|
+
snapshotArtifacts(instance.artifactDir, stateConfig.outputs, visitCount, unversioned);
|
|
1430
|
+
}
|
|
1431
|
+
instance.tab.write(`[agent] Starting "${stateId}" (persona: ${stateConfig.persona})`);
|
|
1432
|
+
const command = buildAgentCommand(stateId, stateConfig, context, definition);
|
|
1433
|
+
const mode = settings.mode === 'builtin'
|
|
1434
|
+
? { kind: 'builtin' }
|
|
1435
|
+
: { kind: 'docker', agent: (settings.dockerAgent ?? 'claude-code') };
|
|
1436
|
+
const effectiveModel = stateConfig.model ?? settings.model;
|
|
1437
|
+
// Shared-container mode only: resolve (or lazily mint) the bundle
|
|
1438
|
+
// for this state's scope, rotate the coordinator's audit stream, and
|
|
1439
|
+
// swap policy before constructing the borrowing session. In
|
|
1440
|
+
// per-state-container mode the new session builds its own
|
|
1441
|
+
// coordinator, so there is nothing to cycle.
|
|
1442
|
+
//
|
|
1443
|
+
// Scope lookup algorithm (docs/designs/workflow-session-identity.md §3):
|
|
1444
|
+
// 1. scope := stateConfig.containerScope ?? "primary"
|
|
1445
|
+
// 2. bundle := instance.bundlesByScope.get(scope)
|
|
1446
|
+
// 3. if absent: lazy-mint via ensureBundleForScope(instance, scope)
|
|
1447
|
+
// 4. borrow bundle via SessionOptions.workflow.infrastructure
|
|
1448
|
+
let bundle;
|
|
1449
|
+
if (this.shouldUseSharedContainer(definition)) {
|
|
1450
|
+
const scope = stateConfig.containerScope ?? DEFAULT_CONTAINER_SCOPE;
|
|
1451
|
+
try {
|
|
1452
|
+
bundle = await this.ensureBundleForScope(instance, scope);
|
|
1453
|
+
}
|
|
1454
|
+
catch (err) {
|
|
1455
|
+
const errMsg = toErrorMessage(err);
|
|
1456
|
+
writeStderr(`[workflow] ensureBundleForScope failed for "${stateId}" (scope=${scope}): ${errMsg}`);
|
|
1457
|
+
instance.tab.write(`[error] Infrastructure setup failed for "${stateId}": ${errMsg}`);
|
|
1458
|
+
instance.messageLog.append({
|
|
1459
|
+
...this.logBase(instance),
|
|
1460
|
+
type: 'error',
|
|
1461
|
+
error: errMsg,
|
|
1462
|
+
context: `ensureBundleForScope "${stateId}" (scope: ${scope})`,
|
|
1463
|
+
});
|
|
1464
|
+
throw err;
|
|
1465
|
+
}
|
|
1466
|
+
try {
|
|
1467
|
+
await this.cyclePolicy(instance, stateConfig.persona, bundle);
|
|
1468
|
+
}
|
|
1469
|
+
catch (err) {
|
|
1470
|
+
const errMsg = toErrorMessage(err);
|
|
1471
|
+
writeStderr(`[workflow] cyclePolicy failed for "${stateId}": ${errMsg}`);
|
|
1472
|
+
instance.tab.write(`[error] Policy cycle failed for "${stateId}": ${errMsg}`);
|
|
1473
|
+
instance.messageLog.append({
|
|
1474
|
+
...this.logBase(instance),
|
|
1475
|
+
type: 'error',
|
|
1476
|
+
error: errMsg,
|
|
1477
|
+
context: `cyclePolicy for "${stateId}" (persona: ${stateConfig.persona})`,
|
|
1478
|
+
});
|
|
1479
|
+
throw err;
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
// Agent-CLI conversation identity for this invocation. Decision table
|
|
1483
|
+
// (docs/designs/workflow-session-identity.md §3):
|
|
1484
|
+
// - freshSession:false AND prior entry in agentConversationsByState -> reuse
|
|
1485
|
+
// - otherwise -> mint fresh via createAgentConversationId()
|
|
1486
|
+
// The Docker adapter decides --session-id vs --resume by probing
|
|
1487
|
+
// conversationStateDir for `<agentConversationId>.jsonl`; a minted-fresh
|
|
1488
|
+
// id has no prior file, a reused one does.
|
|
1489
|
+
const priorConversationId = stateConfig.freshSession === false ? context.agentConversationsByState[stateId] : undefined;
|
|
1490
|
+
const agentConversationId = priorConversationId ?? createAgentConversationId();
|
|
1491
|
+
// Mutable tracker updated by the hard-failure retry loop when the
|
|
1492
|
+
// session rotates its conversation id. The final value is what lands
|
|
1493
|
+
// in the invocation result / error and hence in the checkpoint's
|
|
1494
|
+
// `agentConversationsByState`, so a later `freshSession: false` visit
|
|
1495
|
+
// resumes the id whose transcript actually exists on disk.
|
|
1496
|
+
let currentConversationId = agentConversationId;
|
|
1497
|
+
// In borrow mode, route this invocation's per-state artifacts
|
|
1498
|
+
// (session.log, session-metadata.json) under a slug-keyed directory
|
|
1499
|
+
// inside the workflow run. The slug is the next available `.N` in
|
|
1500
|
+
// the bundle's states dir, so true logical re-visits AND resume legs
|
|
1501
|
+
// both get their own dir — never appending into a prior leg's logs.
|
|
1502
|
+
let stateSlug;
|
|
1503
|
+
let workflowStateDir;
|
|
1504
|
+
if (bundle) {
|
|
1505
|
+
// Thread lane.id into the slug so N same-state fan-out lanes each get a
|
|
1506
|
+
// distinct `{stateId}_lane_{id}.{N}` diagnostics dir. nextStateSlug is
|
|
1507
|
+
// check-then-act, so N concurrent lanes sharing one `stateId` would
|
|
1508
|
+
// otherwise compute the same slug and clobber each other's session.log /
|
|
1509
|
+
// session-metadata.json (§5.4 — the one diagnostics-dir collision).
|
|
1510
|
+
stateSlug = nextStateSlug(getBundleStatesDir(instance.id, bundle.bundleId), stateId, context.lane?.id);
|
|
1511
|
+
workflowStateDir = getInvocationDir(instance.id, bundle.bundleId, stateSlug);
|
|
1512
|
+
mkdirSync(workflowStateDir, { recursive: true });
|
|
1513
|
+
}
|
|
1514
|
+
// Read the cached per-run staged path; never recompute from
|
|
1515
|
+
// `instance.definitionPath` here. Computing it per-state would
|
|
1516
|
+
// re-introduce the resume-fragility bug fixed by staging at start.
|
|
1517
|
+
const workflowSkillsDir = instance.workflowSkillsDir;
|
|
1518
|
+
// Workflow context for the session. Always emitted for workflow
|
|
1519
|
+
// runs (so the orchestrator's identity isn't ambiguous), even when
|
|
1520
|
+
// there's no bundle to borrow — `infrastructure` / `stateDir` /
|
|
1521
|
+
// `stateSlug` opt in only when shared-container mode applies.
|
|
1522
|
+
const workflowOptions = {
|
|
1523
|
+
...(bundle ? { infrastructure: bundle } : {}),
|
|
1524
|
+
...(workflowStateDir ? { stateDir: workflowStateDir, stateSlug } : {}),
|
|
1525
|
+
...resolveWorkflowSkillsOptions(stateConfig.skills, workflowSkillsDir),
|
|
1526
|
+
};
|
|
1527
|
+
let session;
|
|
1528
|
+
try {
|
|
1529
|
+
session = await this.deps.createSession({
|
|
1530
|
+
persona: stateConfig.persona,
|
|
1531
|
+
mode,
|
|
1532
|
+
agentConversationId,
|
|
1533
|
+
workspacePath: instance.workspacePath,
|
|
1534
|
+
systemPromptAugmentation: definition.settings?.systemPrompt,
|
|
1535
|
+
...(effectiveModel != null ? { agentModelOverride: effectiveModel } : {}),
|
|
1536
|
+
...(settings.maxSessionSeconds != null
|
|
1537
|
+
? { resourceBudgetOverrides: { maxSessionSeconds: settings.maxSessionSeconds } }
|
|
1538
|
+
: {}),
|
|
1539
|
+
// The nested record colocates the borrowed bundle, per-state
|
|
1540
|
+
// artifact dir, and workflow-bundled skills. `buildSessionConfig`
|
|
1541
|
+
// enforces the borrow-mode invariant (stateDir requires
|
|
1542
|
+
// infrastructure) at runtime.
|
|
1543
|
+
...(Object.keys(workflowOptions).length > 0 ? { workflow: workflowOptions } : {}),
|
|
1544
|
+
});
|
|
1545
|
+
}
|
|
1546
|
+
catch (err) {
|
|
1547
|
+
const errMsg = toErrorMessage(err);
|
|
1548
|
+
writeStderr(`[workflow] Session creation failed for "${stateId}": ${errMsg}`);
|
|
1549
|
+
instance.tab.write(`[error] Session creation failed for "${stateId}": ${errMsg}`);
|
|
1550
|
+
instance.messageLog.append({
|
|
1551
|
+
...this.logBase(instance),
|
|
1552
|
+
type: 'error',
|
|
1553
|
+
error: errMsg,
|
|
1554
|
+
context: `session creation for "${stateId}"`,
|
|
1555
|
+
});
|
|
1556
|
+
throw new AgentInvocationError({ stateId, agentConversationId: currentConversationId, cause: err });
|
|
1557
|
+
}
|
|
1558
|
+
instance.activeSessions.add(session);
|
|
1559
|
+
try {
|
|
1560
|
+
instance.tab.write(`[agent] Sending command to "${stateId}"...`);
|
|
1561
|
+
const { messageLog } = instance;
|
|
1562
|
+
const statusInstructions = buildStatusInstructions(stateConfig.transitions);
|
|
1563
|
+
const logReceived = (text, output) => {
|
|
1564
|
+
messageLog.append({
|
|
1565
|
+
...this.logBase(instance),
|
|
1566
|
+
type: 'agent_received',
|
|
1567
|
+
role: stateConfig.persona,
|
|
1568
|
+
message: text,
|
|
1569
|
+
verdict: output?.verdict ?? null,
|
|
1570
|
+
// eslint-disable-next-line @typescript-eslint/no-deprecated -- logged for diagnostics
|
|
1571
|
+
confidence: output?.confidence ?? null,
|
|
1572
|
+
});
|
|
1573
|
+
};
|
|
1574
|
+
const logAgentRetry = (reason, details, retryMessage) => {
|
|
1575
|
+
messageLog.append({
|
|
1576
|
+
...this.logBase(instance),
|
|
1577
|
+
type: 'agent_retry',
|
|
1578
|
+
role: stateConfig.persona,
|
|
1579
|
+
reason,
|
|
1580
|
+
details,
|
|
1581
|
+
retryMessage,
|
|
1582
|
+
});
|
|
1583
|
+
};
|
|
1584
|
+
// Uniform agent-turn entrypoint: every prompt and reprompt must flow
|
|
1585
|
+
// through here so the quota-exhaustion / transient-failure
|
|
1586
|
+
// short-circuits apply to ALL of them. Retrying or reprompting a turn
|
|
1587
|
+
// whose adapter reported either signal would only burn more of the
|
|
1588
|
+
// already-exhausted budget (quota) or hammer a stalled upstream
|
|
1589
|
+
// (transient) — instead we log the structured event once and throw
|
|
1590
|
+
// a dedicated error so higher layers can distinguish "paused by
|
|
1591
|
+
// provider" / "stalled upstream" from "aborted by bug" (today all
|
|
1592
|
+
// three abort; M4 turns these into a `paused` terminal phase without
|
|
1593
|
+
// touching here).
|
|
1594
|
+
//
|
|
1595
|
+
// Sessions without `sendMessageDetailed` (e.g., built-in in-process
|
|
1596
|
+
// sessions) degrade cleanly to `sendMessage`, which cannot produce
|
|
1597
|
+
// either signal — the short-circuits simply never fire for them.
|
|
1598
|
+
const sendAgentTurn = async (msg) => {
|
|
1599
|
+
const result = session.sendMessageDetailed
|
|
1600
|
+
? await session.sendMessageDetailed(msg)
|
|
1601
|
+
: { text: await session.sendMessage(msg), hardFailure: false };
|
|
1602
|
+
if (result.quotaExhausted) {
|
|
1603
|
+
const { resetAt, rawMessage } = result.quotaExhausted;
|
|
1604
|
+
logReceived(result.text, undefined);
|
|
1605
|
+
messageLog.append({
|
|
1606
|
+
...this.logBase(instance),
|
|
1607
|
+
type: 'quota_exhausted',
|
|
1608
|
+
role: stateConfig.persona,
|
|
1609
|
+
resetAt: resetAt?.toISOString(),
|
|
1610
|
+
rawMessage,
|
|
1611
|
+
});
|
|
1612
|
+
// Stamp the instance before throwing so `handleWorkflowComplete`
|
|
1613
|
+
// can force an abort-preserving terminal regardless of which
|
|
1614
|
+
// state the onError target resolves to — protects checkpoint
|
|
1615
|
+
// retention for workflow definitions whose error target happens
|
|
1616
|
+
// to be a normal terminal (e.g. `done`).
|
|
1617
|
+
instance.quotaExhausted = { resetAt, rawMessage };
|
|
1618
|
+
throw new WorkflowQuotaExhaustedError({ stateId, resetAt, rawMessage });
|
|
1619
|
+
}
|
|
1620
|
+
if (result.transientFailure) {
|
|
1621
|
+
const { kind, rawMessage } = result.transientFailure;
|
|
1622
|
+
// Same shape as the quota branch — append a structured log
|
|
1623
|
+
// entry, stamp the instance, throw a dedicated error — but
|
|
1624
|
+
// deliberately skip `logReceived`. Quota envelopes carry the
|
|
1625
|
+
// provider's actual rate-limit message in `result.text`, which
|
|
1626
|
+
// is worth recording as the agent's turn. Transient envelopes
|
|
1627
|
+
// carry only the agent's preamble with no real assistant
|
|
1628
|
+
// content, so logging it as `agent_received` would falsely
|
|
1629
|
+
// imply the agent produced a turn.
|
|
1630
|
+
messageLog.append({
|
|
1631
|
+
...this.logBase(instance),
|
|
1632
|
+
type: 'transient_failure',
|
|
1633
|
+
role: stateConfig.persona,
|
|
1634
|
+
kind,
|
|
1635
|
+
rawMessage,
|
|
1636
|
+
});
|
|
1637
|
+
instance.transientFailure = { kind, rawMessage };
|
|
1638
|
+
throw new WorkflowTransientFailureError({ stateId, kind, rawMessage });
|
|
1639
|
+
}
|
|
1640
|
+
return result;
|
|
1641
|
+
};
|
|
1642
|
+
messageLog.append({
|
|
1643
|
+
...this.logBase(instance),
|
|
1644
|
+
type: 'agent_sent',
|
|
1645
|
+
role: stateConfig.persona,
|
|
1646
|
+
message: command,
|
|
1647
|
+
});
|
|
1648
|
+
// Shared-container mode: the MITM is long-lived across agents. Flip
|
|
1649
|
+
// its routing id so this agent's events land under its own session id
|
|
1650
|
+
// (matching what the bridge registers below). Optional-chain because
|
|
1651
|
+
// builtin/per-state-container workflows have no shared `infra`.
|
|
1652
|
+
const agentSessionId = session.getInfo().id;
|
|
1653
|
+
// Per-lane token ATTRIBUTION is best-effort under fan-out: N lanes share one
|
|
1654
|
+
// long-lived MITM, so setTokenSessionId is racy across concurrent lanes and a
|
|
1655
|
+
// given event may be attributed to whichever lane's session id is currently
|
|
1656
|
+
// flipped. The AGGREGATE — instance.tokens.outputTokens, summed over every
|
|
1657
|
+
// registered sessionId below — is authoritative for budget and the
|
|
1658
|
+
// fan-out merge (mergeRecordedFanOutContext); the per-lane split is not.
|
|
1659
|
+
bundle?.setTokenSessionId(agentSessionId);
|
|
1660
|
+
// Trajectory-capture lifecycle: brackets the agent's run with a
|
|
1661
|
+
// begin/end pair on the bundle. The matching `endCaptureSession`
|
|
1662
|
+
// is in the finally block, awaited BEFORE session.close() per §11
|
|
1663
|
+
// so the manifest entry is durable even if close() throws.
|
|
1664
|
+
// `bundle` is genuinely optional (builtin / per-state-container
|
|
1665
|
+
// workflows have no shared bundle); the method is always present.
|
|
1666
|
+
if (bundle) {
|
|
1667
|
+
bundle.beginCaptureSession({
|
|
1668
|
+
sessionId: agentSessionId,
|
|
1669
|
+
persona: stateConfig.persona,
|
|
1670
|
+
fsmState: stateId,
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
instance.tokens.sessionIds.add(agentSessionId);
|
|
1674
|
+
this.emitLifecycleEvent({
|
|
1675
|
+
kind: 'agent_started',
|
|
1676
|
+
workflowId,
|
|
1677
|
+
state: stateId,
|
|
1678
|
+
persona: stateConfig.persona,
|
|
1679
|
+
sessionId: agentSessionId,
|
|
1680
|
+
});
|
|
1681
|
+
// Two-phase retry: (1) hard-failure retries re-send the ORIGINAL
|
|
1682
|
+
// command with a rotated conversation id, (2) soft-failure reprompt
|
|
1683
|
+
// asks the agent to fix a missing/malformed agent_status block.
|
|
1684
|
+
//
|
|
1685
|
+
// Hard failures (exitCode != 0 with empty output, e.g., upstream
|
|
1686
|
+
// provider stall that kills the CLI mid-stream) leave the agent's
|
|
1687
|
+
// session id consumed but no resumable transcript on disk — a retry
|
|
1688
|
+
// with the same id is rejected by the CLI. Rotating and resending the
|
|
1689
|
+
// original prompt is the correct recovery path; a missing-status-block
|
|
1690
|
+
// reprompt into a dead session cannot possibly succeed.
|
|
1691
|
+
const MAX_HARD_RETRIES = 2;
|
|
1692
|
+
let responseText = '';
|
|
1693
|
+
for (let attempt = 0; attempt <= MAX_HARD_RETRIES; attempt++) {
|
|
1694
|
+
const result = await sendAgentTurn(command);
|
|
1695
|
+
responseText = result.text;
|
|
1696
|
+
if (!result.hardFailure)
|
|
1697
|
+
break;
|
|
1698
|
+
logReceived(responseText, undefined);
|
|
1699
|
+
logAgentRetry('upstream_stall', `Agent exited without producing output (attempt ${attempt + 1}/${MAX_HARD_RETRIES + 1})`, command);
|
|
1700
|
+
if (attempt === MAX_HARD_RETRIES) {
|
|
1701
|
+
throw new Error(`Agent failed to produce output after ${MAX_HARD_RETRIES + 1} attempts (upstream stall)`);
|
|
1702
|
+
}
|
|
1703
|
+
const rotated = session.rotateAgentConversationId?.();
|
|
1704
|
+
if (rotated)
|
|
1705
|
+
currentConversationId = rotated;
|
|
1706
|
+
}
|
|
1707
|
+
let parseResult = tryParseAgentStatus(responseText);
|
|
1708
|
+
logReceived(responseText, parseResult.kind === 'ok' ? parseResult.output : undefined);
|
|
1709
|
+
let agentOutput;
|
|
1710
|
+
if (parseResult.kind === 'ok') {
|
|
1711
|
+
agentOutput = parseResult.output;
|
|
1712
|
+
}
|
|
1713
|
+
else {
|
|
1714
|
+
const malformed = parseResult.kind === 'malformed' ? parseResult.error : undefined;
|
|
1715
|
+
const retryMsg = buildStatusBlockReprompt(statusInstructions, malformed);
|
|
1716
|
+
logAgentRetry(malformed ? 'malformed_status_block' : 'missing_status_block', malformed
|
|
1717
|
+
? `Malformed agent_status block: ${malformed.message}`
|
|
1718
|
+
: 'Response did not contain an agent_status block', retryMsg);
|
|
1719
|
+
responseText = (await sendAgentTurn(retryMsg)).text;
|
|
1720
|
+
parseResult = tryParseAgentStatus(responseText);
|
|
1721
|
+
logReceived(responseText, parseResult.kind === 'ok' ? parseResult.output : undefined);
|
|
1722
|
+
if (parseResult.kind !== 'ok') {
|
|
1723
|
+
throw new Error(parseResult.kind === 'malformed'
|
|
1724
|
+
? `Agent produced a malformed agent_status block after retry: ${parseResult.error.message}`
|
|
1725
|
+
: 'Agent failed to provide agent_status block after retry');
|
|
1726
|
+
}
|
|
1727
|
+
agentOutput = parseResult.output;
|
|
1728
|
+
}
|
|
1729
|
+
// Validate verdict against valid transitions before the result reaches XState.
|
|
1730
|
+
// This prevents silent deadlocks when the agent returns a verdict that no
|
|
1731
|
+
// transition's `when` clause matches.
|
|
1732
|
+
const validVerdicts = getValidVerdicts(stateConfig.transitions);
|
|
1733
|
+
if (validVerdicts && !validVerdicts.has(agentOutput.verdict)) {
|
|
1734
|
+
const retryMsg = buildInvalidVerdictReprompt(agentOutput.verdict, stateConfig.transitions);
|
|
1735
|
+
logAgentRetry('invalid_verdict', `Verdict "${agentOutput.verdict}" not in valid set: ${[...validVerdicts].join(', ')}`, retryMsg);
|
|
1736
|
+
responseText = (await sendAgentTurn(retryMsg)).text;
|
|
1737
|
+
const retryParse = tryParseAgentStatus(responseText);
|
|
1738
|
+
logReceived(responseText, retryParse.kind === 'ok' ? retryParse.output : undefined);
|
|
1739
|
+
if (retryParse.kind !== 'ok') {
|
|
1740
|
+
const suffix = retryParse.kind === 'malformed'
|
|
1741
|
+
? `retry produced a malformed status block: ${retryParse.error.message}`
|
|
1742
|
+
: 'retry did not include a status block';
|
|
1743
|
+
throw new Error(`Agent verdict "${agentOutput.verdict}" is not valid for this state ` +
|
|
1744
|
+
`(expected one of: ${[...validVerdicts].join(', ')}), and ${suffix}`);
|
|
1745
|
+
}
|
|
1746
|
+
if (!validVerdicts.has(retryParse.output.verdict)) {
|
|
1747
|
+
throw new Error(`Agent returned invalid verdict "${retryParse.output.verdict}" after retry ` +
|
|
1748
|
+
`(expected one of: ${[...validVerdicts].join(', ')})`);
|
|
1749
|
+
}
|
|
1750
|
+
agentOutput = retryParse.output;
|
|
1751
|
+
}
|
|
1752
|
+
const missingArtifacts = this.findMissingArtifacts(stateConfig, instance.artifactDir);
|
|
1753
|
+
if (missingArtifacts.length > 0) {
|
|
1754
|
+
const artifactRetryMsg = buildArtifactReprompt(missingArtifacts, stateConfig.transitions);
|
|
1755
|
+
logAgentRetry('missing_artifacts', `Missing: ${missingArtifacts.join(', ')}`, artifactRetryMsg);
|
|
1756
|
+
const retryResponse = (await sendAgentTurn(artifactRetryMsg)).text;
|
|
1757
|
+
const retryParse = tryParseAgentStatus(retryResponse);
|
|
1758
|
+
if (retryParse.kind === 'ok') {
|
|
1759
|
+
logReceived(retryResponse, retryParse.output);
|
|
1760
|
+
agentOutput = retryParse.output;
|
|
1761
|
+
}
|
|
1762
|
+
else {
|
|
1763
|
+
logReceived(retryResponse, undefined);
|
|
1764
|
+
}
|
|
1765
|
+
const stillMissing = this.findMissingArtifacts(stateConfig, instance.artifactDir);
|
|
1766
|
+
if (stillMissing.length > 0) {
|
|
1767
|
+
throw new Error(`Missing artifacts after retry: ${stillMissing.join(', ')}`);
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
const outputHash = computeOutputHash(stateConfig.outputs, instance.artifactDir, instance.workspacePath);
|
|
1771
|
+
const artifacts = collectArtifactPaths(stateConfig.outputs, instance.artifactDir);
|
|
1772
|
+
instance.tab.write(`[agent] "${stateId}" completed: verdict=${agentOutput.verdict}, artifacts=${Object.keys(artifacts).join(',') || 'none'}`);
|
|
1773
|
+
this.emitLifecycleEvent({
|
|
1774
|
+
kind: 'agent_completed',
|
|
1775
|
+
workflowId,
|
|
1776
|
+
state: stateId,
|
|
1777
|
+
persona: stateConfig.persona,
|
|
1778
|
+
verdict: agentOutput.verdict,
|
|
1779
|
+
});
|
|
1780
|
+
return {
|
|
1781
|
+
output: agentOutput,
|
|
1782
|
+
agentConversationId: currentConversationId,
|
|
1783
|
+
artifacts,
|
|
1784
|
+
outputHash,
|
|
1785
|
+
responseText,
|
|
1786
|
+
// Snapshot the workflow's cumulative output-token count AT THE
|
|
1787
|
+
// END of this agent's turn. The XState assign action uses this
|
|
1788
|
+
// value to update ctx.totalTokens; it reflects every `message_end`
|
|
1789
|
+
// the bus subscriber has seen so far, including all earlier
|
|
1790
|
+
// agents in the workflow.
|
|
1791
|
+
totalTokens: instance.tokens.outputTokens,
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
catch (err) {
|
|
1795
|
+
// Quota exhaustion / transient upstream failure already produced
|
|
1796
|
+
// their own structured log entries inside `sendAgentTurn`;
|
|
1797
|
+
// appending a generic `error` entry here would double-log the same
|
|
1798
|
+
// event and make `workflow inspect` surface a duplicate red error
|
|
1799
|
+
// line alongside the structured signal. Suppress the generic entry
|
|
1800
|
+
// when the cause is a `WorkflowQuotaExhaustedError` or
|
|
1801
|
+
// `WorkflowTransientFailureError`.
|
|
1802
|
+
if (!isWorkflowQuotaExhaustedError(err) && !isWorkflowTransientFailureError(err)) {
|
|
1803
|
+
instance.messageLog.append({
|
|
1804
|
+
...this.logBase(instance),
|
|
1805
|
+
type: 'error',
|
|
1806
|
+
error: toErrorMessage(err),
|
|
1807
|
+
context: `agent "${stateId}" (persona: ${stateConfig.persona})`,
|
|
1808
|
+
});
|
|
1809
|
+
}
|
|
1810
|
+
throw new AgentInvocationError({ stateId, agentConversationId: currentConversationId, cause: err });
|
|
1811
|
+
}
|
|
1812
|
+
finally {
|
|
1813
|
+
instance.activeSessions.delete(session);
|
|
1814
|
+
const endedSessionId = session.getInfo().id;
|
|
1815
|
+
// Trajectory-capture lifecycle: end the capture session FIRST so
|
|
1816
|
+
// the `session-end` manifest entry is durable even if
|
|
1817
|
+
// session.close() throws. The two-phase drain inside
|
|
1818
|
+
// endCaptureSession waits for in-flight reassembly to settle
|
|
1819
|
+
// before enqueuing the manifest end-marker (§9 / §11).
|
|
1820
|
+
// `bundle` is genuinely optional (builtin / per-state-container
|
|
1821
|
+
// workflows); the method is always present when the bundle is.
|
|
1822
|
+
if (bundle) {
|
|
1823
|
+
await bundle.endCaptureSession(endedSessionId).catch((err) => {
|
|
1824
|
+
writeStderr(`[workflow] endCaptureSession failed for "${stateId}": ${toErrorMessage(err)}`);
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
await session.close().catch((closeErr) => {
|
|
1828
|
+
writeStderr(`[workflow] session.close() failed for "${stateId}": ${toErrorMessage(closeErr)}`);
|
|
1829
|
+
});
|
|
1830
|
+
// session.close() drains in-flight streams that captured the
|
|
1831
|
+
// agent's session id at attach time; they keep posting under it
|
|
1832
|
+
// until they finish, regardless of setTokenSessionId(undefined).
|
|
1833
|
+
// The clear here only governs future attachments.
|
|
1834
|
+
bundle?.setTokenSessionId(undefined);
|
|
1835
|
+
instance.tokens.sessionIds.delete(endedSessionId);
|
|
1836
|
+
// Pairs 1:1 with agent_started; emitted unconditionally in finally
|
|
1837
|
+
// so success, failure, and abort paths all clean up the bridge.
|
|
1838
|
+
this.emitLifecycleEvent({
|
|
1839
|
+
kind: 'agent_session_ended',
|
|
1840
|
+
workflowId,
|
|
1841
|
+
state: stateId,
|
|
1842
|
+
sessionId: endedSessionId,
|
|
1843
|
+
});
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
// -----------------------------------------------------------------------
|
|
1847
|
+
// Fan-out state execution
|
|
1848
|
+
// -----------------------------------------------------------------------
|
|
1849
|
+
/**
|
|
1850
|
+
* Runs one fan-out batch: the invoke body of the real `workers` state,
|
|
1851
|
+
* registered as the `fanOutService` actor (see {@link provideActors}). For
|
|
1852
|
+
* each worker it creates a free-standing child round actor, starts it, and
|
|
1853
|
+
* barrier-joins on every child reaching a `final` state, then folds the lane
|
|
1854
|
+
* verdicts into a single parent-edge result (§6.2).
|
|
1855
|
+
*
|
|
1856
|
+
* WHY free-standing `createActor` + `.start()` and NOT in-context `spawn`:
|
|
1857
|
+
* this is a `fromPromise` invoke body (plain async TS), not an XState entry
|
|
1858
|
+
* action — `spawn` is only callable from inside an action. A free-standing
|
|
1859
|
+
* actor is its OWN root, so the parent's `snapshot.value` stays the string
|
|
1860
|
+
* `"workers"` (single-active-state spine preserved, §6.4); the children's
|
|
1861
|
+
* snapshots are their own, read here via {@link waitForRoundChild}. A
|
|
1862
|
+
* `type:'parallel'` region would instead fold into the parent's `value` and
|
|
1863
|
+
* make it an object — fatal to checkpoint/terminal/gate serialization (§6.4).
|
|
1864
|
+
*
|
|
1865
|
+
* WHY mint the scope bundle BEFORE fan-out: {@link preMintFanOutScopeBundle}
|
|
1866
|
+
* runs `ensureBundleForScope` serially up front so the single shared
|
|
1867
|
+
* container exists before any child's first `container:true` step. N children
|
|
1868
|
+
* racing their first container step would each see an empty `bundlesByScope`
|
|
1869
|
+
* and mint a SEPARATE container, defeating `sharedContainer` (§6.2 / §8.1;
|
|
1870
|
+
* cross-ref the `ensureBundleForScope` check-then-act race comment).
|
|
1871
|
+
*
|
|
1872
|
+
* Never throws: every failure path — workflow gone, misconfigured segment,
|
|
1873
|
+
* unsupported worker count, missing child machine — funnels through
|
|
1874
|
+
* {@link fanOutErrorResult}, which yields a `result_file_error` verdict that
|
|
1875
|
+
* the parent `workers` state routes to its `failed` edge. (The `fanOutService`
|
|
1876
|
+
* wrapper in {@link provideActors} would also catch a thrown error, but the
|
|
1877
|
+
* uniform result keeps the parent transition table verdict-driven.)
|
|
1878
|
+
*/
|
|
1879
|
+
async runFanOutSegment(workflowId, input, definition, roundMachinesByState) {
|
|
1880
|
+
const instance = this.workflows.get(workflowId);
|
|
1881
|
+
if (!instance) {
|
|
1882
|
+
return this.fanOutErrorResult(input, `workflow ${workflowId} not found`);
|
|
1883
|
+
}
|
|
1884
|
+
const fanOut = input.stateConfig.fanOut;
|
|
1885
|
+
const segment = input.stateConfig.segment ?? [];
|
|
1886
|
+
if (fanOut === undefined || segment.length === 0) {
|
|
1887
|
+
return this.fanOutErrorResult(input, `State "${input.stateId}" is not a configured fan-out segment.`);
|
|
1888
|
+
}
|
|
1889
|
+
const workers = resolveFanOutWorkers(fanOut, definition.settings);
|
|
1890
|
+
if (!Number.isInteger(workers) || workers < 1) {
|
|
1891
|
+
return this.fanOutErrorResult(input, `State "${input.stateId}" resolved to invalid worker count ${workers}.`);
|
|
1892
|
+
}
|
|
1893
|
+
const roundMachine = roundMachinesByState.get(input.stateId);
|
|
1894
|
+
if (!roundMachine) {
|
|
1895
|
+
return this.fanOutErrorResult(input, `No child round machine is registered for fan-out state "${input.stateId}".`);
|
|
1896
|
+
}
|
|
1897
|
+
// PRECONDITION: mint the shared-container bundle once, before any child.
|
|
1898
|
+
await this.preMintFanOutScopeBundle(instance, input.stateId, segment);
|
|
1899
|
+
const reconstruction = this.reconstructFanOutBatchFromDb(instance, definition, segment, workers);
|
|
1900
|
+
if (reconstruction.kind === 'error') {
|
|
1901
|
+
return this.fanOutErrorResult(input, reconstruction.error);
|
|
1902
|
+
}
|
|
1903
|
+
const missingLanes = reconstruction.kind === 'ready'
|
|
1904
|
+
? reconstruction.reconstruction.missingLanes
|
|
1905
|
+
: Array.from({ length: workers }, (_, index) => index);
|
|
1906
|
+
const preparedLaneResults = await this.prepareFanOutLaneResults(workflowId, input, definition, segment, workers, {
|
|
1907
|
+
batchIndex: reconstruction.kind === 'ready' && reconstruction.reconstruction.recordedByLane.size > 0
|
|
1908
|
+
? reconstruction.reconstruction.batchIndex
|
|
1909
|
+
: undefined,
|
|
1910
|
+
lanes: missingLanes,
|
|
1911
|
+
});
|
|
1912
|
+
if (preparedLaneResults.error !== undefined) {
|
|
1913
|
+
return this.fanOutErrorResult(input, preparedLaneResults.error);
|
|
1914
|
+
}
|
|
1915
|
+
instance.tab.write(`[fanout] Starting "${input.stateId}" with ${workers} worker${workers === 1 ? '' : 's'}`);
|
|
1916
|
+
const providedRoundMachine = this.provideActors(roundMachine, workflowId, definition, roundMachinesByState);
|
|
1917
|
+
let drainTrigger;
|
|
1918
|
+
const childActors = [];
|
|
1919
|
+
const lanePromises = [];
|
|
1920
|
+
// Drain-on-escalation trigger. WHAT: the FIRST lane to reach `blocked`/
|
|
1921
|
+
// `errored` latches `drainTrigger`; the `!== undefined` guard makes this
|
|
1922
|
+
// fire-once, so a later co-blocker never re-triggers. Every un-settled peer
|
|
1923
|
+
// is then drained (stopped) on the trigger's behalf.
|
|
1924
|
+
//
|
|
1925
|
+
// WHY drain-first / discard healthy peers: a batch escalation re-decides the
|
|
1926
|
+
// WHOLE round (the parent routes the whole batch to one human gate), so an
|
|
1927
|
+
// in-flight peer's eventual `recorded` would be thrown away anyway — draining
|
|
1928
|
+
// stops that wasted LLM spend instead of letting peers run to completion.
|
|
1929
|
+
//
|
|
1930
|
+
// WHY stop-at-boundary, NOT AbortSignal: there is no AbortSignal threaded
|
|
1931
|
+
// into the executor layer (FSM-M3), so a lane can only be stopped at a state
|
|
1932
|
+
// boundary via `actor.stop()`; an already-dispatched external service call
|
|
1933
|
+
// may still finish out-of-band, which is why {@link waitForRoundChild}'s
|
|
1934
|
+
// drain resolves the lane's outcome the moment it stops rather than awaiting
|
|
1935
|
+
// the in-flight call.
|
|
1936
|
+
//
|
|
1937
|
+
// Skip edges: the trigger lane itself is skipped (it already settled with its
|
|
1938
|
+
// own blocked/errored outcome), and any `isSettled()` peer is skipped — a peer
|
|
1939
|
+
// that RECORDED before the drain reached it keeps its real outcome (it is not
|
|
1940
|
+
// rewritten to `drained`). The SINGLE aggregated gate over all
|
|
1941
|
+
// blocked/errored lanes is produced later, at {@link joinFanOutBatch}.
|
|
1942
|
+
const requestDrain = (trigger) => {
|
|
1943
|
+
if (drainTrigger !== undefined)
|
|
1944
|
+
return;
|
|
1945
|
+
drainTrigger = trigger;
|
|
1946
|
+
instance.tab.write(`[fanout] "${input.stateId}" draining peers after lane ${trigger.index} ${trigger.status}: ${trigger.reason}`);
|
|
1947
|
+
for (const child of childActors) {
|
|
1948
|
+
if (child.index === trigger.index || child.isSettled())
|
|
1949
|
+
continue;
|
|
1950
|
+
child.drain(trigger);
|
|
1951
|
+
}
|
|
1952
|
+
};
|
|
1953
|
+
// Two loops on purpose: construct ALL waiters before starting ANY actor, so
|
|
1954
|
+
// an early-settling child's drain reaches peers that already have waiters
|
|
1955
|
+
// (a child started in loop 1 could settle synchronously and call
|
|
1956
|
+
// `requestDrain` before loop 2's later peers existed).
|
|
1957
|
+
for (let index = 0; index < workers; index += 1) {
|
|
1958
|
+
// Synthesize-vs-respawn fork. A lane whose node already recorded before the
|
|
1959
|
+
// crash is DB-authoritative: synthesize its outcome from the durable node
|
|
1960
|
+
// (no re-run, no fresh budget). A missing lane is RE-SPAWNED — and a
|
|
1961
|
+
// re-spawned lane restarts with a fresh visit/token budget (its per-lane
|
|
1962
|
+
// `maxVisits` resets, §6.1 / FSM-M1) and may drift to a different verdict
|
|
1963
|
+
// than its pre-crash run would have produced (§11.2). Synthesized lanes are
|
|
1964
|
+
// intentionally NOT pushed to `childActors`: they have no actor to drain, so
|
|
1965
|
+
// the drain-on-escalation loop must never touch them.
|
|
1966
|
+
const recordedLane = reconstruction.kind === 'ready' ? reconstruction.reconstruction.recordedByLane.get(index) : undefined;
|
|
1967
|
+
if (recordedLane !== undefined) {
|
|
1968
|
+
lanePromises[index] = Promise.resolve(this.synthesizeRecordedFanOutOutcome(input, segment, workers, recordedLane));
|
|
1969
|
+
continue;
|
|
1970
|
+
}
|
|
1971
|
+
const childContext = this.buildFanOutLaneContext(input.context, index, workers, preparedLaneResults.byLane[index]);
|
|
1972
|
+
const child = createActor(providedRoundMachine, { input: { context: childContext } });
|
|
1973
|
+
const waiter = this.waitForRoundChild(child, index, requestDrain);
|
|
1974
|
+
childActors.push(waiter);
|
|
1975
|
+
lanePromises[index] = waiter.promise;
|
|
1976
|
+
}
|
|
1977
|
+
for (const child of childActors) {
|
|
1978
|
+
child.actor.start();
|
|
1979
|
+
}
|
|
1980
|
+
const settled = await Promise.allSettled(lanePromises);
|
|
1981
|
+
let result = this.joinFanOutBatch(instance, input, settled);
|
|
1982
|
+
if (workers > 1 && result.verdict === 'recorded') {
|
|
1983
|
+
const promotion = await this.promoteBarrierCognition(workflowId, definition, segment, result.context, settled);
|
|
1984
|
+
if (promotion.error !== undefined) {
|
|
1985
|
+
return this.fanOutErrorResult(input, promotion.error);
|
|
1986
|
+
}
|
|
1987
|
+
if (promotion.result?.payload !== undefined) {
|
|
1988
|
+
result = {
|
|
1989
|
+
...result,
|
|
1990
|
+
payload: {
|
|
1991
|
+
...(result.payload ?? {}),
|
|
1992
|
+
cognition_promotion: promotion.result.payload,
|
|
1993
|
+
},
|
|
1994
|
+
};
|
|
1995
|
+
}
|
|
1996
|
+
// Barrier-owned canonical stop_signals: with N lanes, each lane wrote only
|
|
1997
|
+
// its lane-scoped stop_signals copy, so the single file the orchestrator
|
|
1998
|
+
// routes on is computed ONCE here, post-barrier, over the N-grown nodes.json
|
|
1999
|
+
// (SHOULD-FIX-1). At workers:1 the lone lane's attach_analysis already wrote
|
|
2000
|
+
// the canonical bare file inline (byte-identical backward compat), so we skip.
|
|
2001
|
+
const stopSignalsError = await this.computeBarrierStopSignals(workflowId, definition, segment, input.context);
|
|
2002
|
+
if (stopSignalsError !== undefined) {
|
|
2003
|
+
return this.fanOutErrorResult(input, stopSignalsError);
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
this.logFanOutJoin(instance, input.stateId, workers, result, settled);
|
|
2007
|
+
instance.tab.write(`[fanout] "${input.stateId}" joined: verdict=${result.verdict ?? 'none'}`);
|
|
2008
|
+
return result;
|
|
2009
|
+
}
|
|
2010
|
+
/**
|
|
2011
|
+
* Promotes recorded lane lessons once, serially, at the fan-out barrier. This
|
|
2012
|
+
* is the workers>1 cognition single-writer point (§9): lane `attach_analysis`
|
|
2013
|
+
* calls only record durable nodes; the parent invokes `promote_cognition`
|
|
2014
|
+
* after every child has recorded and before re-entering `orchestrator`.
|
|
2015
|
+
*
|
|
2016
|
+
* Only `recorded` lanes are promoted — blocked/errored children are filtered
|
|
2017
|
+
* out because they contributed no durable node to seed a lesson from.
|
|
2018
|
+
*
|
|
2019
|
+
* Returns one of three shapes:
|
|
2020
|
+
* - `{ error }` — the promotion ran but did not pass (the bridge returned a
|
|
2021
|
+
* non-`cognition_promoted` verdict / `needs_repair`), OR — a hard error —
|
|
2022
|
+
* `lanes.length === 0` for a workers>1 recorded batch (every recorded child
|
|
2023
|
+
* should carry a lane id, so an empty set means the join is inconsistent).
|
|
2024
|
+
* - `{ result }` — the promotion passed; the caller folds the payload into the
|
|
2025
|
+
* batch result under `cognition_promotion`.
|
|
2026
|
+
* - `{}` — a no-op, when {@link buildEvolvePromoteCognitionInput} returns
|
|
2027
|
+
* undefined (non-evolve segment, or a segment with nothing to promote).
|
|
2028
|
+
*
|
|
2029
|
+
* Mixed-verdict deferral (within §9, intended): this only runs when the batch
|
|
2030
|
+
* verdict is `recorded`. When a batch instead escalates/fails, recorded lanes'
|
|
2031
|
+
* lessons are NOT promoted that turn — on APPROVE→resume the recorded lanes
|
|
2032
|
+
* idempotent-skip then barrier-promote (the ledger dedups), but an
|
|
2033
|
+
* ABORT/FORCE_REVISION never promotes them, by design: an aborted batch must
|
|
2034
|
+
* not seed cognition.
|
|
2035
|
+
*/
|
|
2036
|
+
async promoteBarrierCognition(workflowId, definition, segment, context, settled) {
|
|
2037
|
+
const lanes = settled
|
|
2038
|
+
.filter((entry) => entry.status === 'fulfilled')
|
|
2039
|
+
.map((entry) => entry.value)
|
|
2040
|
+
.filter((child) => child.status === 'recorded')
|
|
2041
|
+
.map((child) => child.context.lane?.id)
|
|
2042
|
+
.filter((lane) => typeof lane === 'number' && Number.isInteger(lane) && lane >= 0)
|
|
2043
|
+
.sort((a, b) => a - b);
|
|
2044
|
+
if (lanes.length === 0) {
|
|
2045
|
+
return { error: 'fan-out cognition promotion had no recorded lanes to promote' };
|
|
2046
|
+
}
|
|
2047
|
+
const attach = this.findEvolveAttachAnalysisState(definition, segment);
|
|
2048
|
+
if (!attach)
|
|
2049
|
+
return {};
|
|
2050
|
+
const input = this.buildEvolvePromoteCognitionInput(attach.stateId, attach.state, attach.command, context, lanes);
|
|
2051
|
+
if (!input)
|
|
2052
|
+
return {};
|
|
2053
|
+
const result = await this.executeDeterministicState(workflowId, input);
|
|
2054
|
+
if (!result.passed || result.verdict !== 'cognition_promoted') {
|
|
2055
|
+
return {
|
|
2056
|
+
error: result.errors ?? `barrier cognition promotion failed with verdict ${result.verdict ?? 'unknown'}`,
|
|
2057
|
+
};
|
|
2058
|
+
}
|
|
2059
|
+
return { result };
|
|
2060
|
+
}
|
|
2061
|
+
/**
|
|
2062
|
+
* Locates the segment's `attach_analysis` deterministic state and its evolve
|
|
2063
|
+
* command (the segment member that carries the `evolve_result.py` path and
|
|
2064
|
+
* `--run-dir`). Returns `undefined` when the segment has no such state — a
|
|
2065
|
+
* non-evolve fan-out has no cognition to promote.
|
|
2066
|
+
*/
|
|
2067
|
+
findEvolveAttachAnalysisState(definition, segment) {
|
|
2068
|
+
for (const stateId of segment) {
|
|
2069
|
+
const state = definition.states[stateId];
|
|
2070
|
+
if (state.type !== 'deterministic')
|
|
2071
|
+
continue;
|
|
2072
|
+
const command = state.run.find((entry) => this.isEvolveAttachAnalysisCommand(entry));
|
|
2073
|
+
if (command)
|
|
2074
|
+
return { stateId, state, command };
|
|
2075
|
+
}
|
|
2076
|
+
return undefined;
|
|
2077
|
+
}
|
|
2078
|
+
/**
|
|
2079
|
+
* Derives the `promote_cognition` invoke input from the segment's already-
|
|
2080
|
+
* resolved `attach_analysis` command: reuses the bridge path, `--run-dir`,
|
|
2081
|
+
* container scope, and timeout; rewrites the subcommand; passes the recorded
|
|
2082
|
+
* lane ids; and writes one barrier-owned result at
|
|
2083
|
+
* `current/cognition_promotion.json`. Returns `undefined` when the command
|
|
2084
|
+
* lacks the bridge path, a `--run-dir`, or a resolvable workspace-relative
|
|
2085
|
+
* result path (the caller then treats promotion as a no-op).
|
|
2086
|
+
*/
|
|
2087
|
+
buildEvolvePromoteCognitionInput(stateId, state, attachCommand, context, lanes) {
|
|
2088
|
+
return this.buildEvolveBarrierInput(stateId, state, attachCommand, context, {
|
|
2089
|
+
stateIdSuffix: '_promote_cognition',
|
|
2090
|
+
subcommand: 'promote_cognition',
|
|
2091
|
+
resultBasename: 'cognition_promotion.json',
|
|
2092
|
+
extraArgs: lanes.flatMap((lane) => ['--lane', String(lane)]),
|
|
2093
|
+
});
|
|
2094
|
+
}
|
|
2095
|
+
/**
|
|
2096
|
+
* Computes the single canonical `current/stop_signals.json` ONCE at the
|
|
2097
|
+
* barrier join by running the bridge's `compute_stop_signals` subcommand over
|
|
2098
|
+
* the batch-grown `nodes.json`. The bridge writes the bare canonical file
|
|
2099
|
+
* atomically (temp+rename); per-lane `attach_analysis` only ever wrote
|
|
2100
|
+
* lane-scoped copies, so this is the sole writer of the routing input the
|
|
2101
|
+
* orchestrator reads each batch (SHOULD-FIX-1).
|
|
2102
|
+
*
|
|
2103
|
+
* Derived from the segment head's declared `sample` command (the only segment
|
|
2104
|
+
* member guaranteed to carry the `evolve_result.py` path and `--run-dir`), so
|
|
2105
|
+
* it inherits the same container scope / timeout. Returns an error string on
|
|
2106
|
+
* failure (the caller folds it into a `result_file_error`), or `undefined` on
|
|
2107
|
+
* success — including the benign case where the segment has no recognizable
|
|
2108
|
+
* evolve sample command (a non-evolve fan-out has no stop signals to compute).
|
|
2109
|
+
*/
|
|
2110
|
+
async computeBarrierStopSignals(workflowId, definition, segment, context) {
|
|
2111
|
+
const sampleStateId = segment[0];
|
|
2112
|
+
const sampleState = definition.states[sampleStateId];
|
|
2113
|
+
if (sampleState.type !== 'deterministic')
|
|
2114
|
+
return undefined;
|
|
2115
|
+
const input = this.buildEvolveStopSignalsInput(sampleStateId, sampleState, context);
|
|
2116
|
+
if (!input)
|
|
2117
|
+
return undefined;
|
|
2118
|
+
const result = await this.executeDeterministicState(workflowId, input);
|
|
2119
|
+
if (!result.passed || result.verdict !== 'stop_signals_computed') {
|
|
2120
|
+
return result.errors ?? `barrier stop_signals computation failed with verdict ${result.verdict ?? 'unknown'}`;
|
|
2121
|
+
}
|
|
2122
|
+
return undefined;
|
|
2123
|
+
}
|
|
2124
|
+
/**
|
|
2125
|
+
* Derives the `compute_stop_signals` invoke input from the segment head's
|
|
2126
|
+
* `sample` command: reuses its `evolve_result.py` path and `--run-dir`,
|
|
2127
|
+
* rewrites the subcommand to `compute_stop_signals`, and points
|
|
2128
|
+
* `--result-file` at the shared `current/stop_signals_compute.json` barrier
|
|
2129
|
+
* artifact. Returns `undefined` when the command lacks the bridge path, a
|
|
2130
|
+
* `--run-dir`, or a resolvable workspace-relative result path.
|
|
2131
|
+
*/
|
|
2132
|
+
buildEvolveStopSignalsInput(stateId, state, context) {
|
|
2133
|
+
const sampleCommand = state.run.find((command) => this.isEvolveSampleCommand(command));
|
|
2134
|
+
if (!sampleCommand)
|
|
2135
|
+
return undefined;
|
|
2136
|
+
return this.buildEvolveBarrierInput(stateId, state, sampleCommand, context, {
|
|
2137
|
+
stateIdSuffix: '_stop_signals',
|
|
2138
|
+
subcommand: 'compute_stop_signals',
|
|
2139
|
+
resultBasename: 'stop_signals_compute.json',
|
|
2140
|
+
});
|
|
2141
|
+
}
|
|
2142
|
+
/**
|
|
2143
|
+
* Mints the fan-out segment's shared-container bundle ONCE, before any child
|
|
2144
|
+
* actor starts (the load-bearing precondition of {@link runFanOutSegment}).
|
|
2145
|
+
*
|
|
2146
|
+
* The mint precondition: only when the workflow actually uses a shared
|
|
2147
|
+
* container (`shouldUseSharedContainer`) AND at least one segment member runs
|
|
2148
|
+
* in-container. A segment with no container members needs no bundle and
|
|
2149
|
+
* returns early.
|
|
2150
|
+
*
|
|
2151
|
+
* `ensureBundleForScope` is an unguarded check-then-act across awaits (its own
|
|
2152
|
+
* doc flags the lazy-mint race): N children each hitting their first
|
|
2153
|
+
* `container:true` step would each see `bundlesByScope.get(scope)` undefined
|
|
2154
|
+
* and mint a SEPARATE container. Calling it once here, serially, closes that
|
|
2155
|
+
* window — every child then borrows the already-minted bundle. Cross-ref the
|
|
2156
|
+
* race comment on `ensureBundleForScope`.
|
|
2157
|
+
*
|
|
2158
|
+
* The multi-scope throw is a guardrail, not a supported path: this slice
|
|
2159
|
+
* keeps the whole segment under one `containerScope` (single-policy
|
|
2160
|
+
* homogeneity, §8.2). A segment spanning >1 scope is out of slice scope and
|
|
2161
|
+
* fails fast rather than minting the wrong bundle.
|
|
2162
|
+
*/
|
|
2163
|
+
async preMintFanOutScopeBundle(instance, fanOutStateId, segment) {
|
|
2164
|
+
if (!this.shouldUseSharedContainer(instance.definition))
|
|
2165
|
+
return;
|
|
2166
|
+
const scopes = new Set();
|
|
2167
|
+
for (const memberId of segment) {
|
|
2168
|
+
const member = instance.definition.states[memberId];
|
|
2169
|
+
if (member.type === 'agent') {
|
|
2170
|
+
scopes.add(member.containerScope ?? DEFAULT_CONTAINER_SCOPE);
|
|
2171
|
+
}
|
|
2172
|
+
else if (member.type === 'deterministic' && member.container === true) {
|
|
2173
|
+
scopes.add(member.containerScope ?? DEFAULT_CONTAINER_SCOPE);
|
|
2174
|
+
}
|
|
2175
|
+
}
|
|
2176
|
+
if (scopes.size === 0)
|
|
2177
|
+
return;
|
|
2178
|
+
if (scopes.size > 1) {
|
|
2179
|
+
throw new Error(`fan-out state "${fanOutStateId}" spans multiple container scopes (${[...scopes].join(', ')}); ` +
|
|
2180
|
+
'multi-scope fan-out is out of scope for this implementation slice.');
|
|
2181
|
+
}
|
|
2182
|
+
// size is exactly 1 here (the 0 and >1 cases returned/threw above), so the
|
|
2183
|
+
// single-element destructure yields the one scope.
|
|
2184
|
+
const [scope] = [...scopes];
|
|
2185
|
+
await this.ensureBundleForScope(instance, scope);
|
|
2186
|
+
}
|
|
2187
|
+
buildFanOutLaneContext(context, index, workers, preparedResults) {
|
|
2188
|
+
if (workers === 1)
|
|
2189
|
+
return context;
|
|
2190
|
+
return {
|
|
2191
|
+
...context,
|
|
2192
|
+
lane: {
|
|
2193
|
+
id: index,
|
|
2194
|
+
dir: `${DEFAULT_EVOLVE_LANE_DIR}/lane_${index}`,
|
|
2195
|
+
relativeDir: `${DEFAULT_EVOLVE_LANE_RELATIVE_DIR}/lane_${index}`,
|
|
2196
|
+
...(preparedResults !== undefined ? { preparedResults } : {}),
|
|
2197
|
+
},
|
|
2198
|
+
};
|
|
2199
|
+
}
|
|
2200
|
+
/**
|
|
2201
|
+
* Reconstructs an in-flight evolve fan-out batch from the durable engine DB.
|
|
2202
|
+
*
|
|
2203
|
+
* This intentionally does NOT use XState child persistence. The checkpoint
|
|
2204
|
+
* stores `{ machineState, context }` and `resolveState()` re-enters `workers`
|
|
2205
|
+
* without live child ActorRefs; free-standing child actors created in the old
|
|
2206
|
+
* process are gone. First-cut crash-resume therefore treats `nodes.json` as the
|
|
2207
|
+
* source of truth: a recorded `step_<batch>_lane_<k>` means lane k is already
|
|
2208
|
+
* done and must be synthesized, while every missing lane is re-spawned fresh
|
|
2209
|
+
* from `sample`. That fresh re-run can produce a different verdict (§11.2
|
|
2210
|
+
* verdict drift), which is acceptable for node-integrity but not a promise of
|
|
2211
|
+
* batch-outcome reproducibility.
|
|
2212
|
+
*
|
|
2213
|
+
* Cross-language seam (deliberate, NOT accidental duplication). The batch-index
|
|
2214
|
+
* reuse rule below re-implements Python `_next_batch_index(run_dir, workers)`.
|
|
2215
|
+
* We re-derive it here rather than consume the bridge's `sample_batch_prepared`
|
|
2216
|
+
* echo because the index is needed BEFORE `sample_batch` runs: TS must know it
|
|
2217
|
+
* to decide synthesize-vs-respawn per lane AND to pin `--batch-index` into the
|
|
2218
|
+
* very `sample_batch` call whose output would otherwise echo it. Having the
|
|
2219
|
+
* bridge be authoritative would invert that ordering. So the two derivations
|
|
2220
|
+
* MUST move together — TS pins `--batch-index` whenever any lane is recorded (so
|
|
2221
|
+
* the bridge cannot diverge), and recomputes the identical number when none is.
|
|
2222
|
+
* Both sides cite the shared `step_<batch>_lane_<k>` format (`evolveStepName` /
|
|
2223
|
+
* the Python f-string) as the contract.
|
|
2224
|
+
*/
|
|
2225
|
+
reconstructFanOutBatchFromDb(instance, definition, segment, workers) {
|
|
2226
|
+
if (workers === 1)
|
|
2227
|
+
return { kind: 'unavailable' };
|
|
2228
|
+
const runDir = this.evolveRunDirForFanOutSegment(definition, segment);
|
|
2229
|
+
if (!runDir)
|
|
2230
|
+
return { kind: 'unavailable' };
|
|
2231
|
+
const runDirRelative = this.containerPathToWorkspaceRelative(runDir);
|
|
2232
|
+
if (!runDirRelative)
|
|
2233
|
+
return { kind: 'unavailable' };
|
|
2234
|
+
const nodesPath = resolve(instance.workspacePath, runDirRelative, 'database_data', 'nodes.json');
|
|
2235
|
+
const nodes = this.readEvolveNodes(nodesPath);
|
|
2236
|
+
if (nodes.error !== undefined)
|
|
2237
|
+
return { kind: 'error', error: nodes.error };
|
|
2238
|
+
const expectedLanes = Array.from({ length: workers }, (_, index) => index);
|
|
2239
|
+
const lanesByBatch = new Map();
|
|
2240
|
+
const recordedByStep = new Map();
|
|
2241
|
+
let highestBatch = 0;
|
|
2242
|
+
for (const [nodeKey, rawNode] of nodes.entries) {
|
|
2243
|
+
const node = this.asRecord(rawNode);
|
|
2244
|
+
if (!node)
|
|
2245
|
+
continue;
|
|
2246
|
+
const meta = this.asRecord(node.meta_info);
|
|
2247
|
+
const stepName = typeof meta?.step_name === 'string' ? meta.step_name : undefined;
|
|
2248
|
+
if (!stepName)
|
|
2249
|
+
continue;
|
|
2250
|
+
const match = EVOLVE_LANE_STEP_RE.exec(stepName);
|
|
2251
|
+
if (!match)
|
|
2252
|
+
continue;
|
|
2253
|
+
const batchIndex = Number.parseInt(match[1], 10);
|
|
2254
|
+
highestBatch = Math.max(highestBatch, batchIndex);
|
|
2255
|
+
// Group 2 (the lane id) is undefined for a legacy non-lane `step_<batch>`
|
|
2256
|
+
// node; only lane-tagged nodes participate in fan-out reconstruction.
|
|
2257
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- an unmatched optional regex group is `undefined` at runtime, but the type is `string`
|
|
2258
|
+
if (match[2] === undefined)
|
|
2259
|
+
continue;
|
|
2260
|
+
const lane = Number.parseInt(match[2], 10);
|
|
2261
|
+
lanesByBatch.set(batchIndex, (lanesByBatch.get(batchIndex) ?? new Set()).add(lane));
|
|
2262
|
+
recordedByStep.set(stepName, {
|
|
2263
|
+
index: lane,
|
|
2264
|
+
batchIndex,
|
|
2265
|
+
stepName,
|
|
2266
|
+
nodeId: this.nodeIdFromRecord(node, nodeKey),
|
|
2267
|
+
parent: this.nodeParentFromRecord(node),
|
|
2268
|
+
stepDir: `${runDir}/steps/${stepName}`,
|
|
2269
|
+
});
|
|
2270
|
+
}
|
|
2271
|
+
// Reuse-vs-advance heuristic — the crux of crash-resume, and the TS twin of
|
|
2272
|
+
// Python `_next_batch_index(run_dir, workers)` (scripts/evolve_result.py).
|
|
2273
|
+
// The highest lane-tagged batch whose CURRENT-worker lanes are only
|
|
2274
|
+
// PARTIALLY present (some expected lane is missing) is the batch that was
|
|
2275
|
+
// in flight when we crashed → REUSE its index, so the re-run lanes write the
|
|
2276
|
+
// SAME `step_<batch>_lane_<k>` names and stay idempotent against the records
|
|
2277
|
+
// that already landed. A fully-recorded highest batch (every expected lane
|
|
2278
|
+
// present) or a fully-drained one (no lane-tagged nodes, so highestLanes is
|
|
2279
|
+
// empty) is finished → advance to `highestBatch + 1`. This must match the
|
|
2280
|
+
// Python reuse rule exactly: when recordedByLane is non-empty we PIN
|
|
2281
|
+
// `--batch-index` to this value (so the bridge cannot diverge); when it is
|
|
2282
|
+
// empty we pass no `--batch-index` and the bridge recomputes the same number.
|
|
2283
|
+
// The shared `step_<batch>_lane_<k>` format (evolveStepName / the Python
|
|
2284
|
+
// f-string) is the contract both sides cite; keep this block and
|
|
2285
|
+
// `_next_batch_index` in lockstep — same seam as STEP_NAME_RE ↔ EVOLVE_LANE_STEP_RE.
|
|
2286
|
+
const highestLanes = lanesByBatch.get(highestBatch) ?? new Set();
|
|
2287
|
+
const batchIndex = highestBatch > 0 && highestLanes.size > 0 && expectedLanes.some((lane) => !highestLanes.has(lane))
|
|
2288
|
+
? highestBatch
|
|
2289
|
+
: highestBatch + 1;
|
|
2290
|
+
const recordedByLane = new Map();
|
|
2291
|
+
for (const lane of expectedLanes) {
|
|
2292
|
+
const recorded = recordedByStep.get(evolveStepName(batchIndex, lane));
|
|
2293
|
+
if (recorded !== undefined)
|
|
2294
|
+
recordedByLane.set(lane, recorded);
|
|
2295
|
+
}
|
|
2296
|
+
const missingLanes = expectedLanes.filter((lane) => !recordedByLane.has(lane));
|
|
2297
|
+
if (recordedByLane.size > 0) {
|
|
2298
|
+
instance.tab.write(`[fanout] Reconstructed "${segment[0]}" batch ${batchIndex}: ` +
|
|
2299
|
+
`${recordedByLane.size} recorded, ${missingLanes.length} to re-run`);
|
|
2300
|
+
}
|
|
2301
|
+
return {
|
|
2302
|
+
kind: 'ready',
|
|
2303
|
+
reconstruction: {
|
|
2304
|
+
batchIndex,
|
|
2305
|
+
recordedByLane,
|
|
2306
|
+
missingLanes,
|
|
2307
|
+
runDir,
|
|
2308
|
+
},
|
|
2309
|
+
};
|
|
2310
|
+
}
|
|
2311
|
+
/**
|
|
2312
|
+
* The evolve `--run-dir` for this fan-out segment's `sample` head, or
|
|
2313
|
+
* `undefined` when the segment head is not a deterministic evolve sample
|
|
2314
|
+
* command. `undefined` makes {@link reconstructFanOutBatchFromDb} return
|
|
2315
|
+
* `unavailable`, so resume falls back to re-spawning all lanes fresh.
|
|
2316
|
+
*/
|
|
2317
|
+
evolveRunDirForFanOutSegment(definition, segment) {
|
|
2318
|
+
const sampleStateId = segment[0];
|
|
2319
|
+
const sampleState = definition.states[sampleStateId];
|
|
2320
|
+
if (sampleState.type !== 'deterministic')
|
|
2321
|
+
return undefined;
|
|
2322
|
+
const sampleCommand = sampleState.run.find((command) => this.isEvolveSampleCommand(command));
|
|
2323
|
+
return sampleCommand ? this.commandFlagValue(sampleCommand, '--run-dir') : undefined;
|
|
2324
|
+
}
|
|
2325
|
+
/**
|
|
2326
|
+
* Reads the engine `nodes.json` node map. Missing file ⇒ `{ entries: [] }`
|
|
2327
|
+
* with NO error — a fresh batch (or a run that has not recorded any node yet)
|
|
2328
|
+
* is a normal pre-crash state and must reconstruct as "nothing recorded". A
|
|
2329
|
+
* read or JSON-parse failure ⇒ an `error`, which aborts the fan-out: a corrupt
|
|
2330
|
+
* DB is NOT safe to treat as empty (we would re-run lanes that may already be
|
|
2331
|
+
* recorded), so the missing-vs-corrupt distinction is load-bearing.
|
|
2332
|
+
*/
|
|
2333
|
+
readEvolveNodes(nodesPath) {
|
|
2334
|
+
if (!existsSync(nodesPath))
|
|
2335
|
+
return { entries: [] };
|
|
2336
|
+
let raw;
|
|
2337
|
+
try {
|
|
2338
|
+
raw = readFileSync(nodesPath, 'utf-8');
|
|
2339
|
+
}
|
|
2340
|
+
catch (err) {
|
|
2341
|
+
return { entries: [], error: `could not read evolve nodes DB ${nodesPath}: ${errorMessage(err)}` };
|
|
2342
|
+
}
|
|
2343
|
+
let parsed;
|
|
2344
|
+
try {
|
|
2345
|
+
parsed = JSON.parse(raw);
|
|
2346
|
+
}
|
|
2347
|
+
catch (err) {
|
|
2348
|
+
return { entries: [], error: `could not parse evolve nodes DB ${nodesPath}: ${errorMessage(err)}` };
|
|
2349
|
+
}
|
|
2350
|
+
const root = this.asRecord(parsed);
|
|
2351
|
+
const nodes = this.asRecord(root?.nodes);
|
|
2352
|
+
return { entries: nodes ? Object.entries(nodes) : [] };
|
|
2353
|
+
}
|
|
2354
|
+
asRecord(value) {
|
|
2355
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
2356
|
+
? value
|
|
2357
|
+
: undefined;
|
|
2358
|
+
}
|
|
2359
|
+
nodeIdFromRecord(node, fallback) {
|
|
2360
|
+
const raw = node.id;
|
|
2361
|
+
if (typeof raw === 'number' && Number.isInteger(raw))
|
|
2362
|
+
return raw;
|
|
2363
|
+
const parsed = Number.parseInt(fallback, 10);
|
|
2364
|
+
return Number.isInteger(parsed) ? parsed : undefined;
|
|
2365
|
+
}
|
|
2366
|
+
/**
|
|
2367
|
+
* The node's parent lineage as a tri-state: `undefined` when the record has no
|
|
2368
|
+
* `parent` array at all (omit the field from the synthesized payload — lineage
|
|
2369
|
+
* unknown), `[]` for a root node (an explicit empty array — root, not omitted),
|
|
2370
|
+
* and `[...ids]` for a child's parent ids. The empty-vs-undefined distinction
|
|
2371
|
+
* is preserved deliberately so {@link synthesizeRecordedFanOutOutcome} can omit
|
|
2372
|
+
* an unknown parent while still echoing an explicit root.
|
|
2373
|
+
*/
|
|
2374
|
+
nodeParentFromRecord(node) {
|
|
2375
|
+
if (!Array.isArray(node.parent))
|
|
2376
|
+
return undefined;
|
|
2377
|
+
const parent = node.parent.filter((item) => typeof item === 'number' && Number.isInteger(item));
|
|
2378
|
+
return parent.length > 0 ? parent : [];
|
|
2379
|
+
}
|
|
2380
|
+
/**
|
|
2381
|
+
* Builds the barrier outcome for a lane that already recorded before a crash.
|
|
2382
|
+
* The durable node proves the lane advanced; the volatile child actor context
|
|
2383
|
+
* (visit counts, token spend, in-flight state) is not recoverable without the
|
|
2384
|
+
* deferred `getPersistedSnapshot()` + in-context spawn migration, so this is a
|
|
2385
|
+
* minimal recorded context sufficient for join, promotion, and duplicate stats.
|
|
2386
|
+
*
|
|
2387
|
+
* Edge cases (the contract callers rely on):
|
|
2388
|
+
* - Called ONLY for a lane the engine DB already recorded — i.e. the
|
|
2389
|
+
* just-before-crash case where the node landed but the process died before
|
|
2390
|
+
* the join. Every NOT-recorded lane is re-spawned instead (see the fork in
|
|
2391
|
+
* {@link runFanOutSegment}).
|
|
2392
|
+
* - Does NOT rebuild on-disk artifacts. The lane's `analysis.md` /
|
|
2393
|
+
* `analysis_record.json` may be absent post-crash, but the durable node is
|
|
2394
|
+
* authoritative, so a fabricated `lastDeterministicResult` with
|
|
2395
|
+
* `{ verdict: 'recorded', idempotent_skip: true }` is sufficient — nothing
|
|
2396
|
+
* downstream re-reads those files for a recorded lane.
|
|
2397
|
+
* - `idempotent_skip: true` is the marker that this outcome was synthesized
|
|
2398
|
+
* (vs produced by a live re-run); the returned `recorded` outcome is
|
|
2399
|
+
* otherwise indistinguishable to {@link joinFanOutBatch} from a live-recorded
|
|
2400
|
+
* lane, so the join, promotion, and duplicate-rate paths need no special case.
|
|
2401
|
+
*/
|
|
2402
|
+
synthesizeRecordedFanOutOutcome(input, segment, workers, recorded) {
|
|
2403
|
+
const base = this.buildFanOutLaneContext(input.context, recorded.index, workers, undefined);
|
|
2404
|
+
const payload = {
|
|
2405
|
+
step_name: recorded.stepName,
|
|
2406
|
+
idempotent_skip: true,
|
|
2407
|
+
...(recorded.nodeId !== undefined ? { node_id: recorded.nodeId } : {}),
|
|
2408
|
+
...(recorded.parent !== undefined ? { parent: [...recorded.parent] } : {}),
|
|
2409
|
+
...(recorded.stepDir !== undefined ? { step_dir: recorded.stepDir } : {}),
|
|
2410
|
+
};
|
|
2411
|
+
return {
|
|
2412
|
+
index: recorded.index,
|
|
2413
|
+
status: 'recorded',
|
|
2414
|
+
context: {
|
|
2415
|
+
...base,
|
|
2416
|
+
round: base.round + 1,
|
|
2417
|
+
previousStateName: segment[segment.length - 1] ?? input.stateId,
|
|
2418
|
+
lastDeterministicResult: {
|
|
2419
|
+
verdict: 'recorded',
|
|
2420
|
+
payload,
|
|
2421
|
+
},
|
|
2422
|
+
},
|
|
2423
|
+
};
|
|
2424
|
+
}
|
|
2425
|
+
/**
|
|
2426
|
+
* Runs the ONE barrier-side `sample_batch(n=workers)` before fan-out and
|
|
2427
|
+
* partitions its result into per-lane pre-computed `sample` results (keyed by
|
|
2428
|
+
* lane id) that {@link executeDeterministicState} replays as each child's
|
|
2429
|
+
* `sample` step.
|
|
2430
|
+
*
|
|
2431
|
+
* WHY one barrier-side `sample_batch(n=N)` and NOT N independent `sample()`
|
|
2432
|
+
* calls: the divergence between lanes is INJECTED at sampling (§7.4). N
|
|
2433
|
+
* independent `sample()` calls each reload the same `nodes.json` and, with a
|
|
2434
|
+
* diversity-maintaining sampler, could draw the SAME parent into multiple
|
|
2435
|
+
* lanes — collapsing the N-wide frontier into "N samples of one distribution"
|
|
2436
|
+
* and producing near-duplicate candidates. A single `sample_batch` over the
|
|
2437
|
+
* whole DB partitions N DISTINCT parents (one per lane) in one serialized,
|
|
2438
|
+
* read-only DB touch, guaranteeing disjoint parent sets up front. The bridge
|
|
2439
|
+
* uses a per-lane seed offset (`base_seed + laneId`) to decorrelate the lanes
|
|
2440
|
+
* while keeping the partition reproducible, with a deterministic hash tiebreak
|
|
2441
|
+
* so the assignment is stable across resume (§7.4).
|
|
2442
|
+
*
|
|
2443
|
+
* `workers === 1` short-circuits: the single lane uses the legacy inline
|
|
2444
|
+
* `sample` step (byte-identical backward compat, gate item 1), so there is no
|
|
2445
|
+
* batch to prepare. A non-deterministic or unrecognized sample state likewise
|
|
2446
|
+
* returns an empty map (the child runs its own `sample`).
|
|
2447
|
+
*/
|
|
2448
|
+
async prepareFanOutLaneResults(workflowId, input, definition, segment, workers, options = {}) {
|
|
2449
|
+
if (workers === 1)
|
|
2450
|
+
return { byLane: {} };
|
|
2451
|
+
const expectedLanes = options.lanes ?? Array.from({ length: workers }, (_, index) => index);
|
|
2452
|
+
if (expectedLanes.length === 0)
|
|
2453
|
+
return { byLane: {} };
|
|
2454
|
+
const sampleStateId = segment[0];
|
|
2455
|
+
const sampleState = definition.states[sampleStateId];
|
|
2456
|
+
if (sampleState.type !== 'deterministic')
|
|
2457
|
+
return { byLane: {} };
|
|
2458
|
+
const batchInput = this.buildEvolveSampleBatchInput(sampleStateId, sampleState, input.context, workers, options);
|
|
2459
|
+
if (!batchInput)
|
|
2460
|
+
return { byLane: {} };
|
|
2461
|
+
const result = await this.executeDeterministicState(workflowId, batchInput);
|
|
2462
|
+
if (!result.passed || result.verdict !== 'sample_batch_prepared') {
|
|
2463
|
+
return {
|
|
2464
|
+
byLane: {},
|
|
2465
|
+
error: result.errors ?? `fan-out sample batch failed with verdict ${result.verdict ?? 'unknown'}`,
|
|
2466
|
+
};
|
|
2467
|
+
}
|
|
2468
|
+
const lanes = Array.isArray(result.payload?.lanes) ? result.payload.lanes : undefined;
|
|
2469
|
+
if (!lanes) {
|
|
2470
|
+
return { byLane: {}, error: 'fan-out sample batch did not return a lanes array' };
|
|
2471
|
+
}
|
|
2472
|
+
const byLane = {};
|
|
2473
|
+
for (const lanePayload of lanes) {
|
|
2474
|
+
if (!lanePayload || typeof lanePayload !== 'object' || Array.isArray(lanePayload))
|
|
2475
|
+
continue;
|
|
2476
|
+
const laneId = lanePayload.lane;
|
|
2477
|
+
if (typeof laneId !== 'number' || !Number.isInteger(laneId) || laneId < 0 || laneId >= workers)
|
|
2478
|
+
continue;
|
|
2479
|
+
if (!expectedLanes.includes(laneId))
|
|
2480
|
+
continue;
|
|
2481
|
+
byLane[laneId] = {
|
|
2482
|
+
[sampleStateId]: {
|
|
2483
|
+
passed: true,
|
|
2484
|
+
verdict: 'sampled',
|
|
2485
|
+
payload: lanePayload,
|
|
2486
|
+
},
|
|
2487
|
+
};
|
|
2488
|
+
}
|
|
2489
|
+
const missing = expectedLanes.filter((index) => !(index in byLane));
|
|
2490
|
+
if (missing.length > 0) {
|
|
2491
|
+
return { byLane: {}, error: `fan-out sample batch omitted lane(s): ${missing.join(', ')}` };
|
|
2492
|
+
}
|
|
2493
|
+
return { byLane };
|
|
2494
|
+
}
|
|
2495
|
+
/**
|
|
2496
|
+
* Derives the deterministic invoke input for the barrier-side `sample_batch`
|
|
2497
|
+
* from the segment head's declared `sample` command: rewrites the `sample`
|
|
2498
|
+
* subcommand to `sample_batch`, adds `--workers <N>`, carries over the
|
|
2499
|
+
* query/top-k flags, and points `--result-file` at the shared
|
|
2500
|
+
* `current/sample_batch.json` (the batch artifact is barrier-owned, not
|
|
2501
|
+
* lane-scoped). Returns `undefined` (the caller then lets each child sample on
|
|
2502
|
+
* its own) when the segment head is not a recognizable evolve `sample`
|
|
2503
|
+
* command, has no `evolve_result.py` entry, or lacks a `--run-dir` / resolvable
|
|
2504
|
+
* workspace-relative result path.
|
|
2505
|
+
*/
|
|
2506
|
+
buildEvolveSampleBatchInput(stateId, state, context, workers, options = {}) {
|
|
2507
|
+
const sampleCommand = state.run.find((command) => this.isEvolveSampleCommand(command));
|
|
2508
|
+
if (!sampleCommand)
|
|
2509
|
+
return undefined;
|
|
2510
|
+
return this.buildEvolveBarrierInput(stateId, state, sampleCommand, context, {
|
|
2511
|
+
stateIdSuffix: '_batch',
|
|
2512
|
+
subcommand: 'sample_batch',
|
|
2513
|
+
resultBasename: 'sample_batch.json',
|
|
2514
|
+
extraArgs: [
|
|
2515
|
+
'--workers',
|
|
2516
|
+
String(workers),
|
|
2517
|
+
...(options.lanes !== undefined ? options.lanes.flatMap((lane) => ['--lane', String(lane)]) : []),
|
|
2518
|
+
...(options.batchIndex !== undefined ? ['--batch-index', String(options.batchIndex)] : []),
|
|
2519
|
+
...this.sampleQueryArgs(sampleCommand),
|
|
2520
|
+
...this.sampleTopKArgs(sampleCommand),
|
|
2521
|
+
],
|
|
2522
|
+
});
|
|
2523
|
+
}
|
|
2524
|
+
/**
|
|
2525
|
+
* Shared skeleton for the three barrier-side `evolve_result.py` invocations
|
|
2526
|
+
* (`sample_batch`, `promote_cognition`, `compute_stop_signals`). Given an
|
|
2527
|
+
* already-resolved `sourceCommand` (the caller does the `state.run.find`), it
|
|
2528
|
+
* locates the bridge path, carries over `--run-dir`, rewrites the subcommand,
|
|
2529
|
+
* splices in the per-caller `extraArgs`, and points `--result-file` at the
|
|
2530
|
+
* shared barrier artifact `current/<resultBasename>`. Returns `undefined` when
|
|
2531
|
+
* the source command lacks the bridge path, a `--run-dir`, or a resolvable
|
|
2532
|
+
* workspace-relative result path — the caller's documented no-op signal.
|
|
2533
|
+
*/
|
|
2534
|
+
buildEvolveBarrierInput(stateId, state, sourceCommand, context, options) {
|
|
2535
|
+
const scriptIndex = evolveResultScriptIndex(sourceCommand);
|
|
2536
|
+
if (scriptIndex < 0)
|
|
2537
|
+
return undefined;
|
|
2538
|
+
const runDir = this.commandFlagValue(sourceCommand, '--run-dir');
|
|
2539
|
+
if (!runDir)
|
|
2540
|
+
return undefined;
|
|
2541
|
+
const resultFile = `${runDir}/current/${options.resultBasename}`;
|
|
2542
|
+
const workspaceRelativeResultFile = this.containerPathToWorkspaceRelative(resultFile);
|
|
2543
|
+
if (!workspaceRelativeResultFile)
|
|
2544
|
+
return undefined;
|
|
2545
|
+
const command = [
|
|
2546
|
+
...sourceCommand.slice(0, scriptIndex + 1),
|
|
2547
|
+
options.subcommand,
|
|
2548
|
+
'--run-dir',
|
|
2549
|
+
runDir,
|
|
2550
|
+
...(options.extraArgs ?? []),
|
|
2551
|
+
'--result-file',
|
|
2552
|
+
resultFile,
|
|
2553
|
+
];
|
|
2554
|
+
return {
|
|
2555
|
+
stateId: `${stateId}${options.stateIdSuffix}`,
|
|
2556
|
+
commands: [command],
|
|
2557
|
+
context,
|
|
2558
|
+
container: state.container ?? false,
|
|
2559
|
+
containerScope: state.containerScope,
|
|
2560
|
+
timeoutMs: state.timeoutMs,
|
|
2561
|
+
resultFile: workspaceRelativeResultFile,
|
|
2562
|
+
};
|
|
2563
|
+
}
|
|
2564
|
+
isEvolveSampleCommand(command) {
|
|
2565
|
+
const scriptIndex = evolveResultScriptIndex(command);
|
|
2566
|
+
return scriptIndex >= 0 && command[scriptIndex + 1] === 'sample';
|
|
2567
|
+
}
|
|
2568
|
+
isEvolveAttachAnalysisCommand(command) {
|
|
2569
|
+
const scriptIndex = evolveResultScriptIndex(command);
|
|
2570
|
+
return scriptIndex >= 0 && command[scriptIndex + 1] === 'attach_analysis';
|
|
2571
|
+
}
|
|
2572
|
+
commandFlagValue(command, flag) {
|
|
2573
|
+
const index = command.indexOf(flag);
|
|
2574
|
+
if (index < 0)
|
|
2575
|
+
return undefined;
|
|
2576
|
+
return command[index + 1];
|
|
2577
|
+
}
|
|
2578
|
+
sampleQueryArgs(command) {
|
|
2579
|
+
if (command.includes('--query-from-spec'))
|
|
2580
|
+
return ['--query-from-spec'];
|
|
2581
|
+
const query = this.commandFlagValue(command, '--query');
|
|
2582
|
+
return query !== undefined ? ['--query', query] : [];
|
|
2583
|
+
}
|
|
2584
|
+
sampleTopKArgs(command) {
|
|
2585
|
+
const topK = this.commandFlagValue(command, '--top-k');
|
|
2586
|
+
return topK !== undefined ? ['--top-k', topK] : [];
|
|
2587
|
+
}
|
|
2588
|
+
containerPathToWorkspaceRelative(path) {
|
|
2589
|
+
const prefix = `${CONTAINER_WORKSPACE_DIR}/`;
|
|
2590
|
+
return path.startsWith(prefix) ? path.slice(prefix.length) : undefined;
|
|
2591
|
+
}
|
|
2592
|
+
/**
|
|
2593
|
+
* Barrier primitive for one lane. Returns a {@link RoundChildWaiter} (NOT a
|
|
2594
|
+
* bare Promise): a two-phase handle whose `promise` resolves by EITHER of two
|
|
2595
|
+
* routes, and whose `drain`/`isSettled` members let the fan-out pump stop and
|
|
2596
|
+
* inspect the lane externally.
|
|
2597
|
+
*
|
|
2598
|
+
* The two resolution routes and the discriminator {@link joinFanOutBatch} reads:
|
|
2599
|
+
* - NATURAL SETTLE — the child's own snapshot reaches a `final` state
|
|
2600
|
+
* (`status === 'done'`), resolving with `recorded`/`blocked`/`errored` and
|
|
2601
|
+
* `drainedBy: undefined`. A free-standing child fires no parent-visible
|
|
2602
|
+
* `onDone`, so the parent watches the child's OWN snapshot (§6.2) via
|
|
2603
|
+
* subscribe-until-done rather than an `xstate.done.actor` event.
|
|
2604
|
+
* - DRAIN — an external `drain(trigger)` call (a peer blocked/errored) stops
|
|
2605
|
+
* the actor and resolves with status `drained` and a populated `drainedBy`.
|
|
2606
|
+
* `drainedBy` being set vs `undefined` is exactly how the join distinguishes
|
|
2607
|
+
* a drained peer from a naturally-settled lane.
|
|
2608
|
+
*
|
|
2609
|
+
* `isSettled` is first-wins: whichever route fires first latches `settled`, so
|
|
2610
|
+
* a lane that RECORDED before the drain reached it keeps its real `recorded`
|
|
2611
|
+
* outcome (the drain is a no-op on an already-settled lane).
|
|
2612
|
+
*
|
|
2613
|
+
* After the child `onError` -> `errored` fix, a segment-member service
|
|
2614
|
+
* rejection becomes a normal `errored` TERMINAL (status `done`, value
|
|
2615
|
+
* `errored`) and resolves here; the `status === 'error'` branch is the
|
|
2616
|
+
* residual safety net for an actor-level throw the `errored` terminal did not
|
|
2617
|
+
* catch (e.g. an entry-action throw), which resolves as an `errored` lane so
|
|
2618
|
+
* drain-on-escalation can still stop peers and the join can report the full
|
|
2619
|
+
* batch.
|
|
2620
|
+
*
|
|
2621
|
+
* WHY a hand-rolled subscribe loop and NOT `xstate`'s `waitFor`: `waitFor`
|
|
2622
|
+
* resolves only when the actor's OWN snapshot stream matches a predicate. The
|
|
2623
|
+
* drain route resolves the SAME promise from OUTSIDE that stream (the pump
|
|
2624
|
+
* synthesizes a `drained` outcome the instant it stops the actor, without
|
|
2625
|
+
* waiting for any in-flight external service call to settle, FSM-M3). `waitFor`
|
|
2626
|
+
* cannot express that external resolver, so the two-phase waiter is kept.
|
|
2627
|
+
*
|
|
2628
|
+
* The trailing `observe(actor.getSnapshot())` after `subscribe` guards the
|
|
2629
|
+
* synchronous-settle-before-subscribe race: a child whose first state
|
|
2630
|
+
* settles synchronously (e.g. a sync entry-action throw) can already be
|
|
2631
|
+
* `done`/`error` by the time `.subscribe` returns, so the subscription would
|
|
2632
|
+
* never deliver that final snapshot. Observing the current snapshot once,
|
|
2633
|
+
* immediately, closes the lane in that case.
|
|
2634
|
+
*/
|
|
2635
|
+
waitForRoundChild(actor, index, onDrainTrigger) {
|
|
2636
|
+
let settled = false;
|
|
2637
|
+
let subscription;
|
|
2638
|
+
let finishOutcome = () => { };
|
|
2639
|
+
const promise = new Promise((resolvePromise) => {
|
|
2640
|
+
const finish = (outcome) => {
|
|
2641
|
+
if (settled)
|
|
2642
|
+
return;
|
|
2643
|
+
settled = true;
|
|
2644
|
+
subscription?.unsubscribe();
|
|
2645
|
+
resolvePromise(outcome);
|
|
2646
|
+
if (outcome.status === 'blocked' || outcome.status === 'errored') {
|
|
2647
|
+
// Compute the surfaced reason ONCE here, at the trigger source, and
|
|
2648
|
+
// thread it on the trigger so the drain log and every drained peer's
|
|
2649
|
+
// `drainedBy` reuse it rather than re-deriving (§ dedup).
|
|
2650
|
+
onDrainTrigger({
|
|
2651
|
+
index: outcome.index,
|
|
2652
|
+
status: outcome.status,
|
|
2653
|
+
context: outcome.context,
|
|
2654
|
+
reason: this.roundChildReason(outcome),
|
|
2655
|
+
});
|
|
2656
|
+
}
|
|
2657
|
+
};
|
|
2658
|
+
finishOutcome = finish;
|
|
2659
|
+
const observe = (snapshot) => {
|
|
2660
|
+
const snap = snapshot;
|
|
2661
|
+
if (snap.status === 'done') {
|
|
2662
|
+
finish({
|
|
2663
|
+
index,
|
|
2664
|
+
status: toRoundChildStatus(snap.value, index),
|
|
2665
|
+
context: snap.context,
|
|
2666
|
+
});
|
|
2667
|
+
}
|
|
2668
|
+
else if (snap.status === 'error') {
|
|
2669
|
+
const reason = snap.error ?? `fan-out child ${index} errored`;
|
|
2670
|
+
finish({
|
|
2671
|
+
index,
|
|
2672
|
+
status: 'errored',
|
|
2673
|
+
context: snap.context,
|
|
2674
|
+
drainedBy: undefined,
|
|
2675
|
+
});
|
|
2676
|
+
writeStderr(`[workflow] fan-out child ${index} actor error: ${toErrorMessage(reason)}`);
|
|
2677
|
+
}
|
|
2678
|
+
};
|
|
2679
|
+
subscription = actor.subscribe(observe);
|
|
2680
|
+
observe(actor.getSnapshot());
|
|
2681
|
+
});
|
|
2682
|
+
return {
|
|
2683
|
+
index,
|
|
2684
|
+
actor,
|
|
2685
|
+
promise,
|
|
2686
|
+
isSettled: () => settled,
|
|
2687
|
+
drain: (trigger) => {
|
|
2688
|
+
if (settled)
|
|
2689
|
+
return;
|
|
2690
|
+
const snapshot = actor.getSnapshot();
|
|
2691
|
+
const context = snapshot.context ?? trigger.context;
|
|
2692
|
+
actor.stop();
|
|
2693
|
+
// Resolve directly after stop(): the stopped actor will not enter its
|
|
2694
|
+
// next state, while any already-started external promise may still finish
|
|
2695
|
+
// out-of-band (there is no AbortSignal in the executor layer, FSM-M3).
|
|
2696
|
+
finishOutcome({
|
|
2697
|
+
index,
|
|
2698
|
+
status: 'drained',
|
|
2699
|
+
context,
|
|
2700
|
+
drainedBy: { index: trigger.index, status: trigger.status, reason: trigger.reason },
|
|
2701
|
+
});
|
|
2702
|
+
},
|
|
2703
|
+
};
|
|
2704
|
+
}
|
|
2705
|
+
/**
|
|
2706
|
+
* Folds the settled lane outcomes into a single parent-edge result, applying
|
|
2707
|
+
* a fixed verdict precedence (worst lane wins):
|
|
2708
|
+
* 1. any rejected settle -> `result_file_error` (a residual actor-level
|
|
2709
|
+
* throw that did not resolve as an `errored` lane);
|
|
2710
|
+
* 2. else any `blocked` lane -> `escalate` (route once to human review,
|
|
2711
|
+
* with every blocked/errored lane reason aggregated);
|
|
2712
|
+
* 3. else any `errored` lane -> `result_file_error` (explicit fail mapping);
|
|
2713
|
+
* 4. else any `drained` lane without an issue trigger -> `result_file_error`;
|
|
2714
|
+
* 5. else all `recorded` -> `recorded` (the batch advanced).
|
|
2715
|
+
*
|
|
2716
|
+
* WHY `Promise.allSettled` (not `Promise.all`) upstream: one lane's rejection
|
|
2717
|
+
* must not abandon its N-1 in-flight peers (FSM-S5). `all` would reject on the
|
|
2718
|
+
* first throw and lose the other lanes' outcomes; `allSettled` lets the join
|
|
2719
|
+
* see the full batch, including parent-synthesized `drained` peer outcomes.
|
|
2720
|
+
*
|
|
2721
|
+
* For `workers: 1`, the recorded path still promotes the single child context
|
|
2722
|
+
* exactly. For N>1, the recorded path folds lane contexts into a parent batch
|
|
2723
|
+
* context: child artifacts/conversations merge, parent visit counts remain
|
|
2724
|
+
* authoritative for the single-active-state spine, and total tokens fan in
|
|
2725
|
+
* from the orchestrator's workflow-level token accumulator.
|
|
2726
|
+
*/
|
|
2727
|
+
joinFanOutBatch(instance, input, settled) {
|
|
2728
|
+
const rejected = settled.find((result) => result.status === 'rejected');
|
|
2729
|
+
if (rejected?.status === 'rejected') {
|
|
2730
|
+
return this.fanOutErrorResult(input, toErrorMessage(rejected.reason));
|
|
2731
|
+
}
|
|
2732
|
+
const children = settled.map((result) => {
|
|
2733
|
+
if (result.status !== 'fulfilled') {
|
|
2734
|
+
throw new Error('unreachable rejected fan-out result after rejection check');
|
|
2735
|
+
}
|
|
2736
|
+
return result.value;
|
|
2737
|
+
});
|
|
2738
|
+
const first = children[0];
|
|
2739
|
+
const childSummaries = children.map((child) => ({
|
|
2740
|
+
index: child.index,
|
|
2741
|
+
status: child.status,
|
|
2742
|
+
// Carry `drainedBy` through to the join log so observability records which
|
|
2743
|
+
// lane's escalation stopped each drained peer (the bare `drained` status
|
|
2744
|
+
// alone does not say). NOTE (drain-first, § consider): `issues` below is the
|
|
2745
|
+
// blocked/errored-AT-DRAIN set only — a peer that WOULD have blocked but was
|
|
2746
|
+
// drained first is reported here as `drained`, NOT as an issue in the gate.
|
|
2747
|
+
...(child.drainedBy !== undefined ? { drainedBy: child.drainedBy } : {}),
|
|
2748
|
+
}));
|
|
2749
|
+
const issues = this.fanOutIssues(children);
|
|
2750
|
+
if (issues.some((issue) => issue.status === 'blocked')) {
|
|
2751
|
+
const source = children.find((child) => child.status === 'blocked') ?? first;
|
|
2752
|
+
const summary = this.formatFanOutIssues('Fan-out batch escalated after blocked lane(s):', issues);
|
|
2753
|
+
return {
|
|
2754
|
+
passed: false,
|
|
2755
|
+
verdict: 'escalate',
|
|
2756
|
+
errors: summary,
|
|
2757
|
+
payload: { issues },
|
|
2758
|
+
context: this.promoteFanOutIssueContext(instance, input.context, input.stateId, source, 'escalate', issues),
|
|
2759
|
+
children: childSummaries,
|
|
2760
|
+
};
|
|
2761
|
+
}
|
|
2762
|
+
if (issues.some((issue) => issue.status === 'errored')) {
|
|
2763
|
+
const source = children.find((child) => child.status === 'errored') ?? first;
|
|
2764
|
+
const summary = this.formatFanOutIssues('Fan-out batch failed after errored lane(s):', issues);
|
|
2765
|
+
return {
|
|
2766
|
+
passed: false,
|
|
2767
|
+
verdict: DETERMINISTIC_RESULT_ERROR_VERDICT,
|
|
2768
|
+
errors: summary,
|
|
2769
|
+
payload: { issues },
|
|
2770
|
+
context: this.promoteFanOutIssueContext(instance, input.context, input.stateId, source, DETERMINISTIC_RESULT_ERROR_VERDICT, issues),
|
|
2771
|
+
children: childSummaries,
|
|
2772
|
+
};
|
|
2773
|
+
}
|
|
2774
|
+
const drained = children.find((child) => child.status === 'drained');
|
|
2775
|
+
if (drained) {
|
|
2776
|
+
return {
|
|
2777
|
+
passed: false,
|
|
2778
|
+
verdict: DETERMINISTIC_RESULT_ERROR_VERDICT,
|
|
2779
|
+
errors: `fan-out child ${drained.index} was drained without a blocked or errored trigger`,
|
|
2780
|
+
context: this.promoteFanOutContext(instance, input.context, drained.context),
|
|
2781
|
+
children: childSummaries,
|
|
2782
|
+
};
|
|
2783
|
+
}
|
|
2784
|
+
return {
|
|
2785
|
+
passed: true,
|
|
2786
|
+
verdict: 'recorded',
|
|
2787
|
+
context: children.length === 1 ? first.context : this.mergeRecordedFanOutContext(instance, input.context, children),
|
|
2788
|
+
children: childSummaries,
|
|
2789
|
+
};
|
|
2790
|
+
}
|
|
2791
|
+
/**
|
|
2792
|
+
* Projects the blocked/errored lanes (the offenders) into the {@link FanOutIssue}
|
|
2793
|
+
* payload that lands in the aggregated gate. Drained peers are intentionally
|
|
2794
|
+
* excluded — they are the drain-first collateral, not independent issues (§ the
|
|
2795
|
+
* reason list is the blocked-at-drain set, not all-would-have-blocked).
|
|
2796
|
+
*/
|
|
2797
|
+
fanOutIssues(children) {
|
|
2798
|
+
return children
|
|
2799
|
+
.filter((child) => child.status === 'blocked' || child.status === 'errored')
|
|
2800
|
+
.map((child) => ({
|
|
2801
|
+
index: child.index,
|
|
2802
|
+
status: child.status,
|
|
2803
|
+
reason: this.roundChildReason(child),
|
|
2804
|
+
}));
|
|
2805
|
+
}
|
|
2806
|
+
/**
|
|
2807
|
+
* Surfaces a one-line human-readable cause for a blocked/errored lane, with an
|
|
2808
|
+
* ASYMMETRIC fallback order per status — THE footgun this method guards:
|
|
2809
|
+
*
|
|
2810
|
+
* - `errored` prefers `lastError` then `previousAgentOutput`: an errored lane
|
|
2811
|
+
* DID trip `storeError`, so `lastError` is the populated, authoritative cause.
|
|
2812
|
+
* - `blocked` prefers `previousAgentOutput` then `lastError`: a blocked lane
|
|
2813
|
+
* (evaluator_blocked) never trips `storeError`, so its `lastError` is null and
|
|
2814
|
+
* the real reason lives on `previousAgentOutput` (the evaluator's message).
|
|
2815
|
+
*
|
|
2816
|
+
* Do NOT collapse the two branches into one fallback chain: a unified order
|
|
2817
|
+
* would surface a stale/null field for one of the two statuses. This rationale
|
|
2818
|
+
* was deleted from the old call sites and is re-homed here on purpose.
|
|
2819
|
+
*/
|
|
2820
|
+
roundChildReason(child) {
|
|
2821
|
+
if (child.status === 'errored') {
|
|
2822
|
+
return (child.context.lastError ??
|
|
2823
|
+
child.context.previousAgentOutput ??
|
|
2824
|
+
`fan-out child ${child.index} ended in "${child.status}"`);
|
|
2825
|
+
}
|
|
2826
|
+
return (child.context.previousAgentOutput ??
|
|
2827
|
+
child.context.lastError ??
|
|
2828
|
+
`fan-out child ${child.index} ended in "${child.status}"`);
|
|
2829
|
+
}
|
|
2830
|
+
/** Renders an issue list as a header line plus one `- lane <i> <status>: <reason>` bullet per offending lane. */
|
|
2831
|
+
formatFanOutIssues(header, issues) {
|
|
2832
|
+
if (issues.length === 0)
|
|
2833
|
+
return header;
|
|
2834
|
+
return [header, ...issues.map((issue) => `- lane ${issue.index} ${issue.status}: ${issue.reason}`)].join('\n');
|
|
2835
|
+
}
|
|
2836
|
+
/**
|
|
2837
|
+
* Builds the parent-spine context for a blocked/errored BATCH (the escalate and
|
|
2838
|
+
* errored-only join branches). WHAT it folds:
|
|
2839
|
+
* - the worst lane's context (`source`) promoted back onto the spine via
|
|
2840
|
+
* {@link promoteFanOutContext} (lane marker stripped, parent visit counts
|
|
2841
|
+
* restored, tokens from the accumulator);
|
|
2842
|
+
* - an AGGREGATED multi-lane summary of every blocked/errored lane written to
|
|
2843
|
+
* `previousAgentOutput` (this is what the orchestrator reads on APPROVE→resume
|
|
2844
|
+
* and the human reads in the gate banner);
|
|
2845
|
+
* - the same lanes as the structured `issues` payload on
|
|
2846
|
+
* `lastDeterministicResult` for the gate.
|
|
2847
|
+
*
|
|
2848
|
+
* WHY ONE gate, not one per offending lane: all blocked/errored reasons fold
|
|
2849
|
+
* into a single payload so the human reviews the batch ONCE. The two verdict
|
|
2850
|
+
* shapes differ only in `lastError`: `escalate` leaves `lastError` null (a
|
|
2851
|
+
* blocked lane is recoverable, not an error), while the errored-only verdict
|
|
2852
|
+
* mirrors the summary into `lastError` (the batch failed).
|
|
2853
|
+
*
|
|
2854
|
+
* SCRATCH-SHAPE TAG (workers:1 vs workers>1 disambiguator): the orchestrator's
|
|
2855
|
+
* recovery prompt has two mutually-exclusive recoveries — legacy `evaluate`-
|
|
2856
|
+
* resume of the one written-but-unscored candidate (bare `current/`) vs discard
|
|
2857
|
+
* the lane scratch and start a fresh `design` batch (lane-scoped
|
|
2858
|
+
* `current/lane_<k>/`). At `workers:1` the lane carries NO `lane` marker and the
|
|
2859
|
+
* scratch is the BARE `current/`, so the legacy resume is the correct recovery;
|
|
2860
|
+
* at `workers>1` the lane scratch is under `current/lane_<k>/` and must be
|
|
2861
|
+
* discarded. The summary message alone ("Fan-out batch escalated…") would push
|
|
2862
|
+
* the LLM toward discard in BOTH cases, so we append an explicit scratch-shape
|
|
2863
|
+
* sentence derived from `source.context.lane` to steer the prompt's branch.
|
|
2864
|
+
*/
|
|
2865
|
+
promoteFanOutIssueContext(instance, parentContext, fanOutStateId, source, verdict, issues) {
|
|
2866
|
+
const summary = this.formatFanOutIssues(verdict === DETERMINISTIC_RESULT_ERROR_VERDICT
|
|
2867
|
+
? 'Fan-out batch failed after errored lane(s):'
|
|
2868
|
+
: 'Fan-out batch escalated after blocked lane(s):', issues);
|
|
2869
|
+
const scratchNote = source.context.lane === undefined
|
|
2870
|
+
? '\nScratch is the bare current/ (a single candidate is written; result.json may be absent) — ' +
|
|
2871
|
+
'resume the round in place; do NOT discard and start a fresh batch.'
|
|
2872
|
+
: '\nScratch is lane-scoped under current/lane_<k>/ — discard the lane scratch and re-decide normally.';
|
|
2873
|
+
return {
|
|
2874
|
+
...this.promoteFanOutContext(instance, parentContext, source.context),
|
|
2875
|
+
previousAgentOutput: `${summary}${scratchNote}`,
|
|
2876
|
+
previousAgentNotes: null,
|
|
2877
|
+
previousStateName: fanOutStateId,
|
|
2878
|
+
lastDeterministicResult: {
|
|
2879
|
+
verdict,
|
|
2880
|
+
payload: { issues: issues.map((issue) => ({ ...issue })) },
|
|
2881
|
+
},
|
|
2882
|
+
...(verdict === DETERMINISTIC_RESULT_ERROR_VERDICT ? { lastError: summary } : { lastError: null }),
|
|
2883
|
+
};
|
|
2884
|
+
}
|
|
2885
|
+
/**
|
|
2886
|
+
* Promotes a SINGLE lane's context back onto the parent spine (the
|
|
2887
|
+
* blocked/errored drain paths, and the `workers: 1` recorded path where there
|
|
2888
|
+
* is exactly one lane). Strips the per-lane `lane` marker so the parent never
|
|
2889
|
+
* carries a lane identity, restores the parent's authoritative `visitCounts`
|
|
2890
|
+
* (the single-active-state spine owns visit accounting; a lane's own counts
|
|
2891
|
+
* are a within-batch detail, §7.5), and reads `totalTokens` from the
|
|
2892
|
+
* instance-level token accumulator (the aggregate over all lanes' registered
|
|
2893
|
+
* sessionIds, which is the budget-authoritative figure, NOT the lane's local
|
|
2894
|
+
* sum). The `lane === undefined` branch is the no-fan-out / `workers: 1` case:
|
|
2895
|
+
* there is no lane marker to strip, so only the token figure is refreshed.
|
|
2896
|
+
*/
|
|
2897
|
+
promoteFanOutContext(instance, parentContext, laneContext) {
|
|
2898
|
+
if (laneContext.lane === undefined) {
|
|
2899
|
+
return {
|
|
2900
|
+
...laneContext,
|
|
2901
|
+
totalTokens: instance.tokens.outputTokens,
|
|
2902
|
+
};
|
|
2903
|
+
}
|
|
2904
|
+
const withoutLane = this.withoutLaneContext(laneContext);
|
|
2905
|
+
return {
|
|
2906
|
+
...withoutLane,
|
|
2907
|
+
visitCounts: parentContext.visitCounts,
|
|
2908
|
+
totalTokens: instance.tokens.outputTokens,
|
|
2909
|
+
};
|
|
2910
|
+
}
|
|
2911
|
+
/**
|
|
2912
|
+
* Folds the N recorded lane contexts back into ONE parent-spine context at the
|
|
2913
|
+
* barrier join (the recorded happy path for `workers > 1`). This is the
|
|
2914
|
+
* highest-stakes merge in the fan-out: the parent FSM is single-active-state,
|
|
2915
|
+
* so after the batch it must hold a single coherent context even though N
|
|
2916
|
+
* child actors each evolved their own copy in parallel. Each field is merged
|
|
2917
|
+
* by a rule chosen for WHY it is correct on the spine, NOT by a uniform
|
|
2918
|
+
* strategy:
|
|
2919
|
+
*
|
|
2920
|
+
* - `visitCounts`: the PARENT's counts are authoritative and copied verbatim.
|
|
2921
|
+
* The spine has a single active state (`workers`), entered once per batch;
|
|
2922
|
+
* the children's per-lane visit counts are a within-batch bookkeeping detail
|
|
2923
|
+
* (§7.5) that must not leak onto the parent, or the global round-limit
|
|
2924
|
+
* backstop would see N× inflated counts.
|
|
2925
|
+
* - `round`: SUM of each lane's POSITIVE delta over the parent baseline
|
|
2926
|
+
* (`max(0, child.round - parent.round)`). A batch advanced the search by as
|
|
2927
|
+
* many rounds as lanes that progressed; summing positive deltas counts each
|
|
2928
|
+
* advancing lane once and ignores any lane that did not move forward.
|
|
2929
|
+
* - `reviewHistory`: SUFFIX-append only — each lane's entries BEYOND the shared
|
|
2930
|
+
* parent baseline are appended. Lanes start from the same parent
|
|
2931
|
+
* reviewHistory, so a naive concat would replay the baseline N times; taking
|
|
2932
|
+
* only the suffix past `parentContext.reviewHistory.length` avoids that N×
|
|
2933
|
+
* duplication while still capturing every lane's new review notes.
|
|
2934
|
+
* - `humanPrompt`: NULLED. A human prompt is a single-turn directive consumed
|
|
2935
|
+
* by one state entry; it has no meaning fanned across N lanes and must not
|
|
2936
|
+
* survive the batch (a stale prompt would mis-route the next orchestrator
|
|
2937
|
+
* turn).
|
|
2938
|
+
* - `totalTokens`: read from the instance-level accumulator (the aggregate over
|
|
2939
|
+
* all lanes' registered MITM sessionIds), NOT summed from the children here —
|
|
2940
|
+
* the accumulator is the budget-authoritative figure and already includes
|
|
2941
|
+
* every lane's spend.
|
|
2942
|
+
* - `artifacts` / `previousOutputHashes` / `agentConversationsByState`: shallow
|
|
2943
|
+
* last-writer-wins merge over the parent baseline (lanes write disjoint,
|
|
2944
|
+
* lane-scoped keys, so collisions are not expected; the parent baseline is
|
|
2945
|
+
* the floor). The remaining scalar fields come from `last` (the
|
|
2946
|
+
* highest-index lane), an arbitrary-but-deterministic representative.
|
|
2947
|
+
*/
|
|
2948
|
+
mergeRecordedFanOutContext(instance, parentContext, children) {
|
|
2949
|
+
const ordered = [...children].sort((a, b) => a.index - b.index);
|
|
2950
|
+
const last = ordered[ordered.length - 1]?.context ?? parentContext;
|
|
2951
|
+
const lastWithoutLane = this.withoutLaneContext(last);
|
|
2952
|
+
const artifacts = ordered.reduce((acc, child) => ({ ...acc, ...child.context.artifacts }), {
|
|
2953
|
+
...parentContext.artifacts,
|
|
2954
|
+
});
|
|
2955
|
+
const previousOutputHashes = ordered.reduce((acc, child) => ({ ...acc, ...child.context.previousOutputHashes }), { ...parentContext.previousOutputHashes });
|
|
2956
|
+
const agentConversationsByState = ordered.reduce((acc, child) => ({ ...acc, ...child.context.agentConversationsByState }), { ...parentContext.agentConversationsByState });
|
|
2957
|
+
const roundDelta = ordered.reduce((sum, child) => sum + Math.max(0, child.context.round - parentContext.round), 0);
|
|
2958
|
+
return {
|
|
2959
|
+
...lastWithoutLane,
|
|
2960
|
+
artifacts,
|
|
2961
|
+
previousOutputHashes,
|
|
2962
|
+
agentConversationsByState,
|
|
2963
|
+
visitCounts: parentContext.visitCounts,
|
|
2964
|
+
reviewHistory: ordered.reduce((acc, child) => child.context.reviewHistory.length > parentContext.reviewHistory.length
|
|
2965
|
+
? [...acc, ...child.context.reviewHistory.slice(parentContext.reviewHistory.length)]
|
|
2966
|
+
: acc, parentContext.reviewHistory),
|
|
2967
|
+
round: parentContext.round + roundDelta,
|
|
2968
|
+
totalTokens: instance.tokens.outputTokens,
|
|
2969
|
+
humanPrompt: null,
|
|
2970
|
+
};
|
|
2971
|
+
}
|
|
2972
|
+
withoutLaneContext(context) {
|
|
2973
|
+
const copy = { ...context };
|
|
2974
|
+
delete copy.lane;
|
|
2975
|
+
return copy;
|
|
2976
|
+
}
|
|
2977
|
+
logFanOutJoin(instance, fanOutState, workers, result, settled) {
|
|
2978
|
+
const fulfilled = settled
|
|
2979
|
+
.filter((entry) => entry.status === 'fulfilled')
|
|
2980
|
+
.map((entry) => entry.value);
|
|
2981
|
+
const duplicateStats = result.verdict === 'recorded' ? this.computeFanOutCandidateDuplicateStats(instance, fulfilled) : undefined;
|
|
2982
|
+
instance.messageLog.append({
|
|
2983
|
+
...this.logBase(instance),
|
|
2984
|
+
type: 'fanout_join',
|
|
2985
|
+
fanOutState,
|
|
2986
|
+
workers,
|
|
2987
|
+
verdict: result.verdict ?? null,
|
|
2988
|
+
...(duplicateStats !== undefined ? duplicateStats : {}),
|
|
2989
|
+
children: result.children ??
|
|
2990
|
+
fulfilled.map((child) => ({
|
|
2991
|
+
index: child.index,
|
|
2992
|
+
status: child.status,
|
|
2993
|
+
})),
|
|
2994
|
+
});
|
|
2995
|
+
if (duplicateStats !== undefined) {
|
|
2996
|
+
instance.tab.write(`[fanout] "${fanOutState}" duplicate_rate=${duplicateStats.duplicateRate.toFixed(3)} ` +
|
|
2997
|
+
`(${duplicateStats.duplicateCount}/${duplicateStats.candidateCount})`);
|
|
2998
|
+
}
|
|
2999
|
+
}
|
|
3000
|
+
/**
|
|
3001
|
+
* Measures effective lane diversity for one recorded batch by content-hashing
|
|
3002
|
+
* each lane's candidate source and counting byte-identical collisions.
|
|
3003
|
+
*
|
|
3004
|
+
* WHY content-hash at fan-in (not assume effective-N = N): distinct sampled
|
|
3005
|
+
* parents do NOT guarantee distinct candidates (§7.4). Convergent lanes — the
|
|
3006
|
+
* researcher LLM collapsing two different parents into the same edit, or a
|
|
3007
|
+
* degenerate sampler handing out overlapping parents — can produce
|
|
3008
|
+
* byte-identical candidates, so "N lanes" can have an effective diversity
|
|
3009
|
+
* below N. Hashing the recorded `steps/<step>/code` files surfaces that as an
|
|
3010
|
+
* observable duplicate rate instead of pretending every lane added a distinct
|
|
3011
|
+
* candidate; the operator reads it as a signal to tune the diversity knobs.
|
|
3012
|
+
* It is a diagnostic, NOT a gate — duplicates are still recorded as distinct
|
|
3013
|
+
* nodes (the engine assigns each its own id, §7.6).
|
|
3014
|
+
*
|
|
3015
|
+
* Every skip is best-effort, NOT an error: a missing step_name, a candidate
|
|
3016
|
+
* file that does not exist yet, or a non-recorded lane is simply omitted from
|
|
3017
|
+
* the sample rather than failing the join — a diagnostic must never block a
|
|
3018
|
+
* batch that otherwise recorded cleanly. The `undefined` return is the
|
|
3019
|
+
* "nothing measurable" contract: when no lane yielded a hashable candidate
|
|
3020
|
+
* (every lane skipped), there is no diversity to report and the caller logs no
|
|
3021
|
+
* duplicate stats at all (distinct from a measured `duplicateRate: 0`).
|
|
3022
|
+
*/
|
|
3023
|
+
computeFanOutCandidateDuplicateStats(instance, children) {
|
|
3024
|
+
const hashes = [];
|
|
3025
|
+
for (const child of children) {
|
|
3026
|
+
if (child.status !== 'recorded')
|
|
3027
|
+
continue;
|
|
3028
|
+
const stepName = child.context.lastDeterministicResult?.payload?.step_name;
|
|
3029
|
+
if (typeof stepName !== 'string' || stepName.length === 0)
|
|
3030
|
+
continue;
|
|
3031
|
+
// Path layout owned by the Python bridge — see EVOLVE_STEP_CODE_PATH_SEGMENTS.
|
|
3032
|
+
const candidatePath = resolve(instance.workspacePath, ...EVOLVE_STEP_CODE_PATH_SEGMENTS, stepName, EVOLVE_STEP_CODE_FILENAME);
|
|
3033
|
+
if (!existsSync(candidatePath))
|
|
3034
|
+
continue;
|
|
3035
|
+
hashes.push(createHash('sha256').update(readFileSync(candidatePath)).digest('hex'));
|
|
3036
|
+
}
|
|
3037
|
+
if (hashes.length === 0)
|
|
3038
|
+
return undefined;
|
|
3039
|
+
const uniqueCandidates = new Set(hashes).size;
|
|
3040
|
+
const duplicateCount = hashes.length - uniqueCandidates;
|
|
3041
|
+
return {
|
|
3042
|
+
duplicateRate: duplicateCount / hashes.length,
|
|
3043
|
+
duplicateCount,
|
|
3044
|
+
uniqueCandidates,
|
|
3045
|
+
candidateCount: hashes.length,
|
|
3046
|
+
};
|
|
3047
|
+
}
|
|
3048
|
+
/**
|
|
3049
|
+
* The single terminal error channel from {@link runFanOutSegment} to the
|
|
3050
|
+
* parent: a `result_file_error` verdict carrying `message`, which the parent
|
|
3051
|
+
* `workers` state routes to its `failed` edge.
|
|
3052
|
+
*/
|
|
3053
|
+
fanOutErrorResult(input, message) {
|
|
3054
|
+
return {
|
|
3055
|
+
passed: false,
|
|
3056
|
+
verdict: DETERMINISTIC_RESULT_ERROR_VERDICT,
|
|
3057
|
+
errors: message,
|
|
3058
|
+
context: input.context,
|
|
3059
|
+
};
|
|
3060
|
+
}
|
|
3061
|
+
// -----------------------------------------------------------------------
|
|
3062
|
+
// Deterministic state execution
|
|
3063
|
+
// -----------------------------------------------------------------------
|
|
3064
|
+
async executeDeterministicState(workflowId, input) {
|
|
3065
|
+
const preparedResult = input.context.lane?.preparedResults?.[input.stateId];
|
|
3066
|
+
if (preparedResult !== undefined) {
|
|
3067
|
+
return preparedResult;
|
|
3068
|
+
}
|
|
3069
|
+
if (!input.container) {
|
|
3070
|
+
return this.runDeterministicHost(input.commands);
|
|
3071
|
+
}
|
|
3072
|
+
const instance = this.workflows.get(workflowId);
|
|
3073
|
+
if (!instance) {
|
|
3074
|
+
return { passed: false, errors: `workflow ${workflowId} not found` };
|
|
3075
|
+
}
|
|
3076
|
+
if (!this.shouldUseSharedContainer(instance.definition)) {
|
|
3077
|
+
return { passed: false, errors: `State "${input.stateId}" requires shared-container Docker execution.` };
|
|
3078
|
+
}
|
|
3079
|
+
const scope = input.containerScope ?? DEFAULT_CONTAINER_SCOPE;
|
|
3080
|
+
const bundleWasLive = instance.bundlesByScope.has(scope);
|
|
3081
|
+
const bundle = await this.ensureBundleForScope(instance, scope);
|
|
3082
|
+
const warning = bundleWasLive
|
|
3083
|
+
? undefined
|
|
3084
|
+
: `container: true state "${input.stateId}": scope "${scope}" had no live container before this state. ` +
|
|
3085
|
+
`On a fresh run this likely means no prior state populated it; on resume this can be expected.`;
|
|
3086
|
+
const base = await this.runDeterministicInContainer(bundle, input, warning);
|
|
3087
|
+
return this.applyResultFile(instance, input, base);
|
|
3088
|
+
}
|
|
3089
|
+
applyResultFile(instance, input, base) {
|
|
3090
|
+
if (input.resultFile === undefined)
|
|
3091
|
+
return base;
|
|
3092
|
+
if (!base.passed)
|
|
3093
|
+
return base;
|
|
3094
|
+
const appendError = (message) => ({
|
|
3095
|
+
...base,
|
|
3096
|
+
passed: false,
|
|
3097
|
+
verdict: DETERMINISTIC_RESULT_ERROR_VERDICT,
|
|
3098
|
+
errors: base.errors ? `${base.errors}\n${message}` : message,
|
|
3099
|
+
});
|
|
3100
|
+
if (!isSafeWorkspaceRelativePath(input.resultFile)) {
|
|
3101
|
+
return appendError(`result file ${input.resultFile} is not a safe workspace-relative path`);
|
|
3102
|
+
}
|
|
3103
|
+
const resultPath = resolve(instance.workspacePath, input.resultFile);
|
|
3104
|
+
// Symlink-safe containment: a container can plant a symlink inside the shared
|
|
3105
|
+
// workspace, and this read happens host-side — so resolve both sides to their
|
|
3106
|
+
// canonical real paths before comparing (same posture as the policy engine,
|
|
3107
|
+
// see resolveRealPath / CLAUDE.md). A lexical resolve()/relative() check would
|
|
3108
|
+
// follow such a symlink and read off-workspace on the host.
|
|
3109
|
+
if (!isWithinDirectory(resultPath, instance.workspacePath)) {
|
|
3110
|
+
return appendError(`result file ${input.resultFile} escapes the workspace`);
|
|
3111
|
+
}
|
|
3112
|
+
if (!existsSync(resultPath)) {
|
|
3113
|
+
return appendError(`result file ${input.resultFile} not found`);
|
|
3114
|
+
}
|
|
3115
|
+
// Split read vs parse so a read failure (EISDIR/EPERM/...) is not mislabeled
|
|
3116
|
+
// as a JSON syntax error.
|
|
3117
|
+
let raw;
|
|
3118
|
+
try {
|
|
3119
|
+
raw = readFileSync(resultPath, 'utf8');
|
|
3120
|
+
}
|
|
3121
|
+
catch (err) {
|
|
3122
|
+
return appendError(`result file ${input.resultFile} could not be read: ${errorMessage(err)}`);
|
|
3123
|
+
}
|
|
3124
|
+
let parsed;
|
|
3125
|
+
try {
|
|
3126
|
+
parsed = JSON.parse(raw);
|
|
3127
|
+
}
|
|
3128
|
+
catch {
|
|
3129
|
+
return appendError(`result file ${input.resultFile} is not valid JSON`);
|
|
3130
|
+
}
|
|
3131
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
|
|
3132
|
+
return appendError(`result file ${input.resultFile} is not a JSON object`);
|
|
3133
|
+
}
|
|
3134
|
+
const resultObject = parsed;
|
|
3135
|
+
if (typeof resultObject.verdict !== 'string' || resultObject.verdict.length === 0) {
|
|
3136
|
+
return appendError(`result file ${input.resultFile} missing verdict`);
|
|
3137
|
+
}
|
|
3138
|
+
const payload = typeof resultObject.payload === 'object' && resultObject.payload !== null && !Array.isArray(resultObject.payload)
|
|
3139
|
+
? resultObject.payload
|
|
3140
|
+
: undefined;
|
|
3141
|
+
return {
|
|
3142
|
+
...base,
|
|
3143
|
+
verdict: resultObject.verdict,
|
|
3144
|
+
passed: typeof resultObject.passed === 'boolean' ? resultObject.passed : base.passed,
|
|
3145
|
+
...(payload !== undefined ? { payload } : {}),
|
|
3146
|
+
};
|
|
3147
|
+
}
|
|
3148
|
+
/**
|
|
3149
|
+
* Shared reduction for both deterministic execution paths: skip empty command
|
|
3150
|
+
* arrays, mine the `N tests pass` heuristic from stdout, accumulate per-command
|
|
3151
|
+
* failures, and shape the `{ passed, testCount, errors }` result. Host vs.
|
|
3152
|
+
* container execution differ only in `runCommand` — keep that the only fork so
|
|
3153
|
+
* the pass/fail and test-count semantics cannot drift between the two paths.
|
|
3154
|
+
*/
|
|
3155
|
+
async reduceDeterministicCommands(commands, runCommand) {
|
|
3156
|
+
let totalTestCount = 0;
|
|
3157
|
+
const allErrors = [];
|
|
3158
|
+
for (const cmdArray of commands) {
|
|
3159
|
+
if (cmdArray.length === 0)
|
|
3160
|
+
continue;
|
|
3161
|
+
const { stdout, error } = await runCommand(cmdArray);
|
|
3162
|
+
if (error !== undefined) {
|
|
3163
|
+
allErrors.push(error);
|
|
3164
|
+
continue;
|
|
3165
|
+
}
|
|
3166
|
+
// Try to extract test count from stdout (simple heuristic)
|
|
3167
|
+
const testMatch = /(\d+)\s+(?:tests?|specs?)\s+pass/i.exec(stdout);
|
|
3168
|
+
if (testMatch) {
|
|
3169
|
+
totalTestCount += parseInt(testMatch[1], 10);
|
|
3170
|
+
}
|
|
3171
|
+
}
|
|
3172
|
+
return {
|
|
3173
|
+
passed: allErrors.length === 0,
|
|
3174
|
+
testCount: totalTestCount > 0 ? totalTestCount : undefined,
|
|
3175
|
+
errors: allErrors.length > 0 ? allErrors.join('\n') : undefined,
|
|
3176
|
+
};
|
|
3177
|
+
}
|
|
3178
|
+
async runDeterministicHost(commands) {
|
|
3179
|
+
return this.reduceDeterministicCommands(commands, async (cmdArray) => {
|
|
3180
|
+
const [binary, ...args] = cmdArray;
|
|
3181
|
+
try {
|
|
3182
|
+
const { stdout } = await execFileAsync(binary, args);
|
|
3183
|
+
return { stdout };
|
|
3184
|
+
}
|
|
3185
|
+
catch (err) {
|
|
3186
|
+
const execErr = err;
|
|
3187
|
+
return { stdout: '', error: execErr.stderr ?? execErr.stdout ?? String(err) };
|
|
3188
|
+
}
|
|
3189
|
+
});
|
|
3190
|
+
}
|
|
3191
|
+
async runDeterministicInContainer(bundle, input, warning) {
|
|
3192
|
+
if (warning)
|
|
3193
|
+
writeStderr(`[workflow] ${warning}`);
|
|
3194
|
+
return this.reduceDeterministicCommands(input.commands, async (cmdArray) => {
|
|
3195
|
+
// Prepend the workflow venv / node bins to the container's live $PATH so
|
|
3196
|
+
// bare `node` / `python` helpers resolve regardless of base-image arch
|
|
3197
|
+
// (the image's own PATH is preserved — see buildWorkflowExecCommand).
|
|
3198
|
+
const result = await bundle.docker.exec(bundle.containerId, buildWorkflowExecCommand(bundle, cmdArray), input.timeoutMs, 'codespace', CONTAINER_WORKSPACE_DIR);
|
|
3199
|
+
if (result.exitCode !== 0) {
|
|
3200
|
+
return { stdout: '', error: result.stderr || result.stdout || `exit ${result.exitCode}` };
|
|
3201
|
+
}
|
|
3202
|
+
return { stdout: result.stdout };
|
|
3203
|
+
});
|
|
3204
|
+
}
|
|
3205
|
+
// -----------------------------------------------------------------------
|
|
3206
|
+
// Gate handling
|
|
3207
|
+
// -----------------------------------------------------------------------
|
|
3208
|
+
handleGateEntry(workflowId, gateName, stateDef) {
|
|
3209
|
+
const instance = this.workflows.get(workflowId);
|
|
3210
|
+
if (!instance)
|
|
3211
|
+
return;
|
|
3212
|
+
const gateRequest = this.buildGateRequest(workflowId, gateName, stateDef);
|
|
3213
|
+
instance.activeGateId = gateRequest.gateId;
|
|
3214
|
+
instance.messageLog.append({
|
|
3215
|
+
...this.logBase(instance),
|
|
3216
|
+
type: 'gate_raised',
|
|
3217
|
+
acceptedEvents: stateDef.acceptedEvents,
|
|
3218
|
+
});
|
|
3219
|
+
this.deps.raiseGate(gateRequest);
|
|
3220
|
+
this.emitLifecycleEvent({
|
|
3221
|
+
kind: 'gate_raised',
|
|
3222
|
+
workflowId,
|
|
3223
|
+
gate: gateRequest,
|
|
3224
|
+
});
|
|
3225
|
+
}
|
|
3226
|
+
buildGateRequest(workflowId, gateName, stateDef) {
|
|
3227
|
+
const instance = this.workflows.get(workflowId);
|
|
3228
|
+
const presentedArtifacts = new Map();
|
|
3229
|
+
for (const artifactRef of stateDef.present ?? []) {
|
|
3230
|
+
if (instance) {
|
|
3231
|
+
const { name } = parseArtifactRef(artifactRef);
|
|
3232
|
+
const dir = resolve(instance.artifactDir, name);
|
|
3233
|
+
if (existsSync(dir)) {
|
|
3234
|
+
presentedArtifacts.set(name, dir);
|
|
3235
|
+
}
|
|
3236
|
+
}
|
|
3237
|
+
}
|
|
3238
|
+
// Surface error context in the gate summary so callers know whether this
|
|
3239
|
+
// gate was reached normally, via an invoke error, or via a deterministic
|
|
3240
|
+
// verdict such as evaluator_blocked.
|
|
3241
|
+
const snapshot = instance?.actor.getSnapshot();
|
|
3242
|
+
const context = snapshot?.context;
|
|
3243
|
+
const previousState = context?.previousStateName && instance ? instance.definition.states[context.previousStateName] : undefined;
|
|
3244
|
+
const previousDeterministicVerdict = previousState?.type === 'deterministic' ? context?.lastDeterministicResult?.verdict : undefined;
|
|
3245
|
+
const previousOutputSummary = context?.previousAgentOutput !== undefined && context.previousAgentOutput !== null
|
|
3246
|
+
? // Collapse the multi-line aggregated fan-out reason to a single line so it
|
|
3247
|
+
// surfaces inline in the one-line gate banner instead of wrapping mid-summary.
|
|
3248
|
+
(truncateForTransition(context.previousAgentOutput) ?? '').replace(/\s+/g, ' ').trim()
|
|
3249
|
+
: undefined;
|
|
3250
|
+
const errorContext = context?.lastError
|
|
3251
|
+
? ` (error: ${context.lastError})`
|
|
3252
|
+
: previousDeterministicVerdict
|
|
3253
|
+
? ` (verdict: ${previousDeterministicVerdict}${previousOutputSummary ? `; ${previousOutputSummary}` : ''})`
|
|
3254
|
+
: '';
|
|
3255
|
+
return {
|
|
3256
|
+
gateId: `${workflowId}-${gateName}`,
|
|
3257
|
+
workflowId,
|
|
3258
|
+
stateName: gateName,
|
|
3259
|
+
acceptedEvents: stateDef.acceptedEvents,
|
|
3260
|
+
presentedArtifacts,
|
|
3261
|
+
summary: `Waiting for human review at ${gateName}${errorContext}`,
|
|
3262
|
+
};
|
|
3263
|
+
}
|
|
3264
|
+
// -----------------------------------------------------------------------
|
|
3265
|
+
// Completion handling
|
|
3266
|
+
// -----------------------------------------------------------------------
|
|
3267
|
+
async handleWorkflowComplete(workflowId, context) {
|
|
3268
|
+
const instance = this.workflows.get(workflowId);
|
|
3269
|
+
if (!instance)
|
|
3270
|
+
return;
|
|
3271
|
+
// Guard against XState emitting a terminal transition multiple times.
|
|
3272
|
+
// `finalStatus` being set is the canonical "already completed" signal;
|
|
3273
|
+
// if it's set, skip re-entering terminal handling so we don't call
|
|
3274
|
+
// destroyWorkflowInfrastructure twice (the teardown itself is idempotent,
|
|
3275
|
+
// but duplicate tab.close() / checkpoint.remove() / lifecycle events
|
|
3276
|
+
// are noisy).
|
|
3277
|
+
if (instance.finalStatus)
|
|
3278
|
+
return;
|
|
3279
|
+
// Check if this is a normal completion or an aborted terminal.
|
|
3280
|
+
// Quota exhaustion takes precedence: the error target may have
|
|
3281
|
+
// resolved to any terminal (including `done`-like ones), but a run
|
|
3282
|
+
// that died on upstream quota MUST be treated as aborted so the
|
|
3283
|
+
// checkpoint is preserved and the user can resume once the provider
|
|
3284
|
+
// window reopens. (M4 will upgrade this to a dedicated `paused`
|
|
3285
|
+
// phase; for now `aborted` gives us the same checkpoint retention.)
|
|
3286
|
+
const stateValue = instance.currentState;
|
|
3287
|
+
if (instance.quotaExhausted) {
|
|
3288
|
+
const resetHint = instance.quotaExhausted.resetAt
|
|
3289
|
+
? ` (resets at ${instance.quotaExhausted.resetAt.toISOString()})`
|
|
3290
|
+
: '';
|
|
3291
|
+
instance.finalStatus = {
|
|
3292
|
+
phase: 'aborted',
|
|
3293
|
+
reason: `Upstream quota exhausted${resetHint}`,
|
|
3294
|
+
};
|
|
3295
|
+
}
|
|
3296
|
+
else if (instance.transientFailure) {
|
|
3297
|
+
// Mirror the quota branch: a transient upstream failure must force
|
|
3298
|
+
// an abort-preserving terminal regardless of which state
|
|
3299
|
+
// `findErrorTarget` resolved to. Without this, a workflow whose
|
|
3300
|
+
// only terminal is `done` would land on `done` and
|
|
3301
|
+
// `handleWorkflowComplete` would mark the run `completed`, breaking
|
|
3302
|
+
// resume.
|
|
3303
|
+
const excerpt = instance.transientFailure.rawMessage.slice(0, 200);
|
|
3304
|
+
instance.finalStatus = {
|
|
3305
|
+
phase: 'aborted',
|
|
3306
|
+
reason: `Transient upstream failure: ${describeTransientFailureKind(instance.transientFailure.kind)} ` +
|
|
3307
|
+
`(resumable — run "ironcurtain workflow resume <baseDir>" once upstream is healthy)\n${excerpt}`,
|
|
3308
|
+
};
|
|
3309
|
+
}
|
|
3310
|
+
else if (stateValue === 'aborted' || stateValue.includes('abort')) {
|
|
3311
|
+
instance.finalStatus = {
|
|
3312
|
+
phase: 'aborted',
|
|
3313
|
+
reason: 'Workflow reached aborted state',
|
|
3314
|
+
};
|
|
3315
|
+
}
|
|
3316
|
+
else {
|
|
3317
|
+
const result = { finalArtifacts: { ...context.artifacts } };
|
|
3318
|
+
instance.finalStatus = {
|
|
3319
|
+
phase: 'completed',
|
|
3320
|
+
result,
|
|
3321
|
+
};
|
|
3322
|
+
}
|
|
3323
|
+
let existing;
|
|
3324
|
+
try {
|
|
3325
|
+
existing = this.deps.checkpointStore.load(workflowId);
|
|
3326
|
+
}
|
|
3327
|
+
catch (err) {
|
|
3328
|
+
writeStderr(`[workflow] Failed to load existing checkpoint for ${workflowId}: ${toErrorMessage(err)}`);
|
|
3329
|
+
}
|
|
3330
|
+
const containerSnapshots = instance.finalStatus.phase === 'aborted' ? await this.snapshotResumableScopes(instance) : undefined;
|
|
3331
|
+
// Persist finalStatus so isCheckpointResumable can later distinguish
|
|
3332
|
+
// completed (excluded) from aborted/failed (still resumable). Preserve
|
|
3333
|
+
// the existing on-disk machineState/context (which points at the last
|
|
3334
|
+
// non-terminal state) so resume-after-abort can re-enter that state
|
|
3335
|
+
// instead of immediately re-completing on a terminal snapshot. The
|
|
3336
|
+
// fallback path covers the unusual case where no prior checkpoint
|
|
3337
|
+
// exists (e.g. a workflow that transitioned straight to terminal
|
|
3338
|
+
// without ever passing through a non-terminal save point).
|
|
3339
|
+
this.saveTerminalCheckpoint(instance, instance.finalStatus, containerSnapshots, existing);
|
|
3340
|
+
instance.tab.write(`[done] ${instance.finalStatus.phase}`);
|
|
3341
|
+
instance.tab.close();
|
|
3342
|
+
// Release the token-stream bus subscription. Done eagerly (before the
|
|
3343
|
+
// async destroyWorkflowInfrastructure) because it's a synchronous
|
|
3344
|
+
// callback — idempotent if already cleared by abort().
|
|
3345
|
+
this.teardownTokenSubscription(instance);
|
|
3346
|
+
const snapshotsToRemove = instance.finalStatus.phase === 'completed'
|
|
3347
|
+
? existing?.containerSnapshots
|
|
3348
|
+
: this.snapshotsSupersededBy(existing?.containerSnapshots, containerSnapshots);
|
|
3349
|
+
const cleanupDocker = snapshotsToRemove ? await this.dockerForSnapshotCleanup(instance) : undefined;
|
|
3350
|
+
// Tear down workflow-scoped Docker infrastructure. Runs asynchronously
|
|
3351
|
+
// because the actor subscription is sync; destroyWorkflowInfrastructure
|
|
3352
|
+
// is error-tolerant so unhandled rejections should not occur, but we
|
|
3353
|
+
// still catch defensively for a belt-and-suspenders guarantee. The
|
|
3354
|
+
// promise is retained on the instance so `shutdownAll` can await it
|
|
3355
|
+
// before the CLI exits (see `teardownPromise`).
|
|
3356
|
+
instance.teardownPromise = (async () => {
|
|
3357
|
+
await this.destroyWorkflowInfrastructure(instance);
|
|
3358
|
+
if (cleanupDocker) {
|
|
3359
|
+
await this.removeSnapshotImagesAfterTeardown(cleanupDocker, snapshotsToRemove);
|
|
3360
|
+
}
|
|
3361
|
+
})().catch((err) => {
|
|
3362
|
+
writeStderr(`[workflow] destroyWorkflowInfrastructure unexpectedly threw for ${workflowId}: ${toErrorMessage(err)}`);
|
|
3363
|
+
});
|
|
3364
|
+
this.emitLifecycleEvent({
|
|
3365
|
+
kind: 'completed',
|
|
3366
|
+
workflowId,
|
|
3367
|
+
});
|
|
3368
|
+
}
|
|
3369
|
+
// -----------------------------------------------------------------------
|
|
3370
|
+
// Artifact helpers
|
|
3371
|
+
// -----------------------------------------------------------------------
|
|
3372
|
+
findMissingArtifacts(stateConfig, artifactDir) {
|
|
3373
|
+
const missing = [];
|
|
3374
|
+
for (const output of stateConfig.outputs) {
|
|
3375
|
+
const dir = resolve(artifactDir, output);
|
|
3376
|
+
if (!hasAnyFiles(dir)) {
|
|
3377
|
+
missing.push(output);
|
|
3378
|
+
}
|
|
3379
|
+
}
|
|
3380
|
+
return missing;
|
|
3381
|
+
}
|
|
3382
|
+
// -----------------------------------------------------------------------
|
|
3383
|
+
// Lifecycle event emission
|
|
3384
|
+
// -----------------------------------------------------------------------
|
|
3385
|
+
emitLifecycleEvent(event) {
|
|
3386
|
+
for (const cb of this.lifecycleCallbacks) {
|
|
3387
|
+
try {
|
|
3388
|
+
cb(event);
|
|
3389
|
+
}
|
|
3390
|
+
catch (cbErr) {
|
|
3391
|
+
// Lifecycle callbacks should not crash the orchestrator, but log the failure
|
|
3392
|
+
console.error(`[workflow] Lifecycle callback threw for "${event.kind}":`, cbErr);
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
// ---------------------------------------------------------------------------
|
|
3398
|
+
// Standalone helpers (exported for testing)
|
|
3399
|
+
// ---------------------------------------------------------------------------
|
|
3400
|
+
// ---------------------------------------------------------------------------
|
|
3401
|
+
// Policy cycling helpers (workflow shared-container mode)
|
|
3402
|
+
// ---------------------------------------------------------------------------
|
|
3403
|
+
/**
|
|
3404
|
+
* Resolves the compiled-policy directory for the given persona name.
|
|
3405
|
+
*
|
|
3406
|
+
* The reserved sentinel `GLOBAL_PERSONA` maps to the package-bundled
|
|
3407
|
+
* generated dir (same one `loadConfig().generatedDir` resolves to for
|
|
3408
|
+
* CLI runs). All other values go through `resolvePersona()`, which
|
|
3409
|
+
* validates the persona exists and has a compiled policy.
|
|
3410
|
+
*/
|
|
3411
|
+
function resolvePersonaPolicyDir(persona) {
|
|
3412
|
+
if (persona === GLOBAL_PERSONA) {
|
|
3413
|
+
return loadConfig().generatedDir;
|
|
3414
|
+
}
|
|
3415
|
+
return resolvePersona(persona).policyDir;
|
|
3416
|
+
}
|
|
3417
|
+
/**
|
|
3418
|
+
* Default `startWorkflowControlServer`: reaches through the infra
|
|
3419
|
+
* bundle's proxy for its policy-swap target and binds a control server
|
|
3420
|
+
* at the workflow-scoped UDS path.
|
|
3421
|
+
*/
|
|
3422
|
+
async function defaultStartWorkflowControlServer(input) {
|
|
3423
|
+
const target = input.infra.proxy.getPolicySwapTarget();
|
|
3424
|
+
if (!target) {
|
|
3425
|
+
throw new Error('Cannot attach workflow control server: the proxy is not yet started. ' +
|
|
3426
|
+
'Call proxy.start() before attaching a control server.');
|
|
3427
|
+
}
|
|
3428
|
+
await target.startControlServer({ socketPath: input.socketPath });
|
|
3429
|
+
}
|
|
3430
|
+
/**
|
|
3431
|
+
* Default `loadPolicyRpc`: speaks HTTP/1.1 over the supplied UDS using
|
|
3432
|
+
* Node's built-in client. Resolves on a 2xx response; rejects on any
|
|
3433
|
+
* non-2xx status, request timeout, or socket error. The coordinator's
|
|
3434
|
+
* reply body is read only on the error path (to surface the error
|
|
3435
|
+
* text) — 2xx bodies are discarded with `res.resume()`.
|
|
3436
|
+
*
|
|
3437
|
+
* Exported so integration tests can drive the wire format without
|
|
3438
|
+
* standing up a full `WorkflowOrchestrator`. Keeping the HTTP request
|
|
3439
|
+
* construction in one place guarantees tests exercise the same bytes
|
|
3440
|
+
* production emits.
|
|
3441
|
+
*/
|
|
3442
|
+
export async function defaultLoadPolicyRpc(input) {
|
|
3443
|
+
const body = JSON.stringify({
|
|
3444
|
+
persona: input.persona,
|
|
3445
|
+
policyDir: input.policyDir,
|
|
3446
|
+
});
|
|
3447
|
+
const timeoutMs = input.timeoutMs ?? LOAD_POLICY_RPC_TIMEOUT_MS;
|
|
3448
|
+
return new Promise((resolveFn, rejectFn) => {
|
|
3449
|
+
let settled = false;
|
|
3450
|
+
const settle = (err) => {
|
|
3451
|
+
if (settled)
|
|
3452
|
+
return;
|
|
3453
|
+
settled = true;
|
|
3454
|
+
if (err)
|
|
3455
|
+
rejectFn(err);
|
|
3456
|
+
else
|
|
3457
|
+
resolveFn();
|
|
3458
|
+
};
|
|
3459
|
+
const req = http.request({
|
|
3460
|
+
socketPath: input.socketPath,
|
|
3461
|
+
method: 'POST',
|
|
3462
|
+
path: POLICY_LOAD_PATH,
|
|
3463
|
+
headers: {
|
|
3464
|
+
'Content-Type': 'application/json',
|
|
3465
|
+
'Content-Length': Buffer.byteLength(body, 'utf-8'),
|
|
3466
|
+
},
|
|
3467
|
+
timeout: timeoutMs,
|
|
3468
|
+
}, (res) => {
|
|
3469
|
+
const status = res.statusCode ?? 0;
|
|
3470
|
+
if (status >= 200 && status < 300) {
|
|
3471
|
+
res.resume();
|
|
3472
|
+
res.on('end', () => settle());
|
|
3473
|
+
res.on('error', settle);
|
|
3474
|
+
return;
|
|
3475
|
+
}
|
|
3476
|
+
const chunks = [];
|
|
3477
|
+
res.on('data', (chunk) => chunks.push(chunk));
|
|
3478
|
+
res.on('end', () => {
|
|
3479
|
+
const text = Buffer.concat(chunks).toString('utf-8');
|
|
3480
|
+
settle(new Error(`loadPolicy RPC failed (status ${status}): ${text.slice(0, 200)}`));
|
|
3481
|
+
});
|
|
3482
|
+
res.on('error', settle);
|
|
3483
|
+
});
|
|
3484
|
+
req.on('timeout', () => {
|
|
3485
|
+
req.destroy(new Error(`loadPolicy RPC timed out after ${timeoutMs}ms`));
|
|
3486
|
+
});
|
|
3487
|
+
req.on('error', settle);
|
|
3488
|
+
req.end(body);
|
|
3489
|
+
});
|
|
3490
|
+
}
|
|
3491
|
+
// ---------------------------------------------------------------------------
|
|
3492
|
+
// Workspace root hashing (file listing + mtime, no content reads)
|
|
3493
|
+
// ---------------------------------------------------------------------------
|
|
3494
|
+
const WORKSPACE_HASH_EXCLUDED_DIRS = new Set([
|
|
3495
|
+
'.git',
|
|
3496
|
+
WORKFLOW_ARTIFACT_DIR,
|
|
3497
|
+
'node_modules',
|
|
3498
|
+
'__pycache__',
|
|
3499
|
+
'.next',
|
|
3500
|
+
'dist',
|
|
3501
|
+
'build',
|
|
3502
|
+
'.cache',
|
|
3503
|
+
'.venv',
|
|
3504
|
+
'venv',
|
|
3505
|
+
]);
|
|
3506
|
+
/**
|
|
3507
|
+
* Recursively collects `relativePath:mtime` entries for all files
|
|
3508
|
+
* under `basePath`, excluding directories in WORKSPACE_HASH_EXCLUDED_DIRS.
|
|
3509
|
+
*/
|
|
3510
|
+
function collectFileEntries(basePath, relativeTo, out) {
|
|
3511
|
+
const dirents = readdirSync(resolve(basePath, relativeTo), { withFileTypes: true });
|
|
3512
|
+
for (const dirent of dirents) {
|
|
3513
|
+
const relPath = relativeTo ? `${relativeTo}/${dirent.name}` : dirent.name;
|
|
3514
|
+
if (dirent.isDirectory()) {
|
|
3515
|
+
if (WORKSPACE_HASH_EXCLUDED_DIRS.has(dirent.name))
|
|
3516
|
+
continue;
|
|
3517
|
+
collectFileEntries(basePath, relPath, out);
|
|
3518
|
+
}
|
|
3519
|
+
else if (dirent.isFile()) {
|
|
3520
|
+
const fullPath = resolve(basePath, relPath);
|
|
3521
|
+
const mtime = statSync(fullPath).mtimeMs;
|
|
3522
|
+
out.push(`${relPath}:${mtime}`);
|
|
3523
|
+
}
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
/** Hashes the workspace root file listing (paths + mtimes) into the given hash. */
|
|
3527
|
+
function hashWorkspaceRoot(hash, workspacePath) {
|
|
3528
|
+
const entries = [];
|
|
3529
|
+
collectFileEntries(workspacePath, '', entries);
|
|
3530
|
+
entries.sort();
|
|
3531
|
+
for (const entry of entries) {
|
|
3532
|
+
hash.update(entry);
|
|
3533
|
+
}
|
|
3534
|
+
}
|
|
3535
|
+
/**
|
|
3536
|
+
* Computes a SHA-256 hash of output artifacts or workspace root file metadata.
|
|
3537
|
+
*
|
|
3538
|
+
* Hashes file metadata (path + size + mtime) — never contents. This bounds cost
|
|
3539
|
+
* regardless of artifact size (fuzz logs, captured stderr, etc. can be many GiB
|
|
3540
|
+
* and would otherwise trip Node's 2 GiB `readFile` ceiling). Any agent rewrite
|
|
3541
|
+
* via fs ops bumps mtime, and same-mtime-same-size collisions in a single ms
|
|
3542
|
+
* window are not a concern for workflow change detection.
|
|
3543
|
+
*/
|
|
3544
|
+
export function computeOutputHash(outputNames, artifactDir, workspacePath) {
|
|
3545
|
+
const hash = createHash('sha256');
|
|
3546
|
+
if (outputNames.length > 0) {
|
|
3547
|
+
for (const output of outputNames) {
|
|
3548
|
+
const dir = resolve(artifactDir, output);
|
|
3549
|
+
const files = collectFilesRecursive(dir);
|
|
3550
|
+
for (const file of files) {
|
|
3551
|
+
const { size, mtimeMs } = statSync(file.fullPath);
|
|
3552
|
+
hash.update(`${file.relativePath}:${size}:${mtimeMs}`);
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
}
|
|
3556
|
+
else {
|
|
3557
|
+
hashWorkspaceRoot(hash, workspacePath);
|
|
3558
|
+
}
|
|
3559
|
+
return hash.digest('hex');
|
|
3560
|
+
}
|
|
3561
|
+
// ---------------------------------------------------------------------------
|
|
3562
|
+
// .gitignore management
|
|
3563
|
+
// ---------------------------------------------------------------------------
|
|
3564
|
+
/** Ensures the workflow artifact directory is listed in the workspace root's .gitignore. */
|
|
3565
|
+
function ensureWorkflowGitignored(workspacePath) {
|
|
3566
|
+
const gitignorePath = resolve(workspacePath, '.gitignore');
|
|
3567
|
+
const dirEntry = `${WORKFLOW_ARTIFACT_DIR}/`;
|
|
3568
|
+
if (existsSync(gitignorePath)) {
|
|
3569
|
+
const content = readFileSync(gitignorePath, 'utf-8');
|
|
3570
|
+
const lines = content.split('\n');
|
|
3571
|
+
const alreadyListed = lines.some((line) => line.trim() === dirEntry || line.trim() === WORKFLOW_ARTIFACT_DIR);
|
|
3572
|
+
if (!alreadyListed) {
|
|
3573
|
+
const suffix = content.endsWith('\n') ? '' : '\n';
|
|
3574
|
+
appendFileSync(gitignorePath, `${suffix}${dirEntry}\n`);
|
|
3575
|
+
}
|
|
3576
|
+
}
|
|
3577
|
+
else {
|
|
3578
|
+
writeFileSync(gitignorePath, `${dirEntry}\n`);
|
|
3579
|
+
}
|
|
3580
|
+
}
|
|
3581
|
+
/**
|
|
3582
|
+
* Collects artifact paths: maps output name -> directory path.
|
|
3583
|
+
*/
|
|
3584
|
+
export function collectArtifactPaths(outputNames, artifactDir) {
|
|
3585
|
+
const result = {};
|
|
3586
|
+
for (const output of outputNames) {
|
|
3587
|
+
const dir = resolve(artifactDir, output);
|
|
3588
|
+
if (existsSync(dir)) {
|
|
3589
|
+
result[output] = dir;
|
|
3590
|
+
}
|
|
3591
|
+
}
|
|
3592
|
+
return result;
|
|
3593
|
+
}
|
|
3594
|
+
//# sourceMappingURL=orchestrator.js.map
|