@aiwg/cli 2026.10.1 → 2026.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/agentic/code/providers/capability-matrix.yaml +270 -0
- package/agentic/code/providers/model-capabilities.v1.json +3183 -24
- package/bin/aiwg.mjs +135 -47
- package/dist/src/a2a/codecs.js +9 -0
- package/dist/src/agents/grounding/knowledge-base.js +54 -6
- package/dist/src/artifacts/browser-export.js +6 -0
- package/dist/src/artifacts/checksum-manifest.js +1 -0
- package/dist/src/artifacts/citation-parser.js +173 -92
- package/dist/src/artifacts/citation-table-schema.js +59 -0
- package/dist/src/artifacts/cli.js +57 -18
- package/dist/src/artifacts/corpus-tools/snapshot.js +1 -1
- package/dist/src/artifacts/corpus-tools/vision-extract.js +107 -15
- package/dist/src/artifacts/corpus-views/ref-parser.js +3 -1
- package/dist/src/artifacts/corpus-views/renderers.js +3 -3
- package/dist/src/artifacts/discovery-eval.js +6 -4
- package/dist/src/artifacts/fortemi-core-query-adapter.js +36 -3
- package/dist/src/artifacts/fortemi-core-sync.js +166 -40
- package/dist/src/artifacts/framework-capabilities.js +171 -0
- package/dist/src/artifacts/framework-index-paths.js +73 -0
- package/dist/src/artifacts/framework-source-digest.js +75 -0
- package/dist/src/artifacts/index-builder.js +285 -131
- package/dist/src/artifacts/index-files.js +132 -18
- package/dist/src/artifacts/index-preparation.js +65 -0
- package/dist/src/artifacts/index-refresh.js +576 -0
- package/dist/src/artifacts/index-worker.mjs +9 -0
- package/dist/src/artifacts/prebuilt-build-lock.js +47 -4
- package/dist/src/artifacts/prebuilt-deployment.js +46 -0
- package/dist/src/artifacts/prebuilt-relations.js +80 -0
- package/dist/src/artifacts/query-engine.js +410 -88
- package/dist/src/artifacts/relations-builder.js +168 -0
- package/dist/src/artifacts/relations.js +148 -0
- package/dist/src/artifacts/repair.js +9 -4
- package/dist/src/artifacts/stats.js +5 -2
- package/dist/src/artifacts/types.js +30 -10
- package/dist/src/artifacts/watcher.js +1 -1
- package/dist/src/auth/credential-store.js +2 -1
- package/dist/src/catalog/cli.d.mts +5 -0
- package/dist/src/channel/manager.d.mts +20 -0
- package/dist/src/channel/manager.mjs +78 -12
- package/dist/src/cli/child-process-result.js +40 -0
- package/dist/src/cli/command-log.js +2 -3
- package/dist/src/cli/env.js +38 -16
- package/dist/src/cli/flag-parsing.js +236 -0
- package/dist/src/cli/handlers/artifacts.js +3 -0
- package/dist/src/cli/handlers/auth.js +3 -0
- package/dist/src/cli/handlers/decision.js +218 -5
- package/dist/src/cli/handlers/diagnose.js +13 -5
- package/dist/src/cli/handlers/effect.js +4 -2
- package/dist/src/cli/handlers/evidence.js +3 -0
- package/dist/src/cli/handlers/feedback.js +14 -5
- package/dist/src/cli/handlers/help.js +1 -0
- package/dist/src/cli/handlers/index.js +3 -1
- package/dist/src/cli/handlers/init.js +248 -30
- package/dist/src/cli/handlers/lint.js +5 -0
- package/dist/src/cli/handlers/mc.js +23 -0
- package/dist/src/cli/handlers/models.js +169 -10
- package/dist/src/cli/handlers/ralph-launcher.js +2 -0
- package/dist/src/cli/handlers/ralph.js +3 -0
- package/dist/src/cli/handlers/refresh.js +28 -0
- package/dist/src/cli/handlers/regenerate.js +14 -0
- package/dist/src/cli/handlers/resource-versions.js +3 -0
- package/dist/src/cli/handlers/rules.js +195 -0
- package/dist/src/cli/handlers/run.js +4 -0
- package/dist/src/cli/handlers/runtime-info.js +25 -11
- package/dist/src/cli/handlers/script-runner.js +19 -8
- package/dist/src/cli/handlers/session.js +406 -62
- package/dist/src/cli/handlers/sessions.js +243 -76
- package/dist/src/cli/handlers/setup-manifest.js +474 -79
- package/dist/src/cli/handlers/setup.js +180 -39
- package/dist/src/cli/handlers/steward.js +11 -0
- package/dist/src/cli/handlers/subcommands.js +60 -3
- package/dist/src/cli/handlers/team.js +18 -8
- package/dist/src/cli/handlers/uhp.js +3 -0
- package/dist/src/cli/handlers/use.js +307 -55
- package/dist/src/cli/handlers/utilities.js +125 -26
- package/dist/src/cli/handlers/version.js +8 -0
- package/dist/src/cli/prompt-utils.js +72 -0
- package/dist/src/cli/provider-resolution.js +11 -0
- package/dist/src/cli/router.js +15 -1
- package/dist/src/cli/services/deployment-verification.js +19 -9
- package/dist/src/cli/services/provider-prepare-coordinator.js +24 -0
- package/dist/src/cli/skill-usage.js +2 -3
- package/dist/src/config/aiwg-config.js +349 -35
- package/dist/src/config/cli.js +256 -23
- package/dist/src/config/decision-config.js +106 -0
- package/dist/src/config/dispatch-thresholds-runtime.d.mts +8 -0
- package/dist/src/config/dispatch-thresholds-runtime.mjs +38 -0
- package/dist/src/config/gitignore.js +50 -0
- package/dist/src/config/index-graph-overrides.js +36 -0
- package/dist/src/config/project-artifacts-health.d.mts +37 -0
- package/dist/src/config/project-artifacts-health.mjs +14 -3
- package/dist/src/config/project-artifacts-runtime.d.mts +16 -0
- package/dist/src/config/user-config-dir.d.mts +9 -0
- package/dist/src/config/user-config.js +38 -11
- package/dist/src/config/user-registry.js +23 -15
- package/dist/src/config/user-state.js +7 -0
- package/dist/src/decision/adapters/jev.js +75 -27
- package/dist/src/decision/ask.js +203 -43
- package/dist/src/decision/context-live-qualification.js +2 -2
- package/dist/src/decision/driver.js +14 -7
- package/dist/src/decision/ensemble/aggregate.js +23 -10
- package/dist/src/decision/ensemble-study/multifact-study.js +6 -2
- package/dist/src/decision/entry.js +25 -6
- package/dist/src/decision/graph-live-qualification.js +1 -1
- package/dist/src/decision/graph-run.js +9 -0
- package/dist/src/decision/heldout/collector.js +15 -14
- package/dist/src/decision/heldout/contract.js +30 -7
- package/dist/src/decision/index.js +1 -0
- package/dist/src/decision/interop.js +60 -8
- package/dist/src/decision/jev-credentials.js +15 -1
- package/dist/src/decision/ledger.js +942 -0
- package/dist/src/decision/offload-activation.js +137 -0
- package/dist/src/decision/offload.js +550 -0
- package/dist/src/decision/qualification/load.js +4 -6
- package/dist/src/decision/savings.js +28 -0
- package/dist/src/decision/validate.js +6 -3
- package/dist/src/effects/dispatch-verifier.js +252 -0
- package/dist/src/effects/index.js +1 -0
- package/dist/src/effects/types.js +5 -1
- package/dist/src/effects/verifiers/git.js +2 -1
- package/dist/src/effects/verifiers/tracker.js +26 -14
- package/dist/src/extensions/commands/definitions.js +19 -0
- package/dist/src/extensions/project-local-remove.js +36 -1
- package/dist/src/features/installer.js +2 -2
- package/dist/src/flow/graph-metadata.js +5 -0
- package/dist/src/gates/providers/loader.js +64 -18
- package/dist/src/installation/manager-command.d.mts +18 -0
- package/dist/src/installation/manager-command.mjs +15 -3
- package/dist/src/installation/manager.d.mts +49 -0
- package/dist/src/installation/manager.mjs +95 -5
- package/dist/src/issues/cli.js +4 -4
- package/dist/src/issues/local.js +24 -5
- package/dist/src/jobs/runner.js +1 -1
- package/dist/src/lint/cli.js +27 -1
- package/dist/src/lint/loader.js +2 -1
- package/dist/src/lint/reporters.js +25 -1
- package/dist/src/lint/rule-activation.js +43 -0
- package/dist/src/lint/runner.js +137 -62
- package/dist/src/mcp/adapters/codex-runtime.js +366 -56
- package/dist/src/mcp/claude-install.d.mts +4 -0
- package/dist/src/mcp/claude-install.mjs +41 -0
- package/dist/src/mcp/cli.d.mts +1 -0
- package/dist/src/mcp/cli.mjs +620 -85
- package/dist/src/mcp/config-file.d.mts +28 -0
- package/dist/src/mcp/config-file.mjs +279 -0
- package/dist/src/mcp/config-layers.d.mts +36 -0
- package/dist/src/mcp/config-layers.mjs +348 -0
- package/dist/src/mcp/credentials.d.mts +51 -0
- package/dist/src/mcp/credentials.mjs +484 -0
- package/dist/src/mcp/grok-build-config.d.mts +6 -0
- package/dist/src/mcp/grok-build-config.mjs +17 -27
- package/dist/src/mcp/helpers.mjs +109 -4
- package/dist/src/mcp/muse-native-config.d.mts +13 -0
- package/dist/src/mcp/muse-native-config.mjs +1 -0
- package/dist/src/mcp/omp-config.d.mts +4 -0
- package/dist/src/mcp/omp-config.mjs +50 -29
- package/dist/src/mcp/profiles.js +97 -23
- package/dist/src/mcp/profiles.mjs +99 -27
- package/dist/src/mcp/registry.js +248 -131
- package/dist/src/mcp/registry.mjs +169 -100
- package/dist/src/mcp/server-config.d.mts +3 -0
- package/dist/src/mcp/server-config.mjs +89 -0
- package/dist/src/mcp/server.mjs +127 -82
- package/dist/src/mcp/test-simple.mjs +2 -1
- package/dist/src/mcp/toml-editor.d.mts +4 -0
- package/dist/src/mcp/toml-string.d.mts +1 -0
- package/dist/src/mcp/toml-string.mjs +9 -0
- package/dist/src/mcp/toml-strip-mcp.d.mts +1 -0
- package/dist/src/mcp/toml-strip-mcp.mjs +135 -0
- package/dist/src/mcp/tool-filters.d.mts +46 -0
- package/dist/src/mcp/tool-filters.mjs +357 -0
- package/dist/src/mcp/tools/agentic-sandbox.mjs +3 -1
- package/dist/src/mcp/tools/decision.mjs +22 -3
- package/dist/src/mcp/tools/discovery.mjs +62 -55
- package/dist/src/mcp/tools/orchestration.mjs +50 -2
- package/dist/src/mcp/tools/subsystems.mjs +39 -8
- package/dist/src/mcp/tools/worker-invoke.mjs +2856 -0
- package/dist/src/mcp/tools/worker-shell-control.mjs +289 -0
- package/dist/src/mcp/tools/worker-stdin-runtime.mjs +79 -0
- package/dist/src/memory/project-registry.js +2 -2
- package/dist/src/mission-protocol/codecs.js +647 -1
- package/dist/src/mission-protocol/types.js +110 -0
- package/dist/src/mission-protocol/worker-status.js +2 -0
- package/dist/src/models/config-loader.js +76 -14
- package/dist/src/models/grok-build-models.js +93 -41
- package/dist/src/models/model-capabilities.v1.json +3183 -24
- package/dist/src/models/model-discovery.js +931 -67
- package/dist/src/models/provider-policy.js +719 -11
- package/dist/src/models/qualification-digest.d.mts +2 -0
- package/dist/src/models/qualification-digest.mjs +17 -0
- package/dist/src/models/resolver.js +261 -0
- package/dist/src/models/worker-direct-readiness.js +550 -0
- package/dist/src/models/worker-dispatch-offer.js +169 -0
- package/dist/src/models/worker-health-readiness.js +144 -0
- package/dist/src/models/worker-installation.d.mts +39 -0
- package/dist/src/models/worker-installation.mjs +986 -0
- package/dist/src/models/wrapper-route.js +18 -0
- package/dist/src/ops/registry.js +9 -14
- package/dist/src/plugins/standalone-packager.js +4 -3
- package/dist/src/process/bounded-child-process.d.mts +84 -0
- package/dist/src/process/bounded-child-process.mjs +430 -0
- package/dist/src/process/cli-command.d.mts +16 -0
- package/dist/src/process/cli-command.mjs +75 -0
- package/dist/src/process/signal-child.d.mts +6 -0
- package/dist/src/process/signal-child.mjs +7 -0
- package/dist/src/providers/capability-matrix.js +8 -0
- package/dist/src/providers/capability-matrix.yaml +270 -0
- package/dist/src/providers/deployed-skill-roots.js +20 -0
- package/dist/src/providers/grok-build-acp.js +3 -2
- package/dist/src/providers/grok-build-headless.js +53 -23
- package/dist/src/providers/grok-build-inspect.js +1 -0
- package/dist/src/providers/omp-agent.d.mts +8 -0
- package/dist/src/providers/omp-diagnostics.d.mts +1 -0
- package/dist/src/providers/omp-paths.d.mts +13 -0
- package/dist/src/providers/provider-definitions.js +9 -1
- package/dist/src/providers/provider-definitions.mjs +9 -2
- package/dist/src/providers/skill-ownership.d.mts +9 -0
- package/dist/src/providers/skill-ownership.mjs +88 -0
- package/dist/src/rules/activation.d.mts +21 -0
- package/dist/src/rules/activation.mjs +62 -0
- package/dist/src/rules/inline-budget.d.mts +26 -0
- package/dist/src/rules/inline-budget.mjs +66 -0
- package/dist/src/rules/inventory.js +129 -0
- package/dist/src/rules/pin-state.js +73 -0
- package/dist/src/rules/pins.d.mts +14 -0
- package/dist/src/rules/pins.mjs +112 -0
- package/dist/src/rules/rule-frontmatter.schema.json +197 -0
- package/dist/src/serve/executor-registry.js +16 -24
- package/dist/src/serve/fleet-mission-conductor.js +9 -0
- package/dist/src/serve/mission-conductor.js +47 -12
- package/dist/src/serve/shared-host-scheduler.js +2 -0
- package/dist/src/serve/worker-invocation.js +558 -0
- package/dist/src/serve/worker-stdin-frame.d.mts +10 -0
- package/dist/src/serve/worker-stdin-frame.mjs +107 -0
- package/dist/src/sessions/adapter-registry.js +49 -0
- package/dist/src/sessions/adapters/claude.js +39 -54
- package/dist/src/sessions/adapters/codex.js +82 -67
- package/dist/src/sessions/adapters/copilot.js +4 -4
- package/dist/src/sessions/adapters/cursor.js +40 -57
- package/dist/src/sessions/adapters/deepseek-harness.js +15 -12
- package/dist/src/sessions/adapters/factory.js +31 -27
- package/dist/src/sessions/adapters/generic.js +43 -9
- package/dist/src/sessions/adapters/grok-build.js +11 -11
- package/dist/src/sessions/adapters/grokbot-locator.js +111 -0
- package/dist/src/sessions/adapters/grokbot.js +434 -24
- package/dist/src/sessions/adapters/hermes.js +27 -29
- package/dist/src/sessions/adapters/muse-workspace-scan.js +128 -0
- package/dist/src/sessions/adapters/muse.js +200 -104
- package/dist/src/sessions/adapters/omp.js +28 -14
- package/dist/src/sessions/adapters/openclaw.js +89 -79
- package/dist/src/sessions/adapters/opencode.js +27 -24
- package/dist/src/sessions/adapters/openhuman.js +18 -60
- package/dist/src/sessions/adapters/pi.js +30 -14
- package/dist/src/sessions/adapters/warp.js +3 -3
- package/dist/src/sessions/adapters/windsurf.js +19 -48
- package/dist/src/sessions/batch-contracts.js +25 -5
- package/dist/src/sessions/batch-import.js +72 -51
- package/dist/src/sessions/candidates.js +47 -11
- package/dist/src/sessions/contracts.js +75 -2
- package/dist/src/sessions/coverage-preflight.js +134 -0
- package/dist/src/sessions/discovery.js +3 -0
- package/dist/src/sessions/export-plan.js +4 -3
- package/dist/src/sessions/grok-build-cli.js +67 -5
- package/dist/src/sessions/import-exclusions.js +72 -0
- package/dist/src/sessions/import-lease.js +34 -18
- package/dist/src/sessions/import-limits.js +37 -0
- package/dist/src/sessions/importer.js +84 -46
- package/dist/src/sessions/index.js +2 -0
- package/dist/src/sessions/lifecycle.js +45 -0
- package/dist/src/sessions/policy.js +133 -25
- package/dist/src/sessions/readers.js +106 -53
- package/dist/src/sessions/refresh.js +106 -0
- package/dist/src/sessions/repository.js +199 -63
- package/dist/src/sessions/workspace-discovery.js +315 -88
- package/dist/src/sessions/workspace-matching.js +107 -0
- package/dist/src/skills/adapters/clawhub.js +10 -8
- package/dist/src/skills/adapters/openclaw.js +10 -7
- package/dist/src/skills/importer.js +1 -0
- package/dist/src/smiths/context-pipeline/aiwg-md.js +11 -0
- package/dist/src/smiths/context-pipeline/claude-hook.js +3 -26
- package/dist/src/smiths/context-pipeline/finalization.js +6 -5
- package/dist/src/smiths/context-pipeline/generator.js +21 -4
- package/dist/src/smiths/context-pipeline/parallelism-section.js +195 -14
- package/dist/src/smiths/context-pipeline/workspace-context.js +116 -9
- package/dist/src/smiths/hook-bridge/codex-translator.js +6 -5
- package/dist/src/smiths/mcpsmith/example.js +1 -1
- package/dist/src/smiths/mcpsmith/generator.js +1 -1
- package/dist/src/smiths/toolsmith/runtime-discovery.d.mts +10 -0
- package/dist/src/smiths/toolsmith/runtime-discovery.mjs +85 -31
- package/dist/src/storage/backends/fs.js +18 -76
- package/dist/src/storage/backends/logseq.js +68 -7
- package/dist/src/storage/backends/obsidian.js +1 -0
- package/dist/src/storage/fortemi-qualification-receipt.js +2 -2
- package/dist/src/testing/generators/test-code-generator.js +57 -30
- package/dist/src/tracker/forge-http.js +2 -1
- package/dist/src/uhp/client.js +11 -1
- package/dist/src/uhp/mission.js +5 -0
- package/dist/src/update/checker.d.mts +2 -0
- package/dist/src/update/checker.mjs +5 -5
- package/dist/src/update/notifier.mjs +54 -2
- package/dist/src/update/service.d.mts +22 -0
- package/dist/src/utils/contained-filesystem.d.mts +37 -0
- package/dist/src/utils/contained-filesystem.mjs +683 -0
- package/dist/src/utils/temporary-directory.js +36 -0
- package/dist/src/writing/example-generator.js +1 -1
- package/dist/src/writing/fidelity.js +18 -1
- package/package.json +32 -2
- package/schemas/mission-v1.schema.json +2429 -0
- package/schemas/models/model-catalog.v1.schema.json +301 -0
- package/schemas/models/model-config-compatibility.v1.schema.json +281 -0
- package/schemas/models/model-policy.v1.schema.json +14 -0
- package/schemas/models/provider-model-capabilities.v1.schema.json +1072 -0
- package/tools/_resolve-impl.mjs +39 -2
- package/tools/agents/deploy-agents.mjs +64 -23
- package/tools/agents/providers/base.mjs +153 -166
- package/tools/agents/providers/claude.mjs +8 -24
- package/tools/agents/providers/codex.mjs +18 -11
- package/tools/agents/providers/cursor.mjs +19 -18
- package/tools/agents/providers/grok-build.mjs +3 -1
- package/tools/agents/providers/grokbot.mjs +11 -12
- package/tools/agents/providers/hermes.mjs +62 -49
- package/tools/agents/providers/omp.mjs +5 -2
- package/tools/agents/providers/openclaw-translator.mjs +19 -5
- package/tools/agents/providers/openclaw.mjs +9 -4
- package/tools/agents/providers/pi.mjs +5 -0
- package/tools/agents/providers/provider-prepare-channel.mjs +5 -0
- package/tools/agents/providers/provider-prepare-coordinator.mjs +247 -0
- package/tools/agents/providers/provider-prepare-worker.mjs +84 -0
- package/tools/agents/providers/rule-delivery.mjs +1270 -0
- package/tools/agents/providers/warp.mjs +20 -26
- package/tools/cli/decision-offload-report.mjs +40 -0
- package/tools/cli/workspace-status.mjs +862 -0
- package/tools/plugin/package-plugins.mjs +186 -8
- package/tools/plugin/plugin-links.mjs +114 -0
- package/tools/providers/antigravity-transport.mjs +3 -1
- package/tools/providers/deepseek-harness-transport.mjs +360 -0
- package/tools/providers/omp-transport.mjs +194 -0
- package/tools/ralph-external/lib/claude-adapter.mjs +389 -0
- package/tools/ralph-external/lib/codex-adapter.mjs +432 -0
- package/tools/ralph-external/lib/consecutive-failure-circuit-breaker.mjs +1748 -0
- package/tools/ralph-external/lib/deepseek-harness-adapter.mjs +397 -0
- package/tools/ralph-external/lib/factory-adapter.mjs +503 -0
- package/tools/ralph-external/lib/launcher-recovery-evidence.mjs +55 -0
- package/tools/ralph-external/lib/muse-adapter.mjs +364 -0
- package/tools/ralph-external/lib/omp-adapter.mjs +315 -0
- package/tools/ralph-external/lib/opencode-adapter.mjs +575 -0
- package/tools/ralph-external/lib/openhuman-adapter.mjs +75 -0
- package/tools/ralph-external/lib/pi-adapter.mjs +329 -0
- package/tools/ralph-external/lib/provider-adapter.mjs +748 -0
- package/tools/ralph-external/lib/recovery-probe.mjs +72 -0
- package/tools/ralph-external/lib/session-environment.mjs +28 -0
- package/tools/ralph-external/lib/systemd-client.mjs +49 -0
- package/tools/ralph-external/lib/worker-health.mjs +439 -0
- package/tools/ralph-external/lib/worker-isolation.mjs +717 -0
- package/tools/ralph-external/lib/worker-overflow.mjs +50 -0
- package/tools/ralph-external/process-monitor.mjs +1017 -0
- package/tools/ralph-external/session-launcher.mjs +5057 -0
- package/tools/scripts/resource-preflight.mjs +541 -0
- package/tools/skills/deploy-skills-codex.mjs +2 -1
- package/vscode-extension/schemas/aiwg.config.v1.json +2916 -0
|
@@ -0,0 +1,2856 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AIWG MCP Worker-Invoke Bridge (#2892)
|
|
3
|
+
*
|
|
4
|
+
* MCP-only conductor: resolve and invoke a worker (run/status/cancel/reconcile_stopped/reset_recovery_scope)
|
|
5
|
+
* WITHOUT any Mission/MC session, daemon, admission store, or activation
|
|
6
|
+
* marker. Reuses, rather than reimplements, the existing Tier 0 primitives:
|
|
7
|
+
*
|
|
8
|
+
* - #2887 resolveWorkerInvocation / prepareWorkerInvocation
|
|
9
|
+
* (src/serve/worker-invocation.ts, via the CLI's own loadWorkerInputs)
|
|
10
|
+
* - #2954 planDirectWorkerLaunch + launchIsolatedWorker
|
|
11
|
+
* (tools/ralph-external/session-launcher.mjs, lib/worker-isolation.mjs)
|
|
12
|
+
* - #2890 ProviderAdapter.parseWorkerFinal + assembleWorkerResult
|
|
13
|
+
* (tools/ralph-external/lib/provider-adapter.mjs, mission-protocol/codecs.ts)
|
|
14
|
+
*
|
|
15
|
+
* This module is the FIRST MCP-side consumer of that pipeline end-to-end,
|
|
16
|
+
* without a shell/CLI round trip and without Mission/MC state. It is
|
|
17
|
+
* intentionally conservative where the pipeline cannot yet produce a fully
|
|
18
|
+
* attested identity: verification is always left `pending` here (never
|
|
19
|
+
* self-accepted) — a separate verifier is required before a direct result is
|
|
20
|
+
* treated as accepted, per the brief's "worker verification ... remains
|
|
21
|
+
* required."
|
|
22
|
+
*
|
|
23
|
+
* ---------------------------------------------------------------------------
|
|
24
|
+
* Hardening vs. the #2896 reference bridge review (this issue moves +
|
|
25
|
+
* redesigns that bridge; see .aiwg/working/s1-scratch/brief-2892.md):
|
|
26
|
+
*
|
|
27
|
+
* 1. BLOCKER (arbitrary path read / config root choice): every path this
|
|
28
|
+
* tool accepts (`brief`, `workdir`, `final_output`) must resolve
|
|
29
|
+
* (realpath, no symlink escape) beneath the served project root the
|
|
30
|
+
* MCP server was started for. `project_dir` is NOT a client-facing
|
|
31
|
+
* field at all — there is exactly one root, fixed at toolset
|
|
32
|
+
* registration time, never per-request. Each path is additionally
|
|
33
|
+
* checked against the existing in-process repo-access semantics
|
|
34
|
+
* (`src/policy/repo-access.ts` `checkRepoAccess`) when a workspace
|
|
35
|
+
* manifest is configured. See `assertServedPath` / `resolveServedProjectRoot`.
|
|
36
|
+
* 2. BLOCKER (client-settable degraded isolation): there is no
|
|
37
|
+
* `allow_degraded` (or similarly named) field in the tool's input
|
|
38
|
+
* schema. Degraded isolation is an operator opt-in read once from the
|
|
39
|
+
* server environment at registration time (`AIWG_MCP_WORKER_ALLOW_DEGRADED=1`)
|
|
40
|
+
* and is reported in every result (`execution.isolation`). See
|
|
41
|
+
* `resolveServerAllowDegraded`.
|
|
42
|
+
* 3. MAJOR (unbounded registry / permanent lockout): `createInvocationRegistry`
|
|
43
|
+
* counts only LIVE (non-terminal) runs against `maxActive`; terminal
|
|
44
|
+
* entries are evicted by TTL and, independently, by an over-capacity
|
|
45
|
+
* oldest-terminal-first sweep. Capacity is therefore never exhausted by
|
|
46
|
+
* the accumulation of *completed* runs — only by concurrently *live* ones.
|
|
47
|
+
* 4. Toolset wildcard: `workers` is registered under its own explicit
|
|
48
|
+
* opt-in toolset name and is excluded from `AIWG_MCP_TOOLSETS=all`
|
|
49
|
+
* (see `src/mcp/tools/subsystems.mjs`).
|
|
50
|
+
* 5. Minor (leaking temp dirs on uncertain runs): the #2896 reference
|
|
51
|
+
* bridge wrote a scratch plan.json to hand an invocation to a spawned
|
|
52
|
+
* launcher subprocess over IPC, and had to remember to clean that
|
|
53
|
+
* directory up on every exit path including `uncertain` ones. This
|
|
54
|
+
* design calls `planDirectWorkerLaunch`/`resolveWorkerInvocation`
|
|
55
|
+
* in-process instead of through a relay subprocess, so there is no
|
|
56
|
+
* scratch plan directory. Shell stdin framing uses a private runtime
|
|
57
|
+
* file, removed on refusal or confirmed termination. Unused
|
|
58
|
+
* final_output reservations are also removed on pre-launch refusal.
|
|
59
|
+
* ---------------------------------------------------------------------------
|
|
60
|
+
*
|
|
61
|
+
* Fix pass (.aiwg/working/s1-scratch/fix-2892-r1.md, round 1 — two
|
|
62
|
+
* independent reviews, probe-confirmed):
|
|
63
|
+
*
|
|
64
|
+
* B1. The served root used to be resolved LAZILY, on the first request
|
|
65
|
+
* (`resolveServedProjectRoot(options.projectRoot)` defaulted its
|
|
66
|
+
* argument to `process.cwd()` at CALL time, and the toolset was always
|
|
67
|
+
* registered regardless). Fixed: the root is resolved ONCE, eagerly,
|
|
68
|
+
* inside `registerWorkerInvokeToolset` itself, from the caller-supplied
|
|
69
|
+
* `options.projectRoot` only; `resolveServedProjectRoot` no longer has
|
|
70
|
+
* a `process.cwd()` default at all, and the toolset refuses to
|
|
71
|
+
* register (logs a diagnostic, returns without calling
|
|
72
|
+
* `server.registerTool`) when no explicit root is supplied. See
|
|
73
|
+
* `src/mcp/server.mjs` (resolves via `findProjectRoot()` before calling
|
|
74
|
+
* `registerOptInToolsets`) and `src/mcp/tools/subsystems.mjs`
|
|
75
|
+
* (`registerOptInToolsets` threads `options` through).
|
|
76
|
+
* B2. The `mcp` transport used to construct a bare SDK `StdioClientTransport`
|
|
77
|
+
* directly, bypassing the #2954 resource gate and `launchIsolatedWorker`
|
|
78
|
+
* entirely, AND the SDK always merges its own `getDefaultEnvironment()`
|
|
79
|
+
* (HOME/LOGNAME/PATH/SHELL/TERM/USER from the CONDUCTOR's own
|
|
80
|
+
* `process.env`) into whatever `env` is supplied — silently widening
|
|
81
|
+
* the qualified allowlist. Fixed: an MCP worker server process now
|
|
82
|
+
* goes through the exact same gate-then-launch pipeline as a shell
|
|
83
|
+
* worker (`planDirectWorkerLaunch` → `launchIsolatedWorker`), using the
|
|
84
|
+
* new `attached: true` mode (`--pipe --wait` under systemd-run; a live
|
|
85
|
+
* stdio pipe in degraded mode) so the conductor gets a real child
|
|
86
|
+
* process whose stdio IS the MCP transport — never the SDK's own
|
|
87
|
+
* spawn. See `ChildProcessTransport` / `defaultLaunchMcpWorkerProcess`.
|
|
88
|
+
* B3. `assertServedPath` used to return the REQUESTED (non-canonical) path
|
|
89
|
+
* after validating containment against the canonical ancestor — a
|
|
90
|
+
* TOCTOU gap between validation and use. Fixed: it now returns the
|
|
91
|
+
* canonical path (canonicalized existing ancestor + literal
|
|
92
|
+
* not-yet-existing tail). `workdir` must now be a strict subdirectory
|
|
93
|
+
* of the served root (never the root itself, never `.git`/`.aiwg`);
|
|
94
|
+
* `final_output` must resolve inside the validated `workdir` and is
|
|
95
|
+
* reserved via an exclusive (`O_CREAT|O_EXCL|O_NOFOLLOW`) create before
|
|
96
|
+
* any launch. `readBoundedFile` opens with `O_NOFOLLOW`, verifies
|
|
97
|
+
* `nlink === 1`/regular-file/size-bound via `fstat`, and re-verifies
|
|
98
|
+
* containment from the opened fd's own `/proc/self/fd/<n>` target
|
|
99
|
+
* (not the pre-open path string) before reading.
|
|
100
|
+
* M1. Admission used to check capacity/identity (reads) BEFORE
|
|
101
|
+
* `await validatedRequest(input)`, but only RESERVE them (the mutating
|
|
102
|
+
* `registry.create()`) AFTER that await — any number of concurrent
|
|
103
|
+
* `run` calls could pass the checks before any of them reserved.
|
|
104
|
+
* Fixed: `request_id` is required for `run`, and capacity+identity
|
|
105
|
+
* reservation (`registry.create()`) happens synchronously, with no
|
|
106
|
+
* `await` between the checks and the reservation — atomic with
|
|
107
|
+
* respect to any other concurrent `run()` invocation.
|
|
108
|
+
* M2. An unconfirmed ("uncertain") worker used to be marked TERMINAL
|
|
109
|
+
* (`markTerminal`), freeing its capacity reservation without any stop
|
|
110
|
+
* proof, and `cancel` only ever flipped a flag without delivering
|
|
111
|
+
* anything anywhere. Fixed: uncertain outcomes use `registry.update`
|
|
112
|
+
* (stays LIVE, capacity held) instead of `markTerminal`; `cancel`
|
|
113
|
+
* invokes a live `entry.control.stop()` handle registered by `run()`
|
|
114
|
+
* for the duration of the in-flight transport call (MCP: sends
|
|
115
|
+
* `notifications/cancelled` over the live client; shell: stops the
|
|
116
|
+
* isolated unit / kills the child) and only reports `cancelDelivered`
|
|
117
|
+
* when that actually succeeds. A later completion for a run whose
|
|
118
|
+
* `cancelRequested` flag is set is recorded as `cancelled`, never
|
|
119
|
+
* `completed`.
|
|
120
|
+
* M3. No live schema-drift check existed before sending a brief to an MCP
|
|
121
|
+
* worker tool. Fixed: on every connect, `defaultCallMcpWorker` lists
|
|
122
|
+
* tools and compares the live tool's input/output schema against the
|
|
123
|
+
* #2887 installation-owner-trusted schema object (`mcpSchemas`),
|
|
124
|
+
* canonically (`canonicalSchemaDigest`, sorted-key — robust to the
|
|
125
|
+
* SDK's own zod-validation round trip reordering keys without
|
|
126
|
+
* changing meaning); any mismatch (or the tool no longer being
|
|
127
|
+
* advertised at all) refuses before any brief is sent.
|
|
128
|
+
* M4. `seenRequestIds` (replay protection) was a plain `Map` that NEVER
|
|
129
|
+
* shrank. Fixed: `createInvocationRegistry` now sweeps it on the same
|
|
130
|
+
* cadence as the main registry (TTL + over-capacity oldest-first),
|
|
131
|
+
* restricted to request ids whose invocation entry has already left
|
|
132
|
+
* the main registry — a request id is never evicted while its
|
|
133
|
+
* invocation is still live, uncertain, or terminal-but-retained.
|
|
134
|
+
* M5. `tryAssembleDirectResult` re-read the brief at result time without
|
|
135
|
+
* ever comparing it to what was pinned at launch (`invocation.briefDigest`),
|
|
136
|
+
* and the `parseWorkerFinal` failure path forwarded the raw parser
|
|
137
|
+
* error message into client-visible diagnostics — a duplicate-key
|
|
138
|
+
* probe showed this can reflect attacker-controlled text straight
|
|
139
|
+
* back out. Fixed: the brief's digest is re-verified against the
|
|
140
|
+
* launch-time pin before assembling a result (mismatch →
|
|
141
|
+
* `OG_BRIEF_EVIDENCE_CONFLICT`, never silently assembled); diagnostics
|
|
142
|
+
* for a parse failure use the error's own bounded `.code` (a
|
|
143
|
+
* fixed-vocabulary `WorkerContractError`/`WorkerPolicyError` token)
|
|
144
|
+
* instead of `.message`.
|
|
145
|
+
* D1. The registry's default `maxActive` is now
|
|
146
|
+
* `min(DEFAULT_MAX_ACTIVE, resolveParallelism(config.parallelism).max_parallel_subagents)`
|
|
147
|
+
* (the existing project-wide parallelism resolver); the terminal TTL
|
|
148
|
+
* is operator-configurable via `AIWG_MCP_WORKER_TTL_MS`, falling back
|
|
149
|
+
* to the code default.
|
|
150
|
+
* D2. The per-`serverRef` MCP call queue (`'per-instance'`/`'serialized'`
|
|
151
|
+
* sessionIsolation) is now explicitly bounded (refuses rather than
|
|
152
|
+
* queuing unboundedly once `MAX_MCP_SERVER_QUEUE_DEPTH` calls are
|
|
153
|
+
* already waiting) and rechecks the server's registry qualification
|
|
154
|
+
* immediately before each queued launch, not just at admission time.
|
|
155
|
+
* ---------------------------------------------------------------------------
|
|
156
|
+
*
|
|
157
|
+
* @issues #2892, #2896 (review reference, not reused as-is)
|
|
158
|
+
*/
|
|
159
|
+
|
|
160
|
+
import { signalChild } from '../../process/signal-child.mjs';
|
|
161
|
+
import { z } from 'zod';
|
|
162
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
163
|
+
import {
|
|
164
|
+
realpath, readlink as fsReadlink, open as fsOpen, lstat, unlink,
|
|
165
|
+
} from 'node:fs/promises';
|
|
166
|
+
import { existsSync, readFileSync, constants as fsConstants } from 'node:fs';
|
|
167
|
+
import { execFile } from 'node:child_process';
|
|
168
|
+
import { promisify } from 'node:util';
|
|
169
|
+
import path from 'node:path';
|
|
170
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
171
|
+
import { mcpError, mcpJson } from '../helpers.mjs';
|
|
172
|
+
import { buildShellStopControl, waitForUnitTermination } from './worker-shell-control.mjs';
|
|
173
|
+
import { launchPreparedShellWorker, sweepWorkerStdin } from './worker-stdin-runtime.mjs';
|
|
174
|
+
|
|
175
|
+
const execFileAsync = promisify(execFile);
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Find the AIWG package root by walking up from THIS file's own location
|
|
179
|
+
* (never from `AIWG_ROOT`/`process.cwd()`, which name the *served project*,
|
|
180
|
+
* not necessarily where this module's own package lives). Depth-independent
|
|
181
|
+
* on purpose: this file ships at `src/mcp/tools/worker-invoke.mjs` in a dev
|
|
182
|
+
* checkout and at `dist/src/mcp/tools/worker-invoke.mjs` in the published
|
|
183
|
+
* package — a fixed-depth relative specifier into `tools/` (which ships
|
|
184
|
+
* unchanged at the package root in both cases) would be correct in one and
|
|
185
|
+
* wrong in the other. Mirrors `tools/_resolve-impl.mjs`'s own `findPackageRoot`.
|
|
186
|
+
*/
|
|
187
|
+
function findOwnPackageRoot() {
|
|
188
|
+
let dir = path.dirname(fileURLToPath(import.meta.url));
|
|
189
|
+
for (let i = 0; i < 12; i += 1) {
|
|
190
|
+
const pkgPath = path.join(dir, 'package.json');
|
|
191
|
+
if (existsSync(pkgPath)) {
|
|
192
|
+
try {
|
|
193
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
|
|
194
|
+
if (pkg.name === 'aiwg' || pkg.name === '@aiwg/cli') return dir;
|
|
195
|
+
} catch { /* keep walking */ }
|
|
196
|
+
}
|
|
197
|
+
const parent = path.dirname(dir);
|
|
198
|
+
if (parent === dir) break;
|
|
199
|
+
dir = parent;
|
|
200
|
+
}
|
|
201
|
+
throw new Error('Could not locate the AIWG package root from worker-invoke.mjs');
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function importImpl(relativeFromSrc) {
|
|
205
|
+
const root = findOwnPackageRoot();
|
|
206
|
+
const { importImpl: delegate } = await import(pathToFileURL(path.join(root, 'tools/_resolve-impl.mjs')).href);
|
|
207
|
+
return delegate(import.meta.url, relativeFromSrc);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
export const WORKERS_TOOLSET_NAME = 'workers';
|
|
211
|
+
export const DEFAULT_MAX_ACTIVE = 8;
|
|
212
|
+
export const DEFAULT_MAX_RETAINED = 128;
|
|
213
|
+
export const DEFAULT_TERMINAL_TTL_MS = 10 * 60 * 1000;
|
|
214
|
+
/** #2892 D2: bound on how many calls may wait behind one 'per-instance'/
|
|
215
|
+
* 'serialized' MCP server's queue before new ones are refused outright. */
|
|
216
|
+
export const MAX_MCP_SERVER_QUEUE_DEPTH = 32;
|
|
217
|
+
/** #2892 B2: `launchIsolatedWorker`'s systemd-unit path requires a positive
|
|
218
|
+
* `memoryMaxGb` ceiling even for an attached (`--pipe --wait`) launch — the
|
|
219
|
+
* MCP worker SERVER process is a separate resource-confined unit from the
|
|
220
|
+
* conductor, and no project-level config currently supplies a per-worker
|
|
221
|
+
* ceiling for it. Shell workers use this fallback in systemd mode only.
|
|
222
|
+
* Operator-overridable via `registerWorkerInvokeToolset`'s
|
|
223
|
+
* `mcpWorkerMemoryMaxGb` option; this is the fallback. */
|
|
224
|
+
export const DEFAULT_MCP_WORKER_MEMORY_MAX_GB = 2;
|
|
225
|
+
/** Raw per-frame budget, including malformed/unterminated frames. */
|
|
226
|
+
export const MAX_MCP_FRAME_BYTES = 10 * 1024 * 1024;
|
|
227
|
+
/** #2892 B3: a worker's `workdir` may never be the conductor-managed control
|
|
228
|
+
* plane or VCS metadata directory, even as a strict subdirectory name. */
|
|
229
|
+
const RESERVED_WORKDIR_NAMES = new Set(['.git', '.aiwg']);
|
|
230
|
+
|
|
231
|
+
// ===========================================================================
|
|
232
|
+
// Defect 2: degraded isolation is a server-side-only operator opt-in.
|
|
233
|
+
// ===========================================================================
|
|
234
|
+
|
|
235
|
+
/** Read once at registration time, from the server's own environment — never
|
|
236
|
+
* from a request. There is no `allow_degraded` field on the tool schema. */
|
|
237
|
+
export function resolveServerAllowDegraded(env = process.env) {
|
|
238
|
+
const raw = env.AIWG_MCP_WORKER_ALLOW_DEGRADED;
|
|
239
|
+
return raw === '1' || raw === 'true';
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** #2892 D1: terminal-entry TTL is operator-configurable (server env, read
|
|
243
|
+
* once at registration — same posture as `resolveServerAllowDegraded`), code
|
|
244
|
+
* default as fallback. */
|
|
245
|
+
export function resolveServerTerminalTtlMs(env = process.env) {
|
|
246
|
+
const raw = Number(env.AIWG_MCP_WORKER_TTL_MS);
|
|
247
|
+
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_TERMINAL_TTL_MS;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ===========================================================================
|
|
251
|
+
// Defect 1 / B1: path containment beneath the served project root, pinned
|
|
252
|
+
// once at registration time — never re-resolved from `process.cwd()`.
|
|
253
|
+
// ===========================================================================
|
|
254
|
+
|
|
255
|
+
/** Canonicalize an EXPLICIT served-root candidate. Throws if it does not
|
|
256
|
+
* exist, or if no candidate is given at all (#2892 B1: there is no implicit
|
|
257
|
+
* `process.cwd()` fallback here — a caller with no explicit root must not
|
|
258
|
+
* call this at all; see `registerWorkerInvokeToolset`'s own refusal path). */
|
|
259
|
+
export async function resolveServedProjectRoot(candidate) {
|
|
260
|
+
if (typeof candidate !== 'string' || candidate.length === 0) {
|
|
261
|
+
throw new Error('resolveServedProjectRoot requires an explicit root; it never falls back to process.cwd()');
|
|
262
|
+
}
|
|
263
|
+
return realpath(path.resolve(candidate));
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Walk up to the nearest existing ancestor, requiring every already-existing
|
|
268
|
+
* segment to be free of symlinks (mirrors #2887's own `inspectPath`), then
|
|
269
|
+
* require that ancestor's canonical path to sit inside `servedRootReal`.
|
|
270
|
+
* A target that does not exist yet (e.g. `final_output`) is allowed as long
|
|
271
|
+
* as its nearest existing ancestor directory is contained and symlink-free;
|
|
272
|
+
* `resolveWorkerInvocation` independently re-validates the target itself.
|
|
273
|
+
*
|
|
274
|
+
* #2892 B3: returns the CANONICAL path (the canonicalized existing ancestor
|
|
275
|
+
* joined with whatever tail segments do not exist yet — those cannot be
|
|
276
|
+
* symlinks, they do not exist, but are preserved literally), not the
|
|
277
|
+
* caller-supplied string. The previous version validated containment against
|
|
278
|
+
* the canonical ancestor but then returned `path.resolve(candidatePath)` —
|
|
279
|
+
* the requested, non-canonical path — leaving a TOCTOU gap between this
|
|
280
|
+
* check and whatever later opens that same path.
|
|
281
|
+
*/
|
|
282
|
+
async function assertServedPath(servedRootReal, candidatePath, label) {
|
|
283
|
+
if (typeof candidatePath !== 'string' || !path.isAbsolute(candidatePath) || candidatePath.includes('\0')) {
|
|
284
|
+
throw new Error(`${label} must be an absolute, NUL-free path`);
|
|
285
|
+
}
|
|
286
|
+
const target = path.resolve(candidatePath);
|
|
287
|
+
let current = target;
|
|
288
|
+
while (true) {
|
|
289
|
+
try {
|
|
290
|
+
const canonical = await realpath(current);
|
|
291
|
+
if (canonical !== current) throw new Error(`${label} resolves through a symlink`);
|
|
292
|
+
const rel = path.relative(servedRootReal, canonical);
|
|
293
|
+
if (rel === '..' || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
|
|
294
|
+
throw new Error(`${label} escapes the served project root`);
|
|
295
|
+
}
|
|
296
|
+
const tail = path.relative(current, target);
|
|
297
|
+
return tail === '' ? canonical : path.join(canonical, tail);
|
|
298
|
+
} catch (error) {
|
|
299
|
+
if (error.code !== 'ENOENT') throw error;
|
|
300
|
+
const parent = path.dirname(current);
|
|
301
|
+
if (parent === current) throw new Error(`${label} has no resolvable ancestor beneath the served root`);
|
|
302
|
+
current = parent;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/** #2892 F1: the default location for conductor-prepared writer worktrees
|
|
308
|
+
* — identical to #2889's own `prepareWorkerWorktree` default
|
|
309
|
+
* (`tools/ralph-external/lib/worktree-preparation.mjs`), so an operator who
|
|
310
|
+
* never overrides either stays consistent between the two. */
|
|
311
|
+
const DEFAULT_WORKER_WORKTREES_SUBDIR = ['.aiwg', 'working', 'worktrees'];
|
|
312
|
+
|
|
313
|
+
/** #2892 F1: resolve the conductor worktrees root ONCE, at registration —
|
|
314
|
+
* same posture as the served root itself (B1). An operator may point it
|
|
315
|
+
* elsewhere via `registerWorkerInvokeToolset`'s `workerWorktreesRoot`
|
|
316
|
+
* option; otherwise it defaults to `<servedRoot>/.aiwg/working/worktrees`.
|
|
317
|
+
* Canonicalized via realpath when the directory already exists (so a
|
|
318
|
+
* symlinked worktrees root cannot be used to smuggle a workdir in from
|
|
319
|
+
* outside it); a not-yet-created root resolves to its literal path —
|
|
320
|
+
* nothing can be "nested under" a directory that does not exist yet, so an
|
|
321
|
+
* unresolvable root never over-grants, it only ever under-grants. */
|
|
322
|
+
async function resolveWorkerWorktreesRoot(servedRootReal, candidate) {
|
|
323
|
+
const resolved = candidate !== undefined ? path.resolve(candidate) : path.join(servedRootReal, ...DEFAULT_WORKER_WORKTREES_SUBDIR);
|
|
324
|
+
try {
|
|
325
|
+
return await realpath(resolved);
|
|
326
|
+
} catch {
|
|
327
|
+
return resolved;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** #2892 F1: list the served repository's own registered git worktrees
|
|
332
|
+
* (absolute, canonical paths), via `git worktree list --porcelain` run
|
|
333
|
+
* with a SANITIZED environment (only `PATH`/`HOME`; never the conductor's
|
|
334
|
+
* own `GIT_*` overrides, credential helpers, or pager settings) and
|
|
335
|
+
* `-c core.fsmonitor=false -c core.hooksPath=/dev/null`, so merely
|
|
336
|
+
* answering this read-only query can never execute a repo-local hook or
|
|
337
|
+
* fsmonitor helper. Never throws: a worktree-less served root (no `.git`
|
|
338
|
+
* at all, or `git` itself unavailable) simply has no registered worktrees
|
|
339
|
+
* — the caller falls back to the `workerWorktreesRoot` check instead. */
|
|
340
|
+
async function defaultListGitWorktrees(servedRootReal) {
|
|
341
|
+
try {
|
|
342
|
+
const { stdout } = await execFileAsync('git', [
|
|
343
|
+
'-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null',
|
|
344
|
+
'-C', servedRootReal, 'worktree', 'list', '--porcelain',
|
|
345
|
+
], {
|
|
346
|
+
env: { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '' },
|
|
347
|
+
timeout: 10000, maxBuffer: 1024 * 1024,
|
|
348
|
+
});
|
|
349
|
+
const entries = stdout.split('\n')
|
|
350
|
+
.filter(line => line.startsWith('worktree '))
|
|
351
|
+
.map(line => line.slice('worktree '.length).trim());
|
|
352
|
+
const canonical = await Promise.all(entries.map(entry => realpath(entry).catch(() => null)));
|
|
353
|
+
return canonical.filter(Boolean);
|
|
354
|
+
} catch {
|
|
355
|
+
return [];
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** #2892 F1 (fix-2892-r2, formerly B3): `workdir` must be a strict
|
|
360
|
+
* subdirectory of the served root AND a conductor-prepared git worktree —
|
|
361
|
+
* never an ordinary in-repo subdirectory, even one that is otherwise
|
|
362
|
+
* strict, symlink-free, and correctly contained. The round-1 fix only
|
|
363
|
+
* enforced containment (not the root itself, not `.git`/`.aiwg`); a probe
|
|
364
|
+
* showed any ordinary strict subdirectory such as `src/` was still
|
|
365
|
+
* accepted. Fixed: `workdir` is accepted only when EITHER (a) it is itself
|
|
366
|
+
* one of the served repository's own registered git worktrees (verified
|
|
367
|
+
* via `git worktree list` / `.git/worktrees` metadata — real git state a
|
|
368
|
+
* caller cannot forge merely by `mkdir`ing a directory), OR (b) it is
|
|
369
|
+
* nested under the explicitly configured (or defaulted)
|
|
370
|
+
* `workerWorktreesRoot` — the same directory #2889's own
|
|
371
|
+
* `prepareWorkerWorktree` already treats as exclusively conductor-managed.
|
|
372
|
+
* `.git`/`.aiwg` remain reserved (never acceptable as a bare workdir)
|
|
373
|
+
* whenever neither acceptance path applies. */
|
|
374
|
+
async function assertWorkdirIsConductorWorktree(servedRootReal, workdirCanonical, { worktreesRoot, listWorktrees = defaultListGitWorktrees } = {}) {
|
|
375
|
+
const rel = path.relative(servedRootReal, workdirCanonical);
|
|
376
|
+
if (rel === '') {
|
|
377
|
+
throw new Error('workdir must be a strict subdirectory of the served project root, not the root itself');
|
|
378
|
+
}
|
|
379
|
+
const registered = await listWorktrees(servedRootReal);
|
|
380
|
+
if (registered.includes(workdirCanonical)) return;
|
|
381
|
+
if (worktreesRoot) {
|
|
382
|
+
const underRoot = path.relative(worktreesRoot, workdirCanonical);
|
|
383
|
+
if (underRoot !== '..' && !underRoot.startsWith(`..${path.sep}`) && !path.isAbsolute(underRoot)) return;
|
|
384
|
+
}
|
|
385
|
+
const [first] = rel.split(path.sep);
|
|
386
|
+
if (RESERVED_WORKDIR_NAMES.has(first)) {
|
|
387
|
+
throw new Error(`workdir must not be inside the reserved '${first}' directory`);
|
|
388
|
+
}
|
|
389
|
+
throw new Error('workdir must be a conductor-prepared git worktree (registered via git worktree, or nested under the configured worker worktrees root), not an ordinary subdirectory');
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** #2892 B3: `final_output` must resolve inside the already-scoped `workdir`
|
|
393
|
+
* — not merely somewhere under the served root — so a worker's own claimed
|
|
394
|
+
* output always lands in its own conductor-prepared worktree. */
|
|
395
|
+
function assertWithinWorkdir(workdirCanonical, candidateCanonical) {
|
|
396
|
+
const rel = path.relative(workdirCanonical, candidateCanonical);
|
|
397
|
+
if (rel === '..' || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
|
|
398
|
+
throw new Error('final_output must resolve inside workdir');
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** #2892 B3: claim `final_output` before any launch via an exclusive create
|
|
403
|
+
* (`O_CREAT|O_EXCL|O_NOFOLLOW`) — refuses if anything (file, symlink, or
|
|
404
|
+
* directory) already sits at that exact path, closing the TOCTOU window
|
|
405
|
+
* between path validation and whatever later writes the worker's result
|
|
406
|
+
* there. The worker's own later `O_TRUNC` write reuses this same inode. */
|
|
407
|
+
async function reserveFinalOutput(finalOutputPath) {
|
|
408
|
+
const handle = await fsOpen(finalOutputPath,
|
|
409
|
+
fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY | fsConstants.O_NOFOLLOW, 0o600);
|
|
410
|
+
try { return await handle.stat(); } finally { await handle.close(); }
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** Defense in depth, on top of the hard root check above: when the project
|
|
414
|
+
* participates in a multi-repo workspace manifest, the path must also be
|
|
415
|
+
* allowed there. Single-project deployments (no workspace manifest) have
|
|
416
|
+
* nothing to consult and rely on the hard root boundary alone. */
|
|
417
|
+
async function assertRepoAccessAllowed(servedRoot, candidatePath, action, repoAccess) {
|
|
418
|
+
const manifestPath = repoAccess.findRepoAccessManifest(servedRoot);
|
|
419
|
+
if (!manifestPath) return;
|
|
420
|
+
const manifest = repoAccess.loadRepoAccessManifest(servedRoot);
|
|
421
|
+
const decision = repoAccess.checkRepoAccess(manifest, candidatePath, action, servedRoot);
|
|
422
|
+
if (!decision.allowed) {
|
|
423
|
+
throw new Error(`repo-access denies ${action} for this path: ${decision.reason}`);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// ===========================================================================
|
|
428
|
+
// Defect 3 / M4: TTL + live-only bounded invocation registry, with a
|
|
429
|
+
// matching bound on replay-protection memory.
|
|
430
|
+
// ===========================================================================
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Terminal entries are pruned by TTL and, independently, by an
|
|
434
|
+
* over-`maxRetained` oldest-terminal-first sweep. `maxActive` is checked
|
|
435
|
+
* against LIVE entries only, so no accumulation of completed runs can ever
|
|
436
|
+
* lock out a new one — only concurrently *live* work can (an "uncertain" run
|
|
437
|
+
* — #2892 M2 — is LIVE, by design, until something proves it stopped).
|
|
438
|
+
*
|
|
439
|
+
* `seenRequestIds` (M4) tracks `{ invocationId, recordedAt }` and is swept on
|
|
440
|
+
* the same cadence, but only ever evicts a request id once its invocation
|
|
441
|
+
* entry has ALREADY left `entries` (pruned away) — replay protection for any
|
|
442
|
+
* live, uncertain, or still-retained-terminal invocation is never weakened
|
|
443
|
+
* by this sweep, only the unbounded-growth tail is.
|
|
444
|
+
*/
|
|
445
|
+
export function createInvocationRegistry({
|
|
446
|
+
maxActive = DEFAULT_MAX_ACTIVE,
|
|
447
|
+
maxRetained = DEFAULT_MAX_RETAINED,
|
|
448
|
+
ttlMs = DEFAULT_TERMINAL_TTL_MS,
|
|
449
|
+
now = () => Date.now(),
|
|
450
|
+
} = {}) {
|
|
451
|
+
const entries = new Map();
|
|
452
|
+
const seenRequestIds = new Map(); // request_id -> { invocationId, recordedAt }
|
|
453
|
+
|
|
454
|
+
// Pin one bounded breaker owner for this registry's lifetime. Replacing
|
|
455
|
+
// it could accumulate protected entries across independently capped owners
|
|
456
|
+
// and orphan the original reconciliation mapping.
|
|
457
|
+
let recoveryOwner;
|
|
458
|
+
function bindRecoveryOwner(breaker) {
|
|
459
|
+
if (!breaker) return;
|
|
460
|
+
if (recoveryOwner && recoveryOwner !== breaker) {
|
|
461
|
+
throw new Error('Invocation registry already has a recovery breaker owner; reuse that owner and reconcile_stopped for unresolved reservations');
|
|
462
|
+
}
|
|
463
|
+
recoveryOwner = breaker;
|
|
464
|
+
}
|
|
465
|
+
// Unresolved entries and their replay identities survive ordinary pruning.
|
|
466
|
+
// Their aggregate bound is the single owner's reservation cap.
|
|
467
|
+
function hasUnresolvedStopMarker(entry) {
|
|
468
|
+
return typeof entry.recoveryBreaker?.hasUnresolvedStop === 'function'
|
|
469
|
+
&& entry.recoveryBreaker.hasUnresolvedStop(entry.invocationId);
|
|
470
|
+
}
|
|
471
|
+
function terminalEntries() {
|
|
472
|
+
return [...entries.values()].filter(entry => entry.terminalAt !== null && !hasUnresolvedStopMarker(entry))
|
|
473
|
+
.sort((a, b) => a.terminalAt - b.terminalAt);
|
|
474
|
+
}
|
|
475
|
+
function pruneSeenRequestIds() {
|
|
476
|
+
const cutoff = now() - ttlMs;
|
|
477
|
+
// Only a request id whose invocation has already left `entries` is even
|
|
478
|
+
// a candidate for eviction — live/uncertain/retained-terminal replay
|
|
479
|
+
// protection is never weakened by this sweep.
|
|
480
|
+
const candidates = [...seenRequestIds.entries()]
|
|
481
|
+
.filter(([, v]) => !entries.has(v.invocationId))
|
|
482
|
+
.sort((a, b) => a[1].recordedAt - b[1].recordedAt);
|
|
483
|
+
for (const [requestId, v] of candidates) {
|
|
484
|
+
if (v.recordedAt <= cutoff) seenRequestIds.delete(requestId);
|
|
485
|
+
}
|
|
486
|
+
let over = seenRequestIds.size - maxRetained;
|
|
487
|
+
if (over > 0) {
|
|
488
|
+
for (const [requestId] of candidates) {
|
|
489
|
+
if (over <= 0) break;
|
|
490
|
+
if (seenRequestIds.delete(requestId)) over -= 1;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
function prune() {
|
|
495
|
+
const cutoff = now() - ttlMs;
|
|
496
|
+
for (const entry of terminalEntries()) if (entry.terminalAt <= cutoff) entries.delete(entry.invocationId);
|
|
497
|
+
let over = entries.size - maxRetained;
|
|
498
|
+
if (over > 0) for (const entry of terminalEntries()) {
|
|
499
|
+
if (over <= 0) break;
|
|
500
|
+
entries.delete(entry.invocationId);
|
|
501
|
+
over -= 1;
|
|
502
|
+
}
|
|
503
|
+
pruneSeenRequestIds();
|
|
504
|
+
}
|
|
505
|
+
function liveCount() {
|
|
506
|
+
return [...entries.values()].filter(entry => entry.terminalAt === null).length;
|
|
507
|
+
}
|
|
508
|
+
return {
|
|
509
|
+
prune,
|
|
510
|
+
liveCount,
|
|
511
|
+
bindRecoveryOwner,
|
|
512
|
+
get recoveryOwner() { return recoveryOwner; },
|
|
513
|
+
get size() { return entries.size; },
|
|
514
|
+
/** Test/diagnostic only — not consulted by any admission decision. */
|
|
515
|
+
get seenCount() { return seenRequestIds.size; },
|
|
516
|
+
canStart() { prune(); return liveCount() < maxActive; },
|
|
517
|
+
/** Never replay the same request_id, terminal or not (#2896 defect: "never blind replay"). */
|
|
518
|
+
claimRequestId(requestId) {
|
|
519
|
+
prune();
|
|
520
|
+
return seenRequestIds.get(requestId)?.invocationId ?? null;
|
|
521
|
+
},
|
|
522
|
+
create(partial) {
|
|
523
|
+
bindRecoveryOwner(partial.recoveryBreaker);
|
|
524
|
+
prune();
|
|
525
|
+
const invocationId = partial.invocationId ?? randomUUID();
|
|
526
|
+
const entry = { invocationId, terminalAt: null, createdAt: now(), ...partial, invocationId };
|
|
527
|
+
entries.set(invocationId, entry);
|
|
528
|
+
if (partial.requestId) seenRequestIds.set(partial.requestId, { invocationId, recordedAt: now() });
|
|
529
|
+
return entry;
|
|
530
|
+
},
|
|
531
|
+
get(invocationId) { prune(); return entries.get(invocationId); },
|
|
532
|
+
update(invocationId, patch) {
|
|
533
|
+
const entry = entries.get(invocationId);
|
|
534
|
+
if (!entry) return undefined;
|
|
535
|
+
bindRecoveryOwner(patch.recoveryBreaker);
|
|
536
|
+
Object.assign(entry, patch);
|
|
537
|
+
return entry;
|
|
538
|
+
},
|
|
539
|
+
markTerminal(invocationId, patch = {}) {
|
|
540
|
+
const entry = entries.get(invocationId);
|
|
541
|
+
if (!entry) return undefined;
|
|
542
|
+
bindRecoveryOwner(patch.recoveryBreaker);
|
|
543
|
+
if (entry.terminalAt === null) entry.terminalAt = now();
|
|
544
|
+
Object.assign(entry, patch);
|
|
545
|
+
// An entry becomes eligible for over-capacity eviction the moment it
|
|
546
|
+
// turns terminal; prune immediately rather than waiting for the next
|
|
547
|
+
// unrelated registry call (otherwise a long idle period after the last
|
|
548
|
+
// run leaves the registry transiently over `maxRetained`).
|
|
549
|
+
prune();
|
|
550
|
+
return entry;
|
|
551
|
+
},
|
|
552
|
+
values() { prune(); return [...entries.values()]; },
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// ===========================================================================
|
|
557
|
+
// Reuse wiring — load TS-authored contract modules through the repo's own
|
|
558
|
+
// src/dist-aware resolver (tools/_resolve-impl.mjs), exactly as the Ralph
|
|
559
|
+
// provider adapter does for the same reason: this file ships unchanged at
|
|
560
|
+
// `(dist/)src/mcp/tools/worker-invoke.mjs` in both dev and the published
|
|
561
|
+
// package, so a bare relative `.js` specifier into `src/serve/...ts` would
|
|
562
|
+
// only resolve once `dist/src/...` has been built.
|
|
563
|
+
// ===========================================================================
|
|
564
|
+
|
|
565
|
+
async function loadServeWorkerInvocation() {
|
|
566
|
+
return importImpl('serve/worker-invocation.js');
|
|
567
|
+
}
|
|
568
|
+
async function loadModelsHandlerInternals() {
|
|
569
|
+
return importImpl('cli/handlers/models.js');
|
|
570
|
+
}
|
|
571
|
+
async function loadMissionProtocol() {
|
|
572
|
+
return importImpl('mission-protocol/index.js');
|
|
573
|
+
}
|
|
574
|
+
async function loadRepoAccessModule() {
|
|
575
|
+
return importImpl('policy/repo-access.js');
|
|
576
|
+
}
|
|
577
|
+
async function loadAiwgConfigModule() {
|
|
578
|
+
return importImpl('config/aiwg-config.js');
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
/** `tools/` ships unchanged at the package root in both dev and the
|
|
582
|
+
* published package (unlike `src/`, which only ships as `dist/src/`), so an
|
|
583
|
+
* AIWG_ROOT-absolute path is the robust way in rather than a relative
|
|
584
|
+
* specifier (whose depth from this file differs between `src/mcp/tools/`
|
|
585
|
+
* and `dist/src/mcp/tools/`). */
|
|
586
|
+
// #2915 final (step 3): the former `resolveRunRecovery` + per-site
|
|
587
|
+
// `canLaunch`/`noteLaunched`/`recordDirectWorkerOutcome` triple is replaced
|
|
588
|
+
// by `run()`'s own single `admitDirectWorker`/`settleDirectWorker` pair
|
|
589
|
+
// (session-launcher.mjs), which `defaultLaunchShellWorker` below BORROWS
|
|
590
|
+
// rather than re-resolving — deleting the duplicate gate that previously
|
|
591
|
+
// rejected its own already-admitted canary (#2915 final review, finding 4).
|
|
592
|
+
|
|
593
|
+
async function loadRalphExternal(relativeFromToolsRoot) {
|
|
594
|
+
const absolute = path.join(findOwnPackageRoot(), 'tools/ralph-external', relativeFromToolsRoot);
|
|
595
|
+
return import(pathToFileURL(absolute).href);
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
/** The CLI's own default loader (`ModelsWorkerServices.load`), reused
|
|
599
|
+
* in-process instead of shelling out to `aiwg models resolve`. */
|
|
600
|
+
async function defaultLoadWorkerInputs(servedRoot) {
|
|
601
|
+
const { loadWorkerInputs } = await loadModelsHandlerInternals();
|
|
602
|
+
return loadWorkerInputs({ cwd: servedRoot, args: ['resolve'], rawArgs: ['models', 'resolve'], frameworkRoot: findOwnPackageRoot() });
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/** #2892 D1: the registry's default `maxActive` respects the project's own
|
|
606
|
+
* parallelism policy (`.aiwg/aiwg.config` `parallelism.max_parallel_subagents`,
|
|
607
|
+
* via the existing `resolveParallelism` resolver) — never more than the code
|
|
608
|
+
* default, possibly fewer. Best-effort: any failure (no config, malformed
|
|
609
|
+
* config, resolver error) falls back to `DEFAULT_MAX_ACTIVE` rather than
|
|
610
|
+
* blocking registration. */
|
|
611
|
+
async function resolveDefaultMaxActive(servedRootReal) {
|
|
612
|
+
try {
|
|
613
|
+
const { readAiwgConfig, resolveParallelism, emptyConfig } = await loadAiwgConfigModule();
|
|
614
|
+
const config = (await readAiwgConfig(servedRootReal)) ?? emptyConfig();
|
|
615
|
+
const resolved = resolveParallelism(config.parallelism);
|
|
616
|
+
const fromConfig = resolved?.max_parallel_subagents;
|
|
617
|
+
return Number.isFinite(fromConfig) && fromConfig > 0 ? Math.min(DEFAULT_MAX_ACTIVE, fromConfig) : DEFAULT_MAX_ACTIVE;
|
|
618
|
+
} catch {
|
|
619
|
+
return DEFAULT_MAX_ACTIVE;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// ===========================================================================
|
|
624
|
+
// Launch — shell transport reuses #2954's gate-then-launch composition.
|
|
625
|
+
// ===========================================================================
|
|
626
|
+
|
|
627
|
+
/** `planDirectWorkerLaunch` returns a blocked gate as `{ failures, gate }`
|
|
628
|
+
* only — it never sets `diagnostic` (that field is reserved for the
|
|
629
|
+
* degraded-isolation-unavailable case; see `worker-isolation.mjs`). Name
|
|
630
|
+
* the specific failing signal(s) here so a dispatch-gate refusal is
|
|
631
|
+
* distinguishable from every other blocked-launch reason in this bridge's
|
|
632
|
+
* own diagnostics, instead of collapsing to one generic string. */
|
|
633
|
+
function describeGateFailures(failures) {
|
|
634
|
+
if (!Array.isArray(failures) || failures.length === 0) return undefined;
|
|
635
|
+
return `worker dispatch gate blocked: ${failures.map(f => `${f.signal} (${f.reason})`).join('; ')}`;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// Audited observer; refresh the historical baseline graph after rebase.
|
|
639
|
+
const waitForUnitTerminationBaseline = waitForUnitTermination;
|
|
640
|
+
export { waitForUnitTerminationBaseline as __testOnlyWaitForUnitTerminationBaseline };
|
|
641
|
+
|
|
642
|
+
// Recovery-disabled dispatch uses the audited preparation path and direct
|
|
643
|
+
// gate/isolation primitives, without entering the current planner's admission.
|
|
644
|
+
// The historical baseline pin/fixture graph must be refreshed after rebase.
|
|
645
|
+
async function defaultLaunchShellWorkerBaseline(invocation, { memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, ...options }) {
|
|
646
|
+
const loadBaselineLaunch = async relative => {
|
|
647
|
+
if (relative !== 'session-launcher.mjs') return loadRalphExternal(relative);
|
|
648
|
+
const { checkWorkerDispatch, launchIsolatedWorker } = await loadRalphExternal('lib/worker-isolation.mjs');
|
|
649
|
+
return {
|
|
650
|
+
planDirectWorkerLaunch: ({ invocation, live, readers, thresholdOverlay, ...launchOptions }) => {
|
|
651
|
+
// Shared preparation must provide the conductor-framed params file;
|
|
652
|
+
// never send the raw brief or launch without it for brief-json.
|
|
653
|
+
if (invocation?.stdin === 'brief-json' && launchOptions.stdinPath === undefined) {
|
|
654
|
+
throw new Error('brief-json stdin requires an explicit conductor-framed params file (stdinPath)');
|
|
655
|
+
}
|
|
656
|
+
const cwd = invocation?.cwd;
|
|
657
|
+
const gate = checkWorkerDispatch({ config: launchOptions.config, live, readers, thresholdOverlay,
|
|
658
|
+
paths: typeof cwd === 'string' && cwd ? [cwd] : undefined });
|
|
659
|
+
if (!gate.ok) return { launched: false, blocked: true, failures: gate.failures ?? [],
|
|
660
|
+
diagnostic: describeGateFailures(gate.failures) };
|
|
661
|
+
return launchIsolatedWorker({
|
|
662
|
+
command: invocation?.argv?.[0], args: invocation?.argv?.slice(1) ?? [], cwd,
|
|
663
|
+
stdoutPath: invocation?.stdoutPath, stderrPath: invocation?.stderrPath, envNames: invocation?.envNames,
|
|
664
|
+
...launchOptions, gate,
|
|
665
|
+
});
|
|
666
|
+
},
|
|
667
|
+
};
|
|
668
|
+
};
|
|
669
|
+
return launchPreparedShellWorker(invocation, { ...options, memoryMaxGb },
|
|
670
|
+
{ loadRalphExternal: loadBaselineLaunch, readBoundedFile, importImpl, describeGateFailures });
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
async function defaultLaunchShellWorker(invocation, { memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, ...options }) {
|
|
674
|
+
if (options.admission === undefined) {
|
|
675
|
+
return defaultLaunchShellWorkerBaseline(invocation, { ...options, memoryMaxGb });
|
|
676
|
+
}
|
|
677
|
+
return launchPreparedShellWorker(invocation, { ...options, memoryMaxGb },
|
|
678
|
+
{ loadRalphExternal, readBoundedFile, importImpl, describeGateFailures });
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
// ===========================================================================
|
|
682
|
+
// Launch — mcp transport (#2892 B2). The MCP worker SERVER process is
|
|
683
|
+
// launched through the SAME gate-then-launch pipeline as a shell worker
|
|
684
|
+
// (`planDirectWorkerLaunch` → `launchIsolatedWorker`, `attached: true`), and
|
|
685
|
+
// the MCP client speaks to it over the launched child's own stdio via
|
|
686
|
+
// `ChildProcessTransport` — never the SDK's `StdioClientTransport`, which
|
|
687
|
+
// spawns its own child and always merges `getDefaultEnvironment()`
|
|
688
|
+
// (HOME/LOGNAME/PATH/SHELL/TERM/USER from the CONDUCTOR's process.env) into
|
|
689
|
+
// whatever `env` is given, silently widening the qualified allowlist.
|
|
690
|
+
//
|
|
691
|
+
// `sessionIsolation: 'per-call'` opens a fresh connection per call; anything
|
|
692
|
+
// else ('per-instance'/'serialized') is additionally serialized through a
|
|
693
|
+
// bounded per-serverRef queue (#2892 D2) so no two concurrent calls ever
|
|
694
|
+
// share one live connection, and each queued call rechecks the server's own
|
|
695
|
+
// registry qualification immediately before ITS launch, not just at the
|
|
696
|
+
// time it was enqueued.
|
|
697
|
+
// ===========================================================================
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* A minimal MCP `Transport` over an already-spawned child process's own
|
|
701
|
+
* stdio, for a child launched by `launchIsolatedWorker({ attached: true })`.
|
|
702
|
+
* Mirrors the SDK's own `StdioClientTransport` framing (newline-delimited
|
|
703
|
+
* JSON-RPC via `ReadBuffer`/`serializeMessage`) but never spawns anything
|
|
704
|
+
* itself and never touches environment variables — the child already exists,
|
|
705
|
+
* already resource-confined, already running with exactly its allowlisted
|
|
706
|
+
* environment.
|
|
707
|
+
*/
|
|
708
|
+
const MCP_TRANSPORT_MAX_BUFFER_BYTES = MAX_MCP_FRAME_BYTES;
|
|
709
|
+
|
|
710
|
+
export class ChildProcessTransport {
|
|
711
|
+
constructor(child, { terminate = () => signalChild(child, 'SIGKILL'),
|
|
712
|
+
deadlineAtMs = null, now = () => Date.now(), isCancelled } = {}) {
|
|
713
|
+
this._child = child;
|
|
714
|
+
this._terminate = terminate;
|
|
715
|
+
this._failure = undefined;
|
|
716
|
+
this._closed = false;
|
|
717
|
+
this._deadlineAtMs = deadlineAtMs;
|
|
718
|
+
this._now = now;
|
|
719
|
+
// #2915 round 8 review 8 (C3): optional cancellation predicate, checked
|
|
720
|
+
// at the write boundary alongside the deadline — `undefined` whenever
|
|
721
|
+
// this call is not gated, matching origin/main's own unbounded
|
|
722
|
+
// transport exactly (see `send()` below).
|
|
723
|
+
this._isCancelled = typeof isCancelled === 'function' ? isCancelled : undefined;
|
|
724
|
+
this.onmessage = undefined;
|
|
725
|
+
this.onerror = undefined;
|
|
726
|
+
this.onclose = undefined;
|
|
727
|
+
// #2892 F2 (fix-2892-r2): the SDK's own `Client.callTool()`/`request()`
|
|
728
|
+
// generates its JSON-RPC request id internally and never returns it to
|
|
729
|
+
// the caller — the only place that real id is ever actually observable
|
|
730
|
+
// is here, at the transport, the instant the framed request is handed
|
|
731
|
+
// off to be written. A `notifications/cancelled` sent with any other
|
|
732
|
+
// value (the caller's own opaque `mcpBinding.requestId`, which has no
|
|
733
|
+
// relationship whatsoever to the wire protocol) means nothing to a
|
|
734
|
+
// worker trying to honor it.
|
|
735
|
+
this.lastRequestId = null;
|
|
736
|
+
// #2892 F3 (fix-2892-r2): the raw (plain `JSON.parse`-only, never
|
|
737
|
+
// zod-round-tripped) response body for each inbound message that has an
|
|
738
|
+
// `id`, keyed by that id. The SDK's own `ToolSchema`/`ListToolsResultSchema`
|
|
739
|
+
// zod validation reconstructs `inputSchema.properties`/
|
|
740
|
+
// `outputSchema.properties` via `z.record()`, whose generated parser
|
|
741
|
+
// writes each member with plain bracket assignment
|
|
742
|
+
// (`result[key] = value`) rather than `Object.defineProperty` —
|
|
743
|
+
// verified empirically against this installed SDK version: a member
|
|
744
|
+
// literally named `__proto__` silently vanishes from the zod-parsed
|
|
745
|
+
// object (the assignment invokes the inherited prototype SETTER
|
|
746
|
+
// instead of creating an own property), while `JSON.parse` itself
|
|
747
|
+
// always creates `__proto__` as an ordinary own data property. Two
|
|
748
|
+
// schemas that differ only in a `__proto__` member therefore digest
|
|
749
|
+
// IDENTICALLY once passed through `client.listTools()` — comparing
|
|
750
|
+
// against this raw map instead closes that gap. Bounded to a small
|
|
751
|
+
// cap since a long-lived connection could otherwise grow it unbounded.
|
|
752
|
+
this._rawResponses = new Map();
|
|
753
|
+
this._buffer = undefined;
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
async start() {
|
|
757
|
+
const { JSONRPCMessageSchema } = await import('@modelcontextprotocol/sdk/types.js');
|
|
758
|
+
this._messageSchema = JSONRPCMessageSchema;
|
|
759
|
+
const child = this._child;
|
|
760
|
+
child.stdout?.on('data', chunk => {
|
|
761
|
+
if (this._failure || this._closed) return;
|
|
762
|
+
try {
|
|
763
|
+
// Check each frame BEFORE concatenation or UTF-8 decoding.
|
|
764
|
+
let offset = 0;
|
|
765
|
+
let frameBytes = this._buffer?.length ?? 0;
|
|
766
|
+
while (offset < chunk.length) {
|
|
767
|
+
const newline = chunk.indexOf(10, offset);
|
|
768
|
+
const end = newline === -1 ? chunk.length : newline;
|
|
769
|
+
frameBytes += end - offset;
|
|
770
|
+
if (frameBytes > MAX_MCP_FRAME_BYTES) {
|
|
771
|
+
this._overflow('OG_WORKER_MCP_FRAME_TOO_LARGE');
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
if (newline === -1) break;
|
|
775
|
+
frameBytes = 0;
|
|
776
|
+
offset = newline + 1;
|
|
777
|
+
}
|
|
778
|
+
this._buffer = this._buffer ? Buffer.concat([this._buffer, chunk]) : chunk;
|
|
779
|
+
this._drain();
|
|
780
|
+
} catch (error) { this.onerror?.(error); }
|
|
781
|
+
});
|
|
782
|
+
child.stdout?.on('error', error => this.onerror?.(error));
|
|
783
|
+
child.stdin?.on('error', error => this.onerror?.(error));
|
|
784
|
+
// The attached child's stderr is never consumed elsewhere; drain it so a
|
|
785
|
+
// chatty worker cannot block on a full pipe buffer.
|
|
786
|
+
child.stderr?.resume();
|
|
787
|
+
child.once('close', () => this._notifyClose());
|
|
788
|
+
child.once('error', error => this.onerror?.(error));
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
_notifyClose() {
|
|
792
|
+
if (this._closed) return;
|
|
793
|
+
this._closed = true;
|
|
794
|
+
this._buffer = undefined;
|
|
795
|
+
this._rawResponses.clear();
|
|
796
|
+
this.onclose?.();
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
_overflow(code) {
|
|
800
|
+
if (this._failure) return;
|
|
801
|
+
this._failure = Object.assign(new Error(code), { code });
|
|
802
|
+
this._buffer = undefined;
|
|
803
|
+
this._rawResponses.clear();
|
|
804
|
+
// Start stopping before pipe closure can exit the leader; onclose rejects
|
|
805
|
+
// pending SDK requests immediately while termination proof is awaited.
|
|
806
|
+
try {
|
|
807
|
+
this._termination = Promise.resolve(this._terminate()).then(result =>
|
|
808
|
+
['confirmed', 'uncertain'].includes(result?.termination) ? result : { termination: 'uncertain', diagnostic: 'OG_WORKER_MCP_STOP_UNCONFIRMED' }).catch(() => ({ termination: 'uncertain', diagnostic: 'OG_WORKER_MCP_STOP_FAILED' }));
|
|
809
|
+
} catch { this._termination = Promise.resolve({ termination: 'uncertain', diagnostic: 'OG_WORKER_MCP_STOP_FAILED' }); }
|
|
810
|
+
this._child.stdout?.destroy();
|
|
811
|
+
this._child.stdin?.destroy();
|
|
812
|
+
try { this.onerror?.(this._failure); } finally { this._notifyClose(); }
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
// Reimplements the SDK's own `ReadBuffer`'s newline-delimited framing
|
|
816
|
+
// directly (rather than delegating to it) so this transport can parse
|
|
817
|
+
// each line TWICE from the exact same bytes: once with plain `JSON.parse`
|
|
818
|
+
// (own-property-preserving, retained in `_rawResponses` — see
|
|
819
|
+
// constructor doc) and once through the SDK's own `JSONRPCMessageSchema`
|
|
820
|
+
// (routed to `onmessage`, unchanged behavior for the Client/Protocol).
|
|
821
|
+
_drain() {
|
|
822
|
+
while (this._buffer) {
|
|
823
|
+
const index = this._buffer.indexOf('\n');
|
|
824
|
+
if (index === -1) return;
|
|
825
|
+
const line = this._buffer.toString('utf8', 0, index).replace(/\r$/, '');
|
|
826
|
+
this._buffer = this._buffer.subarray(index + 1);
|
|
827
|
+
this._handleLine(line);
|
|
828
|
+
if (this._buffer?.length === 0) this._buffer = undefined;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
_handleLine(line) {
|
|
833
|
+
let raw;
|
|
834
|
+
try {
|
|
835
|
+
raw = JSON.parse(line);
|
|
836
|
+
} catch (error) { this.onerror?.(error); return; }
|
|
837
|
+
if (raw && typeof raw === 'object' && !Array.isArray(raw) && raw.id !== undefined && raw.id !== null) {
|
|
838
|
+
this._rawResponses.set(raw.id, raw);
|
|
839
|
+
if (this._rawResponses.size > 64) {
|
|
840
|
+
const oldest = this._rawResponses.keys().next().value;
|
|
841
|
+
this._rawResponses.delete(oldest);
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
let message;
|
|
845
|
+
try {
|
|
846
|
+
message = this._messageSchema.parse(raw);
|
|
847
|
+
} catch (error) { this.onerror?.(error); return; }
|
|
848
|
+
this.onmessage?.(message);
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
async send(message) {
|
|
852
|
+
if (this._failure) throw this._failure;
|
|
853
|
+
if (this._closed) throw new Error('worker MCP transport is closed');
|
|
854
|
+
// #2892 F2: a JSON-RPC REQUEST (never a notification or response) is the
|
|
855
|
+
// only message shape with both `method` and `id` — track the most
|
|
856
|
+
// recently sent one's id so an in-flight `stop()`/error-path cancel can
|
|
857
|
+
// reference the worker's own actual request, not a caller-supplied
|
|
858
|
+
// value unrelated to this connection's wire protocol.
|
|
859
|
+
if (message && typeof message === 'object' && typeof message.method === 'string' && message.id !== undefined) {
|
|
860
|
+
this.lastRequestId = message.id;
|
|
861
|
+
}
|
|
862
|
+
const { serializeMessage } = await import('@modelcontextprotocol/sdk/shared/stdio.js');
|
|
863
|
+
// #2915 round 8 (D5): the write boundary — AFTER the dynamic import's
|
|
864
|
+
// own `await` (review 7 finding 4: "checking before an SDK call can
|
|
865
|
+
// precede the actual send by an await... a request can cross its
|
|
866
|
+
// deadline during that gap"). Stop/cancel cleanup traffic
|
|
867
|
+
// (`notifications/cancelled`) is exempt — it must always be allowed to
|
|
868
|
+
// reach the worker, deadline or not.
|
|
869
|
+
const isStopTraffic = message && typeof message === 'object' && message.method === 'notifications/cancelled';
|
|
870
|
+
if (this._deadlineAtMs !== null && !isStopTraffic && this._now() >= this._deadlineAtMs) {
|
|
871
|
+
throw Object.assign(new Error('worker MCP transport deadline exceeded before write'), { code: -32001 });
|
|
872
|
+
}
|
|
873
|
+
// #2915 round 8 review 8 (C3): cancellation at the write boundary,
|
|
874
|
+
// exempting stop/cleanup traffic for the same reason as the deadline
|
|
875
|
+
// check above — a `notifications/cancelled` notification must always be
|
|
876
|
+
// allowed to reach the worker, cancelled or not.
|
|
877
|
+
if (!isStopTraffic && this._isCancelled?.()) {
|
|
878
|
+
throw Object.assign(new Error('worker MCP transport refused: cancelled before write'), { code: -32800 });
|
|
879
|
+
}
|
|
880
|
+
return new Promise((resolve, reject) => {
|
|
881
|
+
if (!this._child.stdin) { reject(new Error('worker MCP process has no stdin')); return; }
|
|
882
|
+
const json = serializeMessage(message);
|
|
883
|
+
if (this._child.stdin.write(json)) resolve();
|
|
884
|
+
else this._child.stdin.once('drain', resolve);
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
async close() {
|
|
889
|
+
const child = this._child;
|
|
890
|
+
const closed = new Promise(resolve => child.once('close', resolve));
|
|
891
|
+
try { child.stdin?.end(); } catch { /* already gone */ }
|
|
892
|
+
await Promise.race([closed, new Promise(resolve => setTimeout(resolve, 2000).unref?.())]);
|
|
893
|
+
if (child.exitCode === null && child.signalCode === null) {
|
|
894
|
+
try { signalChild(child, 'SIGTERM'); } catch { /* already gone */ }
|
|
895
|
+
await Promise.race([closed, new Promise(resolve => setTimeout(resolve, 2000).unref?.())]);
|
|
896
|
+
}
|
|
897
|
+
if (child.exitCode === null && child.signalCode === null) {
|
|
898
|
+
try { signalChild(child, 'SIGKILL'); } catch { /* already gone */ }
|
|
899
|
+
}
|
|
900
|
+
this._buffer = undefined;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
/** Launch the MCP worker SERVER process through the gate-then-launch
|
|
905
|
+
* pipeline, attached, so its stdio can back a `ChildProcessTransport`.
|
|
906
|
+
* `cwd` is the resolved invocation's own worktree (`invocation.cwd`,
|
|
907
|
+
* #2887's `request.workdir`) — the dispatch gate samples free disk on it
|
|
908
|
+
* (`checkWorkerDispatch({ paths: [cwd] })`, see `planDirectWorkerLaunch`).
|
|
909
|
+
* Without it the gate has no filesystem to sample and fails closed with
|
|
910
|
+
* "free disk not detectable on this host" on EVERY host, regardless of
|
|
911
|
+
* isolation mode, because an MCP server registry entry does not itself
|
|
912
|
+
* carry a worktree path the way a resolved shell invocation does. A
|
|
913
|
+
* registry-declared `server.cwd` (none of today's callers set one) still
|
|
914
|
+
* wins when present, matching the pre-existing precedence for the actual
|
|
915
|
+
* spawn directory. */
|
|
916
|
+
/** #2915 round 8 (D6): the MCP-native counterpart of
|
|
917
|
+
* `defaultLaunchShellWorkerBaseline` — see its doc comment for why this
|
|
918
|
+
* bypasses the CURRENT `planDirectWorkerLaunch` entirely rather than
|
|
919
|
+
* omitting `admission` and letting that function's own standalone
|
|
920
|
+
* admission branch run (review 7 finding 13's second bullet, which names
|
|
921
|
+
* this exact function: "MCP native path, M:941... omitting the disabled
|
|
922
|
+
* admission loses CLI identity before the flat planner. It performs a new
|
|
923
|
+
* admission; at 128 occupied reservations, disabled work returns blocked
|
|
924
|
+
* where baseline launches"). */
|
|
925
|
+
async function defaultLaunchMcpWorkerProcessBaseline(server, { allowDegraded, config, name, cwd, memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, live, readers, thresholdOverlay }) {
|
|
926
|
+
if (server?.type !== 'stdio') {
|
|
927
|
+
return { launched: false, blocked: true, diagnostic: 'Only stdio MCP worker servers are qualified for direct handoff' };
|
|
928
|
+
}
|
|
929
|
+
const { checkWorkerDispatch, launchIsolatedWorker } = await loadRalphExternal('lib/worker-isolation.mjs');
|
|
930
|
+
const envNames = Object.keys(server.env ?? {});
|
|
931
|
+
const resolvedCwd = server.cwd ?? cwd;
|
|
932
|
+
const gate = checkWorkerDispatch({ config, live, readers, thresholdOverlay, paths: typeof resolvedCwd === 'string' && resolvedCwd ? [resolvedCwd] : undefined });
|
|
933
|
+
if (!gate.ok) return { launched: false, blocked: true, failures: gate.failures ?? [], diagnostic: describeGateFailures(gate.failures) };
|
|
934
|
+
const launch = await launchIsolatedWorker({
|
|
935
|
+
name, memoryMaxGb, command: server.command ?? '', args: server.args ?? [], cwd: resolvedCwd,
|
|
936
|
+
envNames, env: server.env ?? {}, gate, config, allowDegraded, attached: true,
|
|
937
|
+
// See `defaultLaunchMcpWorkerProcess`: an MCP stdio server needs a live,
|
|
938
|
+
// writable stdin to receive JSON-RPC requests.
|
|
939
|
+
stdin: 'open',
|
|
940
|
+
});
|
|
941
|
+
if (!launch.launched && launch.diagnostic === undefined) {
|
|
942
|
+
return { ...launch, diagnostic: describeGateFailures(launch.failures) };
|
|
943
|
+
}
|
|
944
|
+
return launch;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
async function defaultLaunchMcpWorkerProcess(server, { allowDegraded, config, name, cwd, memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, live, readers, thresholdOverlay, admission, invocation, invocationId, dispatchGuard }) {
|
|
948
|
+
// #2915 round 8 (D6): see `defaultLaunchShellWorker`'s identical guard —
|
|
949
|
+
// `admission` is only ever present when this attempt is actually gated.
|
|
950
|
+
if (admission === undefined) {
|
|
951
|
+
return defaultLaunchMcpWorkerProcessBaseline(server, { allowDegraded, config, name, cwd, memoryMaxGb, live, readers, thresholdOverlay });
|
|
952
|
+
}
|
|
953
|
+
if (server?.type !== 'stdio') {
|
|
954
|
+
return { launched: false, blocked: true, diagnostic: 'Only stdio MCP worker servers are qualified for direct handoff' };
|
|
955
|
+
}
|
|
956
|
+
const { planDirectWorkerLaunch } = await loadRalphExternal('session-launcher.mjs');
|
|
957
|
+
const envNames = Object.keys(server.env ?? {});
|
|
958
|
+
const launch = await planDirectWorkerLaunch({
|
|
959
|
+
name, memoryMaxGb, command: server.command ?? '', args: server.args ?? [], cwd: server.cwd ?? cwd,
|
|
960
|
+
envNames, env: server.env ?? {}, config, allowDegraded, attached: true,
|
|
961
|
+
// See `defaultLaunchShellWorker`: test-only dispatch-gate passthroughs,
|
|
962
|
+
// always undefined for production registrations.
|
|
963
|
+
live, readers, thresholdOverlay,
|
|
964
|
+
// #2915 review 4 (finding 2): BORROW the caller's own already-admitted
|
|
965
|
+
// reservation — never a second, independent planner admission for the
|
|
966
|
+
// native-MCP launch path (the exact "one reservation-aware admission"
|
|
967
|
+
// requirement `defaultLaunchShellWorker` already honored).
|
|
968
|
+
//
|
|
969
|
+
// #2915 round 8 review 8 (C6): the borrowed admission's dispatch
|
|
970
|
+
// identity must be derived from THIS call's own trusted, resolved
|
|
971
|
+
// `invocation`/`invocationId` — never the borrowed wrapper's own copies.
|
|
972
|
+
// `planDirectWorkerLaunch`'s own top-level parameter destructuring
|
|
973
|
+
// derives `stdinPath`/`stdoutPath`/`stderrPath`/`command`/`args`/`cwd`
|
|
974
|
+
// AS DEFAULTS from a full `invocation` object when a caller omits them
|
|
975
|
+
// — correct for the shell launch path, but `WorkerResolvedInvocation`
|
|
976
|
+
// (src/serve/worker-invocation.ts) sets `stdoutPath`/`stderrPath`
|
|
977
|
+
// unconditionally for EVERY transport, so passing the full object here
|
|
978
|
+
// would leak them in and collide with `attached: true` (which forbids
|
|
979
|
+
// stdio file redirection entirely). Pass only the minimal shape
|
|
980
|
+
// `selectionDigestFor` actually reads (`{ selected }`) so the digest
|
|
981
|
+
// still matches the original admission's own selection without
|
|
982
|
+
// resurrecting any shell-specific field this attached launch never
|
|
983
|
+
// uses.
|
|
984
|
+
admission, invocation: invocation ? { selected: invocation.selected } : invocation, invocationId, dispatchGuard,
|
|
985
|
+
// #2895's `attached` mode defaults `stdin: 'closed'` (stdio[0] 'ignore') —
|
|
986
|
+
// correct for its own verification-runner callers, which only read a
|
|
987
|
+
// worker's output. An MCP stdio server needs a live, writable stdin to
|
|
988
|
+
// receive JSON-RPC requests; 'open' is the one additive, minimal change
|
|
989
|
+
// this bridge needs on top of the reviewed #2895 attached launch (no
|
|
990
|
+
// change to #2895's own default behavior for other callers).
|
|
991
|
+
stdin: 'open',
|
|
992
|
+
});
|
|
993
|
+
if (!launch.launched && launch.diagnostic === undefined) {
|
|
994
|
+
return { ...launch, diagnostic: describeGateFailures(launch.failures) };
|
|
995
|
+
}
|
|
996
|
+
return launch;
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
const mcpServerQueues = new Map(); // serverRef -> { promise: Promise<void>, depth: number }
|
|
1000
|
+
/** Test-only observability into the module-level queue map's size — never
|
|
1001
|
+
* consulted by any admission/launch decision. */
|
|
1002
|
+
export function __testOnlyMcpServerQueueCount() { return mcpServerQueues.size; }
|
|
1003
|
+
export { ChildProcessTransport as __testOnlyChildProcessTransport, MCP_TRANSPORT_MAX_BUFFER_BYTES as __testOnlyMcpTransportMaxBufferBytes };
|
|
1004
|
+
export { defaultCallMcpWorker as __testOnlyDefaultCallMcpWorker };
|
|
1005
|
+
|
|
1006
|
+
/** #2892 D2: bounded per-serverRef serialization queue. Refuses outright
|
|
1007
|
+
* (never silently queues unboundedly) once `MAX_MCP_SERVER_QUEUE_DEPTH`
|
|
1008
|
+
* calls are already waiting on this serverRef. */
|
|
1009
|
+
export function withServerQueue(serverRef, task) {
|
|
1010
|
+
const state = mcpServerQueues.get(serverRef) ?? { promise: Promise.resolve(), depth: 0 };
|
|
1011
|
+
if (state.depth >= MAX_MCP_SERVER_QUEUE_DEPTH) {
|
|
1012
|
+
return Promise.resolve({
|
|
1013
|
+
outcome: 'blocked',
|
|
1014
|
+
diagnostic: `worker MCP server '${serverRef}' serialized queue is at capacity (${MAX_MCP_SERVER_QUEUE_DEPTH}); retry once in-flight calls complete`,
|
|
1015
|
+
});
|
|
1016
|
+
}
|
|
1017
|
+
state.depth += 1;
|
|
1018
|
+
mcpServerQueues.set(serverRef, state);
|
|
1019
|
+
const settled = state.promise.then(task, task);
|
|
1020
|
+
state.promise = settled.then(() => {}, () => {});
|
|
1021
|
+
return settled.finally(() => {
|
|
1022
|
+
state.depth -= 1;
|
|
1023
|
+
// F5 (fix-2892-r2): prune this serverRef's queue entry once nothing is
|
|
1024
|
+
// pending against it — otherwise every DISTINCT serverRef this process
|
|
1025
|
+
// ever sees (e.g. 100 one-off completed servers) permanently retains a
|
|
1026
|
+
// Map entry, even though its `depth` has long since returned to 0.
|
|
1027
|
+
// Guarded by object identity so a racing newer `state` already
|
|
1028
|
+
// installed in the map (depth > 0 again) is never deleted out from
|
|
1029
|
+
// under it.
|
|
1030
|
+
if (state.depth === 0 && mcpServerQueues.get(serverRef) === state) {
|
|
1031
|
+
mcpServerQueues.delete(serverRef);
|
|
1032
|
+
}
|
|
1033
|
+
});
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
/** #2892 F5 (fix-2892-r2, formerly D2): a digest over the server definition
|
|
1037
|
+
* fields that matter to the gate-then-launch pipeline (command, args, env
|
|
1038
|
+
* variable NAMES — never values) — used to detect a changed registry
|
|
1039
|
+
* definition between admission and a queued launch, which the pre-existing
|
|
1040
|
+
* `enabled` check alone did not catch (a registry edit that keeps `enabled:
|
|
1041
|
+
* true` but changes `command`/`args`/`env` still launched the STALE,
|
|
1042
|
+
* admission-time server object). Env names are sorted (a set, not an
|
|
1043
|
+
* ordered list) so insertion-order-only changes never false-positive. */
|
|
1044
|
+
function serverDefinitionDigest(server) {
|
|
1045
|
+
return canonicalSchemaDigest({
|
|
1046
|
+
command: server?.command ?? null,
|
|
1047
|
+
args: Array.isArray(server?.args) ? server.args : [],
|
|
1048
|
+
envNames: Object.keys(server?.env ?? {}).sort(),
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/** M3: canonical (sorted-key) digest for comparing a live MCP tool schema
|
|
1053
|
+
* against the #2887 installation-owner-trusted schema object — robust to
|
|
1054
|
+
* the SDK's own zod-validation round trip, which can reorder keys without
|
|
1055
|
+
* changing meaning. Mirrors `workerConfigDigest`'s canonicalization
|
|
1056
|
+
* (src/config/aiwg-config.ts) applied to a schema specifically. */
|
|
1057
|
+
function canonicalSchemaDigest(schema) {
|
|
1058
|
+
const canonical = node => {
|
|
1059
|
+
if (Array.isArray(node)) return node.map(canonical);
|
|
1060
|
+
if (node !== null && typeof node === 'object') {
|
|
1061
|
+
return Object.fromEntries(Object.keys(node).sort().filter(key => node[key] !== undefined).map(key => [key, canonical(node[key])]));
|
|
1062
|
+
}
|
|
1063
|
+
return node ?? null;
|
|
1064
|
+
};
|
|
1065
|
+
return `sha256:${createHash('sha256').update(JSON.stringify(canonical(schema))).digest('hex')}`;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
// Audited native MCP lifecycle, without recovery admission.
|
|
1069
|
+
async function defaultCallMcpWorkerBaseline(mcpBinding, server, options = {}) {
|
|
1070
|
+
const {
|
|
1071
|
+
sessionIsolation, mcpRegistry, allowDegraded = false, config, name = 'mcp-worker', registerControl,
|
|
1072
|
+
memoryMaxGb, mcpSchemas, qualifiedServerDigest, cwd,
|
|
1073
|
+
// See `defaultLaunchShellWorker`: test-only dispatch-gate passthroughs,
|
|
1074
|
+
// always undefined for production registrations.
|
|
1075
|
+
live, readers, thresholdOverlay,
|
|
1076
|
+
} = options;
|
|
1077
|
+
const run = async () => {
|
|
1078
|
+
// D2/F5: recheck the server's own registry qualification immediately
|
|
1079
|
+
// before THIS launch — a serialized/per-instance queue may have held
|
|
1080
|
+
// this call for a while before its turn came up. Checking `enabled`
|
|
1081
|
+
// alone is not requalification: an operator can edit a registry entry's
|
|
1082
|
+
// command/args/env while leaving `enabled: true`, and the ORIGINAL
|
|
1083
|
+
// (admission-time) `server` object passed into this function would
|
|
1084
|
+
// still be the one that gets launched below unless the FRESH
|
|
1085
|
+
// definition is also compared against what was qualified.
|
|
1086
|
+
if (mcpRegistry) {
|
|
1087
|
+
const requalified = await mcpRegistry.get(mcpBinding.serverRef);
|
|
1088
|
+
if (!requalified || requalified.enabled === false) {
|
|
1089
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP server is no longer registered/enabled' };
|
|
1090
|
+
}
|
|
1091
|
+
if (qualifiedServerDigest && serverDefinitionDigest(requalified) !== qualifiedServerDigest) {
|
|
1092
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP server definition changed since qualification; resolve again' };
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
// Resolve fallible dependencies and construct the client before spawning.
|
|
1096
|
+
const { stopOverflowWorker } = await loadRalphExternal('lib/worker-overflow.mjs');
|
|
1097
|
+
const { Client } = await import('@modelcontextprotocol/sdk/client/index.js');
|
|
1098
|
+
const client = new Client({ name: 'aiwg-mcp-worker-invoke', version: '1.0.0' }, { capabilities: {} });
|
|
1099
|
+
// B2: gate + launch the MCP worker SERVER process through the same
|
|
1100
|
+
// pipeline a shell worker uses — never a bare SDK StdioClientTransport.
|
|
1101
|
+
const launch = await defaultLaunchMcpWorkerProcessBaseline(server, { allowDegraded, config, name, cwd, live, readers, thresholdOverlay, ...(memoryMaxGb !== undefined ? { memoryMaxGb } : {}) });
|
|
1102
|
+
if (!launch.launched) {
|
|
1103
|
+
return { outcome: 'blocked', diagnostic: launch.diagnostic ?? 'worker MCP process launch was refused' };
|
|
1104
|
+
}
|
|
1105
|
+
const child = launch.child;
|
|
1106
|
+
const transport = new ChildProcessTransport(child, { terminate: () => stopOverflowWorker(launch) });
|
|
1107
|
+
let disconnected = false;
|
|
1108
|
+
let retainOverflowControl = false;
|
|
1109
|
+
transport.onclose = () => { disconnected = true; };
|
|
1110
|
+
const control = {
|
|
1111
|
+
stop: async () => {
|
|
1112
|
+
if (transport._failure) {
|
|
1113
|
+
// The connection is closed after overflow. Reuse the owned launch's
|
|
1114
|
+
// stop authority, including its termination proof, on explicit cancel.
|
|
1115
|
+
await transport._termination;
|
|
1116
|
+
return stopOverflowWorker(launch);
|
|
1117
|
+
}
|
|
1118
|
+
try {
|
|
1119
|
+
// F2: the ACTUAL in-flight JSON-RPC request id (captured at the
|
|
1120
|
+
// transport layer — see `ChildProcessTransport.send`), never the
|
|
1121
|
+
// caller's own opaque `mcpBinding.requestId`, which the worker has
|
|
1122
|
+
// no way to relate to anything it is actually processing.
|
|
1123
|
+
await client.notification({ method: 'notifications/cancelled', params: { requestId: transport.lastRequestId, reason: 'operator-cancel' } });
|
|
1124
|
+
return true;
|
|
1125
|
+
} catch {
|
|
1126
|
+
try { return signalChild(child, 'SIGTERM'); } catch { return false; }
|
|
1127
|
+
}
|
|
1128
|
+
},
|
|
1129
|
+
};
|
|
1130
|
+
try {
|
|
1131
|
+
registerControl?.(control);
|
|
1132
|
+
await client.connect(transport);
|
|
1133
|
+
// M3: refuse before any brief is sent if the live tool schema has
|
|
1134
|
+
// drifted from what was qualified. Compared CANONICALLY (sorted
|
|
1135
|
+
// keys) on both sides, not via the raw `#2887 mcpToolSchemaDigest`
|
|
1136
|
+
// pin directly — the SDK's own `listTools()` round-trips the wire
|
|
1137
|
+
// JSON through its zod validators, which can legitimately reorder
|
|
1138
|
+
// object keys without changing meaning; a byte-exact digest of the
|
|
1139
|
+
// POST-validation object against a PRE-validation pin would false-
|
|
1140
|
+
// positive on every call. `mcpSchemas` (the #2887 installation-
|
|
1141
|
+
// owner-trusted schema objects, not a client-supplied digest) is the
|
|
1142
|
+
// comparison's source of truth here.
|
|
1143
|
+
const listed = await client.listTools();
|
|
1144
|
+
const tool = listed?.tools?.find(t => t.name === mcpBinding.tool);
|
|
1145
|
+
if (!tool) {
|
|
1146
|
+
return { outcome: 'blocked', diagnostic: 'qualified MCP tool is no longer advertised by the server' };
|
|
1147
|
+
}
|
|
1148
|
+
// F3 (fix-2892-r2): digest the RAW wire response (plain `JSON.parse`,
|
|
1149
|
+
// retained by the transport — see `ChildProcessTransport`'s doc
|
|
1150
|
+
// comment), never `tool.inputSchema`/`tool.outputSchema` from the
|
|
1151
|
+
// zod-validated `listed` object above — zod's own `z.record()`
|
|
1152
|
+
// reconstruction of `properties` silently drops a member literally
|
|
1153
|
+
// named `__proto__`, so two schemas differing only in that member
|
|
1154
|
+
// would otherwise digest identically and the brief would still be
|
|
1155
|
+
// sent. A raw response this transport cannot account for at all
|
|
1156
|
+
// (should be unreachable — `listed` only resolved because SOME
|
|
1157
|
+
// response arrived) fails closed rather than falling back to the
|
|
1158
|
+
// lossy object.
|
|
1159
|
+
const rawToolsList = transport._rawResponses.get(transport.lastRequestId);
|
|
1160
|
+
const rawTool = rawToolsList?.result?.tools?.find(t => t && t.name === mcpBinding.tool);
|
|
1161
|
+
if (!rawTool) {
|
|
1162
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool schema could not be verified against the raw wire response' };
|
|
1163
|
+
}
|
|
1164
|
+
if (mcpSchemas?.input && canonicalSchemaDigest(rawTool.inputSchema) !== canonicalSchemaDigest(mcpSchemas.input)) {
|
|
1165
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool input schema digest drifted from the qualified pin' };
|
|
1166
|
+
}
|
|
1167
|
+
if (mcpSchemas?.output) {
|
|
1168
|
+
// A qualified descriptor that requires an output schema is never
|
|
1169
|
+
// satisfied by a worker that stopped advertising one at all —
|
|
1170
|
+
// missing is drift, not "nothing to compare".
|
|
1171
|
+
if (!rawTool.outputSchema) {
|
|
1172
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool no longer advertises an output schema the qualified descriptor requires' };
|
|
1173
|
+
}
|
|
1174
|
+
if (canonicalSchemaDigest(rawTool.outputSchema) !== canonicalSchemaDigest(mcpSchemas.output)) {
|
|
1175
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool output schema digest drifted from the qualified pin' };
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
const response = await client.callTool({ name: mcpBinding.tool, arguments: mcpBinding.arguments }, undefined, { timeout: mcpBinding.timeoutSeconds * 1000 });
|
|
1179
|
+
return { outcome: 'responded', response };
|
|
1180
|
+
} catch (error) {
|
|
1181
|
+
const isTimeout = error?.code === -32001 || /timeout/i.test(String(error?.message ?? ''));
|
|
1182
|
+
try { await client.notification({ method: 'notifications/cancelled', params: { requestId: transport.lastRequestId, reason: isTimeout ? 'timeout' : 'error' } }); } catch { /* delivery only, never stop proof */ }
|
|
1183
|
+
const termination = transport._failure ? await transport._termination : await stopOverflowWorker(launch);
|
|
1184
|
+
retainOverflowControl = !!transport._failure && termination?.termination !== 'confirmed';
|
|
1185
|
+
return { outcome: disconnected ? 'disconnected' : isTimeout ? 'timeout' : 'error', error: transport._failure ?? error,
|
|
1186
|
+
...(transport._failure ? { termination: termination?.termination ?? 'uncertain',
|
|
1187
|
+
diagnostic: [transport._failure.code, termination?.diagnostic].filter(Boolean).join('; ') } : {}) };
|
|
1188
|
+
} finally {
|
|
1189
|
+
try { if (!retainOverflowControl) registerControl?.(null); }
|
|
1190
|
+
finally { try { await client.close(); } catch { /* already closed */ } }
|
|
1191
|
+
}
|
|
1192
|
+
};
|
|
1193
|
+
return sessionIsolation === 'per-call' ? run() : withServerQueue(mcpBinding.serverRef, run);
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
/** #2915 round 8 (D6): the GATED implementation — reached only through the
|
|
1198
|
+
* router below when `options.admission` is present (never omitted-but-falsy;
|
|
1199
|
+
* `run()`'s own `continueGated` always supplies a truthily-gated admission).
|
|
1200
|
+
* No branch here needs to re-check `admission.gated`; by construction it is
|
|
1201
|
+
* always true for the duration of this function. */
|
|
1202
|
+
async function defaultCallMcpWorkerGated(mcpBinding, server, options = {}) {
|
|
1203
|
+
const {
|
|
1204
|
+
sessionIsolation, mcpRegistry, allowDegraded = false, config, name = 'mcp-worker', registerControl,
|
|
1205
|
+
memoryMaxGb, mcpSchemas, qualifiedServerDigest, cwd, admission, invocation, invocationId,
|
|
1206
|
+
// #2915 round 8 (D5): the ONE absolute deadline `run()` computed once,
|
|
1207
|
+
// unchanged. Never re-derived here from `mcpBinding.timeoutSeconds` or
|
|
1208
|
+
// `admission.token.mintedAtMs` (that double-subtracted preparation
|
|
1209
|
+
// time — round 8 critique, D5/review 7 finding 4's last bullet).
|
|
1210
|
+
deadlineAtMs = null,
|
|
1211
|
+
// #2915 round 8 review 8 (C3): optional cancellation predicate, checked
|
|
1212
|
+
// alongside every deadline check below (connect/listTools/callTool) and
|
|
1213
|
+
// forwarded into `ChildProcessTransport` for the write boundary. Also
|
|
1214
|
+
// folded into the `dispatchGuard` passed to the launch itself so
|
|
1215
|
+
// cancellation during pre-spawn preparation prevents the spawn too.
|
|
1216
|
+
isCancelled,
|
|
1217
|
+
dispatchGuard,
|
|
1218
|
+
observeStop,
|
|
1219
|
+
// See `defaultLaunchShellWorker`: test-only dispatch-gate passthroughs,
|
|
1220
|
+
// always undefined for production registrations.
|
|
1221
|
+
live, readers, thresholdOverlay,
|
|
1222
|
+
} = options;
|
|
1223
|
+
const run = async () => {
|
|
1224
|
+
try {
|
|
1225
|
+
// D2/F5: recheck the server's own registry qualification immediately
|
|
1226
|
+
// before THIS launch — a serialized/per-instance queue may have held
|
|
1227
|
+
// this call for a while before its turn came up. Checking `enabled`
|
|
1228
|
+
// alone is not requalification: an operator can edit a registry entry's
|
|
1229
|
+
// command/args/env while leaving `enabled: true`, and the ORIGINAL
|
|
1230
|
+
// (admission-time) `server` object passed into this function would
|
|
1231
|
+
// still be the one that gets launched below unless the FRESH
|
|
1232
|
+
// definition is also compared against what was qualified.
|
|
1233
|
+
if (mcpRegistry) {
|
|
1234
|
+
const requalified = await mcpRegistry.get(mcpBinding.serverRef);
|
|
1235
|
+
if (!requalified || requalified.enabled === false) {
|
|
1236
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP server is no longer registered/enabled', dispatchState: 'never-began' };
|
|
1237
|
+
}
|
|
1238
|
+
if (qualifiedServerDigest && serverDefinitionDigest(requalified) !== qualifiedServerDigest) {
|
|
1239
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP server definition changed since qualification; resolve again', dispatchState: 'never-began' };
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
// #2915 review 5 ("a second worker over unresolved execution") /
|
|
1243
|
+
// round 8 (D5): a serialized/per-instance queue (D2, below) may have
|
|
1244
|
+
// held this call long enough that the admitted attempt's deadline —
|
|
1245
|
+
// the ONE absolute `deadlineAtMs` `run()` computed once, which
|
|
1246
|
+
// already folds in the canary's own bounded window (min(task
|
|
1247
|
+
// deadline, admission+300s)) when applicable — has since passed. For
|
|
1248
|
+
// a canary, the breaker's own sweep reopens the scope and installs a
|
|
1249
|
+
// blocking marker the moment anything next calls `admit()`/`settle()`
|
|
1250
|
+
// on it (possibly a sibling request), regardless of whether THIS
|
|
1251
|
+
// queued call ever actually launches. Revalidate immediately before
|
|
1252
|
+
// launch, matching the registry-qualification recheck just above:
|
|
1253
|
+
// an expired admission must never launch. Falls back to the
|
|
1254
|
+
// canary-only `admission.token.deadlineMs` check when no caller
|
|
1255
|
+
// supplies `deadlineAtMs` (back-compat for any caller that does
|
|
1256
|
+
// not thread it through) — review 7 finding 4: "an ordinary request
|
|
1257
|
+
// queued beyond its 900-second budget still spawned and completed"
|
|
1258
|
+
// is exactly the plain (non-canary) task-deadline case this widens
|
|
1259
|
+
// to cover.
|
|
1260
|
+
const nowForExpiry = admission.breaker?.nowMs?.() ?? Date.now();
|
|
1261
|
+
const admissionExpired = deadlineAtMs !== null
|
|
1262
|
+
? nowForExpiry >= deadlineAtMs
|
|
1263
|
+
: !!(admission.token && nowForExpiry > admission.token.deadlineMs);
|
|
1264
|
+
if (admissionExpired) {
|
|
1265
|
+
return { outcome: 'blocked', diagnostic: 'worker admission expired while queued; recovery scope may have reopened', dispatchState: 'never-began' };
|
|
1266
|
+
}
|
|
1267
|
+
// #2915 round 8 review 8 (C3): cancellation requested while this call
|
|
1268
|
+
// was queued (or during the checks above) must prevent the spawn too,
|
|
1269
|
+
// exactly like an expired deadline — never dispatch after the operator
|
|
1270
|
+
// already asked to stop.
|
|
1271
|
+
if (typeof isCancelled === 'function' && isCancelled()) {
|
|
1272
|
+
return { outcome: 'blocked', diagnostic: 'worker dispatch refused: cancelled before process start', dispatchState: 'never-began' };
|
|
1273
|
+
}
|
|
1274
|
+
} catch (error) {
|
|
1275
|
+
// #2915 review 6 (finding 2, "leaked capacity"): a thrown pre-spawn
|
|
1276
|
+
// requalification/expiry check — nothing has been dispatched yet — must
|
|
1277
|
+
// settle the same way an ordinary blocked requalification does, never
|
|
1278
|
+
// propagate as an exception. `continueGated`'s own caller sets
|
|
1279
|
+
// `spawnAttempted` BEFORE invoking this function (it cannot know in
|
|
1280
|
+
// advance whether a real spawn is about to happen), so letting this
|
|
1281
|
+
// throw escape would be treated as genuinely post-spawn uncertainty
|
|
1282
|
+
// and leak the registry's run-slot forever — the exact gap a
|
|
1283
|
+
// native-MCP requalification error (`mcpRegistry.get` throwing rather
|
|
1284
|
+
// than returning falsy/disabled) left open.
|
|
1285
|
+
return { outcome: 'blocked', diagnostic: `worker MCP pre-spawn requalification failed: ${error?.message ?? String(error)}`, dispatchState: 'never-began' };
|
|
1286
|
+
}
|
|
1287
|
+
let stopOverflowWorker;
|
|
1288
|
+
let client;
|
|
1289
|
+
try {
|
|
1290
|
+
// Resolve fallible dependencies and construct the client before spawning.
|
|
1291
|
+
({ stopOverflowWorker } = await loadRalphExternal('lib/worker-overflow.mjs'));
|
|
1292
|
+
const { Client } = await import('@modelcontextprotocol/sdk/client/index.js');
|
|
1293
|
+
client = new Client({ name: 'aiwg-mcp-worker-invoke', version: '1.0.0' }, { capabilities: {} });
|
|
1294
|
+
// Async setup may consume the remaining admitted canary budget.
|
|
1295
|
+
if (admission?.gated && admission.token
|
|
1296
|
+
&& (admission.breaker?.nowMs?.() ?? Date.now()) > admission.token.deadlineMs) {
|
|
1297
|
+
return { outcome: 'blocked', diagnostic: 'worker admission expired while queued; recovery scope may have reopened' };
|
|
1298
|
+
}
|
|
1299
|
+
} catch (error) {
|
|
1300
|
+
return { outcome: 'blocked', diagnostic: `worker MCP pre-spawn setup failed: ${error?.message ?? String(error)}`, dispatchState: 'never-began' };
|
|
1301
|
+
}
|
|
1302
|
+
// B2: gate + launch the MCP worker SERVER process through the same
|
|
1303
|
+
// pipeline a shell worker uses — never a bare SDK StdioClientTransport.
|
|
1304
|
+
const launch = await defaultLaunchMcpWorkerProcess(server, { allowDegraded, config, name, cwd, live, readers, thresholdOverlay, admission, invocation, invocationId, dispatchGuard, ...(memoryMaxGb !== undefined ? { memoryMaxGb } : {}) });
|
|
1305
|
+
if (!launch.launched) {
|
|
1306
|
+
// #2915 round 8 review 8 (C2): a failed/timed-out `systemd-run`
|
|
1307
|
+
// submission does not prove nothing dispatched — carry `launch`'s own
|
|
1308
|
+
// `dispatchState: 'submission-uncertain'` forward so the caller's stop
|
|
1309
|
+
// evidence resolution never treats this as a proven zero-dispatch
|
|
1310
|
+
// refusal. Every other `!launched` case (gate blocked, dispatch-commit
|
|
1311
|
+
// hook abort, degraded-without-allowDegraded) is a proven pre-spawn
|
|
1312
|
+
// refusal: `never-began`.
|
|
1313
|
+
return { outcome: 'blocked', diagnostic: launch.diagnostic ?? 'worker MCP process launch was refused',
|
|
1314
|
+
dispatchState: launch.dispatchState === 'submission-uncertain' ? 'submission-uncertain' : 'never-began',
|
|
1315
|
+
isolationMode: launch.mode ?? null, unitName: launch.unit ?? null, child: launch.child ?? null };
|
|
1316
|
+
}
|
|
1317
|
+
const child = launch.child;
|
|
1318
|
+
// #2915 round 8 review 8 (C2): from here on a real process-start call
|
|
1319
|
+
// happened — every outcome below (including a post-connect 'blocked'
|
|
1320
|
+
// schema-drift refusal) carries this evidence forward so settlement
|
|
1321
|
+
// never manufactures a confirmed stop from an outcome label alone.
|
|
1322
|
+
const isolationEvidence = { dispatchState: 'dispatched', isolationMode: launch.mode ?? null, unitName: launch.unit ?? null, child: child ?? null };
|
|
1323
|
+
// #2915 review 6 (finding 3, "one absolute deadline") / round 8 (D5):
|
|
1324
|
+
// `deadlineAtMs` is the ONE fixed absolute wall-clock deadline `run()`
|
|
1325
|
+
// computed once at admission — never re-derived here. `nowMsFn` is
|
|
1326
|
+
// needed before constructing the transport below, so it moves up from
|
|
1327
|
+
// its former position just before the `connect()` call.
|
|
1328
|
+
const nowMsFn = () => admission.breaker?.nowMs?.() ?? Date.now();
|
|
1329
|
+
const absoluteDeadlineMs = deadlineAtMs;
|
|
1330
|
+
// #2915 review 6 (finding 6b, "changed disabled behavior"): the 10 MiB
|
|
1331
|
+
// unterminated-frame transport bound AND the deadline write-boundary
|
|
1332
|
+
// check (D5) are both enforced unconditionally here — this function is
|
|
1333
|
+
// reached only when gated (see `ChildProcessTransport`'s own doc
|
|
1334
|
+
// comment).
|
|
1335
|
+
const transport = new ChildProcessTransport(child, { terminate: () => stopOverflowWorker(launch),
|
|
1336
|
+
deadlineAtMs: absoluteDeadlineMs, now: nowMsFn, isCancelled });
|
|
1337
|
+
let disconnected = false;
|
|
1338
|
+
let retainOverflowControl = false;
|
|
1339
|
+
transport.onclose = () => { disconnected = true; };
|
|
1340
|
+
const control = {
|
|
1341
|
+
stop: async () => {
|
|
1342
|
+
if (transport._failure) {
|
|
1343
|
+
// The connection is closed after overflow. Reuse the owned launch's
|
|
1344
|
+
// stop authority, including its termination proof, on explicit cancel.
|
|
1345
|
+
await transport._termination;
|
|
1346
|
+
return stopOverflowWorker(launch);
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
// F2: the ACTUAL in-flight JSON-RPC request id (captured at the
|
|
1350
|
+
// transport layer — see `ChildProcessTransport.send`), never the
|
|
1351
|
+
// caller's own opaque `mcpBinding.requestId`, which the worker has
|
|
1352
|
+
// no way to relate to anything it is actually processing.
|
|
1353
|
+
await client.notification({ method: 'notifications/cancelled', params: { requestId: transport.lastRequestId, reason: 'operator-cancel' } });
|
|
1354
|
+
return true;
|
|
1355
|
+
} catch {
|
|
1356
|
+
try { return signalChild(child, 'SIGTERM'); } catch { return false; }
|
|
1357
|
+
}
|
|
1358
|
+
},
|
|
1359
|
+
};
|
|
1360
|
+
// #2915 review 6 (finding 3, "one absolute deadline"): derive EVERY
|
|
1361
|
+
// stage's timeout (connect/initialize, list, call) from ONE fixed
|
|
1362
|
+
// absolute wall-clock deadline, recomputed fresh immediately before
|
|
1363
|
+
// each stage from the time ACTUALLY remaining — never a per-stage
|
|
1364
|
+
// default (the SDK's own 60s `DEFAULT_REQUEST_TIMEOUT_MSEC`) and never a
|
|
1365
|
+
// budget computed once before a serialized/per-instance queue wait that
|
|
1366
|
+
// can itself consume most of it. Reproduced: a native request queued
|
|
1367
|
+
// 250s into a 300s canary window still received a fresh 300-SECOND SDK
|
|
1368
|
+
// call timeout with only 50s actually remaining, while connect/list each
|
|
1369
|
+
// separately spent up to another 60s on top of that — vastly overrunning
|
|
1370
|
+
// the breaker's own canary window.
|
|
1371
|
+
//
|
|
1372
|
+
// #2915 round 8 (D5): `remaining <= 0` REFUSES outright (client-side)
|
|
1373
|
+
// rather than merely passing `timeout: 0` through to the SDK and
|
|
1374
|
+
// trusting its own timer delivery — review 7 finding 4: "at exactly the
|
|
1375
|
+
// canary deadline, connect/list/call all received timeout 0, sent
|
|
1376
|
+
// requests, and completed successfully."
|
|
1377
|
+
// #2915 round 8 review 8 (C3): checked alongside every deadline check
|
|
1378
|
+
// below — cancellation requested while a request is in flight (or
|
|
1379
|
+
// between stages) must prevent the NEXT request, never retroactively
|
|
1380
|
+
// the one already sent.
|
|
1381
|
+
const cancelledNow = () => typeof isCancelled === 'function' && isCancelled();
|
|
1382
|
+
try {
|
|
1383
|
+
registerControl?.(control);
|
|
1384
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) throw Object.assign(new Error('worker MCP deadline already passed before connect'), { code: -32001 });
|
|
1385
|
+
if (cancelledNow()) throw Object.assign(new Error('worker MCP call cancelled before connect'), { code: -32800 });
|
|
1386
|
+
if (absoluteDeadlineMs !== null) {
|
|
1387
|
+
await client.connect(transport, { timeout: Math.max(0, absoluteDeadlineMs - nowMsFn()) });
|
|
1388
|
+
} else {
|
|
1389
|
+
await client.connect(transport);
|
|
1390
|
+
}
|
|
1391
|
+
// M3: refuse before any brief is sent if the live tool schema has
|
|
1392
|
+
// drifted from what was qualified. Compared CANONICALLY (sorted
|
|
1393
|
+
// keys) on both sides, not via the raw `#2887 mcpToolSchemaDigest`
|
|
1394
|
+
// pin directly — the SDK's own `listTools()` round-trips the wire
|
|
1395
|
+
// JSON through its zod validators, which can legitimately reorder
|
|
1396
|
+
// object keys without changing meaning; a byte-exact digest of the
|
|
1397
|
+
// POST-validation object against a PRE-validation pin would false-
|
|
1398
|
+
// positive on every call. `mcpSchemas` (the #2887 installation-
|
|
1399
|
+
// owner-trusted schema objects, not a client-supplied digest) is the
|
|
1400
|
+
// comparison's source of truth here.
|
|
1401
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) {
|
|
1402
|
+
throw Object.assign(new Error('worker MCP deadline already passed before listTools'), { code: -32001 });
|
|
1403
|
+
}
|
|
1404
|
+
if (cancelledNow()) throw Object.assign(new Error('worker MCP call cancelled before listTools'), { code: -32800 });
|
|
1405
|
+
const listed = absoluteDeadlineMs !== null
|
|
1406
|
+
? await client.listTools(undefined, { timeout: Math.max(0, absoluteDeadlineMs - nowMsFn()) })
|
|
1407
|
+
: await client.listTools();
|
|
1408
|
+
const tool = listed?.tools?.find(t => t.name === mcpBinding.tool);
|
|
1409
|
+
if (!tool) {
|
|
1410
|
+
return { outcome: 'blocked', diagnostic: 'qualified MCP tool is no longer advertised by the server', ...isolationEvidence };
|
|
1411
|
+
}
|
|
1412
|
+
// F3 (fix-2892-r2): digest the RAW wire response (plain `JSON.parse`,
|
|
1413
|
+
// retained by the transport — see `ChildProcessTransport`'s doc
|
|
1414
|
+
// comment), never `tool.inputSchema`/`tool.outputSchema` from the
|
|
1415
|
+
// zod-validated `listed` object above — zod's own `z.record()`
|
|
1416
|
+
// reconstruction of `properties` silently drops a member literally
|
|
1417
|
+
// named `__proto__`, so two schemas differing only in that member
|
|
1418
|
+
// would otherwise digest identically and the brief would still be
|
|
1419
|
+
// sent. A raw response this transport cannot account for at all
|
|
1420
|
+
// (should be unreachable — `listed` only resolved because SOME
|
|
1421
|
+
// response arrived) fails closed rather than falling back to the
|
|
1422
|
+
// lossy object.
|
|
1423
|
+
const rawToolsList = transport._rawResponses.get(transport.lastRequestId);
|
|
1424
|
+
const rawTool = rawToolsList?.result?.tools?.find(t => t && t.name === mcpBinding.tool);
|
|
1425
|
+
if (!rawTool) {
|
|
1426
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool schema could not be verified against the raw wire response', ...isolationEvidence };
|
|
1427
|
+
}
|
|
1428
|
+
if (mcpSchemas?.input && canonicalSchemaDigest(rawTool.inputSchema) !== canonicalSchemaDigest(mcpSchemas.input)) {
|
|
1429
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool input schema digest drifted from the qualified pin', ...isolationEvidence };
|
|
1430
|
+
}
|
|
1431
|
+
if (mcpSchemas?.output) {
|
|
1432
|
+
// A qualified descriptor that requires an output schema is never
|
|
1433
|
+
// satisfied by a worker that stopped advertising one at all —
|
|
1434
|
+
// missing is drift, not "nothing to compare".
|
|
1435
|
+
if (!rawTool.outputSchema) {
|
|
1436
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool no longer advertises an output schema the qualified descriptor requires', ...isolationEvidence };
|
|
1437
|
+
}
|
|
1438
|
+
if (canonicalSchemaDigest(rawTool.outputSchema) !== canonicalSchemaDigest(mcpSchemas.output)) {
|
|
1439
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool output schema digest drifted from the qualified pin', ...isolationEvidence };
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) {
|
|
1443
|
+
throw Object.assign(new Error('worker MCP deadline already passed before callTool'), { code: -32001 });
|
|
1444
|
+
}
|
|
1445
|
+
if (cancelledNow()) throw Object.assign(new Error('worker MCP call cancelled before callTool'), { code: -32800 });
|
|
1446
|
+
const callTimeoutMs = absoluteDeadlineMs !== null
|
|
1447
|
+
? Math.max(0, absoluteDeadlineMs - nowMsFn())
|
|
1448
|
+
: mcpBinding.timeoutSeconds * 1000;
|
|
1449
|
+
const response = await client.callTool({ name: mcpBinding.tool, arguments: mcpBinding.arguments }, undefined, { timeout: callTimeoutMs });
|
|
1450
|
+
// #2915 round 8 (D5): "a response arriving after deadlineAt is a
|
|
1451
|
+
// timeout regardless of SDK timer delivery" — review 7 finding 4: "a
|
|
1452
|
+
// response arriving after the deadline could still return completed."
|
|
1453
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) {
|
|
1454
|
+
return { outcome: 'timeout', error: new Error('worker MCP response arrived after the deadline'), ...isolationEvidence };
|
|
1455
|
+
}
|
|
1456
|
+
return { outcome: 'responded', response, ...isolationEvidence };
|
|
1457
|
+
} catch (error) {
|
|
1458
|
+
const isTimeout = error?.code === -32001 || /timeout/i.test(String(error?.message ?? ''));
|
|
1459
|
+
const isCancelledError = error?.code === -32800;
|
|
1460
|
+
try { await client.notification({ method: 'notifications/cancelled', params: { requestId: transport.lastRequestId, reason: isTimeout ? 'timeout' : isCancelledError ? 'cancelled' : 'error' } }); } catch { /* delivery only, never stop proof */ }
|
|
1461
|
+
const termination = transport._failure ? await transport._termination : await stopOverflowWorker(launch);
|
|
1462
|
+
retainOverflowControl = !!transport._failure && termination?.termination !== 'confirmed';
|
|
1463
|
+
return { outcome: disconnected ? 'disconnected' : isTimeout ? 'timeout' : 'error', error: transport._failure ?? error,
|
|
1464
|
+
...isolationEvidence,
|
|
1465
|
+
...(transport._failure ? { termination: termination?.termination ?? 'uncertain',
|
|
1466
|
+
diagnostic: [transport._failure.code, termination?.diagnostic].filter(Boolean).join('; ') } : {}) };
|
|
1467
|
+
} finally {
|
|
1468
|
+
try { if (!retainOverflowControl) registerControl?.(null); }
|
|
1469
|
+
finally {
|
|
1470
|
+
try {
|
|
1471
|
+
const stopping = client.close();
|
|
1472
|
+
observeStop?.(stopping, isolationEvidence);
|
|
1473
|
+
await stopping;
|
|
1474
|
+
} catch { /* already closed */ }
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
};
|
|
1478
|
+
return sessionIsolation === 'per-call' ? run() : withServerQueue(mcpBinding.serverRef, run);
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
/** #2915 round 8 (D6): thin router, mirroring
|
|
1482
|
+
* `defaultLaunchShellWorker`/`defaultLaunchMcpWorkerProcess`'s identical
|
|
1483
|
+
* pattern — `options.admission` is only ever present when this call is
|
|
1484
|
+
* actually gated (`continueGated` always supplies one; `continueBaseline`
|
|
1485
|
+
* never includes the key at all). Absent routes to the literal baseline
|
|
1486
|
+
* reimplementation; present routes to the gated implementation. */
|
|
1487
|
+
async function defaultCallMcpWorker(mcpBinding, server, options = {}) {
|
|
1488
|
+
if (options.admission === undefined) {
|
|
1489
|
+
return defaultCallMcpWorkerBaseline(mcpBinding, server, options);
|
|
1490
|
+
}
|
|
1491
|
+
return defaultCallMcpWorkerGated(mcpBinding, server, options);
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
// ===========================================================================
|
|
1495
|
+
// Health hook extension point (documented, no dependency on #2896 code).
|
|
1496
|
+
// ===========================================================================
|
|
1497
|
+
|
|
1498
|
+
/**
|
|
1499
|
+
* `healthHook`, when supplied to `registerWorkerInvokeToolset`, is called as
|
|
1500
|
+
* `healthHook(entry)` while building a `status` response and may return a
|
|
1501
|
+
* richer health object to merge in. This bridge reports only process/
|
|
1502
|
+
* transport-observed state (`entry.health` below); richer liveness signals
|
|
1503
|
+
* (quiet/stalled/hung classification) are intentionally out of scope here —
|
|
1504
|
+
* #2896 (or any later health-observer work) can supply them through this
|
|
1505
|
+
* hook without this module depending on that package.
|
|
1506
|
+
*/
|
|
1507
|
+
function publicStatus(entry, healthHook) {
|
|
1508
|
+
const base = {
|
|
1509
|
+
invocationId: entry.invocationId,
|
|
1510
|
+
requestId: entry.requestId,
|
|
1511
|
+
status: entry.status ?? (entry.terminalAt === null ? 'running' : 'uncertain'),
|
|
1512
|
+
termination: entry.termination ?? 'running',
|
|
1513
|
+
cancelRequested: !!entry.cancelRequested,
|
|
1514
|
+
cancelDelivered: !!entry.cancelDelivered,
|
|
1515
|
+
isolationMode: entry.isolationMode ?? null,
|
|
1516
|
+
createdAt: entry.createdAt,
|
|
1517
|
+
terminalAt: entry.terminalAt,
|
|
1518
|
+
diagnostics: entry.diagnostics ?? [],
|
|
1519
|
+
result: entry.result ?? null,
|
|
1520
|
+
health: entry.health ?? null,
|
|
1521
|
+
};
|
|
1522
|
+
if (typeof healthHook === 'function') {
|
|
1523
|
+
try {
|
|
1524
|
+
const extra = healthHook(entry);
|
|
1525
|
+
if (extra && typeof extra === 'object') base.health = { ...base.health, ...extra };
|
|
1526
|
+
} catch { /* extension-point failures never break status reporting */ }
|
|
1527
|
+
}
|
|
1528
|
+
return base;
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
// ===========================================================================
|
|
1532
|
+
// Tool registration.
|
|
1533
|
+
// ===========================================================================
|
|
1534
|
+
|
|
1535
|
+
const inputSchema = z.object({
|
|
1536
|
+
// #2915 round 8 (D4): `reconcile_stopped` is an explicit operator-
|
|
1537
|
+
// authorized reconciliation operation on this SAME handler (never a
|
|
1538
|
+
// mailbox file, never a new breaker) — see `reconcileStopped` below.
|
|
1539
|
+
operation: z.enum(['run', 'status', 'cancel', 'reconcile_stopped', 'reset_recovery_scope']),
|
|
1540
|
+
invocation_id: z.string().uuid().optional(),
|
|
1541
|
+
scope_id: z.string().min(1).max(512).optional(),
|
|
1542
|
+
request_id: z.string().min(1).max(128).regex(/^[a-zA-Z0-9_.:-]+$/).optional(),
|
|
1543
|
+
role: z.enum(['efficiency', 'coding', 'reasoning']).optional(),
|
|
1544
|
+
task: z.string().min(1).max(256).optional(),
|
|
1545
|
+
group: z.string().min(1).regex(/^[a-z][a-z0-9-]{0,62}$/).optional(),
|
|
1546
|
+
brief: z.string().min(1).optional(),
|
|
1547
|
+
workdir: z.string().min(1).optional(),
|
|
1548
|
+
final_output: z.string().min(1).optional(),
|
|
1549
|
+
transport: z.enum(['shell', 'mcp']).default('shell'),
|
|
1550
|
+
// #2915 step 4 (retry linkage): conductor-declared predecessor invocation
|
|
1551
|
+
// id. Accepted and shape-validated here; `run()` below refuses explicitly
|
|
1552
|
+
// rather than silently ignoring it (no server-owned capture-root
|
|
1553
|
+
// configuration exists yet for the MCP shell-launch path — see
|
|
1554
|
+
// session-launcher.mjs `verifyRetryPredecessor`, which the CLI entry
|
|
1555
|
+
// point already uses). Omission is unrelated work, as always.
|
|
1556
|
+
retry_of: z.string().uuid().optional(),
|
|
1557
|
+
}).strict();
|
|
1558
|
+
|
|
1559
|
+
/**
|
|
1560
|
+
* #2892 B1: `options.projectRoot` is REQUIRED — the caller (`server.mjs`,
|
|
1561
|
+
* via `subsystems.mjs`'s `registerOptInToolsets`) must resolve the served
|
|
1562
|
+
* project root itself (e.g. `findProjectRoot()`) before registration. This
|
|
1563
|
+
* function never falls back to `process.cwd()`, and refuses to register the
|
|
1564
|
+
* `worker-invoke` tool at all when no explicit root is supplied — a
|
|
1565
|
+
* worker-invoke tool with no real root to bind to is a bug, not a degraded
|
|
1566
|
+
* mode. `registerOptInToolsets` awaits this function's returned Promise, so
|
|
1567
|
+
* callers that need the tool registered before proceeding must `await` it.
|
|
1568
|
+
*/
|
|
1569
|
+
export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
1570
|
+
const allowDegraded = options.allowDegraded ?? resolveServerAllowDegraded();
|
|
1571
|
+
if (!options.projectRoot) {
|
|
1572
|
+
console.error("[AIWG MCP] 'workers' toolset refused: no explicit served project root was supplied at registration time (never falls back to process.cwd()).");
|
|
1573
|
+
return;
|
|
1574
|
+
}
|
|
1575
|
+
const servedRootReal = await resolveServedProjectRoot(options.projectRoot);
|
|
1576
|
+
await sweepWorkerStdin(options.stdinRuntimeRoot);
|
|
1577
|
+
// Pinned once, here, at registration — never re-resolved per request, and
|
|
1578
|
+
// never re-derived from `process.cwd()` (#2892 B1).
|
|
1579
|
+
const servedRoot = async () => servedRootReal;
|
|
1580
|
+
// #2892 F1: resolved once, at registration, same posture as the served
|
|
1581
|
+
// root itself.
|
|
1582
|
+
const workerWorktreesRootReal = await resolveWorkerWorktreesRoot(servedRootReal, options.workerWorktreesRoot);
|
|
1583
|
+
const listWorktreesImpl = options.listWorktrees ?? defaultListGitWorktrees;
|
|
1584
|
+
const maxActive = options.maxActive ?? await resolveDefaultMaxActive(servedRootReal);
|
|
1585
|
+
const ttlMs = options.ttlMs ?? resolveServerTerminalTtlMs();
|
|
1586
|
+
const registry = options.registry ?? createInvocationRegistry({ now: options.now, maxActive, ttlMs });
|
|
1587
|
+
const loadWorkerInputsImpl = options.loadWorkerInputs ?? defaultLoadWorkerInputs;
|
|
1588
|
+
const launchShellWorkerImpl = options.launchShellWorker ?? defaultLaunchShellWorker;
|
|
1589
|
+
const callMcpWorkerImpl = options.callMcpWorker ?? defaultCallMcpWorker;
|
|
1590
|
+
// Both continuations use the audited observer and hardened stop controls.
|
|
1591
|
+
const waitForUnitTerminationOverride = options.waitForUnitTermination;
|
|
1592
|
+
// #2915 review 5 ("a second worker over unresolved execution"): the
|
|
1593
|
+
// degraded-isolation shell wait path below shares the launcher's own
|
|
1594
|
+
// bounded SIGTERM-then-SIGKILL escalation (`terminateChildWithGrace`)
|
|
1595
|
+
// rather than sending an unobserved SIGTERM alone.
|
|
1596
|
+
const terminateChildImpl = options.terminateChildWithGrace
|
|
1597
|
+
?? (async child => (await loadRalphExternal('session-launcher.mjs')).terminateChildWithGrace(child));
|
|
1598
|
+
const mcpWorkerMemoryMaxGb = options.mcpWorkerMemoryMaxGb; // undefined -> defaultLaunchMcpWorkerProcess's own fallback
|
|
1599
|
+
// Test-only: `planDirectWorkerLaunch`'s own dispatch-gate passthroughs
|
|
1600
|
+
// (`checkWorkerDispatch({ live, readers, thresholdOverlay })`), forwarded
|
|
1601
|
+
// through to the default shell/mcp launchers below so the REAL
|
|
1602
|
+
// gate-then-launch pipeline still runs against fixed synthetic host
|
|
1603
|
+
// signals instead of the current host's actual load/memory/PSI/disk.
|
|
1604
|
+
// Production registrations never set `options.gateInputs` — omitted,
|
|
1605
|
+
// every field here is `undefined` and the gate samples the real host.
|
|
1606
|
+
const gateInputs = options.gateInputs ?? {};
|
|
1607
|
+
const healthHook = options.healthHook;
|
|
1608
|
+
const now = options.now ?? (() => new Date());
|
|
1609
|
+
const repoAccess = options.repoAccess; // injected lazily below when absent
|
|
1610
|
+
const repoAccessModule = async () => repoAccess ?? loadRepoAccessModule();
|
|
1611
|
+
|
|
1612
|
+
async function validatedRequest(input) {
|
|
1613
|
+
const root = await servedRoot();
|
|
1614
|
+
const access = await repoAccessModule();
|
|
1615
|
+
const brief = await assertServedPath(root, input.brief, 'brief');
|
|
1616
|
+
await assertRepoAccessAllowed(root, brief, 'read', access);
|
|
1617
|
+
const workdir = await assertServedPath(root, input.workdir, 'workdir');
|
|
1618
|
+
await assertWorkdirIsConductorWorktree(root, workdir, { worktreesRoot: workerWorktreesRootReal, listWorktrees: listWorktreesImpl });
|
|
1619
|
+
await assertRepoAccessAllowed(root, workdir, 'write', access);
|
|
1620
|
+
const finalOutput = await assertServedPath(root, input.final_output, 'final_output');
|
|
1621
|
+
assertWithinWorkdir(workdir, finalOutput);
|
|
1622
|
+
await assertRepoAccessAllowed(root, finalOutput, 'write', access);
|
|
1623
|
+
return { role: input.role, task: input.task, group: input.group, brief, workdir, final_output: finalOutput, transport: input.transport };
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
/**
|
|
1627
|
+
* #2915 round 8 (D5, review 7 finding 9): "MCP discards the settlement
|
|
1628
|
+
* promise and its diagnostic." `runInner` is the full implementation
|
|
1629
|
+
* (unchanged below); `captureSettlementPromise`, when called, hands this
|
|
1630
|
+
* run's ONE cached ledger-settlement promise (`settleDirectWorker`'s own
|
|
1631
|
+
* return value, see `settleRecovery` below) up to `run()`'s thin wrapper,
|
|
1632
|
+
* which awaits it at this function's one true return boundary and folds
|
|
1633
|
+
* any refund-failure diagnostic it carries into the result — instead of
|
|
1634
|
+
* letting that promise, and whatever it resolves to, simply vanish.
|
|
1635
|
+
*/
|
|
1636
|
+
async function run(input) {
|
|
1637
|
+
let settlementPromise = null;
|
|
1638
|
+
const transitions = [];
|
|
1639
|
+
let result = await runInner(input, promise => { settlementPromise = promise; }, event => {
|
|
1640
|
+
transitions.push(event.summary);
|
|
1641
|
+
(options.onRecoveryTransition ?? (event => console.error(event.summary)))(event);
|
|
1642
|
+
});
|
|
1643
|
+
if (settlementPromise) {
|
|
1644
|
+
let diagnostic = null;
|
|
1645
|
+
try { diagnostic = await settlementPromise; } catch { /* never fail a run on diagnostic retrieval */ }
|
|
1646
|
+
if (diagnostic && result?.invocationId) {
|
|
1647
|
+
const current = registry.get(result.invocationId);
|
|
1648
|
+
if (current) {
|
|
1649
|
+
result = registry.update(result.invocationId, { diagnostics: [...(current.diagnostics ?? []), diagnostic] });
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
if (transitions.length && result?.invocationId) {
|
|
1654
|
+
result = registry.update(result.invocationId, { diagnostics: [...(result.diagnostics ?? []), ...transitions] });
|
|
1655
|
+
}
|
|
1656
|
+
return result;
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
async function runInner(input, captureSettlementPromise, onTransition) {
|
|
1660
|
+
// M1: request_id is required, and admission (capacity + replay check +
|
|
1661
|
+
// reservation) is entirely synchronous — no `await` between the checks
|
|
1662
|
+
// and `registry.create()` — so no concurrent `run()` call can observe a
|
|
1663
|
+
// stale "capacity available" / "request_id unused" read.
|
|
1664
|
+
if (!input.request_id) throw new Error('Worker invocation requires request_id (identity-bound atomic admission)');
|
|
1665
|
+
if (!input.role || !input.task || !input.group || !input.brief || !input.workdir || !input.final_output) {
|
|
1666
|
+
throw new Error('Incomplete worker invocation');
|
|
1667
|
+
}
|
|
1668
|
+
// #2915 step 4: `retry_of` has no server-owned capture-root
|
|
1669
|
+
// configuration to verify a predecessor against on this bridge yet
|
|
1670
|
+
// (see the schema comment above) — refuse explicitly, before any
|
|
1671
|
+
// capacity is reserved, rather than silently accepting and ignoring a
|
|
1672
|
+
// conductor's retry declaration. Predates round 8 and applies
|
|
1673
|
+
// unconditionally (not a gating concern): `retry_of` support is simply
|
|
1674
|
+
// not implemented on this bridge yet, for gated or not-gated callers
|
|
1675
|
+
// alike, so origin/main's own schema (which has no `retry_of` field at
|
|
1676
|
+
// all) never had an equivalent case to diverge from.
|
|
1677
|
+
if (input.retry_of !== undefined) throw new Error('retry_of is not yet supported on the MCP worker-invoke bridge (OG_RETRY_UNSUPPORTED_MCP)');
|
|
1678
|
+
if (!registry.canStart()) throw new Error('Worker invocation capacity is reserved; no live run slot is available');
|
|
1679
|
+
const existing = registry.claimRequestId(input.request_id);
|
|
1680
|
+
if (existing) throw new Error('Worker request_id was already used; mint a fresh request_id instead of replaying');
|
|
1681
|
+
const entry = registry.create({ requestId: input.request_id, transport: input.transport, status: 'running', termination: 'running' });
|
|
1682
|
+
let request;
|
|
1683
|
+
let outputReservation;
|
|
1684
|
+
const execution = { mayHaveLaunched: false };
|
|
1685
|
+
let reservationCleanup;
|
|
1686
|
+
const cleanupReservation = () => reservationCleanup ??= (async () => {
|
|
1687
|
+
if (!outputReservation) return;
|
|
1688
|
+
try {
|
|
1689
|
+
const current = await lstat(request.final_output);
|
|
1690
|
+
// Best effort: the pathname can change between lstat and unlink.
|
|
1691
|
+
if (current.dev === outputReservation.dev && current.ino === outputReservation.ino && current.size === 0) {
|
|
1692
|
+
await unlink(request.final_output);
|
|
1693
|
+
}
|
|
1694
|
+
} catch (error) {
|
|
1695
|
+
if (error.code !== 'ENOENT') console.error('[worker-invoke] output reservation cleanup failed');
|
|
1696
|
+
}
|
|
1697
|
+
})();
|
|
1698
|
+
const markTerminal = async update => {
|
|
1699
|
+
if (!execution.mayHaveLaunched) await cleanupReservation();
|
|
1700
|
+
return registry.markTerminal(entry.invocationId, update);
|
|
1701
|
+
};
|
|
1702
|
+
try {
|
|
1703
|
+
request = await validatedRequest(input);
|
|
1704
|
+
} catch (error) {
|
|
1705
|
+
// M1: release the capacity/identity reservation on validation failure
|
|
1706
|
+
// ("release on validation failure"), but preserve the pre-existing
|
|
1707
|
+
// malformed/escaping-input contract (an MCP tool error, not a status
|
|
1708
|
+
// envelope) by rethrowing after releasing — the outer handler below
|
|
1709
|
+
// converts this into `mcpError(...)`.
|
|
1710
|
+
registry.markTerminal(entry.invocationId, { status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
1711
|
+
throw error;
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
// Shared pre-dispatch preparation owns capacity and output cleanup.
|
|
1715
|
+
let prefix;
|
|
1716
|
+
try {
|
|
1717
|
+
try {
|
|
1718
|
+
outputReservation = await reserveFinalOutput(request.final_output);
|
|
1719
|
+
} catch {
|
|
1720
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['final_output already exists or is not reservable as a fresh plain file'] });
|
|
1721
|
+
}
|
|
1722
|
+
const root = await servedRoot();
|
|
1723
|
+
const { snapshot, drivers } = await loadWorkerInputsImpl(root);
|
|
1724
|
+
const { resolveWorkerInvocation } = await loadServeWorkerInvocation();
|
|
1725
|
+
const plan = await resolveWorkerInvocation(snapshot, request, drivers, now().toISOString());
|
|
1726
|
+
if (plan.invocation?.readiness !== 'launchable') {
|
|
1727
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: plan.diagnostics ?? [] });
|
|
1728
|
+
}
|
|
1729
|
+
prefix = { root, snapshot, drivers, invocation: plan.invocation };
|
|
1730
|
+
} catch (error) {
|
|
1731
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
1732
|
+
}
|
|
1733
|
+
const { root, snapshot, drivers, invocation } = prefix;
|
|
1734
|
+
|
|
1735
|
+
// =========================================================================
|
|
1736
|
+
// #2915 round 8 (D6): ONE decision, immediately after the prefix, from
|
|
1737
|
+
// the launcher's own pure eligibility predicate,
|
|
1738
|
+
// `directRecoveryEligibility` (session-launcher.mjs) — the SAME
|
|
1739
|
+
// function `launchOneWorker`'s own not-gated/gated split and
|
|
1740
|
+
// `admitDirectWorker` itself both call, so this decision cannot drift
|
|
1741
|
+
// out of sync with either (round 8 critique, D6; the decision is
|
|
1742
|
+
// structurally one function, not three matching conditions). It
|
|
1743
|
+
// consults, in order, the global switch, the per-CLI switch,
|
|
1744
|
+
// `canVerifyAttempt`, and an explicit breaker override, and reserves
|
|
1745
|
+
// nothing — every ineligible answer returns `{ gated: false }` before
|
|
1746
|
+
// ever touching the breaker or the ledger. The only check made BEFORE
|
|
1747
|
+
// even importing the recovery module is the global switch
|
|
1748
|
+
// (`config.parallelism.worker_dispatch`) — matching the SAME field
|
|
1749
|
+
// `session-launcher.mjs`'s own call sites resolve as `gate.enabled`
|
|
1750
|
+
// (`checkWorkerDispatch`) — so a globally disabled deployment never
|
|
1751
|
+
// imports the recovery module at all (#2915 review 4 finding 7),
|
|
1752
|
+
// exactly like origin/main never having this feature.
|
|
1753
|
+
// =========================================================================
|
|
1754
|
+
const dispatchGateEnabled = !!snapshot.config?.parallelism?.worker_dispatch;
|
|
1755
|
+
let admission;
|
|
1756
|
+
let recoveryModule;
|
|
1757
|
+
let admissionGuard;
|
|
1758
|
+
if (!dispatchGateEnabled) {
|
|
1759
|
+
admission = { gated: false };
|
|
1760
|
+
} else {
|
|
1761
|
+
try {
|
|
1762
|
+
recoveryModule = await loadRalphExternal('session-launcher.mjs');
|
|
1763
|
+
const eligibility = recoveryModule.directRecoveryEligibility({
|
|
1764
|
+
gateEnabled: dispatchGateEnabled, config: snapshot.config, invocation, drivers, scopedBreaker: options.scopedBreaker,
|
|
1765
|
+
});
|
|
1766
|
+
if (!eligibility.gated) {
|
|
1767
|
+
// Not eligible (per-CLI switch, `canVerifyAttempt`, or an
|
|
1768
|
+
// explicit breaker override) — nothing was reserved; `admission`
|
|
1769
|
+
// is exactly the predicate's own `{ gated: false, reason? }`.
|
|
1770
|
+
admission = eligibility;
|
|
1771
|
+
} else {
|
|
1772
|
+
// #2915 review 4 (finding 8): consult the same per-task-chain
|
|
1773
|
+
// allowance ledger the launcher path uses, keyed off the SAME
|
|
1774
|
+
// stable digest and the SAME resolved `invocation.limits.
|
|
1775
|
+
// maxAttempts` -- no new configuration, just the existing
|
|
1776
|
+
// ceiling already resolved above by `resolveWorkerInvocation`.
|
|
1777
|
+
// MCP has no server-owned capture root to persist a
|
|
1778
|
+
// cross-process ledger file under, so `captureRoot` is
|
|
1779
|
+
// intentionally omitted: the ledger falls back to this
|
|
1780
|
+
// process's own in-memory map (see `reserveDirectWorkerAllowance`
|
|
1781
|
+
// in session-launcher.mjs). That still enforces the allowance
|
|
1782
|
+
// correctly WITHIN this server process -- including across
|
|
1783
|
+
// concurrent sibling retries, which serialize on the same
|
|
1784
|
+
// in-process reservation check -- but it is explicitly NOT
|
|
1785
|
+
// cross-process-safe: a second MCP server process, or a restart
|
|
1786
|
+
// of this one, starts the count over for the same task chain.
|
|
1787
|
+
const breaker = options.scopedBreaker ?? recoveryModule.getDirectRecoveryBreaker();
|
|
1788
|
+
registry.bindRecoveryOwner(breaker);
|
|
1789
|
+
const startedAtMs = breaker.nowMs();
|
|
1790
|
+
admissionGuard = {
|
|
1791
|
+
nowMs: () => breaker.nowMs(),
|
|
1792
|
+
deadlineAtMs: startedAtMs + Math.min(invocation.health?.hardTimeoutSeconds ?? Infinity,
|
|
1793
|
+
invocation.limits.wallSeconds, invocation.transport === 'mcp' ? invocation.mcp.timeoutSeconds : Infinity) * 1000,
|
|
1794
|
+
isCancelled: () => !!registry.get(entry.invocationId)?.cancelRequested,
|
|
1795
|
+
};
|
|
1796
|
+
const taskChainDigest = await recoveryModule.stableTaskChainDigest(request, invocation.briefDigest ?? null);
|
|
1797
|
+
admission = await recoveryModule.admitDirectWorker({
|
|
1798
|
+
gateEnabled: dispatchGateEnabled, config: snapshot.config,
|
|
1799
|
+
invocation, invocationId: entry.invocationId, drivers, scopedBreaker: breaker, dispatchGuard: admissionGuard,
|
|
1800
|
+
taskChainDigest, maxAttempts: invocation.limits?.maxAttempts, onTransition,
|
|
1801
|
+
});
|
|
1802
|
+
}
|
|
1803
|
+
} catch (error) {
|
|
1804
|
+
// #2915 P0 (fail closed, no leaked slot): this admission call runs
|
|
1805
|
+
// AFTER `registry.create()` already reserved this run's capacity/
|
|
1806
|
+
// identity slot. Any exception here (bad breaker state, a thrown
|
|
1807
|
+
// import, a malformed gate result) must refuse the launch the same
|
|
1808
|
+
// way an ordinary recovery-scope refusal does — `markTerminal`,
|
|
1809
|
+
// freeing the reservation. A pre-spawn breaker error is not an
|
|
1810
|
+
// uncertain worker outcome: no worker ever ran.
|
|
1811
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed',
|
|
1812
|
+
diagnostics: [`recovery admission failed closed: ${error?.message ?? String(error)}`] });
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
if (admission.refused) {
|
|
1816
|
+
// #2915 review 4 (finding 8): a ledger-exhaustion refusal has no
|
|
1817
|
+
// covering scope (`blockedBy` is empty) -- fall back to the
|
|
1818
|
+
// refusal's own code/reason so the diagnostic is never an empty
|
|
1819
|
+
// "recovery scope open: " for that case.
|
|
1820
|
+
const blockedBy = admission.refused.blockedBy?.length ? admission.refused.blockedBy.join(', ')
|
|
1821
|
+
: (admission.refused.code ?? admission.refused.reason ?? 'unknown');
|
|
1822
|
+
// #2915 round 8 (D4, C4): a `blocking-markers:<N>` component names
|
|
1823
|
+
// genuinely unresolved execution occupying reservation slots (never
|
|
1824
|
+
// mere backoff) — point at the exact reachable reconciliation route
|
|
1825
|
+
// for it, this tool's own `reconcile_stopped` operation, rather than
|
|
1826
|
+
// leaving the operator to go looking for one.
|
|
1827
|
+
const markerNote = admission.refused.blockedBy?.some(b => b.startsWith('blocking-markers:'))
|
|
1828
|
+
? "; unresolved reservations can be reconciled via this tool's 'reconcile_stopped' operation (see 'status' with no invocation_id for the pending list)"
|
|
1829
|
+
: '';
|
|
1830
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed',
|
|
1831
|
+
diagnostics: [`recovery scope open: ${blockedBy}${markerNote}${admission.refused.reason ? `; ${admission.refused.reason}` : ''}`] });
|
|
1832
|
+
}
|
|
1833
|
+
// #2915 round 8 (D4): track which breaker instance actually owns this
|
|
1834
|
+
// invocation's reservation (when gated) — the `reconcile_stopped`
|
|
1835
|
+
// operation and the no-id `status` listing both need to reach the
|
|
1836
|
+
// SAME breaker a later `admitDirectWorker` call used, not a fresh
|
|
1837
|
+
// default. Internal bookkeeping only — never surfaced by `publicStatus`.
|
|
1838
|
+
if (admission.breaker) {
|
|
1839
|
+
try {
|
|
1840
|
+
registry.update(entry.invocationId, { recoveryBreaker: admission.breaker });
|
|
1841
|
+
} catch (error) {
|
|
1842
|
+
const diagnostic = await recoveryModule.settleDirectWorker(admission, { kind: 'uncertain', confirmedStop: true });
|
|
1843
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed',
|
|
1844
|
+
diagnostics: [error.message, diagnostic].filter(Boolean) });
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
// Share reservation ownership; recovery accounting belongs only to the gated continuation.
|
|
1849
|
+
const ctx = { entry, request, root, snapshot, drivers, invocation, execution, cleanupReservation, markTerminal };
|
|
1850
|
+
if (admission.gated) {
|
|
1851
|
+
return continueGated(ctx, { admission, recoveryModule, captureSettlementPromise, admissionGuard });
|
|
1852
|
+
}
|
|
1853
|
+
return continueBaseline(ctx);
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
// Recovery-disabled execution follows current main's audited lifecycle.
|
|
1857
|
+
// Only recovery admission/accounting is omitted; stop and cleanup stay active.
|
|
1858
|
+
async function continueBaseline(ctx) {
|
|
1859
|
+
const { entry, request, root, snapshot, drivers, invocation, execution, cleanupReservation, markTerminal } = ctx;
|
|
1860
|
+
try {
|
|
1861
|
+
// F4 (fix-2892-r2, formerly part of M5): per operator decision, the
|
|
1862
|
+
// MCP path REQUIRES the structured WorkerDirectBrief — a probe showed
|
|
1863
|
+
// a plaintext brief (or a JSON brief swapped to plaintext between
|
|
1864
|
+
// launch and result assembly) was accepted, bypassing digest
|
|
1865
|
+
// verification entirely and completing anyway, because
|
|
1866
|
+
// `tryAssembleDirectResult` below treats a non-JSON brief as "nothing
|
|
1867
|
+
// to assemble against" and best-effort no-ops rather than refusing.
|
|
1868
|
+
// Fixed: for `mcp` transport ONLY, validate — AT ADMISSION, BEFORE any
|
|
1869
|
+
// launch — that the brief parses as a WorkerDirectBrief (#2890/#2879
|
|
1870
|
+
// codec), and pin ITS digest ourselves (never relying on whatever
|
|
1871
|
+
// `invocation.briefDigest` #2887's own resolver may or may not have
|
|
1872
|
+
// set for this transport). A non-structured or schema-invalid brief
|
|
1873
|
+
// refuses before any worker ever runs.
|
|
1874
|
+
let mcpBriefDigest;
|
|
1875
|
+
if (invocation.transport === 'mcp') {
|
|
1876
|
+
let briefBytes;
|
|
1877
|
+
try {
|
|
1878
|
+
briefBytes = await readBoundedFile(request.brief, 262144, root);
|
|
1879
|
+
} catch {
|
|
1880
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['OG_BRIEF_UNREADABLE'] });
|
|
1881
|
+
}
|
|
1882
|
+
try {
|
|
1883
|
+
const { assertValidWorkerPayload } = await loadMissionProtocol();
|
|
1884
|
+
assertValidWorkerPayload('workerDirectBrief', JSON.parse(briefBytes.toString('utf8')));
|
|
1885
|
+
} catch (error) {
|
|
1886
|
+
// Fixed-vocabulary diagnostic only — NEVER the raw JSON.parse/
|
|
1887
|
+
// validator message, exactly like M5's own `parseWorkerFinal`
|
|
1888
|
+
// diagnostic below. A duplicate-key/planted-marker probe showed
|
|
1889
|
+
// this exact kind of path can otherwise reflect attacker/worker-
|
|
1890
|
+
// controlled input text straight back out to the caller.
|
|
1891
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [typeof error?.code === 'string' ? error.code : 'OG_BRIEF_NOT_STRUCTURED'] });
|
|
1892
|
+
}
|
|
1893
|
+
mcpBriefDigest = `sha256:${createHash('sha256').update(briefBytes).digest('hex')}`;
|
|
1894
|
+
}
|
|
1895
|
+
const deadlineMs = Math.min(invocation.health?.hardTimeoutSeconds ?? invocation.limits.wallSeconds, invocation.limits.wallSeconds) * 1000;
|
|
1896
|
+
let observation;
|
|
1897
|
+
let rawOutput;
|
|
1898
|
+
if (invocation.transport === 'shell') {
|
|
1899
|
+
execution.mayHaveLaunched = true;
|
|
1900
|
+
const launch = await launchShellWorkerImpl(invocation, { allowDegraded, config: snapshot.config,
|
|
1901
|
+
name: `mcp-${entry.invocationId.slice(0, 8)}`, memoryMaxGb: options.shellWorkerMemoryMaxGb,
|
|
1902
|
+
driver: drivers[invocation.selected.cli], servedRoot: root, stdinRuntimeRoot: options.stdinRuntimeRoot, ...gateInputs });
|
|
1903
|
+
registry.update(entry.invocationId, { isolationMode: launch.mode ?? null });
|
|
1904
|
+
if (!launch.launched && !launch.launchAttempted) {
|
|
1905
|
+
execution.mayHaveLaunched = false;
|
|
1906
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [launch.diagnostic ?? 'worker launch was refused'] });
|
|
1907
|
+
}
|
|
1908
|
+
// M2: a live stop handle for the duration of this call, so an
|
|
1909
|
+
// explicit `cancel` can actually stop the isolated unit / kill the
|
|
1910
|
+
// child, not just record that cancellation was requested.
|
|
1911
|
+
const control = buildShellStopControl(launch, {
|
|
1912
|
+
waitForUnit: (waitForUnitTerminationOverride ?? waitForUnitTerminationBaseline),
|
|
1913
|
+
stopUnit: options.stopWorkerUnit,
|
|
1914
|
+
killUnit: options.killWorkerUnit,
|
|
1915
|
+
graceMs: options.shellStopGraceMs ?? (invocation.health?.terminateGraceSeconds ?? 1) * 1000,
|
|
1916
|
+
onConfirmed: async () => {
|
|
1917
|
+
if (!launch.launched) await cleanupReservation();
|
|
1918
|
+
await Promise.resolve().then(() => launch.cleanup?.()).catch(() => {});
|
|
1919
|
+
const live = registry.get(entry.invocationId);
|
|
1920
|
+
if (live?.terminalAt !== null) return;
|
|
1921
|
+
if (live.status === 'uncertain' || live.status === 'blocked') {
|
|
1922
|
+
registry.markTerminal(entry.invocationId, { control: null, termination: 'confirmed',
|
|
1923
|
+
status: live.cancelRequested ? 'cancelled' : 'failed' });
|
|
1924
|
+
} else registry.update(entry.invocationId, { control: null, termination: 'confirmed' });
|
|
1925
|
+
},
|
|
1926
|
+
});
|
|
1927
|
+
if (registry.get(entry.invocationId)?.termination !== 'confirmed') {
|
|
1928
|
+
registry.update(entry.invocationId, { control });
|
|
1929
|
+
}
|
|
1930
|
+
// Honor a cancel that arrived while the launcher was still pending.
|
|
1931
|
+
if (registry.get(entry.invocationId)?.cancelRequested) await control.stop();
|
|
1932
|
+
const outcome = launch.launched ? await control.wait(deadlineMs) : await control.stopAndConfirm();
|
|
1933
|
+
observation = { transport: 'shell', status: outcome.termination === 'confirmed' && outcome.exitCode === 0 ? 'completed' : 'failed', termination: outcome.termination, exitCode: outcome.exitCode };
|
|
1934
|
+
if (outcome.termination !== 'confirmed') {
|
|
1935
|
+
// M2: stays LIVE — capacity is held until a stop is proven or the
|
|
1936
|
+
// conductor disposes of it explicitly, never freed on a mere
|
|
1937
|
+
// unconfirmed-termination timeout.
|
|
1938
|
+
return registry.update(entry.invocationId, { status: 'uncertain', termination: 'uncertain', diagnostics: ['worker termination could not be confirmed before the deadline'] });
|
|
1939
|
+
}
|
|
1940
|
+
await Promise.resolve().then(() => launch.cleanup?.()).catch(() => {
|
|
1941
|
+
console.error('[worker-invoke] stdin cleanup failed after confirmed termination');
|
|
1942
|
+
});
|
|
1943
|
+
if (!launch.launched) {
|
|
1944
|
+
execution.mayHaveLaunched = false;
|
|
1945
|
+
return markTerminal({ control: null, status: 'failed', termination: 'confirmed',
|
|
1946
|
+
diagnostics: [launch.diagnostic ?? 'worker launch failed; termination confirmed'] });
|
|
1947
|
+
}
|
|
1948
|
+
if (outcome.timedOut || outcome.stopped) {
|
|
1949
|
+
return registry.markTerminal(entry.invocationId, { control: null, termination: 'confirmed',
|
|
1950
|
+
status: registry.get(entry.invocationId)?.cancelRequested ? 'cancelled' : 'failed',
|
|
1951
|
+
diagnostics: [outcome.timedOut ? 'worker deadline exceeded; termination confirmed' : 'worker stopped; termination confirmed'] });
|
|
1952
|
+
}
|
|
1953
|
+
try { rawOutput = await readBoundedFile(invocation.finalPath, invocation.maxOutputBytes, root); } catch {
|
|
1954
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['worker final output was not written'] });
|
|
1955
|
+
}
|
|
1956
|
+
} else {
|
|
1957
|
+
const { mcpRegistry, qualification } = drivers[invocation.selected.cli];
|
|
1958
|
+
const mcpServer = await mcpRegistry.get(invocation.mcp.serverRef);
|
|
1959
|
+
if (!mcpServer) return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['worker MCP server is not registered'] });
|
|
1960
|
+
// F5: pin the qualified server's definition digest NOW, at
|
|
1961
|
+
// admission, so a queued launch can detect a registry edit that
|
|
1962
|
+
// happened while it was waiting (see `serverDefinitionDigest`).
|
|
1963
|
+
const qualifiedServerDigest = serverDefinitionDigest(mcpServer);
|
|
1964
|
+
execution.mayHaveLaunched = true;
|
|
1965
|
+
const result = await callMcpWorkerImpl(invocation.mcp, mcpServer, {
|
|
1966
|
+
sessionIsolation: invocation.mcp.sessionIsolation, mcpRegistry, allowDegraded, config: snapshot.config,
|
|
1967
|
+
// #2892 (CI fix): the resolved invocation's own worktree
|
|
1968
|
+
// (`invocation.cwd`, #2887's `request.workdir`) — without it the
|
|
1969
|
+
// dispatch gate's free-disk sample has no filesystem to read and
|
|
1970
|
+
// fails closed on every host, never reaching isolation at all.
|
|
1971
|
+
// See `defaultLaunchMcpWorkerProcess`'s doc comment.
|
|
1972
|
+
cwd: invocation.cwd,
|
|
1973
|
+
name: `mcp-${entry.invocationId.slice(0, 8)}`, memoryMaxGb: mcpWorkerMemoryMaxGb, mcpSchemas: qualification?.mcpSchemas,
|
|
1974
|
+
qualifiedServerDigest, registerControl: control => registry.update(entry.invocationId, { control }),
|
|
1975
|
+
...gateInputs,
|
|
1976
|
+
});
|
|
1977
|
+
if (result.outcome === 'blocked') {
|
|
1978
|
+
return registry.markTerminal(entry.invocationId, { status: 'blocked', termination: 'confirmed', diagnostics: [result.diagnostic ?? 'worker MCP launch was refused'] });
|
|
1979
|
+
}
|
|
1980
|
+
if (result.outcome !== 'responded') {
|
|
1981
|
+
// Overflow holds capacity and stop control until termination is proven.
|
|
1982
|
+
if (result.diagnostic?.includes('OG_WORKER_MCP_FRAME_TOO_LARGE')) {
|
|
1983
|
+
// An explicit cancel can have confirmed termination while this run
|
|
1984
|
+
// was awaiting its initial overflow evidence.
|
|
1985
|
+
if (entry.terminalAt !== null) return entry;
|
|
1986
|
+
const patch = { status: result.termination === 'confirmed' ? 'failed' : 'uncertain',
|
|
1987
|
+
termination: result.termination ?? 'uncertain', diagnostics: [result.diagnostic] };
|
|
1988
|
+
return result.termination === 'confirmed'
|
|
1989
|
+
? registry.markTerminal(entry.invocationId, { ...patch, control: null })
|
|
1990
|
+
: registry.update(entry.invocationId, patch);
|
|
1991
|
+
}
|
|
1992
|
+
// M2: stays LIVE for the same reason as the shell timeout path above.
|
|
1993
|
+
return registry.update(entry.invocationId, { status: 'uncertain', termination: 'uncertain', diagnostics: [`worker ${result.outcome}; cancellation delivered, not confirmed stopped`] });
|
|
1994
|
+
}
|
|
1995
|
+
rawOutput = result.response;
|
|
1996
|
+
observation = { transport: 'mcp', status: result.response.isError ? 'failed' : 'completed', termination: 'confirmed', exitCode: result.response.isError ? 1 : null, isError: !!result.response.isError };
|
|
1997
|
+
if (result.response.isError) {
|
|
1998
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['worker reported isError'] });
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
const { ProviderAdapter } = await loadRalphExternal('lib/provider-adapter.mjs');
|
|
2002
|
+
let claims;
|
|
2003
|
+
try {
|
|
2004
|
+
claims = await new (class extends ProviderAdapter {})().parseWorkerFinal(rawOutput, {
|
|
2005
|
+
format: invocation.transport === 'shell' ? 'jsonl' : 'mcp', observation, maxOutputBytes: invocation.maxOutputBytes,
|
|
2006
|
+
});
|
|
2007
|
+
} catch (error) {
|
|
2008
|
+
// M5: bounded, fixed-vocabulary diagnostic only — never the raw
|
|
2009
|
+
// parser error message, which can echo attacker/worker-controlled
|
|
2010
|
+
// input text back out (a duplicate-key probe reflected a planted
|
|
2011
|
+
// marker through this exact path before the fix).
|
|
2012
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: [typeof error?.code === 'string' ? error.code : 'OG_WORKER_FINAL_INVALID'] });
|
|
2013
|
+
}
|
|
2014
|
+
const assembled = await tryAssembleDirectResult({ request, invocation, observation, claims, root,
|
|
2015
|
+
briefDigest: mcpBriefDigest, requireStructuredBrief: invocation.transport === 'mcp' });
|
|
2016
|
+
if (assembled?.conflict) {
|
|
2017
|
+
// M5: the brief changed between launch and result assembly — never
|
|
2018
|
+
// silently assemble a result against a different brief than the one
|
|
2019
|
+
// the worker was actually launched against.
|
|
2020
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['OG_BRIEF_EVIDENCE_CONFLICT'] });
|
|
2021
|
+
}
|
|
2022
|
+
// M2: a cancel requested at ANY point during this run's lifetime —
|
|
2023
|
+
// even before a live `control` handle existed to deliver it to — means
|
|
2024
|
+
// this result is never trusted as `completed`, regardless of what the
|
|
2025
|
+
// worker itself claims.
|
|
2026
|
+
const liveEntry = registry.get(entry.invocationId);
|
|
2027
|
+
return registry.markTerminal(entry.invocationId, {
|
|
2028
|
+
status: liveEntry?.cancelRequested ? 'cancelled' : (claims.status === 'completed' ? 'completed' : 'failed'),
|
|
2029
|
+
termination: 'confirmed',
|
|
2030
|
+
result: assembled?.result ?? { claims },
|
|
2031
|
+
});
|
|
2032
|
+
} catch (error) {
|
|
2033
|
+
if (error.workerNotLaunched) execution.mayHaveLaunched = false;
|
|
2034
|
+
if (!execution.mayHaveLaunched) {
|
|
2035
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
2036
|
+
}
|
|
2037
|
+
if (registry.get(entry.invocationId)?.termination === 'confirmed') {
|
|
2038
|
+
return registry.markTerminal(entry.invocationId, { control: null,
|
|
2039
|
+
status: registry.get(entry.invocationId)?.cancelRequested ? 'cancelled' : 'failed',
|
|
2040
|
+
termination: 'confirmed', diagnostics: [error.message] });
|
|
2041
|
+
}
|
|
2042
|
+
// An exception after launch is not proof that a worker stopped.
|
|
2043
|
+
return registry.update(entry.invocationId, { status: 'uncertain', termination: 'uncertain', diagnostics: [error.message] });
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
/**
|
|
2048
|
+
* #2915 round 8 (D6): the GATED continuation — reached only when the
|
|
2049
|
+
* decision point in `runInner` already admitted (or is borrowing) a
|
|
2050
|
+
* breaker reservation. `admission.gated` is therefore always true for the
|
|
2051
|
+
* duration of this function; no branch here needs to re-check it (the
|
|
2052
|
+
* previous worker's scattered `admission?.gated` conditionals are gone by
|
|
2053
|
+
* construction, not by review).
|
|
2054
|
+
*/
|
|
2055
|
+
async function continueGated(ctx, { admission, recoveryModule, captureSettlementPromise, admissionGuard }) {
|
|
2056
|
+
const { entry, request, root, snapshot, drivers, invocation, execution, cleanupReservation, markTerminal } = ctx;
|
|
2057
|
+
// #2915 final: `recoverySettled`/`spawnAttempted` are function-scoped
|
|
2058
|
+
// (not inside the try block below) so the catch-all at the end of this
|
|
2059
|
+
// function can also settle this run's admission (#2915 final review,
|
|
2060
|
+
// finding 2: a thrown error reaching that catch previously left the
|
|
2061
|
+
// reservation live with zero capacity recovered — the exact "slot leak"
|
|
2062
|
+
// the final review named).
|
|
2063
|
+
let recoverySettled = false;
|
|
2064
|
+
// #2915 review 5 (finding 2b, "leaked capacity"): distinguishes PROVEN
|
|
2065
|
+
// pre-spawn exceptions (nothing ever ran -- the registry run-slot must
|
|
2066
|
+
// be released) from post-spawn uncertainty (a worker process/call
|
|
2067
|
+
// genuinely started and may still be running -- the slot correctly
|
|
2068
|
+
// stays live per M2). Set true only once a shell launch or MCP call is
|
|
2069
|
+
// confirmed to have actually started (past every "never launched"/
|
|
2070
|
+
// "blocked" refusal below), never merely attempted.
|
|
2071
|
+
let spawnAttempted = false;
|
|
2072
|
+
let isolationEvidence = null;
|
|
2073
|
+
let settlementPromise;
|
|
2074
|
+
let settlementStarted;
|
|
2075
|
+
let cancelFollowUp;
|
|
2076
|
+
let neverExecutedStop = false;
|
|
2077
|
+
const settlementReady = new Promise(resolve => { settlementStarted = resolve; });
|
|
2078
|
+
const resolveEvidence = evidence => neverExecutedStop && evidence?.isolationMode === 'systemd-unit'
|
|
2079
|
+
&& evidence.unitName === isolationEvidence?.unitName
|
|
2080
|
+
? Promise.resolve({ stop: 'confirmed', confirmedStop: true }) : recoveryModule.resolveStopEvidence({
|
|
2081
|
+
...evidence, getStatus: options.isolationStatus, stopUnit: options.stopUnit,
|
|
2082
|
+
terminateChild: terminateChildImpl,
|
|
2083
|
+
});
|
|
2084
|
+
const reconcileEvidence = async evidence => {
|
|
2085
|
+
await settlementReady;
|
|
2086
|
+
await settlementPromise;
|
|
2087
|
+
if (!admission.breaker?.hasUnresolvedStop?.(entry.invocationId)) return true;
|
|
2088
|
+
const resolved = await resolveEvidence({ ...evidence, stopFirst: false });
|
|
2089
|
+
if (resolved.confirmedStop) {
|
|
2090
|
+
cancelFollowUp?.();
|
|
2091
|
+
const reconciled = admission.breaker?.reconcileStopEvidence?.({ invocationId: entry.invocationId, confirmedStop: true });
|
|
2092
|
+
if ((reconciled?.released ?? 0) > 0) registry.markTerminal(entry.invocationId, {});
|
|
2093
|
+
}
|
|
2094
|
+
return resolved.confirmedStop;
|
|
2095
|
+
};
|
|
2096
|
+
// The outward result and execution protection have independent lifetimes.
|
|
2097
|
+
// Retain pending escalation and a fixed, bounded sequence of unit observations.
|
|
2098
|
+
const observeStop = (stopping, evidence) => {
|
|
2099
|
+
Promise.resolve(stopping).then(() => reconcileEvidence(evidence)).catch(() => {});
|
|
2100
|
+
};
|
|
2101
|
+
let followUpScheduled = false;
|
|
2102
|
+
const observeLater = evidence => {
|
|
2103
|
+
if (followUpScheduled || evidence?.isolationMode !== 'systemd-unit' || !evidence.unitName) return;
|
|
2104
|
+
followUpScheduled = true;
|
|
2105
|
+
cancelFollowUp = recoveryModule.observeLateStopEvidence(() => reconcileEvidence(evidence));
|
|
2106
|
+
};
|
|
2107
|
+
const settleRecovery = (outcome) => {
|
|
2108
|
+
if (recoverySettled) return;
|
|
2109
|
+
recoverySettled = true;
|
|
2110
|
+
try {
|
|
2111
|
+
// #2915 round 8 (D5, review 7 finding 9): `settleDirectWorker`
|
|
2112
|
+
// RETURNS a promise (resolving to a bounded refund-failure
|
|
2113
|
+
// diagnostic string, or `null`) — capture it instead of discarding
|
|
2114
|
+
// it, so `run()`'s wrapper above can await it and surface the
|
|
2115
|
+
// diagnostic in the result.
|
|
2116
|
+
settlementPromise = recoveryModule.settleDirectWorker(admission, {
|
|
2117
|
+
...outcome, callerAborted: !!registry.get(entry.invocationId)?.cancelRequested,
|
|
2118
|
+
});
|
|
2119
|
+
captureSettlementPromise?.(settlementPromise);
|
|
2120
|
+
} catch { /* recovery accounting must never fail a run */ }
|
|
2121
|
+
settlementStarted();
|
|
2122
|
+
if (!outcome.confirmedStop) observeLater(isolationEvidence);
|
|
2123
|
+
};
|
|
2124
|
+
let recoveryDeadlineAtMs = 0;
|
|
2125
|
+
const settleObservedFailure = async (result, failureHint = '') => {
|
|
2126
|
+
await recoveryModule.observeLauncherRecoveryOutcome({ invocation, result, failureHint,
|
|
2127
|
+
installation: drivers[invocation.selected.cli]?.qualification?.installation, breaker: admission.breaker,
|
|
2128
|
+
timeoutMs: () => Math.min(10_000, recoveryDeadlineAtMs - admission.breaker.nowMs()),
|
|
2129
|
+
beforeRequest: async () => !registry.get(entry.invocationId)?.cancelRequested,
|
|
2130
|
+
onDispatch: () => recoveryModule.chargeDirectWorkerDiagnostic(admission, {
|
|
2131
|
+
deadlineAtMs: recoveryDeadlineAtMs, nowMs: admission.breaker.nowMs,
|
|
2132
|
+
isCancelled: () => !!registry.get(entry.invocationId)?.cancelRequested }) });
|
|
2133
|
+
settleRecovery({ ...recoveryModule.classifyAttemptForSettlement(invocation, result, admission),
|
|
2134
|
+
confirmedStop: result.confirmedStop });
|
|
2135
|
+
};
|
|
2136
|
+
try {
|
|
2137
|
+
// #2915 round 8 review 8 (C3): ONE absolute wall-clock deadline for
|
|
2138
|
+
// this attempt, computed HERE — at the very top of this function,
|
|
2139
|
+
// BEFORE any brief preparation — as the minimum of the task/health
|
|
2140
|
+
// budget, the transport (mcp per-call) budget when this is an
|
|
2141
|
+
// mcp-transport attempt, and the canary's own bounded window when
|
|
2142
|
+
// this admission is the breaker's canary, on the admission's own
|
|
2143
|
+
// monotonic clock. Passed down UNCHANGED from here on; NEVER
|
|
2144
|
+
// re-derived from an already-shortened duration. Computing this AFTER
|
|
2145
|
+
// brief preparation (the former position, right before the
|
|
2146
|
+
// transport-specific branches below) let preparation time elapse for
|
|
2147
|
+
// free: reproduced spending 60s in post-admission brief
|
|
2148
|
+
// validation/digest computation while a 30s native MCP call still
|
|
2149
|
+
// spawned and completed, because the deadline was derived from a
|
|
2150
|
+
// fresh `deadlineClockNow()` reading taken only after that
|
|
2151
|
+
// preparation finished (round 8 review 8, C3; review 7 finding 4's
|
|
2152
|
+
// last bullet).
|
|
2153
|
+
const taskDeadlineMs = Math.min(invocation.health?.hardTimeoutSeconds ?? invocation.limits.wallSeconds, invocation.limits.wallSeconds) * 1000;
|
|
2154
|
+
const deadlineClockNow = () => admission.breaker?.nowMs?.() ?? Date.now();
|
|
2155
|
+
const deadlineCandidatesMs = [admissionGuard?.deadlineAtMs ?? ((admission.startedAtMs ?? deadlineClockNow()) + taskDeadlineMs)];
|
|
2156
|
+
if (invocation.transport === 'mcp') deadlineCandidatesMs.push((admission.startedAtMs ?? deadlineClockNow()) + invocation.mcp.timeoutSeconds * 1000);
|
|
2157
|
+
if (admission.token?.canaryKeys?.length > 0) deadlineCandidatesMs.push(admission.token.deadlineMs);
|
|
2158
|
+
const deadlineAtMs = Math.min(...deadlineCandidatesMs);
|
|
2159
|
+
recoveryDeadlineAtMs = deadlineAtMs;
|
|
2160
|
+
// #2915 round 8 review 8 (C3): cancellation predicate, read from the
|
|
2161
|
+
// registry's own live `cancelRequested` flag — set by `cancel()`
|
|
2162
|
+
// (M2) at any point during this run's lifetime, including before any
|
|
2163
|
+
// launch/control handle exists. Folded into `dispatchGuard` below so
|
|
2164
|
+
// the leaf dispatch-commit hook (D2) refuses to spawn a cancelled
|
|
2165
|
+
// attempt, and threaded into the native-MCP per-request/write-boundary
|
|
2166
|
+
// checks so cancellation during pre-spawn preparation prevents the
|
|
2167
|
+
// spawn and any request (round 8 review 8, C3).
|
|
2168
|
+
const isCancelled = () => !!registry.get(entry.invocationId)?.cancelRequested;
|
|
2169
|
+
// #2915 round 8 review 8 (C3): passed to both launch paths so the D2
|
|
2170
|
+
// leaf hook re-checks deadline/cancellation immediately before the
|
|
2171
|
+
// real process-start call, after every preparation step has run.
|
|
2172
|
+
const dispatchGuard = { deadlineAtMs, nowMs: deadlineClockNow, isCancelled };
|
|
2173
|
+
// F4 (fix-2892-r2, formerly part of M5): per operator decision, the
|
|
2174
|
+
// MCP path REQUIRES the structured WorkerDirectBrief — a probe showed
|
|
2175
|
+
// a plaintext brief (or a JSON brief swapped to plaintext between
|
|
2176
|
+
// launch and result assembly) was accepted, bypassing digest
|
|
2177
|
+
// verification entirely and completing anyway, because
|
|
2178
|
+
// `tryAssembleDirectResult` below treats a non-JSON brief as "nothing
|
|
2179
|
+
// to assemble against" and best-effort no-ops rather than refusing.
|
|
2180
|
+
// Fixed: for `mcp` transport ONLY, validate — AT ADMISSION, BEFORE any
|
|
2181
|
+
// launch — that the brief parses as a WorkerDirectBrief (#2890/#2879
|
|
2182
|
+
// codec), and pin ITS digest ourselves (never relying on whatever
|
|
2183
|
+
// `invocation.briefDigest` #2887's own resolver may or may not have
|
|
2184
|
+
// set for this transport). A non-structured or schema-invalid brief
|
|
2185
|
+
// refuses before any worker ever runs.
|
|
2186
|
+
let mcpBriefDigest;
|
|
2187
|
+
if (invocation.transport === 'mcp') {
|
|
2188
|
+
let briefBytes;
|
|
2189
|
+
try {
|
|
2190
|
+
briefBytes = await readBoundedFile(request.brief, 262144, root);
|
|
2191
|
+
} catch {
|
|
2192
|
+
// #2915 review 4 (finding 2): this pre-spawn refusal happens
|
|
2193
|
+
// AFTER admission (above) but was never settling it — a leaked
|
|
2194
|
+
// reservation on every unreadable MCP brief.
|
|
2195
|
+
//
|
|
2196
|
+
// #2915 review 6 (finding 4, "stop evidence on every settlement"):
|
|
2197
|
+
// zero workers were ever dispatched for this reservation — settle
|
|
2198
|
+
// with `confirmedStop: true` so a canary whose own deadline has
|
|
2199
|
+
// already passed (e.g. an expired queued request) releases its
|
|
2200
|
+
// reservation outright instead of leaving a permanent blocking
|
|
2201
|
+
// marker behind for an attempt that never ran at all.
|
|
2202
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2203
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['OG_BRIEF_UNREADABLE'] });
|
|
2204
|
+
}
|
|
2205
|
+
try {
|
|
2206
|
+
const { assertValidWorkerPayload } = await loadMissionProtocol();
|
|
2207
|
+
assertValidWorkerPayload('workerDirectBrief', JSON.parse(briefBytes.toString('utf8')));
|
|
2208
|
+
} catch (error) {
|
|
2209
|
+
// Fixed-vocabulary diagnostic only — NEVER the raw JSON.parse/
|
|
2210
|
+
// validator message, exactly like M5's own `parseWorkerFinal`
|
|
2211
|
+
// diagnostic below. A duplicate-key/planted-marker probe showed
|
|
2212
|
+
// this exact kind of path can otherwise reflect attacker/worker-
|
|
2213
|
+
// controlled input text straight back out to the caller.
|
|
2214
|
+
//
|
|
2215
|
+
// #2915 review 6 (finding 4): zero workers dispatched — same
|
|
2216
|
+
// proven-never-dispatched settlement as above.
|
|
2217
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2218
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [typeof error?.code === 'string' ? error.code : 'OG_BRIEF_NOT_STRUCTURED'] });
|
|
2219
|
+
}
|
|
2220
|
+
mcpBriefDigest = `sha256:${createHash('sha256').update(briefBytes).digest('hex')}`;
|
|
2221
|
+
}
|
|
2222
|
+
let observation;
|
|
2223
|
+
let rawOutput;
|
|
2224
|
+
// #2915 review 5 ("a second worker over unresolved execution"): a
|
|
2225
|
+
// SEPARATE fact from `kind` threaded into every `settleRecovery` call
|
|
2226
|
+
// below — whether THIS attempt's own descendant tree is launcher-
|
|
2227
|
+
// confirmed stopped, so a settlement arriving against an already-
|
|
2228
|
+
// canary-deadline-expired reservation can resolve that reservation's
|
|
2229
|
+
// blocking marker (see `ScopedFailureBreaker.settle()`). Stays false
|
|
2230
|
+
// for native MCP (response completion is not stop proof).
|
|
2231
|
+
let confirmedStop = false;
|
|
2232
|
+
if (invocation.transport === 'shell') {
|
|
2233
|
+
// #2915 round 8 (D2, replaces the former `await debitRecovery()`
|
|
2234
|
+
// call here): the ledger debit now happens INSIDE the dispatch-
|
|
2235
|
+
// commit hook `planDirectWorkerLaunch`'s borrowed-admission branch
|
|
2236
|
+
// constructs (session-launcher.mjs), invoked by `launchIsolatedWorker`
|
|
2237
|
+
// itself immediately before the real process-start call — never
|
|
2238
|
+
// here, before the dispatch resource gate inside `launchShellWorkerImpl`
|
|
2239
|
+
// has even run. This closes the "MCP debits before queue/resource
|
|
2240
|
+
// refusals" gap (#2915 round 8 critique, D2): a resource-gate
|
|
2241
|
+
// refusal below now spends nothing.
|
|
2242
|
+
//
|
|
2243
|
+
// #2915 round 8 (D5): checked before spawn. A deadline already
|
|
2244
|
+
// passed before this bridge even attempted to spawn is a proven
|
|
2245
|
+
// zero-dispatch refusal, exactly like any other pre-spawn gate
|
|
2246
|
+
// refusal.
|
|
2247
|
+
//
|
|
2248
|
+
// #2915 round 8 review 8 (C3): cancellation itself is deliberately
|
|
2249
|
+
// NOT re-checked here (unlike the deadline) -- the authoritative
|
|
2250
|
+
// "prevent the spawn" enforcement for cancellation lives in
|
|
2251
|
+
// `dispatchGuard` below, re-checked inside the LEAF dispatch-commit
|
|
2252
|
+
// hook (D2) after every real preparation step the actual launch
|
|
2253
|
+
// performs. Adding a second, earlier check at this bridge level,
|
|
2254
|
+
// ahead of even invoking `launchShellWorkerImpl`, would race an
|
|
2255
|
+
// operator cancel arriving while an in-flight (but not yet
|
|
2256
|
+
// resolved) launch call is still legitimately starting, and
|
|
2257
|
+
// mis-refuse a launch that was already under way.
|
|
2258
|
+
if (deadlineClockNow() >= deadlineAtMs) {
|
|
2259
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2260
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['worker dispatch deadline already passed before spawn'] });
|
|
2261
|
+
}
|
|
2262
|
+
// `spawnAttempted` is still set BEFORE calling, not after inspecting
|
|
2263
|
+
// the result -- `launchShellWorkerImpl` itself may already have side
|
|
2264
|
+
// effects (dispatch gate sampling, resource checks, or a genuine
|
|
2265
|
+
// spawn) before it returns or throws, so a throw from WITHIN it can
|
|
2266
|
+
// never be proven to be pre-spawn. From here on, any exception
|
|
2267
|
+
// reaching the outer catch-all is post-spawn uncertainty.
|
|
2268
|
+
spawnAttempted = true;
|
|
2269
|
+
execution.mayHaveLaunched = true;
|
|
2270
|
+
const launch = await launchShellWorkerImpl(invocation, { allowDegraded, config: snapshot.config,
|
|
2271
|
+
name: `mcp-${entry.invocationId.slice(0, 8)}`, memoryMaxGb: options.shellWorkerMemoryMaxGb,
|
|
2272
|
+
driver: drivers[invocation.selected.cli], servedRoot: root, stdinRuntimeRoot: options.stdinRuntimeRoot, admission, invocationId: entry.invocationId, dispatchGuard, ...gateInputs });
|
|
2273
|
+
let failureHint = Buffer.alloc(0);
|
|
2274
|
+
const observeHint = chunk => {
|
|
2275
|
+
if (failureHint.length < 8192) failureHint = Buffer.concat([failureHint, Buffer.from(chunk).subarray(0, 8192 - failureHint.length)]);
|
|
2276
|
+
};
|
|
2277
|
+
launch.child?.stderr?.on('data', observeHint);
|
|
2278
|
+
isolationEvidence = { isolationMode: launch.mode ?? null, unitName: launch.unit ?? null, child: launch.child ?? null,
|
|
2279
|
+
dispatchState: launch.dispatchState === 'submission-uncertain' ? 'submission-uncertain' : (launch.launched || launch.launchAttempted) ? 'dispatched' : 'never-began' };
|
|
2280
|
+
registry.update(entry.invocationId, { isolationMode: launch.mode ?? null });
|
|
2281
|
+
if (!launch.launched && !launch.launchAttempted && launch.dispatchState !== 'submission-uncertain') {
|
|
2282
|
+
execution.mayHaveLaunched = false;
|
|
2283
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2284
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [launch.diagnostic ?? 'worker launch was refused'] });
|
|
2285
|
+
}
|
|
2286
|
+
// M2: a live stop handle for the duration of this call, so an
|
|
2287
|
+
// explicit `cancel` can actually stop the isolated unit / kill the
|
|
2288
|
+
// child, not just record that cancellation was requested.
|
|
2289
|
+
// #2915 review 4 (finding 3) / round 8 (D5): clipped to the ONE
|
|
2290
|
+
// absolute `deadlineAtMs` computed above (which already folds in the
|
|
2291
|
+
// admitted canary's own 300s window) — never re-derived from
|
|
2292
|
+
// `taskDeadlineMs` independently of it, and always measured as
|
|
2293
|
+
// "remaining from right now" so time already spent launching is
|
|
2294
|
+
// never double-counted.
|
|
2295
|
+
let deadlineMs = Math.max(0, deadlineAtMs - deadlineClockNow());
|
|
2296
|
+
const control = buildShellStopControl(launch, {
|
|
2297
|
+
waitForUnit: (unit, budgetMs, observerOptions) => (waitForUnitTerminationOverride ?? waitForUnitTermination)(unit,
|
|
2298
|
+
budgetMs, { ...observerOptions, clock: deadlineClockNow }),
|
|
2299
|
+
nowMs: deadlineClockNow,
|
|
2300
|
+
stopUnit: options.stopWorkerUnit ?? options.stopUnit,
|
|
2301
|
+
killUnit: options.killWorkerUnit,
|
|
2302
|
+
graceMs: options.shellStopGraceMs ?? (invocation.health?.terminateGraceSeconds ?? 1) * 1000,
|
|
2303
|
+
onConfirmed: async outcome => {
|
|
2304
|
+
// The shell observer proved an accepted stop plus a fresh inactive
|
|
2305
|
+
// unit that never exec'd; ExecMainCode alone cannot retain it.
|
|
2306
|
+
neverExecutedStop = outcome.neverExecuted === true;
|
|
2307
|
+
const stopEvidence = await resolveEvidence({ ...isolationEvidence, stopFirst: false });
|
|
2308
|
+
confirmedStop = stopEvidence.confirmedStop;
|
|
2309
|
+
observeStop(Promise.resolve(), isolationEvidence);
|
|
2310
|
+
if (!launch.launched) await cleanupReservation();
|
|
2311
|
+
await Promise.resolve().then(() => launch.cleanup?.()).catch(() => {});
|
|
2312
|
+
const live = registry.get(entry.invocationId);
|
|
2313
|
+
if (live?.terminalAt !== null) return;
|
|
2314
|
+
if (live.status === 'uncertain' || live.status === 'blocked') {
|
|
2315
|
+
registry.markTerminal(entry.invocationId, { control: null, termination: 'confirmed',
|
|
2316
|
+
status: live.cancelRequested ? 'cancelled' : 'failed' });
|
|
2317
|
+
} else registry.update(entry.invocationId, { control: null, termination: 'confirmed' });
|
|
2318
|
+
},
|
|
2319
|
+
});
|
|
2320
|
+
if (registry.get(entry.invocationId)?.termination !== 'confirmed') {
|
|
2321
|
+
registry.update(entry.invocationId, { control });
|
|
2322
|
+
}
|
|
2323
|
+
// Honor a cancel that arrived while the launcher was still pending.
|
|
2324
|
+
if (registry.get(entry.invocationId)?.cancelRequested) await control.stop();
|
|
2325
|
+
let outcome = launch.launched ? await control.wait(deadlineMs, { deadlineAtMs, nowMs: deadlineClockNow }) : await control.stopAndConfirm();
|
|
2326
|
+
// Preserve the control's timeout latch and refuse a late clean outcome.
|
|
2327
|
+
if (outcome.termination === 'confirmed' && !outcome.timedOut && deadlineClockNow() >= deadlineAtMs) {
|
|
2328
|
+
outcome = { ...outcome, timedOut: true };
|
|
2329
|
+
}
|
|
2330
|
+
const stopEvidence = await resolveEvidence({ ...isolationEvidence, stopFirst: false });
|
|
2331
|
+
confirmedStop = stopEvidence.confirmedStop;
|
|
2332
|
+
// #2915 review 6 (finding 1, "false recovery after timeout"): a
|
|
2333
|
+
// forced stop at the attempt's own deadline is never treated as a
|
|
2334
|
+
// candidate success, regardless of the exit code the forced stop
|
|
2335
|
+
// happened to observe (a systemd unit whose main process had
|
|
2336
|
+
// already legitimately exited 0 right before the stop escalation
|
|
2337
|
+
// fired reports exactly that exit code even though the attempt
|
|
2338
|
+
// itself was never allowed to run to its own completion). Preserved
|
|
2339
|
+
// independently of `outcome.exitCode`/`outcome.termination` through
|
|
2340
|
+
// to the candidate-success gate below (`strictlyCompleted`).
|
|
2341
|
+
observation = { transport: 'shell', status: outcome.termination === 'confirmed' && outcome.exitCode === 0 && !outcome.timedOut ? 'completed' : 'failed', termination: outcome.termination, exitCode: outcome.exitCode, timedOut: !!outcome.timedOut };
|
|
2342
|
+
// #2915 final (evidence contract): the attempt-level fact is derived
|
|
2343
|
+
// from this bridge's own launcher-observed wait status — never a
|
|
2344
|
+
// worker claim — independent of `termination`/`descendants` above.
|
|
2345
|
+
// `termination === 'confirmed'` here already reflects THIS bridge's
|
|
2346
|
+
// own trusted observation (systemd-unit query, or this process's own
|
|
2347
|
+
// `child.close` event).
|
|
2348
|
+
//
|
|
2349
|
+
// #2915 review 5 (finding 1a, "false recovery"): a definite BAD exit
|
|
2350
|
+
// is trustworthy failure evidence regardless of what happens
|
|
2351
|
+
// downstream, so it settles immediately. A confirmed CLEAN exit
|
|
2352
|
+
// (`exitCode === 0`) is only a CANDIDATE success -- settlement is
|
|
2353
|
+
// deferred past strict final-claims validation, brief-evidence
|
|
2354
|
+
// matching and the cancellation check below (common code, shared
|
|
2355
|
+
// with the MCP branch), so a worker that exits 0 but never produces
|
|
2356
|
+
// a valid, matching, uncancelled completed claim can never close a
|
|
2357
|
+
// scope through this path. Uncertain termination never implicates
|
|
2358
|
+
// anything either way.
|
|
2359
|
+
launch.child?.stderr?.removeListener('data', observeHint);
|
|
2360
|
+
if (outcome.termination !== 'confirmed') {
|
|
2361
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2362
|
+
// M2: stays LIVE — capacity is held until a stop is proven or the
|
|
2363
|
+
// conductor disposes of it explicitly, never freed on a mere
|
|
2364
|
+
// unconfirmed-termination timeout.
|
|
2365
|
+
return (confirmedStop ? registry.markTerminal : registry.update)(entry.invocationId, {
|
|
2366
|
+
status: launch.launched ? 'uncertain' : 'blocked', termination: confirmedStop ? 'confirmed' : 'uncertain',
|
|
2367
|
+
diagnostics: ['worker termination could not be confirmed before the deadline'],
|
|
2368
|
+
});
|
|
2369
|
+
}
|
|
2370
|
+
if (outcome.exitCode !== 0) {
|
|
2371
|
+
if (!failureHint.length && invocation.stderrPath) {
|
|
2372
|
+
try { failureHint = await readBoundedFile(invocation.stderrPath, 8192, root); } catch { /* no usable hint */ }
|
|
2373
|
+
}
|
|
2374
|
+
await settleObservedFailure({ ...outcome, invocationId: entry.invocationId, attemptOutcome: 'failed',
|
|
2375
|
+
cancelled: !!registry.get(entry.invocationId)?.cancelRequested, confirmedStop }, failureHint.toString('utf8'));
|
|
2376
|
+
}
|
|
2377
|
+
await Promise.resolve().then(() => launch.cleanup?.()).catch(() => {
|
|
2378
|
+
console.error('[worker-invoke] stdin cleanup failed after confirmed termination');
|
|
2379
|
+
});
|
|
2380
|
+
if (!launch.launched) {
|
|
2381
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2382
|
+
execution.mayHaveLaunched = false;
|
|
2383
|
+
return markTerminal({ control: null, status: 'failed', termination: 'confirmed',
|
|
2384
|
+
diagnostics: [launch.diagnostic ?? 'worker launch failed; termination confirmed'] });
|
|
2385
|
+
}
|
|
2386
|
+
if (outcome.timedOut || outcome.stopped) {
|
|
2387
|
+
settleRecovery({ kind: registry.get(entry.invocationId)?.cancelRequested ? 'uncertain' : 'failed', confirmedStop });
|
|
2388
|
+
return registry.markTerminal(entry.invocationId, { control: null, termination: 'confirmed',
|
|
2389
|
+
status: registry.get(entry.invocationId)?.cancelRequested ? 'cancelled' : 'failed',
|
|
2390
|
+
diagnostics: [outcome.timedOut ? 'worker deadline exceeded; termination confirmed' : 'worker stopped; termination confirmed'] });
|
|
2391
|
+
}
|
|
2392
|
+
try { rawOutput = await readBoundedFile(invocation.finalPath, invocation.maxOutputBytes, root); } catch {
|
|
2393
|
+
// #2915 review 5 (finding 1a): a confirmed clean exit that never
|
|
2394
|
+
// produced a readable final output is not a success -- settle
|
|
2395
|
+
// failed now rather than leaving it candidate-success forever
|
|
2396
|
+
// (the bad-exit branch above already settled; this is a no-op
|
|
2397
|
+
// there).
|
|
2398
|
+
settleRecovery({ kind: 'failed', confirmedStop });
|
|
2399
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['worker final output was not written'] });
|
|
2400
|
+
}
|
|
2401
|
+
} else {
|
|
2402
|
+
const { mcpRegistry, qualification } = drivers[invocation.selected.cli];
|
|
2403
|
+
const mcpServer = await mcpRegistry.get(invocation.mcp.serverRef);
|
|
2404
|
+
if (!mcpServer) {
|
|
2405
|
+
// #2915 review 4 (finding 2): another pre-spawn refusal after
|
|
2406
|
+
// admission that was never settling it.
|
|
2407
|
+
//
|
|
2408
|
+
// #2915 review 6 (finding 4): proven zero dispatch.
|
|
2409
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2410
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['worker MCP server is not registered'] });
|
|
2411
|
+
}
|
|
2412
|
+
// F5: pin the qualified server's definition digest NOW, at
|
|
2413
|
+
// admission, so a queued launch can detect a registry edit that
|
|
2414
|
+
// happened while it was waiting (see `serverDefinitionDigest`).
|
|
2415
|
+
const qualifiedServerDigest = serverDefinitionDigest(mcpServer);
|
|
2416
|
+
// #2915 review 5 ("a second worker over unresolved execution") /
|
|
2417
|
+
// round 8 (D5): clipped to the ONE absolute `deadlineAtMs` computed
|
|
2418
|
+
// above — never re-derived independently of it (that re-derivation,
|
|
2419
|
+
// against `admission.token.deadlineMs` alone, used to miss the
|
|
2420
|
+
// plain task-deadline case entirely and was also the source of the
|
|
2421
|
+
// "subtracts preparation time twice" defect once `defaultCallMcpWorker`
|
|
2422
|
+
// added its OWN mint-time-relative re-derivation on top — round 8
|
|
2423
|
+
// critique, D5/review 7 finding 4's last bullet). This clip exists
|
|
2424
|
+
// for callers that only ever see `mcpBinding.timeoutSeconds` (every
|
|
2425
|
+
// existing injected `callMcpWorker` test double, and the ACTUAL
|
|
2426
|
+
// enforcement inside `defaultCallMcpWorker` uses the separately
|
|
2427
|
+
// threaded `deadlineAtMs` below, never re-adding onto
|
|
2428
|
+
// `admission.token.mintedAtMs`.
|
|
2429
|
+
let mcpBinding = invocation.mcp;
|
|
2430
|
+
const remainingSeconds = Math.max(0, (deadlineAtMs - deadlineClockNow()) / 1000);
|
|
2431
|
+
if (remainingSeconds < mcpBinding.timeoutSeconds) mcpBinding = { ...mcpBinding, timeoutSeconds: remainingSeconds };
|
|
2432
|
+
// #2915 round 8 (D2, replaces the former `await debitRecovery()`
|
|
2433
|
+
// call here): same reasoning as the shell path above — the ledger
|
|
2434
|
+
// debit now happens inside the dispatch-commit hook, deep inside
|
|
2435
|
+
// `defaultLaunchMcpWorkerProcess` -> `planDirectWorkerLaunch` ->
|
|
2436
|
+
// `launchIsolatedWorker`, immediately before the real MCP worker
|
|
2437
|
+
// SERVER process actually starts.
|
|
2438
|
+
//
|
|
2439
|
+
// `spawnAttempted` is still set BEFORE calling, not after
|
|
2440
|
+
// inspecting `result.outcome` -- `callMcpWorkerImpl` itself may
|
|
2441
|
+
// already have launched a real MCP server process and sent bytes
|
|
2442
|
+
// before it returns or throws (#2892 F2's own "thrown MCP transport
|
|
2443
|
+
// error stays LIVE" case is exactly this), so a throw from WITHIN
|
|
2444
|
+
// it can never be proven to be pre-spawn.
|
|
2445
|
+
spawnAttempted = true;
|
|
2446
|
+
execution.mayHaveLaunched = true;
|
|
2447
|
+
const result = await callMcpWorkerImpl(mcpBinding, mcpServer, {
|
|
2448
|
+
sessionIsolation: invocation.mcp.sessionIsolation, mcpRegistry, allowDegraded, config: snapshot.config,
|
|
2449
|
+
// #2915 round 8 (D5): the ONE absolute deadline for this call,
|
|
2450
|
+
// computed once above.
|
|
2451
|
+
deadlineAtMs,
|
|
2452
|
+
// #2892 (CI fix): the resolved invocation's own worktree
|
|
2453
|
+
// (`invocation.cwd`, #2887's `request.workdir`) — without it the
|
|
2454
|
+
// dispatch gate's free-disk sample has no filesystem to read and
|
|
2455
|
+
// fails closed on every host, never reaching isolation at all.
|
|
2456
|
+
// See `defaultLaunchMcpWorkerProcess`'s doc comment.
|
|
2457
|
+
cwd: invocation.cwd,
|
|
2458
|
+
name: `mcp-${entry.invocationId.slice(0, 8)}`, memoryMaxGb: mcpWorkerMemoryMaxGb, mcpSchemas: qualification?.mcpSchemas,
|
|
2459
|
+
qualifiedServerDigest, registerControl: control => registry.update(entry.invocationId, { control }),
|
|
2460
|
+
// #2915 review 4 (finding 2): the SAME top-level admission this
|
|
2461
|
+
// run() already holds, borrowed through to the native-MCP launch
|
|
2462
|
+
// path — never a second, independent planner reservation. See
|
|
2463
|
+
// `defaultLaunchMcpWorkerProcess`.
|
|
2464
|
+
admission,
|
|
2465
|
+
// #2915 round 8 review 8 (C6/C3): independently resolved dispatch
|
|
2466
|
+
// identity (never derived from the borrowed admission wrapper) and
|
|
2467
|
+
// the one dispatch-commit deadline/cancellation guard, threaded
|
|
2468
|
+
// all the way to the leaf hook.
|
|
2469
|
+
invocation, invocationId: entry.invocationId, dispatchGuard, isCancelled, observeStop,
|
|
2470
|
+
...gateInputs,
|
|
2471
|
+
});
|
|
2472
|
+
isolationEvidence = { isolationMode: result.isolationMode ?? null, unitName: result.unitName ?? null,
|
|
2473
|
+
child: result.child ?? null, dispatchState: result.dispatchState ?? 'dispatched' };
|
|
2474
|
+
if (result.outcome === 'blocked') {
|
|
2475
|
+
// Never launched (dispatch gate or queue refusal): settle with no
|
|
2476
|
+
// failure evidence.
|
|
2477
|
+
//
|
|
2478
|
+
// #2915 round 8 review 8 (C2): a `blocked` outcome does NOT always
|
|
2479
|
+
// mean proven zero dispatch -- a schema-drift refusal (or any
|
|
2480
|
+
// other post-connect `blocked` return) happens AFTER a real MCP
|
|
2481
|
+
// worker server process was spawned and connected to
|
|
2482
|
+
// (`result.dispatchState === 'dispatched'`). Resolve REAL stop
|
|
2483
|
+
// evidence for that case through the shared `resolveStopEvidence`
|
|
2484
|
+
// (a server that ignores close/TERM/KILL must never be reported
|
|
2485
|
+
// confirmed-stopped) rather than hardcoding `confirmedStop: true`
|
|
2486
|
+
// for every `blocked` label (review 8, C section finding 2: "an
|
|
2487
|
+
// MCP blocked result AFTER a process was spawned is dispatched").
|
|
2488
|
+
// Every pre-spawn `blocked` path inside `callMcpWorkerImpl`
|
|
2489
|
+
// (registry requalification, admission-expiry, a thrown
|
|
2490
|
+
// pre-launch exception, or a proven pre-spawn launch refusal)
|
|
2491
|
+
// carries `dispatchState: 'never-began'` (or `'submission-
|
|
2492
|
+
// uncertain'` for a failed/timed-out systemd-run submission) and
|
|
2493
|
+
// is handled accordingly — ALWAYS proven zero dispatch only for
|
|
2494
|
+
// `'never-began'`.
|
|
2495
|
+
let blockedConfirmedStop;
|
|
2496
|
+
if (result.dispatchState === 'dispatched' || result.dispatchState === 'submission-uncertain') {
|
|
2497
|
+
const evidence = await resolveEvidence(isolationEvidence);
|
|
2498
|
+
blockedConfirmedStop = evidence.confirmedStop;
|
|
2499
|
+
} else {
|
|
2500
|
+
blockedConfirmedStop = true;
|
|
2501
|
+
execution.mayHaveLaunched = false;
|
|
2502
|
+
}
|
|
2503
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: blockedConfirmedStop });
|
|
2504
|
+
return blockedConfirmedStop ? markTerminal({
|
|
2505
|
+
status: 'blocked', termination: 'confirmed',
|
|
2506
|
+
diagnostics: [result.diagnostic ?? 'worker MCP launch was refused'],
|
|
2507
|
+
}) : registry.update(entry.invocationId, {
|
|
2508
|
+
status: 'blocked', termination: blockedConfirmedStop ? 'confirmed' : 'uncertain',
|
|
2509
|
+
diagnostics: [result.diagnostic ?? 'worker MCP launch was refused'],
|
|
2510
|
+
});
|
|
2511
|
+
}
|
|
2512
|
+
confirmedStop = (await resolveEvidence({ ...isolationEvidence, stopFirst: false })).confirmedStop;
|
|
2513
|
+
if (result.outcome !== 'responded') {
|
|
2514
|
+
// Overflow holds capacity and stop control until termination is proven.
|
|
2515
|
+
if (result.diagnostic?.includes('OG_WORKER_MCP_FRAME_TOO_LARGE')) {
|
|
2516
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2517
|
+
// An explicit cancel can have confirmed termination while this run
|
|
2518
|
+
// was awaiting its initial overflow evidence.
|
|
2519
|
+
if (entry.terminalAt !== null) return entry;
|
|
2520
|
+
const patch = { status: result.termination === 'confirmed' ? 'failed' : 'uncertain',
|
|
2521
|
+
termination: result.termination ?? 'uncertain', diagnostics: [result.diagnostic] };
|
|
2522
|
+
return result.termination === 'confirmed'
|
|
2523
|
+
? registry.markTerminal(entry.invocationId, { ...patch, control: null })
|
|
2524
|
+
: registry.update(entry.invocationId, patch);
|
|
2525
|
+
}
|
|
2526
|
+
// M2: stays LIVE for the same reason as the shell timeout path above.
|
|
2527
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2528
|
+
return (confirmedStop ? registry.markTerminal : registry.update)(entry.invocationId, {
|
|
2529
|
+
status: 'uncertain', termination: confirmedStop ? 'confirmed' : 'uncertain',
|
|
2530
|
+
diagnostics: [`worker ${result.outcome}; cancellation delivered`],
|
|
2531
|
+
});
|
|
2532
|
+
}
|
|
2533
|
+
rawOutput = result.response;
|
|
2534
|
+
observation = { transport: 'mcp', status: result.response.isError ? 'failed' : 'completed', termination: 'confirmed', exitCode: result.response.isError ? 1 : null, isError: !!result.response.isError };
|
|
2535
|
+
// #2915 final (evidence contract): a transport-correlated response
|
|
2536
|
+
// that is not `isError` is a CANDIDATE successful attempt even
|
|
2537
|
+
// though `exitCode` is legitimately `null` for MCP (#2915 final
|
|
2538
|
+
// review, finding 4: "successful responses produce exitCode:null...
|
|
2539
|
+
// recorded one invocation failure and left the shared canary
|
|
2540
|
+
// active" — the former exitCode-based classification treated every
|
|
2541
|
+
// successful MCP response as a failure). Descendant-stop evidence is
|
|
2542
|
+
// a separate, independent fact this bridge never claims here.
|
|
2543
|
+
//
|
|
2544
|
+
// #2915 review 5 (finding 1a): an `isError` response IS trustworthy
|
|
2545
|
+
// failure evidence and settles immediately, exactly like a
|
|
2546
|
+
// confirmed bad exit above. A non-error response is only a
|
|
2547
|
+
// CANDIDATE -- settlement is deferred to the common code below,
|
|
2548
|
+
// past strict final-claims validation, brief-evidence matching and
|
|
2549
|
+
// the cancellation check, so a transport-correlated response alone
|
|
2550
|
+
// can never close a scope.
|
|
2551
|
+
if (result.response.isError) {
|
|
2552
|
+
await settleObservedFailure({ invocationId: entry.invocationId, attemptOutcome: 'failed',
|
|
2553
|
+
termination: 'confirmed', exitCode: 1, confirmedStop,
|
|
2554
|
+
cancelled: !!registry.get(entry.invocationId)?.cancelRequested }, JSON.stringify(result.response).slice(0, 8192));
|
|
2555
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['worker reported isError'] });
|
|
2556
|
+
}
|
|
2557
|
+
}
|
|
2558
|
+
const { ProviderAdapter } = await loadRalphExternal('lib/provider-adapter.mjs');
|
|
2559
|
+
let claims;
|
|
2560
|
+
try {
|
|
2561
|
+
claims = await new (class extends ProviderAdapter {})().parseWorkerFinal(rawOutput, {
|
|
2562
|
+
format: invocation.transport === 'shell' ? 'jsonl' : 'mcp', observation, maxOutputBytes: invocation.maxOutputBytes,
|
|
2563
|
+
});
|
|
2564
|
+
} catch (error) {
|
|
2565
|
+
// #2915 review 5 (finding 1a): the strict parse itself rejected the
|
|
2566
|
+
// claims -- the candidate success from above is settled failed
|
|
2567
|
+
// here, never left to default to an earlier premature success.
|
|
2568
|
+
//
|
|
2569
|
+
// #2915 review 6 (finding 4, "stop evidence on every settlement"):
|
|
2570
|
+
// thread the shell path's own launcher-confirmed termination fact
|
|
2571
|
+
// (`confirmedStop`) through here too — a late systemd completion
|
|
2572
|
+
// with confirmed termination but invalid claims must not retain a
|
|
2573
|
+
// permanent blocking marker merely because this rejection path
|
|
2574
|
+
// omitted it. Stays `false` for native MCP (unaffected).
|
|
2575
|
+
settleRecovery({ kind: 'failed', confirmedStop });
|
|
2576
|
+
// M5: bounded, fixed-vocabulary diagnostic only — never the raw
|
|
2577
|
+
// parser error message, which can echo attacker/worker-controlled
|
|
2578
|
+
// input text back out (a duplicate-key probe reflected a planted
|
|
2579
|
+
// marker through this exact path before the fix).
|
|
2580
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: [typeof error?.code === 'string' ? error.code : 'OG_WORKER_FINAL_INVALID'] });
|
|
2581
|
+
}
|
|
2582
|
+
const assembled = await tryAssembleDirectResult({ request, invocation, observation, claims, root,
|
|
2583
|
+
briefDigest: mcpBriefDigest, requireStructuredBrief: invocation.transport === 'mcp' });
|
|
2584
|
+
if (assembled?.conflict) {
|
|
2585
|
+
// #2915 review 5 (finding 1a): the brief changed between launch and
|
|
2586
|
+
// result assembly -- a candidate success settles failed here, never
|
|
2587
|
+
// silently assembled (or recorded recovered) against a different
|
|
2588
|
+
// brief than the one the worker was actually launched against.
|
|
2589
|
+
//
|
|
2590
|
+
// #2915 review 6 (finding 4): same `confirmedStop` threading as the
|
|
2591
|
+
// parse-failure path above.
|
|
2592
|
+
settleRecovery({ kind: 'failed', confirmedStop });
|
|
2593
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['OG_BRIEF_EVIDENCE_CONFLICT'] });
|
|
2594
|
+
}
|
|
2595
|
+
// M2: a cancel requested at ANY point during this run's lifetime —
|
|
2596
|
+
// even before a live `control` handle existed to deliver it to — means
|
|
2597
|
+
// this result is never trusted as `completed`, regardless of what the
|
|
2598
|
+
// worker itself claims.
|
|
2599
|
+
const liveEntry = registry.get(entry.invocationId);
|
|
2600
|
+
const cancelled = !!liveEntry?.cancelRequested;
|
|
2601
|
+
// #2915 review 6 (finding 1, "false recovery after timeout"): a
|
|
2602
|
+
// timed-out attempt (the shell path's own forced-stop-at-deadline
|
|
2603
|
+
// observation, see `observation.timedOut` above) is never treated as
|
|
2604
|
+
// strictly completed, regardless of what a worker claim says or what
|
|
2605
|
+
// exit code the forced stop happened to observe.
|
|
2606
|
+
const strictlyCompleted = claims.status === 'completed' && !observation?.timedOut;
|
|
2607
|
+
// #2915 review 5 (finding 1a): this is the ONE place a candidate
|
|
2608
|
+
// success (deferred above) is finally settled -- only after strict
|
|
2609
|
+
// completed claims, matching brief evidence (no conflict, checked
|
|
2610
|
+
// above) and the cancellation check all agree. A cancelled run never
|
|
2611
|
+
// settles 'succeeded' even with valid claims (the operator asked to
|
|
2612
|
+
// stop it); an invalid/non-completed claim settles 'failed', exactly
|
|
2613
|
+
// like every other claim-rejection path above.
|
|
2614
|
+
settleRecovery({ kind: cancelled ? 'uncertain' : strictlyCompleted ? 'succeeded' : 'failed', confirmedStop });
|
|
2615
|
+
return registry.markTerminal(entry.invocationId, {
|
|
2616
|
+
status: cancelled ? 'cancelled' : (strictlyCompleted ? 'completed' : 'failed'),
|
|
2617
|
+
termination: 'confirmed',
|
|
2618
|
+
result: assembled?.result ?? { claims },
|
|
2619
|
+
});
|
|
2620
|
+
} catch (error) {
|
|
2621
|
+
// Registry status and an error label do not establish descendant stop.
|
|
2622
|
+
if (error.workerNotLaunched) execution.mayHaveLaunched = false;
|
|
2623
|
+
const neverDispatched = !spawnAttempted || !!error.workerNotLaunched;
|
|
2624
|
+
const confirmedStop = neverDispatched || (await resolveEvidence({ ...isolationEvidence, stopFirst: false })).confirmedStop;
|
|
2625
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2626
|
+
if (neverDispatched) {
|
|
2627
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
2628
|
+
}
|
|
2629
|
+
if (confirmedStop) {
|
|
2630
|
+
return registry.markTerminal(entry.invocationId, { control: null,
|
|
2631
|
+
status: registry.get(entry.invocationId)?.cancelRequested ? 'cancelled' : 'failed',
|
|
2632
|
+
termination: 'confirmed', diagnostics: [error.message] });
|
|
2633
|
+
}
|
|
2634
|
+
// Keep capacity and stop control while execution remains unresolved.
|
|
2635
|
+
return registry.update(entry.invocationId, { status: 'uncertain', termination: 'uncertain', diagnostics: [error.message] });
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
async function tryAssembleDirectResult({ request, invocation, observation, claims, root, briefDigest, requireStructuredBrief }) {
|
|
2640
|
+
// M5 (and F4 for mcp): `briefDigest` (fix-2892-r2) is THIS run's own
|
|
2641
|
+
// admission-time pin for the `mcp` transport — computed and validated
|
|
2642
|
+
// BEFORE launch, in `run()` — and always takes priority over
|
|
2643
|
+
// `invocation.briefDigest` (which #2887's own resolver populates for
|
|
2644
|
+
// EVERY transport, structured brief or not); falling back to the
|
|
2645
|
+
// latter preserves the shell path's pre-existing behavior unchanged.
|
|
2646
|
+
const pinnedDigest = briefDigest ?? invocation.briefDigest;
|
|
2647
|
+
try {
|
|
2648
|
+
const briefBytes = await readBoundedFile(request.brief, 262144, root);
|
|
2649
|
+
const text = briefBytes.toString('utf8');
|
|
2650
|
+
if (!text.trimStart().startsWith('{')) {
|
|
2651
|
+
// F4: `requireStructuredBrief` (true only for `mcp`, where THIS
|
|
2652
|
+
// run's own admission already proved the brief was structured
|
|
2653
|
+
// JSON) is the signal here — NOT merely "a digest happens to be
|
|
2654
|
+
// pinned", since #2887's resolver pins `invocation.briefDigest` for
|
|
2655
|
+
// every transport, plain-text brief or not. Without that
|
|
2656
|
+
// distinction, an ordinary plain-text SHELL brief (never required
|
|
2657
|
+
// to be structured) would be misreported as a changed-brief
|
|
2658
|
+
// conflict merely because a digest existed. For `mcp`, a brief
|
|
2659
|
+
// that is no longer JSON at all by result-assembly time IS itself
|
|
2660
|
+
// evidence of a changed brief — without this, swapping a JSON
|
|
2661
|
+
// brief to plaintext between launch and assembly bypassed digest
|
|
2662
|
+
// verification entirely and still completed.
|
|
2663
|
+
if (requireStructuredBrief) return { conflict: true };
|
|
2664
|
+
return null; // plain-text brief; no WorkerDirectBrief to assemble against
|
|
2665
|
+
}
|
|
2666
|
+
if (pinnedDigest) {
|
|
2667
|
+
const liveDigest = `sha256:${createHash('sha256').update(briefBytes).digest('hex')}`;
|
|
2668
|
+
if (liveDigest !== pinnedDigest) return { conflict: true };
|
|
2669
|
+
}
|
|
2670
|
+
const { assertValidWorkerPayload, assembleWorkerResult } = await loadMissionProtocol();
|
|
2671
|
+
const brief = JSON.parse(text);
|
|
2672
|
+
assertValidWorkerPayload('workerDirectBrief', brief);
|
|
2673
|
+
const execution = {
|
|
2674
|
+
status: observation.status === 'completed' ? 'completed' : 'failed',
|
|
2675
|
+
exitCode: observation.exitCode, requested: invocation.selected,
|
|
2676
|
+
observedModel: null, observedFamily: null, observedEffort: null,
|
|
2677
|
+
identityEvidence: [], identityQuality: 'unavailable',
|
|
2678
|
+
startedAt: new Date().toISOString(), endedAt: new Date().toISOString(), termination: observation.termination,
|
|
2679
|
+
usage: { inputTokens: null, outputTokens: null, cacheTokens: null, reasoningTokens: null, costUsd: null, quality: 'unknown', reservedTokens: 0, reservedUsd: null },
|
|
2680
|
+
failure: observation.status === 'completed' ? null : { class: 'unknown', message: 'Worker did not report a completed claim', retryable: false },
|
|
2681
|
+
};
|
|
2682
|
+
// Never self-accept: acceptance requires a separate verifier with an
|
|
2683
|
+
// independently-verified artifact digest. This bridge only proves
|
|
2684
|
+
// observed process/transport termination and parsed claims.
|
|
2685
|
+
const verification = { status: 'pending', verifierId: null, runnerRef: null, artifactDigest: null, evidence: [], findings: [], controlledContext: null };
|
|
2686
|
+
return { result: assembleWorkerResult({ brief, execution, worker: claims, verification }) };
|
|
2687
|
+
} catch {
|
|
2688
|
+
// F4: the same `requireStructuredBrief` reasoning as above — a brief
|
|
2689
|
+
// that was provably structured at admission (mcp only) but now fails
|
|
2690
|
+
// to re-parse/re-validate at all is a conflict, not a silent no-op.
|
|
2691
|
+
if (requireStructuredBrief) return { conflict: true };
|
|
2692
|
+
return null; // best-effort enrichment only; the bounded claims result above always stands
|
|
2693
|
+
}
|
|
2694
|
+
}
|
|
2695
|
+
|
|
2696
|
+
async function status(input) {
|
|
2697
|
+
if (!input.invocation_id) {
|
|
2698
|
+
// #2915 round 8 (D4): ordinary status, read-only, no id — lists every
|
|
2699
|
+
// invocation this server currently tracks as a `dispatched` reservation
|
|
2700
|
+
// with `stop: 'unresolved'`, across every distinct breaker instance
|
|
2701
|
+
// any tracked invocation's gated admission used. Never mutates
|
|
2702
|
+
// anything (`listUnresolvedStops` is a pure read accessor on the
|
|
2703
|
+
// breaker); this is the "reachable for every long-lived owner" route
|
|
2704
|
+
// C4 requires, surfaced here rather than printed-only.
|
|
2705
|
+
const breakers = new Set(registry.recoveryOwner ? [registry.recoveryOwner] : []);
|
|
2706
|
+
for (const entry of registry.values()) if (entry.recoveryBreaker) breakers.add(entry.recoveryBreaker);
|
|
2707
|
+
const unresolvedReservations = [];
|
|
2708
|
+
for (const breaker of breakers) {
|
|
2709
|
+
if (typeof breaker.listUnresolvedStops === 'function') unresolvedReservations.push(...breaker.listUnresolvedStops());
|
|
2710
|
+
}
|
|
2711
|
+
const interventionScopes = [...breakers].flatMap(breaker => (breaker.snapshot?.().scopes ?? [])
|
|
2712
|
+
.filter(scope => scope.state === 'operator-intervention')
|
|
2713
|
+
.map(scope => ({ scopeId: scope.scopeKey, reason: scope.operatorReason,
|
|
2714
|
+
resetInstruction: { operation: 'reset_recovery_scope', scope_id: scope.scopeKey } })));
|
|
2715
|
+
return { unresolvedReservations, interventionScopes };
|
|
2716
|
+
}
|
|
2717
|
+
const entry = registry.get(input.invocation_id);
|
|
2718
|
+
if (!entry) throw new Error('Unknown invocation_id');
|
|
2719
|
+
return publicStatus(entry, healthHook);
|
|
2720
|
+
}
|
|
2721
|
+
|
|
2722
|
+
/**
|
|
2723
|
+
* #2915 round 8 (D4, "reconciliation route"): the explicit operator-
|
|
2724
|
+
* authorized reconciliation operation on this SAME `worker-invoke`
|
|
2725
|
+
* status/cancel handler — it already runs inside the owning server and
|
|
2726
|
+
* closes over its registry. Accepted ONLY for an invocation whose stored
|
|
2727
|
+
* reservation is `dispatched` with `stop: 'unresolved'` (verified via
|
|
2728
|
+
* `ScopedFailureBreaker.hasUnresolvedStop`, a pure read accessor — never
|
|
2729
|
+
* inferred from this registry's own, separately-maintained status
|
|
2730
|
+
* fields). Applied to THAT invocation's actual breaker (`entry.
|
|
2731
|
+
* recoveryBreaker`, pinned at admission — never a fresh default breaker).
|
|
2732
|
+
* Idempotent: a second call finds nothing left to reconcile and reports
|
|
2733
|
+
* why, rather than throwing or re-applying anything. No mailbox files;
|
|
2734
|
+
* nothing is written anywhere other than the breaker's own in-memory
|
|
2735
|
+
* reservation table.
|
|
2736
|
+
*/
|
|
2737
|
+
async function reconcileStopped(input) {
|
|
2738
|
+
if (!input.invocation_id) throw new Error('reconcile_stopped requires invocation_id');
|
|
2739
|
+
const entry = registry.get(input.invocation_id);
|
|
2740
|
+
if (!entry) throw new Error('Unknown invocation_id');
|
|
2741
|
+
const breaker = entry.recoveryBreaker ?? null;
|
|
2742
|
+
const base = publicStatus(entry, healthHook);
|
|
2743
|
+
if (!breaker || typeof breaker.hasUnresolvedStop !== 'function') {
|
|
2744
|
+
return { ...base, reconciled: false, reconcileReason: 'this invocation has no tracked gated reservation to reconcile' };
|
|
2745
|
+
}
|
|
2746
|
+
if (!breaker.hasUnresolvedStop(entry.invocationId)) {
|
|
2747
|
+
return { ...base, reconciled: false, reconcileReason: 'no dispatched reservation with unresolved stop evidence exists for this invocation (already reconciled, never dispatched, or already confirmed stopped)' };
|
|
2748
|
+
}
|
|
2749
|
+
const result = breaker.reconcileStopEvidence({ invocationId: entry.invocationId, confirmedStop: true });
|
|
2750
|
+
const applied = (result?.released ?? 0) > 0;
|
|
2751
|
+
// #2915 round 8 review 8 (C7): applying a confirmed stop must ALSO
|
|
2752
|
+
// finalize this invocation's registry capacity -- a dispatched
|
|
2753
|
+
// reservation with unresolved stop kept this entry's run-slot
|
|
2754
|
+
// effectively pinned (the registry's own `hasUnresolvedStopMarker`
|
|
2755
|
+
// exemption, above, retains it past the ordinary terminal sweep for
|
|
2756
|
+
// exactly this reason). `markTerminal({})` sets `terminalAt` without
|
|
2757
|
+
// touching `status` -- a truthful terminal state that never invents
|
|
2758
|
+
// task success for an attempt whose own outcome this bridge still
|
|
2759
|
+
// never confirmed. Idempotent: a no-op if the entry was already
|
|
2760
|
+
// terminal.
|
|
2761
|
+
if (applied) registry.markTerminal(entry.invocationId, {});
|
|
2762
|
+
return {
|
|
2763
|
+
...publicStatus(entry, healthHook),
|
|
2764
|
+
reconciled: applied,
|
|
2765
|
+
...(applied ? {} : { reconcileReason: 'reservation was already reconciled or no longer exists' }),
|
|
2766
|
+
};
|
|
2767
|
+
}
|
|
2768
|
+
|
|
2769
|
+
async function resetRecoveryScope(input) {
|
|
2770
|
+
if (!input.scope_id) throw new Error('reset_recovery_scope requires scope_id');
|
|
2771
|
+
const breaker = registry.recoveryOwner;
|
|
2772
|
+
if (!breaker) throw new Error('No recovery breaker owner is bound to this registry');
|
|
2773
|
+
return { scopeId: input.scope_id, ...breaker.reset(input.scope_id) };
|
|
2774
|
+
}
|
|
2775
|
+
|
|
2776
|
+
async function cancel(input) {
|
|
2777
|
+
const entry = registry.get(input.invocation_id);
|
|
2778
|
+
if (!entry) throw new Error('Unknown invocation_id');
|
|
2779
|
+
registry.update(entry.invocationId, { cancelRequested: true });
|
|
2780
|
+
// M2: actually deliver the cancellation when a live control handle
|
|
2781
|
+
// exists (the run has an in-flight shell/MCP transport call right now);
|
|
2782
|
+
// `cancelDelivered` reports delivery accurately — it is set ONLY when
|
|
2783
|
+
// the stop signal was actually transmitted, never unconditionally.
|
|
2784
|
+
if (entry.terminalAt === null && entry.control) {
|
|
2785
|
+
let delivered = false;
|
|
2786
|
+
let evidence;
|
|
2787
|
+
try {
|
|
2788
|
+
const result = await entry.control.stop();
|
|
2789
|
+
evidence = result && typeof result === 'object' ? result : undefined;
|
|
2790
|
+
delivered = evidence ? evidence.delivered === true : !!result;
|
|
2791
|
+
} catch { delivered = false; }
|
|
2792
|
+
registry.update(entry.invocationId, { cancelDelivered: delivered });
|
|
2793
|
+
if (evidence?.termination === 'confirmed') {
|
|
2794
|
+
registry.markTerminal(entry.invocationId, { status: 'cancelled', termination: 'confirmed', control: null });
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
// Cancellation delivery is recorded; it is NEVER treated as stop proof.
|
|
2798
|
+
// Overflow's retained control can separately prove termination through the
|
|
2799
|
+
// same stop authority used by the run; delivery alone never frees capacity.
|
|
2800
|
+
return publicStatus(registry.get(entry.invocationId), healthHook);
|
|
2801
|
+
}
|
|
2802
|
+
|
|
2803
|
+
server.registerTool('worker-invoke', {
|
|
2804
|
+
title: 'Invoke or Observe a Direct-Handoff Worker',
|
|
2805
|
+
description: "Run, check, cancel, or reconcile stop evidence for a qualified worker without any Mission/MC session — same policy/brief/claims path as the shell direct-handoff tier. Degraded isolation is a server-side operator setting only; there is no client-facing switch. Cancel delivery is not stop confirmation. 'status' with no invocation_id lists invocations still awaiting stop-evidence reconciliation; 'reconcile_stopped' applies that reconciliation for one invocation_id once the operator has independently established its worker actually stopped. 'status' also lists interventionScopes with their reason and reset instruction; 'reset_recovery_scope' requires scope_id and resets only that owning scope after all active or unresolved executions stop, preserving cumulative allowance.",
|
|
2806
|
+
inputSchema,
|
|
2807
|
+
annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
|
|
2808
|
+
}, async input => {
|
|
2809
|
+
try {
|
|
2810
|
+
if (input.operation === 'run') return mcpJson(publicStatus(await run(input), healthHook));
|
|
2811
|
+
if (input.operation === 'status') return mcpJson(await status(input));
|
|
2812
|
+
if (input.operation === 'reset_recovery_scope') return mcpJson(await resetRecoveryScope(input));
|
|
2813
|
+
if (input.operation === 'reconcile_stopped') return mcpJson(await reconcileStopped(input));
|
|
2814
|
+
return mcpJson(await cancel(input));
|
|
2815
|
+
} catch (error) {
|
|
2816
|
+
return mcpError(`worker-invoke: ${error.message}`);
|
|
2817
|
+
}
|
|
2818
|
+
});
|
|
2819
|
+
}
|
|
2820
|
+
|
|
2821
|
+
/**
|
|
2822
|
+
* #2892 B3: open with `O_NOFOLLOW` (never follow a symlink planted at the
|
|
2823
|
+
* leaf after `assertServedPath` validated it), verify via `fstat` that the
|
|
2824
|
+
* opened file is a regular, unaliased (`nlink === 1`) file within the byte
|
|
2825
|
+
* bound, and re-verify containment from the OPENED fd's own
|
|
2826
|
+
* `/proc/self/fd/<n>` target — not the pre-open path string — before
|
|
2827
|
+
* reading. This closes the gap where a path swap between validation and
|
|
2828
|
+
* this read could otherwise exfiltrate an arbitrary file's bytes into the
|
|
2829
|
+
* parsed worker claims.
|
|
2830
|
+
*/
|
|
2831
|
+
async function readBoundedFile(filePath, maxBytes, servedRootReal) {
|
|
2832
|
+
const handle = await fsOpen(filePath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
2833
|
+
try {
|
|
2834
|
+
const info = await handle.stat();
|
|
2835
|
+
if (!info.isFile() || info.nlink !== 1 || info.size > maxBytes) {
|
|
2836
|
+
throw new Error('worker output exceeds the bounded size or is not a regular unaliased file');
|
|
2837
|
+
}
|
|
2838
|
+
if (servedRootReal !== undefined) {
|
|
2839
|
+
const target = await fsReadlink(`/proc/self/fd/${handle.fd}`);
|
|
2840
|
+
const rel = path.relative(servedRootReal, target);
|
|
2841
|
+
if (rel === '..' || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
|
|
2842
|
+
throw new Error('worker output resolved outside the served project root after open');
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
const buffer = Buffer.alloc(info.size);
|
|
2846
|
+
let length = 0;
|
|
2847
|
+
while (length < buffer.length) {
|
|
2848
|
+
const { bytesRead } = await handle.read(buffer, length, buffer.length - length, null);
|
|
2849
|
+
if (!bytesRead) break;
|
|
2850
|
+
length += bytesRead;
|
|
2851
|
+
}
|
|
2852
|
+
return buffer.subarray(0, length);
|
|
2853
|
+
} finally {
|
|
2854
|
+
await handle.close();
|
|
2855
|
+
}
|
|
2856
|
+
}
|