@aiwg/cli 2026.10.3 → 2026.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/agentic/code/providers/model-capabilities.v1.json +96 -4
- package/bin/aiwg.mjs +50 -27
- package/dist/src/agents/grounding/knowledge-base.js +54 -6
- package/dist/src/artifacts/browser-export.js +6 -0
- package/dist/src/artifacts/checksum-manifest.js +1 -0
- package/dist/src/artifacts/citation-parser.js +173 -92
- package/dist/src/artifacts/citation-table-schema.js +59 -0
- package/dist/src/artifacts/cli.js +69 -18
- package/dist/src/artifacts/corpus-tools/snapshot.js +1 -1
- package/dist/src/artifacts/corpus-tools/vision-extract.js +2 -0
- package/dist/src/artifacts/corpus-views/ref-parser.js +3 -1
- package/dist/src/artifacts/corpus-views/renderers.js +3 -3
- package/dist/src/artifacts/discovery-eval.js +6 -4
- package/dist/src/artifacts/fortemi-core-query-adapter.js +36 -3
- package/dist/src/artifacts/fortemi-core-sync.js +166 -40
- package/dist/src/artifacts/framework-capabilities.js +171 -0
- package/dist/src/artifacts/framework-index-paths.js +73 -0
- package/dist/src/artifacts/framework-source-digest.js +75 -0
- package/dist/src/artifacts/index-builder.js +286 -131
- package/dist/src/artifacts/index-files.js +132 -18
- package/dist/src/artifacts/index-preparation.js +130 -0
- package/dist/src/artifacts/index-refresh.js +591 -0
- package/dist/src/artifacts/index-worker.mjs +11 -0
- package/dist/src/artifacts/prebuilt-build-lock.js +47 -4
- package/dist/src/artifacts/prebuilt-deployment.js +46 -0
- package/dist/src/artifacts/prebuilt-relations.js +80 -0
- package/dist/src/artifacts/query-engine.js +410 -88
- package/dist/src/artifacts/relations-builder.js +168 -0
- package/dist/src/artifacts/relations.js +148 -0
- package/dist/src/artifacts/stats.js +5 -2
- package/dist/src/artifacts/types.js +30 -10
- package/dist/src/artifacts/watcher.js +1 -1
- package/dist/src/auth/credential-store.js +2 -1
- package/dist/src/catalog/cli.d.mts +5 -0
- package/dist/src/channel/manager.d.mts +20 -0
- package/dist/src/channel/manager.mjs +19 -2
- package/dist/src/cli/command-log.js +2 -3
- package/dist/src/cli/handlers/artifacts.js +3 -0
- package/dist/src/cli/handlers/auth.js +3 -0
- package/dist/src/cli/handlers/decision.js +19 -12
- package/dist/src/cli/handlers/diagnose.js +4 -3
- package/dist/src/cli/handlers/effect.js +4 -2
- package/dist/src/cli/handlers/evidence.js +3 -0
- package/dist/src/cli/handlers/help.js +1 -0
- package/dist/src/cli/handlers/index.js +3 -1
- package/dist/src/cli/handlers/init.js +4 -1
- package/dist/src/cli/handlers/lint.js +5 -0
- package/dist/src/cli/handlers/models.js +104 -8
- package/dist/src/cli/handlers/ralph-launcher.js +2 -0
- package/dist/src/cli/handlers/refresh.js +12 -0
- package/dist/src/cli/handlers/resource-versions.js +3 -0
- package/dist/src/cli/handlers/rules.js +195 -0
- package/dist/src/cli/handlers/run.js +4 -0
- package/dist/src/cli/handlers/runtime-info.js +1 -2
- package/dist/src/cli/handlers/script-runner.js +3 -2
- package/dist/src/cli/handlers/session.js +19 -9
- package/dist/src/cli/handlers/sessions.js +137 -43
- package/dist/src/cli/handlers/setup-manifest.js +39 -44
- package/dist/src/cli/handlers/setup.js +41 -0
- package/dist/src/cli/handlers/steward.js +11 -0
- package/dist/src/cli/handlers/subcommands.js +57 -2
- package/dist/src/cli/handlers/uhp.js +3 -0
- package/dist/src/cli/handlers/use.js +294 -55
- package/dist/src/cli/handlers/utilities.js +26 -10
- package/dist/src/cli/handlers/version.js +8 -0
- package/dist/src/cli/router.js +15 -1
- package/dist/src/cli/services/deployment-verification.js +19 -9
- package/dist/src/cli/services/provider-prepare-coordinator.js +24 -0
- package/dist/src/cli/skill-usage.js +2 -3
- package/dist/src/config/aiwg-config.js +87 -37
- package/dist/src/config/cli.js +111 -5
- package/dist/src/config/decision-config.js +106 -0
- package/dist/src/config/dispatch-thresholds-runtime.d.mts +8 -0
- package/dist/src/config/gitignore.js +50 -0
- package/dist/src/config/index-graph-overrides.js +36 -0
- package/dist/src/config/project-artifacts-health.d.mts +37 -0
- package/dist/src/config/project-artifacts-runtime.d.mts +16 -0
- package/dist/src/config/user-config-dir.d.mts +9 -0
- package/dist/src/config/user-config.js +2 -1
- package/dist/src/config/user-registry.js +23 -15
- package/dist/src/config/user-state.js +7 -0
- package/dist/src/decision/ask.js +203 -43
- package/dist/src/decision/context-live-qualification.js +2 -2
- package/dist/src/decision/driver.js +13 -8
- package/dist/src/decision/ensemble/aggregate.js +23 -10
- package/dist/src/decision/ensemble-study/multifact-study.js +6 -2
- package/dist/src/decision/entry.js +2 -15
- package/dist/src/decision/graph-live-qualification.js +1 -1
- package/dist/src/decision/graph-run.js +9 -0
- package/dist/src/decision/heldout/collector.js +5 -13
- package/dist/src/decision/heldout/contract.js +30 -7
- package/dist/src/decision/jev-credentials.js +15 -1
- package/dist/src/decision/ledger.js +131 -28
- package/dist/src/decision/offload-activation.js +137 -0
- package/dist/src/decision/offload.js +550 -0
- package/dist/src/decision/qualification/load.js +4 -6
- package/dist/src/decision/savings.js +28 -0
- package/dist/src/effects/types.js +5 -1
- package/dist/src/effects/verifiers/git.js +2 -1
- package/dist/src/effects/verifiers/tracker.js +26 -14
- package/dist/src/extensions/commands/definitions.js +19 -0
- package/dist/src/extensions/project-local-remove.js +36 -1
- package/dist/src/gates/providers/loader.js +61 -17
- package/dist/src/installation/manager-command.d.mts +18 -0
- package/dist/src/installation/manager-command.mjs +14 -2
- package/dist/src/installation/manager.d.mts +49 -0
- package/dist/src/installation/manager.mjs +78 -3
- package/dist/src/issues/cli.js +4 -4
- package/dist/src/issues/local.js +24 -5
- package/dist/src/jobs/runner.js +1 -1
- package/dist/src/lint/cli.js +27 -1
- package/dist/src/lint/loader.js +2 -1
- package/dist/src/lint/reporters.js +25 -1
- package/dist/src/lint/rule-activation.js +43 -0
- package/dist/src/lint/runner.js +137 -62
- package/dist/src/mcp/adapters/codex-runtime.js +7 -1
- package/dist/src/mcp/claude-install.d.mts +4 -0
- package/dist/src/mcp/claude-install.mjs +41 -0
- package/dist/src/mcp/cli.d.mts +1 -0
- package/dist/src/mcp/cli.mjs +36 -11
- package/dist/src/mcp/config-file.d.mts +28 -0
- package/dist/src/mcp/config-layers.d.mts +36 -0
- package/dist/src/mcp/credentials.d.mts +51 -0
- package/dist/src/mcp/grok-build-config.d.mts +6 -0
- package/dist/src/mcp/grok-build-config.mjs +1 -1
- package/dist/src/mcp/helpers.mjs +3 -2
- package/dist/src/mcp/muse-native-config.d.mts +13 -0
- package/dist/src/mcp/muse-native-config.mjs +1 -0
- package/dist/src/mcp/omp-config.d.mts +4 -0
- package/dist/src/mcp/registry.js +3 -93
- package/dist/src/mcp/registry.mjs +3 -87
- package/dist/src/mcp/server-config.d.mts +3 -0
- package/dist/src/mcp/server-config.mjs +89 -0
- package/dist/src/mcp/server.mjs +37 -375
- package/dist/src/mcp/test-simple.mjs +2 -1
- package/dist/src/mcp/toml-editor.d.mts +4 -0
- package/dist/src/mcp/toml-string.d.mts +1 -0
- package/dist/src/mcp/toml-strip-mcp.d.mts +1 -0
- package/dist/src/mcp/tool-filters.d.mts +46 -0
- package/dist/src/mcp/tools/decision.mjs +12 -11
- package/dist/src/mcp/tools/discovery.mjs +62 -55
- package/dist/src/mcp/tools/orchestration.mjs +12 -2
- package/dist/src/mcp/tools/worker-invoke.mjs +1340 -23
- package/dist/src/mcp/tools/worker-shell-control.mjs +52 -21
- package/dist/src/mcp/tools/worker-stdin-runtime.mjs +3 -3
- package/dist/src/mission-protocol/codecs.js +8 -0
- package/dist/src/models/config-loader.js +7 -2
- package/dist/src/models/model-capabilities.v1.json +96 -4
- package/dist/src/models/model-discovery.js +76 -11
- package/dist/src/models/provider-policy.js +101 -7
- package/dist/src/models/qualification-digest.d.mts +2 -0
- package/dist/src/models/resolver.js +12 -10
- package/dist/src/models/worker-direct-readiness.js +65 -44
- package/dist/src/models/worker-dispatch-offer.js +3 -3
- package/dist/src/models/worker-health-readiness.js +16 -3
- package/dist/src/models/worker-installation.d.mts +39 -0
- package/dist/src/models/worker-installation.mjs +135 -28
- package/dist/src/process/bounded-child-process.d.mts +84 -0
- package/dist/src/process/bounded-child-process.mjs +43 -0
- package/dist/src/process/cli-command.d.mts +16 -0
- package/dist/src/process/signal-child.d.mts +6 -0
- package/dist/src/process/signal-child.mjs +7 -0
- package/dist/src/providers/deployed-skill-roots.js +20 -0
- package/dist/src/providers/grok-build-acp.js +3 -2
- package/dist/src/providers/grok-build-headless.js +12 -4
- package/dist/src/providers/grok-build-inspect.js +1 -0
- package/dist/src/providers/omp-agent.d.mts +8 -0
- package/dist/src/providers/omp-diagnostics.d.mts +1 -0
- package/dist/src/providers/omp-paths.d.mts +13 -0
- package/dist/src/providers/skill-ownership.d.mts +9 -0
- package/dist/src/providers/skill-ownership.mjs +88 -0
- package/dist/src/rules/activation.d.mts +21 -0
- package/dist/src/rules/activation.mjs +62 -0
- package/dist/src/rules/inline-budget.d.mts +26 -0
- package/dist/src/rules/inline-budget.mjs +66 -0
- package/dist/src/rules/inventory.js +129 -0
- package/dist/src/rules/pin-state.js +73 -0
- package/dist/src/rules/pins.d.mts +14 -0
- package/dist/src/rules/pins.mjs +112 -0
- package/dist/src/rules/rule-frontmatter.schema.json +197 -0
- package/dist/src/serve/mission-conductor.js +38 -12
- package/dist/src/serve/worker-invocation.js +30 -10
- package/dist/src/serve/worker-stdin-frame.d.mts +10 -0
- package/dist/src/sessions/adapter-registry.js +23 -19
- package/dist/src/sessions/adapters/claude.js +39 -54
- package/dist/src/sessions/adapters/codex.js +95 -67
- package/dist/src/sessions/adapters/copilot.js +4 -4
- package/dist/src/sessions/adapters/cursor.js +40 -57
- package/dist/src/sessions/adapters/deepseek-harness.js +15 -12
- package/dist/src/sessions/adapters/factory.js +31 -27
- package/dist/src/sessions/adapters/generic.js +10 -10
- package/dist/src/sessions/adapters/grok-build.js +11 -11
- package/dist/src/sessions/adapters/grokbot.js +24 -13
- package/dist/src/sessions/adapters/hermes.js +27 -29
- package/dist/src/sessions/adapters/muse-workspace-scan.js +31 -11
- package/dist/src/sessions/adapters/muse.js +150 -93
- package/dist/src/sessions/adapters/omp.js +28 -14
- package/dist/src/sessions/adapters/openclaw.js +89 -79
- package/dist/src/sessions/adapters/opencode.js +27 -24
- package/dist/src/sessions/adapters/openhuman.js +18 -60
- package/dist/src/sessions/adapters/pi.js +29 -14
- package/dist/src/sessions/adapters/warp.js +3 -3
- package/dist/src/sessions/adapters/windsurf.js +19 -48
- package/dist/src/sessions/batch-contracts.js +16 -8
- package/dist/src/sessions/batch-import.js +252 -12
- package/dist/src/sessions/contracts.js +50 -3
- package/dist/src/sessions/coverage-preflight.js +27 -2
- package/dist/src/sessions/export-plan.js +4 -3
- package/dist/src/sessions/import-exclusions.js +72 -0
- package/dist/src/sessions/import-lease.js +34 -18
- package/dist/src/sessions/import-limits.js +37 -0
- package/dist/src/sessions/importer.js +60 -38
- package/dist/src/sessions/lifecycle.js +45 -0
- package/dist/src/sessions/policy.js +145 -26
- package/dist/src/sessions/readers.js +106 -53
- package/dist/src/sessions/refresh.js +105 -0
- package/dist/src/sessions/repository.js +151 -54
- package/dist/src/sessions/workspace-discovery.js +292 -60
- package/dist/src/sessions/workspace-matching.js +107 -0
- package/dist/src/skills/importer.js +1 -0
- package/dist/src/smiths/context-pipeline/aiwg-md.js +11 -0
- package/dist/src/smiths/context-pipeline/claude-hook.js +3 -26
- package/dist/src/smiths/context-pipeline/finalization.js +6 -5
- package/dist/src/smiths/context-pipeline/generator.js +21 -4
- package/dist/src/smiths/context-pipeline/parallelism-section.js +7 -15
- package/dist/src/smiths/mcpsmith/generator.js +1 -1
- package/dist/src/smiths/toolsmith/runtime-discovery.d.mts +10 -0
- package/dist/src/smiths/toolsmith/runtime-discovery.mjs +76 -18
- package/dist/src/storage/backends/fs.js +18 -76
- package/dist/src/storage/backends/obsidian.js +1 -0
- package/dist/src/tracker/forge-http.js +2 -1
- package/dist/src/update/checker.d.mts +2 -0
- package/dist/src/update/service.d.mts +22 -0
- package/dist/src/utils/contained-filesystem.d.mts +37 -0
- package/dist/src/utils/contained-filesystem.mjs +683 -0
- package/dist/src/utils/temporary-directory.js +36 -0
- package/dist/src/writing/example-generator.js +1 -1
- package/package.json +15 -2
- package/schemas/models/model-catalog.v1.schema.json +1 -1
- package/schemas/models/provider-model-capabilities.v1.schema.json +6 -0
- package/tools/_resolve-impl.mjs +13 -15
- package/tools/agents/deploy-agents.mjs +49 -21
- package/tools/agents/providers/base.mjs +104 -134
- package/tools/agents/providers/claude.mjs +8 -24
- package/tools/agents/providers/codex.mjs +18 -11
- package/tools/agents/providers/cursor.mjs +19 -18
- package/tools/agents/providers/grok-build.mjs +3 -1
- package/tools/agents/providers/hermes.mjs +57 -47
- package/tools/agents/providers/openclaw.mjs +9 -4
- package/tools/agents/providers/pi.mjs +5 -0
- package/tools/agents/providers/provider-prepare-channel.mjs +5 -0
- package/tools/agents/providers/provider-prepare-coordinator.mjs +247 -0
- package/tools/agents/providers/provider-prepare-worker.mjs +84 -0
- package/tools/agents/providers/rule-delivery.mjs +1270 -0
- package/tools/agents/providers/warp.mjs +20 -26
- package/tools/cli/decision-offload-report.mjs +40 -0
- package/tools/cli/workspace-status.mjs +862 -0
- package/tools/plugin/package-plugins.mjs +186 -8
- package/tools/plugin/plugin-links.mjs +114 -0
- package/tools/providers/antigravity-transport.mjs +3 -1
- package/tools/providers/deepseek-harness-transport.mjs +6 -4
- package/tools/providers/omp-transport.mjs +4 -2
- package/tools/ralph-external/lib/consecutive-failure-circuit-breaker.mjs +1748 -0
- package/tools/ralph-external/lib/launcher-recovery-evidence.mjs +55 -0
- package/tools/ralph-external/lib/omp-adapter.mjs +1 -1
- package/tools/ralph-external/lib/pi-adapter.mjs +0 -4
- package/tools/ralph-external/lib/provider-adapter.mjs +59 -1
- package/tools/ralph-external/lib/recovery-probe.mjs +72 -0
- package/tools/ralph-external/lib/session-environment.mjs +28 -0
- package/tools/ralph-external/lib/systemd-client.mjs +49 -0
- package/tools/ralph-external/lib/worker-health.mjs +1 -1
- package/tools/ralph-external/lib/worker-isolation.mjs +717 -0
- package/tools/ralph-external/lib/worker-overflow.mjs +50 -0
- package/tools/ralph-external/process-monitor.mjs +1017 -0
- package/tools/ralph-external/session-launcher.mjs +5057 -0
- package/tools/scripts/resource-preflight.mjs +541 -0
- package/tools/skills/deploy-skills-codex.mjs +2 -1
- package/vscode-extension/schemas/aiwg.config.v1.json +1194 -231
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* AIWG MCP Worker-Invoke Bridge (#2892)
|
|
3
3
|
*
|
|
4
|
-
* MCP-only conductor: resolve and invoke a worker (run/status/cancel)
|
|
4
|
+
* MCP-only conductor: resolve and invoke a worker (run/status/cancel/reconcile_stopped/reset_recovery_scope)
|
|
5
5
|
* WITHOUT any Mission/MC session, daemon, admission store, or activation
|
|
6
6
|
* marker. Reuses, rather than reimplements, the existing Tier 0 primitives:
|
|
7
7
|
*
|
|
@@ -157,6 +157,7 @@
|
|
|
157
157
|
* @issues #2892, #2896 (review reference, not reused as-is)
|
|
158
158
|
*/
|
|
159
159
|
|
|
160
|
+
import { signalChild } from '../../process/signal-child.mjs';
|
|
160
161
|
import { z } from 'zod';
|
|
161
162
|
import { createHash, randomUUID } from 'node:crypto';
|
|
162
163
|
import {
|
|
@@ -450,8 +451,26 @@ export function createInvocationRegistry({
|
|
|
450
451
|
const entries = new Map();
|
|
451
452
|
const seenRequestIds = new Map(); // request_id -> { invocationId, recordedAt }
|
|
452
453
|
|
|
454
|
+
// Pin one bounded breaker owner for this registry's lifetime. Replacing
|
|
455
|
+
// it could accumulate protected entries across independently capped owners
|
|
456
|
+
// and orphan the original reconciliation mapping.
|
|
457
|
+
let recoveryOwner;
|
|
458
|
+
function bindRecoveryOwner(breaker) {
|
|
459
|
+
if (!breaker) return;
|
|
460
|
+
if (recoveryOwner && recoveryOwner !== breaker) {
|
|
461
|
+
throw new Error('Invocation registry already has a recovery breaker owner; reuse that owner and reconcile_stopped for unresolved reservations');
|
|
462
|
+
}
|
|
463
|
+
recoveryOwner = breaker;
|
|
464
|
+
}
|
|
465
|
+
// Unresolved entries and their replay identities survive ordinary pruning.
|
|
466
|
+
// Their aggregate bound is the single owner's reservation cap.
|
|
467
|
+
function hasUnresolvedStopMarker(entry) {
|
|
468
|
+
return typeof entry.recoveryBreaker?.hasUnresolvedStop === 'function'
|
|
469
|
+
&& entry.recoveryBreaker.hasUnresolvedStop(entry.invocationId);
|
|
470
|
+
}
|
|
453
471
|
function terminalEntries() {
|
|
454
|
-
return [...entries.values()].filter(entry => entry.terminalAt !== null
|
|
472
|
+
return [...entries.values()].filter(entry => entry.terminalAt !== null && !hasUnresolvedStopMarker(entry))
|
|
473
|
+
.sort((a, b) => a.terminalAt - b.terminalAt);
|
|
455
474
|
}
|
|
456
475
|
function pruneSeenRequestIds() {
|
|
457
476
|
const cutoff = now() - ttlMs;
|
|
@@ -489,6 +508,8 @@ export function createInvocationRegistry({
|
|
|
489
508
|
return {
|
|
490
509
|
prune,
|
|
491
510
|
liveCount,
|
|
511
|
+
bindRecoveryOwner,
|
|
512
|
+
get recoveryOwner() { return recoveryOwner; },
|
|
492
513
|
get size() { return entries.size; },
|
|
493
514
|
/** Test/diagnostic only — not consulted by any admission decision. */
|
|
494
515
|
get seenCount() { return seenRequestIds.size; },
|
|
@@ -499,6 +520,7 @@ export function createInvocationRegistry({
|
|
|
499
520
|
return seenRequestIds.get(requestId)?.invocationId ?? null;
|
|
500
521
|
},
|
|
501
522
|
create(partial) {
|
|
523
|
+
bindRecoveryOwner(partial.recoveryBreaker);
|
|
502
524
|
prune();
|
|
503
525
|
const invocationId = partial.invocationId ?? randomUUID();
|
|
504
526
|
const entry = { invocationId, terminalAt: null, createdAt: now(), ...partial, invocationId };
|
|
@@ -510,12 +532,14 @@ export function createInvocationRegistry({
|
|
|
510
532
|
update(invocationId, patch) {
|
|
511
533
|
const entry = entries.get(invocationId);
|
|
512
534
|
if (!entry) return undefined;
|
|
535
|
+
bindRecoveryOwner(patch.recoveryBreaker);
|
|
513
536
|
Object.assign(entry, patch);
|
|
514
537
|
return entry;
|
|
515
538
|
},
|
|
516
539
|
markTerminal(invocationId, patch = {}) {
|
|
517
540
|
const entry = entries.get(invocationId);
|
|
518
541
|
if (!entry) return undefined;
|
|
542
|
+
bindRecoveryOwner(patch.recoveryBreaker);
|
|
519
543
|
if (entry.terminalAt === null) entry.terminalAt = now();
|
|
520
544
|
Object.assign(entry, patch);
|
|
521
545
|
// An entry becomes eligible for over-capacity eviction the moment it
|
|
@@ -559,6 +583,13 @@ async function loadAiwgConfigModule() {
|
|
|
559
583
|
* AIWG_ROOT-absolute path is the robust way in rather than a relative
|
|
560
584
|
* specifier (whose depth from this file differs between `src/mcp/tools/`
|
|
561
585
|
* and `dist/src/mcp/tools/`). */
|
|
586
|
+
// #2915 final (step 3): the former `resolveRunRecovery` + per-site
|
|
587
|
+
// `canLaunch`/`noteLaunched`/`recordDirectWorkerOutcome` triple is replaced
|
|
588
|
+
// by `run()`'s own single `admitDirectWorker`/`settleDirectWorker` pair
|
|
589
|
+
// (session-launcher.mjs), which `defaultLaunchShellWorker` below BORROWS
|
|
590
|
+
// rather than re-resolving — deleting the duplicate gate that previously
|
|
591
|
+
// rejected its own already-admitted canary (#2915 final review, finding 4).
|
|
592
|
+
|
|
562
593
|
async function loadRalphExternal(relativeFromToolsRoot) {
|
|
563
594
|
const absolute = path.join(findOwnPackageRoot(), 'tools/ralph-external', relativeFromToolsRoot);
|
|
564
595
|
return import(pathToFileURL(absolute).href);
|
|
@@ -604,7 +635,45 @@ function describeGateFailures(failures) {
|
|
|
604
635
|
return `worker dispatch gate blocked: ${failures.map(f => `${f.signal} (${f.reason})`).join('; ')}`;
|
|
605
636
|
}
|
|
606
637
|
|
|
638
|
+
// Audited observer; refresh the historical baseline graph after rebase.
|
|
639
|
+
const waitForUnitTerminationBaseline = waitForUnitTermination;
|
|
640
|
+
export { waitForUnitTerminationBaseline as __testOnlyWaitForUnitTerminationBaseline };
|
|
641
|
+
|
|
642
|
+
// Recovery-disabled dispatch uses the audited preparation path and direct
|
|
643
|
+
// gate/isolation primitives, without entering the current planner's admission.
|
|
644
|
+
// The historical baseline pin/fixture graph must be refreshed after rebase.
|
|
645
|
+
async function defaultLaunchShellWorkerBaseline(invocation, { memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, ...options }) {
|
|
646
|
+
const loadBaselineLaunch = async relative => {
|
|
647
|
+
if (relative !== 'session-launcher.mjs') return loadRalphExternal(relative);
|
|
648
|
+
const { checkWorkerDispatch, launchIsolatedWorker } = await loadRalphExternal('lib/worker-isolation.mjs');
|
|
649
|
+
return {
|
|
650
|
+
planDirectWorkerLaunch: ({ invocation, live, readers, thresholdOverlay, ...launchOptions }) => {
|
|
651
|
+
// Shared preparation must provide the conductor-framed params file;
|
|
652
|
+
// never send the raw brief or launch without it for brief-json.
|
|
653
|
+
if (invocation?.stdin === 'brief-json' && launchOptions.stdinPath === undefined) {
|
|
654
|
+
throw new Error('brief-json stdin requires an explicit conductor-framed params file (stdinPath)');
|
|
655
|
+
}
|
|
656
|
+
const cwd = invocation?.cwd;
|
|
657
|
+
const gate = checkWorkerDispatch({ config: launchOptions.config, live, readers, thresholdOverlay,
|
|
658
|
+
paths: typeof cwd === 'string' && cwd ? [cwd] : undefined });
|
|
659
|
+
if (!gate.ok) return { launched: false, blocked: true, failures: gate.failures ?? [],
|
|
660
|
+
diagnostic: describeGateFailures(gate.failures) };
|
|
661
|
+
return launchIsolatedWorker({
|
|
662
|
+
command: invocation?.argv?.[0], args: invocation?.argv?.slice(1) ?? [], cwd,
|
|
663
|
+
stdoutPath: invocation?.stdoutPath, stderrPath: invocation?.stderrPath, envNames: invocation?.envNames,
|
|
664
|
+
...launchOptions, gate,
|
|
665
|
+
});
|
|
666
|
+
},
|
|
667
|
+
};
|
|
668
|
+
};
|
|
669
|
+
return launchPreparedShellWorker(invocation, { ...options, memoryMaxGb },
|
|
670
|
+
{ loadRalphExternal: loadBaselineLaunch, readBoundedFile, importImpl, describeGateFailures });
|
|
671
|
+
}
|
|
672
|
+
|
|
607
673
|
async function defaultLaunchShellWorker(invocation, { memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, ...options }) {
|
|
674
|
+
if (options.admission === undefined) {
|
|
675
|
+
return defaultLaunchShellWorkerBaseline(invocation, { ...options, memoryMaxGb });
|
|
676
|
+
}
|
|
608
677
|
return launchPreparedShellWorker(invocation, { ...options, memoryMaxGb },
|
|
609
678
|
{ loadRalphExternal, readBoundedFile, importImpl, describeGateFailures });
|
|
610
679
|
}
|
|
@@ -636,12 +705,22 @@ async function defaultLaunchShellWorker(invocation, { memoryMaxGb = DEFAULT_MCP_
|
|
|
636
705
|
* already resource-confined, already running with exactly its allowlisted
|
|
637
706
|
* environment.
|
|
638
707
|
*/
|
|
708
|
+
const MCP_TRANSPORT_MAX_BUFFER_BYTES = MAX_MCP_FRAME_BYTES;
|
|
709
|
+
|
|
639
710
|
export class ChildProcessTransport {
|
|
640
|
-
constructor(child, { terminate = () => child
|
|
711
|
+
constructor(child, { terminate = () => signalChild(child, 'SIGKILL'),
|
|
712
|
+
deadlineAtMs = null, now = () => Date.now(), isCancelled } = {}) {
|
|
641
713
|
this._child = child;
|
|
642
714
|
this._terminate = terminate;
|
|
643
715
|
this._failure = undefined;
|
|
644
716
|
this._closed = false;
|
|
717
|
+
this._deadlineAtMs = deadlineAtMs;
|
|
718
|
+
this._now = now;
|
|
719
|
+
// #2915 round 8 review 8 (C3): optional cancellation predicate, checked
|
|
720
|
+
// at the write boundary alongside the deadline — `undefined` whenever
|
|
721
|
+
// this call is not gated, matching origin/main's own unbounded
|
|
722
|
+
// transport exactly (see `send()` below).
|
|
723
|
+
this._isCancelled = typeof isCancelled === 'function' ? isCancelled : undefined;
|
|
645
724
|
this.onmessage = undefined;
|
|
646
725
|
this.onerror = undefined;
|
|
647
726
|
this.onclose = undefined;
|
|
@@ -781,6 +860,23 @@ export class ChildProcessTransport {
|
|
|
781
860
|
this.lastRequestId = message.id;
|
|
782
861
|
}
|
|
783
862
|
const { serializeMessage } = await import('@modelcontextprotocol/sdk/shared/stdio.js');
|
|
863
|
+
// #2915 round 8 (D5): the write boundary — AFTER the dynamic import's
|
|
864
|
+
// own `await` (review 7 finding 4: "checking before an SDK call can
|
|
865
|
+
// precede the actual send by an await... a request can cross its
|
|
866
|
+
// deadline during that gap"). Stop/cancel cleanup traffic
|
|
867
|
+
// (`notifications/cancelled`) is exempt — it must always be allowed to
|
|
868
|
+
// reach the worker, deadline or not.
|
|
869
|
+
const isStopTraffic = message && typeof message === 'object' && message.method === 'notifications/cancelled';
|
|
870
|
+
if (this._deadlineAtMs !== null && !isStopTraffic && this._now() >= this._deadlineAtMs) {
|
|
871
|
+
throw Object.assign(new Error('worker MCP transport deadline exceeded before write'), { code: -32001 });
|
|
872
|
+
}
|
|
873
|
+
// #2915 round 8 review 8 (C3): cancellation at the write boundary,
|
|
874
|
+
// exempting stop/cleanup traffic for the same reason as the deadline
|
|
875
|
+
// check above — a `notifications/cancelled` notification must always be
|
|
876
|
+
// allowed to reach the worker, cancelled or not.
|
|
877
|
+
if (!isStopTraffic && this._isCancelled?.()) {
|
|
878
|
+
throw Object.assign(new Error('worker MCP transport refused: cancelled before write'), { code: -32800 });
|
|
879
|
+
}
|
|
784
880
|
return new Promise((resolve, reject) => {
|
|
785
881
|
if (!this._child.stdin) { reject(new Error('worker MCP process has no stdin')); return; }
|
|
786
882
|
const json = serializeMessage(message);
|
|
@@ -795,11 +891,11 @@ export class ChildProcessTransport {
|
|
|
795
891
|
try { child.stdin?.end(); } catch { /* already gone */ }
|
|
796
892
|
await Promise.race([closed, new Promise(resolve => setTimeout(resolve, 2000).unref?.())]);
|
|
797
893
|
if (child.exitCode === null && child.signalCode === null) {
|
|
798
|
-
try { child
|
|
894
|
+
try { signalChild(child, 'SIGTERM'); } catch { /* already gone */ }
|
|
799
895
|
await Promise.race([closed, new Promise(resolve => setTimeout(resolve, 2000).unref?.())]);
|
|
800
896
|
}
|
|
801
897
|
if (child.exitCode === null && child.signalCode === null) {
|
|
802
|
-
try { child
|
|
898
|
+
try { signalChild(child, 'SIGKILL'); } catch { /* already gone */ }
|
|
803
899
|
}
|
|
804
900
|
this._buffer = undefined;
|
|
805
901
|
}
|
|
@@ -817,7 +913,43 @@ export class ChildProcessTransport {
|
|
|
817
913
|
* registry-declared `server.cwd` (none of today's callers set one) still
|
|
818
914
|
* wins when present, matching the pre-existing precedence for the actual
|
|
819
915
|
* spawn directory. */
|
|
820
|
-
|
|
916
|
+
/** #2915 round 8 (D6): the MCP-native counterpart of
|
|
917
|
+
* `defaultLaunchShellWorkerBaseline` — see its doc comment for why this
|
|
918
|
+
* bypasses the CURRENT `planDirectWorkerLaunch` entirely rather than
|
|
919
|
+
* omitting `admission` and letting that function's own standalone
|
|
920
|
+
* admission branch run (review 7 finding 13's second bullet, which names
|
|
921
|
+
* this exact function: "MCP native path, M:941... omitting the disabled
|
|
922
|
+
* admission loses CLI identity before the flat planner. It performs a new
|
|
923
|
+
* admission; at 128 occupied reservations, disabled work returns blocked
|
|
924
|
+
* where baseline launches"). */
|
|
925
|
+
async function defaultLaunchMcpWorkerProcessBaseline(server, { allowDegraded, config, name, cwd, memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, live, readers, thresholdOverlay }) {
|
|
926
|
+
if (server?.type !== 'stdio') {
|
|
927
|
+
return { launched: false, blocked: true, diagnostic: 'Only stdio MCP worker servers are qualified for direct handoff' };
|
|
928
|
+
}
|
|
929
|
+
const { checkWorkerDispatch, launchIsolatedWorker } = await loadRalphExternal('lib/worker-isolation.mjs');
|
|
930
|
+
const envNames = Object.keys(server.env ?? {});
|
|
931
|
+
const resolvedCwd = server.cwd ?? cwd;
|
|
932
|
+
const gate = checkWorkerDispatch({ config, live, readers, thresholdOverlay, paths: typeof resolvedCwd === 'string' && resolvedCwd ? [resolvedCwd] : undefined });
|
|
933
|
+
if (!gate.ok) return { launched: false, blocked: true, failures: gate.failures ?? [], diagnostic: describeGateFailures(gate.failures) };
|
|
934
|
+
const launch = await launchIsolatedWorker({
|
|
935
|
+
name, memoryMaxGb, command: server.command ?? '', args: server.args ?? [], cwd: resolvedCwd,
|
|
936
|
+
envNames, env: server.env ?? {}, gate, config, allowDegraded, attached: true,
|
|
937
|
+
// See `defaultLaunchMcpWorkerProcess`: an MCP stdio server needs a live,
|
|
938
|
+
// writable stdin to receive JSON-RPC requests.
|
|
939
|
+
stdin: 'open',
|
|
940
|
+
});
|
|
941
|
+
if (!launch.launched && launch.diagnostic === undefined) {
|
|
942
|
+
return { ...launch, diagnostic: describeGateFailures(launch.failures) };
|
|
943
|
+
}
|
|
944
|
+
return launch;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
async function defaultLaunchMcpWorkerProcess(server, { allowDegraded, config, name, cwd, memoryMaxGb = DEFAULT_MCP_WORKER_MEMORY_MAX_GB, live, readers, thresholdOverlay, admission, invocation, invocationId, dispatchGuard }) {
|
|
948
|
+
// #2915 round 8 (D6): see `defaultLaunchShellWorker`'s identical guard —
|
|
949
|
+
// `admission` is only ever present when this attempt is actually gated.
|
|
950
|
+
if (admission === undefined) {
|
|
951
|
+
return defaultLaunchMcpWorkerProcessBaseline(server, { allowDegraded, config, name, cwd, memoryMaxGb, live, readers, thresholdOverlay });
|
|
952
|
+
}
|
|
821
953
|
if (server?.type !== 'stdio') {
|
|
822
954
|
return { launched: false, blocked: true, diagnostic: 'Only stdio MCP worker servers are qualified for direct handoff' };
|
|
823
955
|
}
|
|
@@ -829,6 +961,27 @@ async function defaultLaunchMcpWorkerProcess(server, { allowDegraded, config, na
|
|
|
829
961
|
// See `defaultLaunchShellWorker`: test-only dispatch-gate passthroughs,
|
|
830
962
|
// always undefined for production registrations.
|
|
831
963
|
live, readers, thresholdOverlay,
|
|
964
|
+
// #2915 review 4 (finding 2): BORROW the caller's own already-admitted
|
|
965
|
+
// reservation — never a second, independent planner admission for the
|
|
966
|
+
// native-MCP launch path (the exact "one reservation-aware admission"
|
|
967
|
+
// requirement `defaultLaunchShellWorker` already honored).
|
|
968
|
+
//
|
|
969
|
+
// #2915 round 8 review 8 (C6): the borrowed admission's dispatch
|
|
970
|
+
// identity must be derived from THIS call's own trusted, resolved
|
|
971
|
+
// `invocation`/`invocationId` — never the borrowed wrapper's own copies.
|
|
972
|
+
// `planDirectWorkerLaunch`'s own top-level parameter destructuring
|
|
973
|
+
// derives `stdinPath`/`stdoutPath`/`stderrPath`/`command`/`args`/`cwd`
|
|
974
|
+
// AS DEFAULTS from a full `invocation` object when a caller omits them
|
|
975
|
+
// — correct for the shell launch path, but `WorkerResolvedInvocation`
|
|
976
|
+
// (src/serve/worker-invocation.ts) sets `stdoutPath`/`stderrPath`
|
|
977
|
+
// unconditionally for EVERY transport, so passing the full object here
|
|
978
|
+
// would leak them in and collide with `attached: true` (which forbids
|
|
979
|
+
// stdio file redirection entirely). Pass only the minimal shape
|
|
980
|
+
// `selectionDigestFor` actually reads (`{ selected }`) so the digest
|
|
981
|
+
// still matches the original admission's own selection without
|
|
982
|
+
// resurrecting any shell-specific field this attached launch never
|
|
983
|
+
// uses.
|
|
984
|
+
admission, invocation: invocation ? { selected: invocation.selected } : invocation, invocationId, dispatchGuard,
|
|
832
985
|
// #2895's `attached` mode defaults `stdin: 'closed'` (stdio[0] 'ignore') —
|
|
833
986
|
// correct for its own verification-runner callers, which only read a
|
|
834
987
|
// worker's output. An MCP stdio server needs a live, writable stdin to
|
|
@@ -847,6 +1000,8 @@ const mcpServerQueues = new Map(); // serverRef -> { promise: Promise<void>, dep
|
|
|
847
1000
|
/** Test-only observability into the module-level queue map's size — never
|
|
848
1001
|
* consulted by any admission/launch decision. */
|
|
849
1002
|
export function __testOnlyMcpServerQueueCount() { return mcpServerQueues.size; }
|
|
1003
|
+
export { ChildProcessTransport as __testOnlyChildProcessTransport, MCP_TRANSPORT_MAX_BUFFER_BYTES as __testOnlyMcpTransportMaxBufferBytes };
|
|
1004
|
+
export { defaultCallMcpWorker as __testOnlyDefaultCallMcpWorker };
|
|
850
1005
|
|
|
851
1006
|
/** #2892 D2: bounded per-serverRef serialization queue. Refuses outright
|
|
852
1007
|
* (never silently queues unboundedly) once `MAX_MCP_SERVER_QUEUE_DEPTH`
|
|
@@ -910,7 +1065,8 @@ function canonicalSchemaDigest(schema) {
|
|
|
910
1065
|
return `sha256:${createHash('sha256').update(JSON.stringify(canonical(schema))).digest('hex')}`;
|
|
911
1066
|
}
|
|
912
1067
|
|
|
913
|
-
|
|
1068
|
+
// Audited native MCP lifecycle, without recovery admission.
|
|
1069
|
+
async function defaultCallMcpWorkerBaseline(mcpBinding, server, options = {}) {
|
|
914
1070
|
const {
|
|
915
1071
|
sessionIsolation, mcpRegistry, allowDegraded = false, config, name = 'mcp-worker', registerControl,
|
|
916
1072
|
memoryMaxGb, mcpSchemas, qualifiedServerDigest, cwd,
|
|
@@ -942,7 +1098,7 @@ async function defaultCallMcpWorker(mcpBinding, server, options = {}) {
|
|
|
942
1098
|
const client = new Client({ name: 'aiwg-mcp-worker-invoke', version: '1.0.0' }, { capabilities: {} });
|
|
943
1099
|
// B2: gate + launch the MCP worker SERVER process through the same
|
|
944
1100
|
// pipeline a shell worker uses — never a bare SDK StdioClientTransport.
|
|
945
|
-
const launch = await
|
|
1101
|
+
const launch = await defaultLaunchMcpWorkerProcessBaseline(server, { allowDegraded, config, name, cwd, live, readers, thresholdOverlay, ...(memoryMaxGb !== undefined ? { memoryMaxGb } : {}) });
|
|
946
1102
|
if (!launch.launched) {
|
|
947
1103
|
return { outcome: 'blocked', diagnostic: launch.diagnostic ?? 'worker MCP process launch was refused' };
|
|
948
1104
|
}
|
|
@@ -967,7 +1123,7 @@ async function defaultCallMcpWorker(mcpBinding, server, options = {}) {
|
|
|
967
1123
|
await client.notification({ method: 'notifications/cancelled', params: { requestId: transport.lastRequestId, reason: 'operator-cancel' } });
|
|
968
1124
|
return true;
|
|
969
1125
|
} catch {
|
|
970
|
-
try { child
|
|
1126
|
+
try { return signalChild(child, 'SIGTERM'); } catch { return false; }
|
|
971
1127
|
}
|
|
972
1128
|
},
|
|
973
1129
|
};
|
|
@@ -1037,6 +1193,304 @@ async function defaultCallMcpWorker(mcpBinding, server, options = {}) {
|
|
|
1037
1193
|
return sessionIsolation === 'per-call' ? run() : withServerQueue(mcpBinding.serverRef, run);
|
|
1038
1194
|
}
|
|
1039
1195
|
|
|
1196
|
+
|
|
1197
|
+
/** #2915 round 8 (D6): the GATED implementation — reached only through the
|
|
1198
|
+
* router below when `options.admission` is present (never omitted-but-falsy;
|
|
1199
|
+
* `run()`'s own `continueGated` always supplies a truthily-gated admission).
|
|
1200
|
+
* No branch here needs to re-check `admission.gated`; by construction it is
|
|
1201
|
+
* always true for the duration of this function. */
|
|
1202
|
+
async function defaultCallMcpWorkerGated(mcpBinding, server, options = {}) {
|
|
1203
|
+
const {
|
|
1204
|
+
sessionIsolation, mcpRegistry, allowDegraded = false, config, name = 'mcp-worker', registerControl,
|
|
1205
|
+
memoryMaxGb, mcpSchemas, qualifiedServerDigest, cwd, admission, invocation, invocationId,
|
|
1206
|
+
// #2915 round 8 (D5): the ONE absolute deadline `run()` computed once,
|
|
1207
|
+
// unchanged. Never re-derived here from `mcpBinding.timeoutSeconds` or
|
|
1208
|
+
// `admission.token.mintedAtMs` (that double-subtracted preparation
|
|
1209
|
+
// time — round 8 critique, D5/review 7 finding 4's last bullet).
|
|
1210
|
+
deadlineAtMs = null,
|
|
1211
|
+
// #2915 round 8 review 8 (C3): optional cancellation predicate, checked
|
|
1212
|
+
// alongside every deadline check below (connect/listTools/callTool) and
|
|
1213
|
+
// forwarded into `ChildProcessTransport` for the write boundary. Also
|
|
1214
|
+
// folded into the `dispatchGuard` passed to the launch itself so
|
|
1215
|
+
// cancellation during pre-spawn preparation prevents the spawn too.
|
|
1216
|
+
isCancelled,
|
|
1217
|
+
dispatchGuard,
|
|
1218
|
+
observeStop,
|
|
1219
|
+
// See `defaultLaunchShellWorker`: test-only dispatch-gate passthroughs,
|
|
1220
|
+
// always undefined for production registrations.
|
|
1221
|
+
live, readers, thresholdOverlay,
|
|
1222
|
+
} = options;
|
|
1223
|
+
const run = async () => {
|
|
1224
|
+
try {
|
|
1225
|
+
// D2/F5: recheck the server's own registry qualification immediately
|
|
1226
|
+
// before THIS launch — a serialized/per-instance queue may have held
|
|
1227
|
+
// this call for a while before its turn came up. Checking `enabled`
|
|
1228
|
+
// alone is not requalification: an operator can edit a registry entry's
|
|
1229
|
+
// command/args/env while leaving `enabled: true`, and the ORIGINAL
|
|
1230
|
+
// (admission-time) `server` object passed into this function would
|
|
1231
|
+
// still be the one that gets launched below unless the FRESH
|
|
1232
|
+
// definition is also compared against what was qualified.
|
|
1233
|
+
if (mcpRegistry) {
|
|
1234
|
+
const requalified = await mcpRegistry.get(mcpBinding.serverRef);
|
|
1235
|
+
if (!requalified || requalified.enabled === false) {
|
|
1236
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP server is no longer registered/enabled', dispatchState: 'never-began' };
|
|
1237
|
+
}
|
|
1238
|
+
if (qualifiedServerDigest && serverDefinitionDigest(requalified) !== qualifiedServerDigest) {
|
|
1239
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP server definition changed since qualification; resolve again', dispatchState: 'never-began' };
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
// #2915 review 5 ("a second worker over unresolved execution") /
|
|
1243
|
+
// round 8 (D5): a serialized/per-instance queue (D2, below) may have
|
|
1244
|
+
// held this call long enough that the admitted attempt's deadline —
|
|
1245
|
+
// the ONE absolute `deadlineAtMs` `run()` computed once, which
|
|
1246
|
+
// already folds in the canary's own bounded window (min(task
|
|
1247
|
+
// deadline, admission+300s)) when applicable — has since passed. For
|
|
1248
|
+
// a canary, the breaker's own sweep reopens the scope and installs a
|
|
1249
|
+
// blocking marker the moment anything next calls `admit()`/`settle()`
|
|
1250
|
+
// on it (possibly a sibling request), regardless of whether THIS
|
|
1251
|
+
// queued call ever actually launches. Revalidate immediately before
|
|
1252
|
+
// launch, matching the registry-qualification recheck just above:
|
|
1253
|
+
// an expired admission must never launch. Falls back to the
|
|
1254
|
+
// canary-only `admission.token.deadlineMs` check when no caller
|
|
1255
|
+
// supplies `deadlineAtMs` (back-compat for any caller that does
|
|
1256
|
+
// not thread it through) — review 7 finding 4: "an ordinary request
|
|
1257
|
+
// queued beyond its 900-second budget still spawned and completed"
|
|
1258
|
+
// is exactly the plain (non-canary) task-deadline case this widens
|
|
1259
|
+
// to cover.
|
|
1260
|
+
const nowForExpiry = admission.breaker?.nowMs?.() ?? Date.now();
|
|
1261
|
+
const admissionExpired = deadlineAtMs !== null
|
|
1262
|
+
? nowForExpiry >= deadlineAtMs
|
|
1263
|
+
: !!(admission.token && nowForExpiry > admission.token.deadlineMs);
|
|
1264
|
+
if (admissionExpired) {
|
|
1265
|
+
return { outcome: 'blocked', diagnostic: 'worker admission expired while queued; recovery scope may have reopened', dispatchState: 'never-began' };
|
|
1266
|
+
}
|
|
1267
|
+
// #2915 round 8 review 8 (C3): cancellation requested while this call
|
|
1268
|
+
// was queued (or during the checks above) must prevent the spawn too,
|
|
1269
|
+
// exactly like an expired deadline — never dispatch after the operator
|
|
1270
|
+
// already asked to stop.
|
|
1271
|
+
if (typeof isCancelled === 'function' && isCancelled()) {
|
|
1272
|
+
return { outcome: 'blocked', diagnostic: 'worker dispatch refused: cancelled before process start', dispatchState: 'never-began' };
|
|
1273
|
+
}
|
|
1274
|
+
} catch (error) {
|
|
1275
|
+
// #2915 review 6 (finding 2, "leaked capacity"): a thrown pre-spawn
|
|
1276
|
+
// requalification/expiry check — nothing has been dispatched yet — must
|
|
1277
|
+
// settle the same way an ordinary blocked requalification does, never
|
|
1278
|
+
// propagate as an exception. `continueGated`'s own caller sets
|
|
1279
|
+
// `spawnAttempted` BEFORE invoking this function (it cannot know in
|
|
1280
|
+
// advance whether a real spawn is about to happen), so letting this
|
|
1281
|
+
// throw escape would be treated as genuinely post-spawn uncertainty
|
|
1282
|
+
// and leak the registry's run-slot forever — the exact gap a
|
|
1283
|
+
// native-MCP requalification error (`mcpRegistry.get` throwing rather
|
|
1284
|
+
// than returning falsy/disabled) left open.
|
|
1285
|
+
return { outcome: 'blocked', diagnostic: `worker MCP pre-spawn requalification failed: ${error?.message ?? String(error)}`, dispatchState: 'never-began' };
|
|
1286
|
+
}
|
|
1287
|
+
let stopOverflowWorker;
|
|
1288
|
+
let client;
|
|
1289
|
+
try {
|
|
1290
|
+
// Resolve fallible dependencies and construct the client before spawning.
|
|
1291
|
+
({ stopOverflowWorker } = await loadRalphExternal('lib/worker-overflow.mjs'));
|
|
1292
|
+
const { Client } = await import('@modelcontextprotocol/sdk/client/index.js');
|
|
1293
|
+
client = new Client({ name: 'aiwg-mcp-worker-invoke', version: '1.0.0' }, { capabilities: {} });
|
|
1294
|
+
// Async setup may consume the remaining admitted canary budget.
|
|
1295
|
+
if (admission?.gated && admission.token
|
|
1296
|
+
&& (admission.breaker?.nowMs?.() ?? Date.now()) > admission.token.deadlineMs) {
|
|
1297
|
+
return { outcome: 'blocked', diagnostic: 'worker admission expired while queued; recovery scope may have reopened' };
|
|
1298
|
+
}
|
|
1299
|
+
} catch (error) {
|
|
1300
|
+
return { outcome: 'blocked', diagnostic: `worker MCP pre-spawn setup failed: ${error?.message ?? String(error)}`, dispatchState: 'never-began' };
|
|
1301
|
+
}
|
|
1302
|
+
// B2: gate + launch the MCP worker SERVER process through the same
|
|
1303
|
+
// pipeline a shell worker uses — never a bare SDK StdioClientTransport.
|
|
1304
|
+
const launch = await defaultLaunchMcpWorkerProcess(server, { allowDegraded, config, name, cwd, live, readers, thresholdOverlay, admission, invocation, invocationId, dispatchGuard, ...(memoryMaxGb !== undefined ? { memoryMaxGb } : {}) });
|
|
1305
|
+
if (!launch.launched) {
|
|
1306
|
+
// #2915 round 8 review 8 (C2): a failed/timed-out `systemd-run`
|
|
1307
|
+
// submission does not prove nothing dispatched — carry `launch`'s own
|
|
1308
|
+
// `dispatchState: 'submission-uncertain'` forward so the caller's stop
|
|
1309
|
+
// evidence resolution never treats this as a proven zero-dispatch
|
|
1310
|
+
// refusal. Every other `!launched` case (gate blocked, dispatch-commit
|
|
1311
|
+
// hook abort, degraded-without-allowDegraded) is a proven pre-spawn
|
|
1312
|
+
// refusal: `never-began`.
|
|
1313
|
+
return { outcome: 'blocked', diagnostic: launch.diagnostic ?? 'worker MCP process launch was refused',
|
|
1314
|
+
dispatchState: launch.dispatchState === 'submission-uncertain' ? 'submission-uncertain' : 'never-began',
|
|
1315
|
+
isolationMode: launch.mode ?? null, unitName: launch.unit ?? null, child: launch.child ?? null };
|
|
1316
|
+
}
|
|
1317
|
+
const child = launch.child;
|
|
1318
|
+
// #2915 round 8 review 8 (C2): from here on a real process-start call
|
|
1319
|
+
// happened — every outcome below (including a post-connect 'blocked'
|
|
1320
|
+
// schema-drift refusal) carries this evidence forward so settlement
|
|
1321
|
+
// never manufactures a confirmed stop from an outcome label alone.
|
|
1322
|
+
const isolationEvidence = { dispatchState: 'dispatched', isolationMode: launch.mode ?? null, unitName: launch.unit ?? null, child: child ?? null };
|
|
1323
|
+
// #2915 review 6 (finding 3, "one absolute deadline") / round 8 (D5):
|
|
1324
|
+
// `deadlineAtMs` is the ONE fixed absolute wall-clock deadline `run()`
|
|
1325
|
+
// computed once at admission — never re-derived here. `nowMsFn` is
|
|
1326
|
+
// needed before constructing the transport below, so it moves up from
|
|
1327
|
+
// its former position just before the `connect()` call.
|
|
1328
|
+
const nowMsFn = () => admission.breaker?.nowMs?.() ?? Date.now();
|
|
1329
|
+
const absoluteDeadlineMs = deadlineAtMs;
|
|
1330
|
+
// #2915 review 6 (finding 6b, "changed disabled behavior"): the 10 MiB
|
|
1331
|
+
// unterminated-frame transport bound AND the deadline write-boundary
|
|
1332
|
+
// check (D5) are both enforced unconditionally here — this function is
|
|
1333
|
+
// reached only when gated (see `ChildProcessTransport`'s own doc
|
|
1334
|
+
// comment).
|
|
1335
|
+
const transport = new ChildProcessTransport(child, { terminate: () => stopOverflowWorker(launch),
|
|
1336
|
+
deadlineAtMs: absoluteDeadlineMs, now: nowMsFn, isCancelled });
|
|
1337
|
+
let disconnected = false;
|
|
1338
|
+
let retainOverflowControl = false;
|
|
1339
|
+
transport.onclose = () => { disconnected = true; };
|
|
1340
|
+
const control = {
|
|
1341
|
+
stop: async () => {
|
|
1342
|
+
if (transport._failure) {
|
|
1343
|
+
// The connection is closed after overflow. Reuse the owned launch's
|
|
1344
|
+
// stop authority, including its termination proof, on explicit cancel.
|
|
1345
|
+
await transport._termination;
|
|
1346
|
+
return stopOverflowWorker(launch);
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
// F2: the ACTUAL in-flight JSON-RPC request id (captured at the
|
|
1350
|
+
// transport layer — see `ChildProcessTransport.send`), never the
|
|
1351
|
+
// caller's own opaque `mcpBinding.requestId`, which the worker has
|
|
1352
|
+
// no way to relate to anything it is actually processing.
|
|
1353
|
+
await client.notification({ method: 'notifications/cancelled', params: { requestId: transport.lastRequestId, reason: 'operator-cancel' } });
|
|
1354
|
+
return true;
|
|
1355
|
+
} catch {
|
|
1356
|
+
try { return signalChild(child, 'SIGTERM'); } catch { return false; }
|
|
1357
|
+
}
|
|
1358
|
+
},
|
|
1359
|
+
};
|
|
1360
|
+
// #2915 review 6 (finding 3, "one absolute deadline"): derive EVERY
|
|
1361
|
+
// stage's timeout (connect/initialize, list, call) from ONE fixed
|
|
1362
|
+
// absolute wall-clock deadline, recomputed fresh immediately before
|
|
1363
|
+
// each stage from the time ACTUALLY remaining — never a per-stage
|
|
1364
|
+
// default (the SDK's own 60s `DEFAULT_REQUEST_TIMEOUT_MSEC`) and never a
|
|
1365
|
+
// budget computed once before a serialized/per-instance queue wait that
|
|
1366
|
+
// can itself consume most of it. Reproduced: a native request queued
|
|
1367
|
+
// 250s into a 300s canary window still received a fresh 300-SECOND SDK
|
|
1368
|
+
// call timeout with only 50s actually remaining, while connect/list each
|
|
1369
|
+
// separately spent up to another 60s on top of that — vastly overrunning
|
|
1370
|
+
// the breaker's own canary window.
|
|
1371
|
+
//
|
|
1372
|
+
// #2915 round 8 (D5): `remaining <= 0` REFUSES outright (client-side)
|
|
1373
|
+
// rather than merely passing `timeout: 0` through to the SDK and
|
|
1374
|
+
// trusting its own timer delivery — review 7 finding 4: "at exactly the
|
|
1375
|
+
// canary deadline, connect/list/call all received timeout 0, sent
|
|
1376
|
+
// requests, and completed successfully."
|
|
1377
|
+
// #2915 round 8 review 8 (C3): checked alongside every deadline check
|
|
1378
|
+
// below — cancellation requested while a request is in flight (or
|
|
1379
|
+
// between stages) must prevent the NEXT request, never retroactively
|
|
1380
|
+
// the one already sent.
|
|
1381
|
+
const cancelledNow = () => typeof isCancelled === 'function' && isCancelled();
|
|
1382
|
+
try {
|
|
1383
|
+
registerControl?.(control);
|
|
1384
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) throw Object.assign(new Error('worker MCP deadline already passed before connect'), { code: -32001 });
|
|
1385
|
+
if (cancelledNow()) throw Object.assign(new Error('worker MCP call cancelled before connect'), { code: -32800 });
|
|
1386
|
+
if (absoluteDeadlineMs !== null) {
|
|
1387
|
+
await client.connect(transport, { timeout: Math.max(0, absoluteDeadlineMs - nowMsFn()) });
|
|
1388
|
+
} else {
|
|
1389
|
+
await client.connect(transport);
|
|
1390
|
+
}
|
|
1391
|
+
// M3: refuse before any brief is sent if the live tool schema has
|
|
1392
|
+
// drifted from what was qualified. Compared CANONICALLY (sorted
|
|
1393
|
+
// keys) on both sides, not via the raw `#2887 mcpToolSchemaDigest`
|
|
1394
|
+
// pin directly — the SDK's own `listTools()` round-trips the wire
|
|
1395
|
+
// JSON through its zod validators, which can legitimately reorder
|
|
1396
|
+
// object keys without changing meaning; a byte-exact digest of the
|
|
1397
|
+
// POST-validation object against a PRE-validation pin would false-
|
|
1398
|
+
// positive on every call. `mcpSchemas` (the #2887 installation-
|
|
1399
|
+
// owner-trusted schema objects, not a client-supplied digest) is the
|
|
1400
|
+
// comparison's source of truth here.
|
|
1401
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) {
|
|
1402
|
+
throw Object.assign(new Error('worker MCP deadline already passed before listTools'), { code: -32001 });
|
|
1403
|
+
}
|
|
1404
|
+
if (cancelledNow()) throw Object.assign(new Error('worker MCP call cancelled before listTools'), { code: -32800 });
|
|
1405
|
+
const listed = absoluteDeadlineMs !== null
|
|
1406
|
+
? await client.listTools(undefined, { timeout: Math.max(0, absoluteDeadlineMs - nowMsFn()) })
|
|
1407
|
+
: await client.listTools();
|
|
1408
|
+
const tool = listed?.tools?.find(t => t.name === mcpBinding.tool);
|
|
1409
|
+
if (!tool) {
|
|
1410
|
+
return { outcome: 'blocked', diagnostic: 'qualified MCP tool is no longer advertised by the server', ...isolationEvidence };
|
|
1411
|
+
}
|
|
1412
|
+
// F3 (fix-2892-r2): digest the RAW wire response (plain `JSON.parse`,
|
|
1413
|
+
// retained by the transport — see `ChildProcessTransport`'s doc
|
|
1414
|
+
// comment), never `tool.inputSchema`/`tool.outputSchema` from the
|
|
1415
|
+
// zod-validated `listed` object above — zod's own `z.record()`
|
|
1416
|
+
// reconstruction of `properties` silently drops a member literally
|
|
1417
|
+
// named `__proto__`, so two schemas differing only in that member
|
|
1418
|
+
// would otherwise digest identically and the brief would still be
|
|
1419
|
+
// sent. A raw response this transport cannot account for at all
|
|
1420
|
+
// (should be unreachable — `listed` only resolved because SOME
|
|
1421
|
+
// response arrived) fails closed rather than falling back to the
|
|
1422
|
+
// lossy object.
|
|
1423
|
+
const rawToolsList = transport._rawResponses.get(transport.lastRequestId);
|
|
1424
|
+
const rawTool = rawToolsList?.result?.tools?.find(t => t && t.name === mcpBinding.tool);
|
|
1425
|
+
if (!rawTool) {
|
|
1426
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool schema could not be verified against the raw wire response', ...isolationEvidence };
|
|
1427
|
+
}
|
|
1428
|
+
if (mcpSchemas?.input && canonicalSchemaDigest(rawTool.inputSchema) !== canonicalSchemaDigest(mcpSchemas.input)) {
|
|
1429
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool input schema digest drifted from the qualified pin', ...isolationEvidence };
|
|
1430
|
+
}
|
|
1431
|
+
if (mcpSchemas?.output) {
|
|
1432
|
+
// A qualified descriptor that requires an output schema is never
|
|
1433
|
+
// satisfied by a worker that stopped advertising one at all —
|
|
1434
|
+
// missing is drift, not "nothing to compare".
|
|
1435
|
+
if (!rawTool.outputSchema) {
|
|
1436
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool no longer advertises an output schema the qualified descriptor requires', ...isolationEvidence };
|
|
1437
|
+
}
|
|
1438
|
+
if (canonicalSchemaDigest(rawTool.outputSchema) !== canonicalSchemaDigest(mcpSchemas.output)) {
|
|
1439
|
+
return { outcome: 'blocked', diagnostic: 'worker MCP tool output schema digest drifted from the qualified pin', ...isolationEvidence };
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) {
|
|
1443
|
+
throw Object.assign(new Error('worker MCP deadline already passed before callTool'), { code: -32001 });
|
|
1444
|
+
}
|
|
1445
|
+
if (cancelledNow()) throw Object.assign(new Error('worker MCP call cancelled before callTool'), { code: -32800 });
|
|
1446
|
+
const callTimeoutMs = absoluteDeadlineMs !== null
|
|
1447
|
+
? Math.max(0, absoluteDeadlineMs - nowMsFn())
|
|
1448
|
+
: mcpBinding.timeoutSeconds * 1000;
|
|
1449
|
+
const response = await client.callTool({ name: mcpBinding.tool, arguments: mcpBinding.arguments }, undefined, { timeout: callTimeoutMs });
|
|
1450
|
+
// #2915 round 8 (D5): "a response arriving after deadlineAt is a
|
|
1451
|
+
// timeout regardless of SDK timer delivery" — review 7 finding 4: "a
|
|
1452
|
+
// response arriving after the deadline could still return completed."
|
|
1453
|
+
if (absoluteDeadlineMs !== null && nowMsFn() >= absoluteDeadlineMs) {
|
|
1454
|
+
return { outcome: 'timeout', error: new Error('worker MCP response arrived after the deadline'), ...isolationEvidence };
|
|
1455
|
+
}
|
|
1456
|
+
return { outcome: 'responded', response, ...isolationEvidence };
|
|
1457
|
+
} catch (error) {
|
|
1458
|
+
const isTimeout = error?.code === -32001 || /timeout/i.test(String(error?.message ?? ''));
|
|
1459
|
+
const isCancelledError = error?.code === -32800;
|
|
1460
|
+
try { await client.notification({ method: 'notifications/cancelled', params: { requestId: transport.lastRequestId, reason: isTimeout ? 'timeout' : isCancelledError ? 'cancelled' : 'error' } }); } catch { /* delivery only, never stop proof */ }
|
|
1461
|
+
const termination = transport._failure ? await transport._termination : await stopOverflowWorker(launch);
|
|
1462
|
+
retainOverflowControl = !!transport._failure && termination?.termination !== 'confirmed';
|
|
1463
|
+
return { outcome: disconnected ? 'disconnected' : isTimeout ? 'timeout' : 'error', error: transport._failure ?? error,
|
|
1464
|
+
...isolationEvidence,
|
|
1465
|
+
...(transport._failure ? { termination: termination?.termination ?? 'uncertain',
|
|
1466
|
+
diagnostic: [transport._failure.code, termination?.diagnostic].filter(Boolean).join('; ') } : {}) };
|
|
1467
|
+
} finally {
|
|
1468
|
+
try { if (!retainOverflowControl) registerControl?.(null); }
|
|
1469
|
+
finally {
|
|
1470
|
+
try {
|
|
1471
|
+
const stopping = client.close();
|
|
1472
|
+
observeStop?.(stopping, isolationEvidence);
|
|
1473
|
+
await stopping;
|
|
1474
|
+
} catch { /* already closed */ }
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
};
|
|
1478
|
+
return sessionIsolation === 'per-call' ? run() : withServerQueue(mcpBinding.serverRef, run);
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
/** #2915 round 8 (D6): thin router, mirroring
|
|
1482
|
+
* `defaultLaunchShellWorker`/`defaultLaunchMcpWorkerProcess`'s identical
|
|
1483
|
+
* pattern — `options.admission` is only ever present when this call is
|
|
1484
|
+
* actually gated (`continueGated` always supplies one; `continueBaseline`
|
|
1485
|
+
* never includes the key at all). Absent routes to the literal baseline
|
|
1486
|
+
* reimplementation; present routes to the gated implementation. */
|
|
1487
|
+
async function defaultCallMcpWorker(mcpBinding, server, options = {}) {
|
|
1488
|
+
if (options.admission === undefined) {
|
|
1489
|
+
return defaultCallMcpWorkerBaseline(mcpBinding, server, options);
|
|
1490
|
+
}
|
|
1491
|
+
return defaultCallMcpWorkerGated(mcpBinding, server, options);
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1040
1494
|
// ===========================================================================
|
|
1041
1495
|
// Health hook extension point (documented, no dependency on #2896 code).
|
|
1042
1496
|
// ===========================================================================
|
|
@@ -1079,8 +1533,12 @@ function publicStatus(entry, healthHook) {
|
|
|
1079
1533
|
// ===========================================================================
|
|
1080
1534
|
|
|
1081
1535
|
const inputSchema = z.object({
|
|
1082
|
-
|
|
1536
|
+
// #2915 round 8 (D4): `reconcile_stopped` is an explicit operator-
|
|
1537
|
+
// authorized reconciliation operation on this SAME handler (never a
|
|
1538
|
+
// mailbox file, never a new breaker) — see `reconcileStopped` below.
|
|
1539
|
+
operation: z.enum(['run', 'status', 'cancel', 'reconcile_stopped', 'reset_recovery_scope']),
|
|
1083
1540
|
invocation_id: z.string().uuid().optional(),
|
|
1541
|
+
scope_id: z.string().min(1).max(512).optional(),
|
|
1084
1542
|
request_id: z.string().min(1).max(128).regex(/^[a-zA-Z0-9_.:-]+$/).optional(),
|
|
1085
1543
|
role: z.enum(['efficiency', 'coding', 'reasoning']).optional(),
|
|
1086
1544
|
task: z.string().min(1).max(256).optional(),
|
|
@@ -1089,6 +1547,13 @@ const inputSchema = z.object({
|
|
|
1089
1547
|
workdir: z.string().min(1).optional(),
|
|
1090
1548
|
final_output: z.string().min(1).optional(),
|
|
1091
1549
|
transport: z.enum(['shell', 'mcp']).default('shell'),
|
|
1550
|
+
// #2915 step 4 (retry linkage): conductor-declared predecessor invocation
|
|
1551
|
+
// id. Accepted and shape-validated here; `run()` below refuses explicitly
|
|
1552
|
+
// rather than silently ignoring it (no server-owned capture-root
|
|
1553
|
+
// configuration exists yet for the MCP shell-launch path — see
|
|
1554
|
+
// session-launcher.mjs `verifyRetryPredecessor`, which the CLI entry
|
|
1555
|
+
// point already uses). Omission is unrelated work, as always.
|
|
1556
|
+
retry_of: z.string().uuid().optional(),
|
|
1092
1557
|
}).strict();
|
|
1093
1558
|
|
|
1094
1559
|
/**
|
|
@@ -1122,7 +1587,14 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1122
1587
|
const loadWorkerInputsImpl = options.loadWorkerInputs ?? defaultLoadWorkerInputs;
|
|
1123
1588
|
const launchShellWorkerImpl = options.launchShellWorker ?? defaultLaunchShellWorker;
|
|
1124
1589
|
const callMcpWorkerImpl = options.callMcpWorker ?? defaultCallMcpWorker;
|
|
1125
|
-
|
|
1590
|
+
// Both continuations use the audited observer and hardened stop controls.
|
|
1591
|
+
const waitForUnitTerminationOverride = options.waitForUnitTermination;
|
|
1592
|
+
// #2915 review 5 ("a second worker over unresolved execution"): the
|
|
1593
|
+
// degraded-isolation shell wait path below shares the launcher's own
|
|
1594
|
+
// bounded SIGTERM-then-SIGKILL escalation (`terminateChildWithGrace`)
|
|
1595
|
+
// rather than sending an unobserved SIGTERM alone.
|
|
1596
|
+
const terminateChildImpl = options.terminateChildWithGrace
|
|
1597
|
+
?? (async child => (await loadRalphExternal('session-launcher.mjs')).terminateChildWithGrace(child));
|
|
1126
1598
|
const mcpWorkerMemoryMaxGb = options.mcpWorkerMemoryMaxGb; // undefined -> defaultLaunchMcpWorkerProcess's own fallback
|
|
1127
1599
|
// Test-only: `planDirectWorkerLaunch`'s own dispatch-gate passthroughs
|
|
1128
1600
|
// (`checkWorkerDispatch({ live, readers, thresholdOverlay })`), forwarded
|
|
@@ -1151,7 +1623,40 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1151
1623
|
return { role: input.role, task: input.task, group: input.group, brief, workdir, final_output: finalOutput, transport: input.transport };
|
|
1152
1624
|
}
|
|
1153
1625
|
|
|
1626
|
+
/**
|
|
1627
|
+
* #2915 round 8 (D5, review 7 finding 9): "MCP discards the settlement
|
|
1628
|
+
* promise and its diagnostic." `runInner` is the full implementation
|
|
1629
|
+
* (unchanged below); `captureSettlementPromise`, when called, hands this
|
|
1630
|
+
* run's ONE cached ledger-settlement promise (`settleDirectWorker`'s own
|
|
1631
|
+
* return value, see `settleRecovery` below) up to `run()`'s thin wrapper,
|
|
1632
|
+
* which awaits it at this function's one true return boundary and folds
|
|
1633
|
+
* any refund-failure diagnostic it carries into the result — instead of
|
|
1634
|
+
* letting that promise, and whatever it resolves to, simply vanish.
|
|
1635
|
+
*/
|
|
1154
1636
|
async function run(input) {
|
|
1637
|
+
let settlementPromise = null;
|
|
1638
|
+
const transitions = [];
|
|
1639
|
+
let result = await runInner(input, promise => { settlementPromise = promise; }, event => {
|
|
1640
|
+
transitions.push(event.summary);
|
|
1641
|
+
(options.onRecoveryTransition ?? (event => console.error(event.summary)))(event);
|
|
1642
|
+
});
|
|
1643
|
+
if (settlementPromise) {
|
|
1644
|
+
let diagnostic = null;
|
|
1645
|
+
try { diagnostic = await settlementPromise; } catch { /* never fail a run on diagnostic retrieval */ }
|
|
1646
|
+
if (diagnostic && result?.invocationId) {
|
|
1647
|
+
const current = registry.get(result.invocationId);
|
|
1648
|
+
if (current) {
|
|
1649
|
+
result = registry.update(result.invocationId, { diagnostics: [...(current.diagnostics ?? []), diagnostic] });
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
if (transitions.length && result?.invocationId) {
|
|
1654
|
+
result = registry.update(result.invocationId, { diagnostics: [...(result.diagnostics ?? []), ...transitions] });
|
|
1655
|
+
}
|
|
1656
|
+
return result;
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
async function runInner(input, captureSettlementPromise, onTransition) {
|
|
1155
1660
|
// M1: request_id is required, and admission (capacity + replay check +
|
|
1156
1661
|
// reservation) is entirely synchronous — no `await` between the checks
|
|
1157
1662
|
// and `registry.create()` — so no concurrent `run()` call can observe a
|
|
@@ -1160,13 +1665,23 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1160
1665
|
if (!input.role || !input.task || !input.group || !input.brief || !input.workdir || !input.final_output) {
|
|
1161
1666
|
throw new Error('Incomplete worker invocation');
|
|
1162
1667
|
}
|
|
1668
|
+
// #2915 step 4: `retry_of` has no server-owned capture-root
|
|
1669
|
+
// configuration to verify a predecessor against on this bridge yet
|
|
1670
|
+
// (see the schema comment above) — refuse explicitly, before any
|
|
1671
|
+
// capacity is reserved, rather than silently accepting and ignoring a
|
|
1672
|
+
// conductor's retry declaration. Predates round 8 and applies
|
|
1673
|
+
// unconditionally (not a gating concern): `retry_of` support is simply
|
|
1674
|
+
// not implemented on this bridge yet, for gated or not-gated callers
|
|
1675
|
+
// alike, so origin/main's own schema (which has no `retry_of` field at
|
|
1676
|
+
// all) never had an equivalent case to diverge from.
|
|
1677
|
+
if (input.retry_of !== undefined) throw new Error('retry_of is not yet supported on the MCP worker-invoke bridge (OG_RETRY_UNSUPPORTED_MCP)');
|
|
1163
1678
|
if (!registry.canStart()) throw new Error('Worker invocation capacity is reserved; no live run slot is available');
|
|
1164
1679
|
const existing = registry.claimRequestId(input.request_id);
|
|
1165
1680
|
if (existing) throw new Error('Worker request_id was already used; mint a fresh request_id instead of replaying');
|
|
1166
1681
|
const entry = registry.create({ requestId: input.request_id, transport: input.transport, status: 'running', termination: 'running' });
|
|
1167
1682
|
let request;
|
|
1168
1683
|
let outputReservation;
|
|
1169
|
-
|
|
1684
|
+
const execution = { mayHaveLaunched: false };
|
|
1170
1685
|
let reservationCleanup;
|
|
1171
1686
|
const cleanupReservation = () => reservationCleanup ??= (async () => {
|
|
1172
1687
|
if (!outputReservation) return;
|
|
@@ -1181,7 +1696,7 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1181
1696
|
}
|
|
1182
1697
|
})();
|
|
1183
1698
|
const markTerminal = async update => {
|
|
1184
|
-
if (!mayHaveLaunched) await cleanupReservation();
|
|
1699
|
+
if (!execution.mayHaveLaunched) await cleanupReservation();
|
|
1185
1700
|
return registry.markTerminal(entry.invocationId, update);
|
|
1186
1701
|
};
|
|
1187
1702
|
try {
|
|
@@ -1195,6 +1710,9 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1195
1710
|
registry.markTerminal(entry.invocationId, { status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
1196
1711
|
throw error;
|
|
1197
1712
|
}
|
|
1713
|
+
|
|
1714
|
+
// Shared pre-dispatch preparation owns capacity and output cleanup.
|
|
1715
|
+
let prefix;
|
|
1198
1716
|
try {
|
|
1199
1717
|
try {
|
|
1200
1718
|
outputReservation = await reserveFinalOutput(request.final_output);
|
|
@@ -1208,7 +1726,138 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1208
1726
|
if (plan.invocation?.readiness !== 'launchable') {
|
|
1209
1727
|
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: plan.diagnostics ?? [] });
|
|
1210
1728
|
}
|
|
1211
|
-
|
|
1729
|
+
prefix = { root, snapshot, drivers, invocation: plan.invocation };
|
|
1730
|
+
} catch (error) {
|
|
1731
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
1732
|
+
}
|
|
1733
|
+
const { root, snapshot, drivers, invocation } = prefix;
|
|
1734
|
+
|
|
1735
|
+
// =========================================================================
|
|
1736
|
+
// #2915 round 8 (D6): ONE decision, immediately after the prefix, from
|
|
1737
|
+
// the launcher's own pure eligibility predicate,
|
|
1738
|
+
// `directRecoveryEligibility` (session-launcher.mjs) — the SAME
|
|
1739
|
+
// function `launchOneWorker`'s own not-gated/gated split and
|
|
1740
|
+
// `admitDirectWorker` itself both call, so this decision cannot drift
|
|
1741
|
+
// out of sync with either (round 8 critique, D6; the decision is
|
|
1742
|
+
// structurally one function, not three matching conditions). It
|
|
1743
|
+
// consults, in order, the global switch, the per-CLI switch,
|
|
1744
|
+
// `canVerifyAttempt`, and an explicit breaker override, and reserves
|
|
1745
|
+
// nothing — every ineligible answer returns `{ gated: false }` before
|
|
1746
|
+
// ever touching the breaker or the ledger. The only check made BEFORE
|
|
1747
|
+
// even importing the recovery module is the global switch
|
|
1748
|
+
// (`config.parallelism.worker_dispatch`) — matching the SAME field
|
|
1749
|
+
// `session-launcher.mjs`'s own call sites resolve as `gate.enabled`
|
|
1750
|
+
// (`checkWorkerDispatch`) — so a globally disabled deployment never
|
|
1751
|
+
// imports the recovery module at all (#2915 review 4 finding 7),
|
|
1752
|
+
// exactly like origin/main never having this feature.
|
|
1753
|
+
// =========================================================================
|
|
1754
|
+
const dispatchGateEnabled = !!snapshot.config?.parallelism?.worker_dispatch;
|
|
1755
|
+
let admission;
|
|
1756
|
+
let recoveryModule;
|
|
1757
|
+
let admissionGuard;
|
|
1758
|
+
if (!dispatchGateEnabled) {
|
|
1759
|
+
admission = { gated: false };
|
|
1760
|
+
} else {
|
|
1761
|
+
try {
|
|
1762
|
+
recoveryModule = await loadRalphExternal('session-launcher.mjs');
|
|
1763
|
+
const eligibility = recoveryModule.directRecoveryEligibility({
|
|
1764
|
+
gateEnabled: dispatchGateEnabled, config: snapshot.config, invocation, drivers, scopedBreaker: options.scopedBreaker,
|
|
1765
|
+
});
|
|
1766
|
+
if (!eligibility.gated) {
|
|
1767
|
+
// Not eligible (per-CLI switch, `canVerifyAttempt`, or an
|
|
1768
|
+
// explicit breaker override) — nothing was reserved; `admission`
|
|
1769
|
+
// is exactly the predicate's own `{ gated: false, reason? }`.
|
|
1770
|
+
admission = eligibility;
|
|
1771
|
+
} else {
|
|
1772
|
+
// #2915 review 4 (finding 8): consult the same per-task-chain
|
|
1773
|
+
// allowance ledger the launcher path uses, keyed off the SAME
|
|
1774
|
+
// stable digest and the SAME resolved `invocation.limits.
|
|
1775
|
+
// maxAttempts` -- no new configuration, just the existing
|
|
1776
|
+
// ceiling already resolved above by `resolveWorkerInvocation`.
|
|
1777
|
+
// MCP has no server-owned capture root to persist a
|
|
1778
|
+
// cross-process ledger file under, so `captureRoot` is
|
|
1779
|
+
// intentionally omitted: the ledger falls back to this
|
|
1780
|
+
// process's own in-memory map (see `reserveDirectWorkerAllowance`
|
|
1781
|
+
// in session-launcher.mjs). That still enforces the allowance
|
|
1782
|
+
// correctly WITHIN this server process -- including across
|
|
1783
|
+
// concurrent sibling retries, which serialize on the same
|
|
1784
|
+
// in-process reservation check -- but it is explicitly NOT
|
|
1785
|
+
// cross-process-safe: a second MCP server process, or a restart
|
|
1786
|
+
// of this one, starts the count over for the same task chain.
|
|
1787
|
+
const breaker = options.scopedBreaker ?? recoveryModule.getDirectRecoveryBreaker();
|
|
1788
|
+
registry.bindRecoveryOwner(breaker);
|
|
1789
|
+
const startedAtMs = breaker.nowMs();
|
|
1790
|
+
admissionGuard = {
|
|
1791
|
+
nowMs: () => breaker.nowMs(),
|
|
1792
|
+
deadlineAtMs: startedAtMs + Math.min(invocation.health?.hardTimeoutSeconds ?? Infinity,
|
|
1793
|
+
invocation.limits.wallSeconds, invocation.transport === 'mcp' ? invocation.mcp.timeoutSeconds : Infinity) * 1000,
|
|
1794
|
+
isCancelled: () => !!registry.get(entry.invocationId)?.cancelRequested,
|
|
1795
|
+
};
|
|
1796
|
+
const taskChainDigest = await recoveryModule.stableTaskChainDigest(request, invocation.briefDigest ?? null);
|
|
1797
|
+
admission = await recoveryModule.admitDirectWorker({
|
|
1798
|
+
gateEnabled: dispatchGateEnabled, config: snapshot.config,
|
|
1799
|
+
invocation, invocationId: entry.invocationId, drivers, scopedBreaker: breaker, dispatchGuard: admissionGuard,
|
|
1800
|
+
taskChainDigest, maxAttempts: invocation.limits?.maxAttempts, onTransition,
|
|
1801
|
+
});
|
|
1802
|
+
}
|
|
1803
|
+
} catch (error) {
|
|
1804
|
+
// #2915 P0 (fail closed, no leaked slot): this admission call runs
|
|
1805
|
+
// AFTER `registry.create()` already reserved this run's capacity/
|
|
1806
|
+
// identity slot. Any exception here (bad breaker state, a thrown
|
|
1807
|
+
// import, a malformed gate result) must refuse the launch the same
|
|
1808
|
+
// way an ordinary recovery-scope refusal does — `markTerminal`,
|
|
1809
|
+
// freeing the reservation. A pre-spawn breaker error is not an
|
|
1810
|
+
// uncertain worker outcome: no worker ever ran.
|
|
1811
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed',
|
|
1812
|
+
diagnostics: [`recovery admission failed closed: ${error?.message ?? String(error)}`] });
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
if (admission.refused) {
|
|
1816
|
+
// #2915 review 4 (finding 8): a ledger-exhaustion refusal has no
|
|
1817
|
+
// covering scope (`blockedBy` is empty) -- fall back to the
|
|
1818
|
+
// refusal's own code/reason so the diagnostic is never an empty
|
|
1819
|
+
// "recovery scope open: " for that case.
|
|
1820
|
+
const blockedBy = admission.refused.blockedBy?.length ? admission.refused.blockedBy.join(', ')
|
|
1821
|
+
: (admission.refused.code ?? admission.refused.reason ?? 'unknown');
|
|
1822
|
+
// #2915 round 8 (D4, C4): a `blocking-markers:<N>` component names
|
|
1823
|
+
// genuinely unresolved execution occupying reservation slots (never
|
|
1824
|
+
// mere backoff) — point at the exact reachable reconciliation route
|
|
1825
|
+
// for it, this tool's own `reconcile_stopped` operation, rather than
|
|
1826
|
+
// leaving the operator to go looking for one.
|
|
1827
|
+
const markerNote = admission.refused.blockedBy?.some(b => b.startsWith('blocking-markers:'))
|
|
1828
|
+
? "; unresolved reservations can be reconciled via this tool's 'reconcile_stopped' operation (see 'status' with no invocation_id for the pending list)"
|
|
1829
|
+
: '';
|
|
1830
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed',
|
|
1831
|
+
diagnostics: [`recovery scope open: ${blockedBy}${markerNote}${admission.refused.reason ? `; ${admission.refused.reason}` : ''}`] });
|
|
1832
|
+
}
|
|
1833
|
+
// #2915 round 8 (D4): track which breaker instance actually owns this
|
|
1834
|
+
// invocation's reservation (when gated) — the `reconcile_stopped`
|
|
1835
|
+
// operation and the no-id `status` listing both need to reach the
|
|
1836
|
+
// SAME breaker a later `admitDirectWorker` call used, not a fresh
|
|
1837
|
+
// default. Internal bookkeeping only — never surfaced by `publicStatus`.
|
|
1838
|
+
if (admission.breaker) {
|
|
1839
|
+
try {
|
|
1840
|
+
registry.update(entry.invocationId, { recoveryBreaker: admission.breaker });
|
|
1841
|
+
} catch (error) {
|
|
1842
|
+
const diagnostic = await recoveryModule.settleDirectWorker(admission, { kind: 'uncertain', confirmedStop: true });
|
|
1843
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed',
|
|
1844
|
+
diagnostics: [error.message, diagnostic].filter(Boolean) });
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
// Share reservation ownership; recovery accounting belongs only to the gated continuation.
|
|
1849
|
+
const ctx = { entry, request, root, snapshot, drivers, invocation, execution, cleanupReservation, markTerminal };
|
|
1850
|
+
if (admission.gated) {
|
|
1851
|
+
return continueGated(ctx, { admission, recoveryModule, captureSettlementPromise, admissionGuard });
|
|
1852
|
+
}
|
|
1853
|
+
return continueBaseline(ctx);
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
// Recovery-disabled execution follows current main's audited lifecycle.
|
|
1857
|
+
// Only recovery admission/accounting is omitted; stop and cleanup stay active.
|
|
1858
|
+
async function continueBaseline(ctx) {
|
|
1859
|
+
const { entry, request, root, snapshot, drivers, invocation, execution, cleanupReservation, markTerminal } = ctx;
|
|
1860
|
+
try {
|
|
1212
1861
|
// F4 (fix-2892-r2, formerly part of M5): per operator decision, the
|
|
1213
1862
|
// MCP path REQUIRES the structured WorkerDirectBrief — a probe showed
|
|
1214
1863
|
// a plaintext brief (or a JSON brief swapped to plaintext between
|
|
@@ -1247,20 +1896,20 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1247
1896
|
let observation;
|
|
1248
1897
|
let rawOutput;
|
|
1249
1898
|
if (invocation.transport === 'shell') {
|
|
1250
|
-
mayHaveLaunched = true;
|
|
1899
|
+
execution.mayHaveLaunched = true;
|
|
1251
1900
|
const launch = await launchShellWorkerImpl(invocation, { allowDegraded, config: snapshot.config,
|
|
1252
1901
|
name: `mcp-${entry.invocationId.slice(0, 8)}`, memoryMaxGb: options.shellWorkerMemoryMaxGb,
|
|
1253
1902
|
driver: drivers[invocation.selected.cli], servedRoot: root, stdinRuntimeRoot: options.stdinRuntimeRoot, ...gateInputs });
|
|
1254
1903
|
registry.update(entry.invocationId, { isolationMode: launch.mode ?? null });
|
|
1255
1904
|
if (!launch.launched && !launch.launchAttempted) {
|
|
1256
|
-
mayHaveLaunched = false;
|
|
1905
|
+
execution.mayHaveLaunched = false;
|
|
1257
1906
|
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [launch.diagnostic ?? 'worker launch was refused'] });
|
|
1258
1907
|
}
|
|
1259
1908
|
// M2: a live stop handle for the duration of this call, so an
|
|
1260
1909
|
// explicit `cancel` can actually stop the isolated unit / kill the
|
|
1261
1910
|
// child, not just record that cancellation was requested.
|
|
1262
1911
|
const control = buildShellStopControl(launch, {
|
|
1263
|
-
waitForUnit:
|
|
1912
|
+
waitForUnit: (waitForUnitTerminationOverride ?? waitForUnitTerminationBaseline),
|
|
1264
1913
|
stopUnit: options.stopWorkerUnit,
|
|
1265
1914
|
killUnit: options.killWorkerUnit,
|
|
1266
1915
|
graceMs: options.shellStopGraceMs ?? (invocation.health?.terminateGraceSeconds ?? 1) * 1000,
|
|
@@ -1269,7 +1918,7 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1269
1918
|
await Promise.resolve().then(() => launch.cleanup?.()).catch(() => {});
|
|
1270
1919
|
const live = registry.get(entry.invocationId);
|
|
1271
1920
|
if (live?.terminalAt !== null) return;
|
|
1272
|
-
if (live.status === 'uncertain') {
|
|
1921
|
+
if (live.status === 'uncertain' || live.status === 'blocked') {
|
|
1273
1922
|
registry.markTerminal(entry.invocationId, { control: null, termination: 'confirmed',
|
|
1274
1923
|
status: live.cancelRequested ? 'cancelled' : 'failed' });
|
|
1275
1924
|
} else registry.update(entry.invocationId, { control: null, termination: 'confirmed' });
|
|
@@ -1292,7 +1941,7 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1292
1941
|
console.error('[worker-invoke] stdin cleanup failed after confirmed termination');
|
|
1293
1942
|
});
|
|
1294
1943
|
if (!launch.launched) {
|
|
1295
|
-
mayHaveLaunched = false;
|
|
1944
|
+
execution.mayHaveLaunched = false;
|
|
1296
1945
|
return markTerminal({ control: null, status: 'failed', termination: 'confirmed',
|
|
1297
1946
|
diagnostics: [launch.diagnostic ?? 'worker launch failed; termination confirmed'] });
|
|
1298
1947
|
}
|
|
@@ -1312,7 +1961,7 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1312
1961
|
// admission, so a queued launch can detect a registry edit that
|
|
1313
1962
|
// happened while it was waiting (see `serverDefinitionDigest`).
|
|
1314
1963
|
const qualifiedServerDigest = serverDefinitionDigest(mcpServer);
|
|
1315
|
-
mayHaveLaunched = true;
|
|
1964
|
+
execution.mayHaveLaunched = true;
|
|
1316
1965
|
const result = await callMcpWorkerImpl(invocation.mcp, mcpServer, {
|
|
1317
1966
|
sessionIsolation: invocation.mcp.sessionIsolation, mcpRegistry, allowDegraded, config: snapshot.config,
|
|
1318
1967
|
// #2892 (CI fix): the resolved invocation's own worktree
|
|
@@ -1381,8 +2030,8 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1381
2030
|
result: assembled?.result ?? { claims },
|
|
1382
2031
|
});
|
|
1383
2032
|
} catch (error) {
|
|
1384
|
-
if (error.workerNotLaunched) mayHaveLaunched = false;
|
|
1385
|
-
if (!mayHaveLaunched) {
|
|
2033
|
+
if (error.workerNotLaunched) execution.mayHaveLaunched = false;
|
|
2034
|
+
if (!execution.mayHaveLaunched) {
|
|
1386
2035
|
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
1387
2036
|
}
|
|
1388
2037
|
if (registry.get(entry.invocationId)?.termination === 'confirmed') {
|
|
@@ -1395,6 +2044,598 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1395
2044
|
}
|
|
1396
2045
|
}
|
|
1397
2046
|
|
|
2047
|
+
/**
|
|
2048
|
+
* #2915 round 8 (D6): the GATED continuation — reached only when the
|
|
2049
|
+
* decision point in `runInner` already admitted (or is borrowing) a
|
|
2050
|
+
* breaker reservation. `admission.gated` is therefore always true for the
|
|
2051
|
+
* duration of this function; no branch here needs to re-check it (the
|
|
2052
|
+
* previous worker's scattered `admission?.gated` conditionals are gone by
|
|
2053
|
+
* construction, not by review).
|
|
2054
|
+
*/
|
|
2055
|
+
async function continueGated(ctx, { admission, recoveryModule, captureSettlementPromise, admissionGuard }) {
|
|
2056
|
+
const { entry, request, root, snapshot, drivers, invocation, execution, cleanupReservation, markTerminal } = ctx;
|
|
2057
|
+
// #2915 final: `recoverySettled`/`spawnAttempted` are function-scoped
|
|
2058
|
+
// (not inside the try block below) so the catch-all at the end of this
|
|
2059
|
+
// function can also settle this run's admission (#2915 final review,
|
|
2060
|
+
// finding 2: a thrown error reaching that catch previously left the
|
|
2061
|
+
// reservation live with zero capacity recovered — the exact "slot leak"
|
|
2062
|
+
// the final review named).
|
|
2063
|
+
let recoverySettled = false;
|
|
2064
|
+
// #2915 review 5 (finding 2b, "leaked capacity"): distinguishes PROVEN
|
|
2065
|
+
// pre-spawn exceptions (nothing ever ran -- the registry run-slot must
|
|
2066
|
+
// be released) from post-spawn uncertainty (a worker process/call
|
|
2067
|
+
// genuinely started and may still be running -- the slot correctly
|
|
2068
|
+
// stays live per M2). Set true only once a shell launch or MCP call is
|
|
2069
|
+
// confirmed to have actually started (past every "never launched"/
|
|
2070
|
+
// "blocked" refusal below), never merely attempted.
|
|
2071
|
+
let spawnAttempted = false;
|
|
2072
|
+
let isolationEvidence = null;
|
|
2073
|
+
let settlementPromise;
|
|
2074
|
+
let settlementStarted;
|
|
2075
|
+
let cancelFollowUp;
|
|
2076
|
+
let neverExecutedStop = false;
|
|
2077
|
+
const settlementReady = new Promise(resolve => { settlementStarted = resolve; });
|
|
2078
|
+
const resolveEvidence = evidence => neverExecutedStop && evidence?.isolationMode === 'systemd-unit'
|
|
2079
|
+
&& evidence.unitName === isolationEvidence?.unitName
|
|
2080
|
+
? Promise.resolve({ stop: 'confirmed', confirmedStop: true }) : recoveryModule.resolveStopEvidence({
|
|
2081
|
+
...evidence, getStatus: options.isolationStatus, stopUnit: options.stopUnit,
|
|
2082
|
+
terminateChild: terminateChildImpl,
|
|
2083
|
+
});
|
|
2084
|
+
const reconcileEvidence = async evidence => {
|
|
2085
|
+
await settlementReady;
|
|
2086
|
+
await settlementPromise;
|
|
2087
|
+
if (!admission.breaker?.hasUnresolvedStop?.(entry.invocationId)) return true;
|
|
2088
|
+
const resolved = await resolveEvidence({ ...evidence, stopFirst: false });
|
|
2089
|
+
if (resolved.confirmedStop) {
|
|
2090
|
+
cancelFollowUp?.();
|
|
2091
|
+
const reconciled = admission.breaker?.reconcileStopEvidence?.({ invocationId: entry.invocationId, confirmedStop: true });
|
|
2092
|
+
if ((reconciled?.released ?? 0) > 0) registry.markTerminal(entry.invocationId, {});
|
|
2093
|
+
}
|
|
2094
|
+
return resolved.confirmedStop;
|
|
2095
|
+
};
|
|
2096
|
+
// The outward result and execution protection have independent lifetimes.
|
|
2097
|
+
// Retain pending escalation and a fixed, bounded sequence of unit observations.
|
|
2098
|
+
const observeStop = (stopping, evidence) => {
|
|
2099
|
+
Promise.resolve(stopping).then(() => reconcileEvidence(evidence)).catch(() => {});
|
|
2100
|
+
};
|
|
2101
|
+
let followUpScheduled = false;
|
|
2102
|
+
const observeLater = evidence => {
|
|
2103
|
+
if (followUpScheduled || evidence?.isolationMode !== 'systemd-unit' || !evidence.unitName) return;
|
|
2104
|
+
followUpScheduled = true;
|
|
2105
|
+
cancelFollowUp = recoveryModule.observeLateStopEvidence(() => reconcileEvidence(evidence));
|
|
2106
|
+
};
|
|
2107
|
+
const settleRecovery = (outcome) => {
|
|
2108
|
+
if (recoverySettled) return;
|
|
2109
|
+
recoverySettled = true;
|
|
2110
|
+
try {
|
|
2111
|
+
// #2915 round 8 (D5, review 7 finding 9): `settleDirectWorker`
|
|
2112
|
+
// RETURNS a promise (resolving to a bounded refund-failure
|
|
2113
|
+
// diagnostic string, or `null`) — capture it instead of discarding
|
|
2114
|
+
// it, so `run()`'s wrapper above can await it and surface the
|
|
2115
|
+
// diagnostic in the result.
|
|
2116
|
+
settlementPromise = recoveryModule.settleDirectWorker(admission, {
|
|
2117
|
+
...outcome, callerAborted: !!registry.get(entry.invocationId)?.cancelRequested,
|
|
2118
|
+
});
|
|
2119
|
+
captureSettlementPromise?.(settlementPromise);
|
|
2120
|
+
} catch { /* recovery accounting must never fail a run */ }
|
|
2121
|
+
settlementStarted();
|
|
2122
|
+
if (!outcome.confirmedStop) observeLater(isolationEvidence);
|
|
2123
|
+
};
|
|
2124
|
+
let recoveryDeadlineAtMs = 0;
|
|
2125
|
+
const settleObservedFailure = async (result, failureHint = '') => {
|
|
2126
|
+
await recoveryModule.observeLauncherRecoveryOutcome({ invocation, result, failureHint,
|
|
2127
|
+
installation: drivers[invocation.selected.cli]?.qualification?.installation, breaker: admission.breaker,
|
|
2128
|
+
timeoutMs: () => Math.min(10_000, recoveryDeadlineAtMs - admission.breaker.nowMs()),
|
|
2129
|
+
beforeRequest: async () => !registry.get(entry.invocationId)?.cancelRequested,
|
|
2130
|
+
onDispatch: () => recoveryModule.chargeDirectWorkerDiagnostic(admission, {
|
|
2131
|
+
deadlineAtMs: recoveryDeadlineAtMs, nowMs: admission.breaker.nowMs,
|
|
2132
|
+
isCancelled: () => !!registry.get(entry.invocationId)?.cancelRequested }) });
|
|
2133
|
+
settleRecovery({ ...recoveryModule.classifyAttemptForSettlement(invocation, result, admission),
|
|
2134
|
+
confirmedStop: result.confirmedStop });
|
|
2135
|
+
};
|
|
2136
|
+
try {
|
|
2137
|
+
// #2915 round 8 review 8 (C3): ONE absolute wall-clock deadline for
|
|
2138
|
+
// this attempt, computed HERE — at the very top of this function,
|
|
2139
|
+
// BEFORE any brief preparation — as the minimum of the task/health
|
|
2140
|
+
// budget, the transport (mcp per-call) budget when this is an
|
|
2141
|
+
// mcp-transport attempt, and the canary's own bounded window when
|
|
2142
|
+
// this admission is the breaker's canary, on the admission's own
|
|
2143
|
+
// monotonic clock. Passed down UNCHANGED from here on; NEVER
|
|
2144
|
+
// re-derived from an already-shortened duration. Computing this AFTER
|
|
2145
|
+
// brief preparation (the former position, right before the
|
|
2146
|
+
// transport-specific branches below) let preparation time elapse for
|
|
2147
|
+
// free: reproduced spending 60s in post-admission brief
|
|
2148
|
+
// validation/digest computation while a 30s native MCP call still
|
|
2149
|
+
// spawned and completed, because the deadline was derived from a
|
|
2150
|
+
// fresh `deadlineClockNow()` reading taken only after that
|
|
2151
|
+
// preparation finished (round 8 review 8, C3; review 7 finding 4's
|
|
2152
|
+
// last bullet).
|
|
2153
|
+
const taskDeadlineMs = Math.min(invocation.health?.hardTimeoutSeconds ?? invocation.limits.wallSeconds, invocation.limits.wallSeconds) * 1000;
|
|
2154
|
+
const deadlineClockNow = () => admission.breaker?.nowMs?.() ?? Date.now();
|
|
2155
|
+
const deadlineCandidatesMs = [admissionGuard?.deadlineAtMs ?? ((admission.startedAtMs ?? deadlineClockNow()) + taskDeadlineMs)];
|
|
2156
|
+
if (invocation.transport === 'mcp') deadlineCandidatesMs.push((admission.startedAtMs ?? deadlineClockNow()) + invocation.mcp.timeoutSeconds * 1000);
|
|
2157
|
+
if (admission.token?.canaryKeys?.length > 0) deadlineCandidatesMs.push(admission.token.deadlineMs);
|
|
2158
|
+
const deadlineAtMs = Math.min(...deadlineCandidatesMs);
|
|
2159
|
+
recoveryDeadlineAtMs = deadlineAtMs;
|
|
2160
|
+
// #2915 round 8 review 8 (C3): cancellation predicate, read from the
|
|
2161
|
+
// registry's own live `cancelRequested` flag — set by `cancel()`
|
|
2162
|
+
// (M2) at any point during this run's lifetime, including before any
|
|
2163
|
+
// launch/control handle exists. Folded into `dispatchGuard` below so
|
|
2164
|
+
// the leaf dispatch-commit hook (D2) refuses to spawn a cancelled
|
|
2165
|
+
// attempt, and threaded into the native-MCP per-request/write-boundary
|
|
2166
|
+
// checks so cancellation during pre-spawn preparation prevents the
|
|
2167
|
+
// spawn and any request (round 8 review 8, C3).
|
|
2168
|
+
const isCancelled = () => !!registry.get(entry.invocationId)?.cancelRequested;
|
|
2169
|
+
// #2915 round 8 review 8 (C3): passed to both launch paths so the D2
|
|
2170
|
+
// leaf hook re-checks deadline/cancellation immediately before the
|
|
2171
|
+
// real process-start call, after every preparation step has run.
|
|
2172
|
+
const dispatchGuard = { deadlineAtMs, nowMs: deadlineClockNow, isCancelled };
|
|
2173
|
+
// F4 (fix-2892-r2, formerly part of M5): per operator decision, the
|
|
2174
|
+
// MCP path REQUIRES the structured WorkerDirectBrief — a probe showed
|
|
2175
|
+
// a plaintext brief (or a JSON brief swapped to plaintext between
|
|
2176
|
+
// launch and result assembly) was accepted, bypassing digest
|
|
2177
|
+
// verification entirely and completing anyway, because
|
|
2178
|
+
// `tryAssembleDirectResult` below treats a non-JSON brief as "nothing
|
|
2179
|
+
// to assemble against" and best-effort no-ops rather than refusing.
|
|
2180
|
+
// Fixed: for `mcp` transport ONLY, validate — AT ADMISSION, BEFORE any
|
|
2181
|
+
// launch — that the brief parses as a WorkerDirectBrief (#2890/#2879
|
|
2182
|
+
// codec), and pin ITS digest ourselves (never relying on whatever
|
|
2183
|
+
// `invocation.briefDigest` #2887's own resolver may or may not have
|
|
2184
|
+
// set for this transport). A non-structured or schema-invalid brief
|
|
2185
|
+
// refuses before any worker ever runs.
|
|
2186
|
+
let mcpBriefDigest;
|
|
2187
|
+
if (invocation.transport === 'mcp') {
|
|
2188
|
+
let briefBytes;
|
|
2189
|
+
try {
|
|
2190
|
+
briefBytes = await readBoundedFile(request.brief, 262144, root);
|
|
2191
|
+
} catch {
|
|
2192
|
+
// #2915 review 4 (finding 2): this pre-spawn refusal happens
|
|
2193
|
+
// AFTER admission (above) but was never settling it — a leaked
|
|
2194
|
+
// reservation on every unreadable MCP brief.
|
|
2195
|
+
//
|
|
2196
|
+
// #2915 review 6 (finding 4, "stop evidence on every settlement"):
|
|
2197
|
+
// zero workers were ever dispatched for this reservation — settle
|
|
2198
|
+
// with `confirmedStop: true` so a canary whose own deadline has
|
|
2199
|
+
// already passed (e.g. an expired queued request) releases its
|
|
2200
|
+
// reservation outright instead of leaving a permanent blocking
|
|
2201
|
+
// marker behind for an attempt that never ran at all.
|
|
2202
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2203
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['OG_BRIEF_UNREADABLE'] });
|
|
2204
|
+
}
|
|
2205
|
+
try {
|
|
2206
|
+
const { assertValidWorkerPayload } = await loadMissionProtocol();
|
|
2207
|
+
assertValidWorkerPayload('workerDirectBrief', JSON.parse(briefBytes.toString('utf8')));
|
|
2208
|
+
} catch (error) {
|
|
2209
|
+
// Fixed-vocabulary diagnostic only — NEVER the raw JSON.parse/
|
|
2210
|
+
// validator message, exactly like M5's own `parseWorkerFinal`
|
|
2211
|
+
// diagnostic below. A duplicate-key/planted-marker probe showed
|
|
2212
|
+
// this exact kind of path can otherwise reflect attacker/worker-
|
|
2213
|
+
// controlled input text straight back out to the caller.
|
|
2214
|
+
//
|
|
2215
|
+
// #2915 review 6 (finding 4): zero workers dispatched — same
|
|
2216
|
+
// proven-never-dispatched settlement as above.
|
|
2217
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2218
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [typeof error?.code === 'string' ? error.code : 'OG_BRIEF_NOT_STRUCTURED'] });
|
|
2219
|
+
}
|
|
2220
|
+
mcpBriefDigest = `sha256:${createHash('sha256').update(briefBytes).digest('hex')}`;
|
|
2221
|
+
}
|
|
2222
|
+
let observation;
|
|
2223
|
+
let rawOutput;
|
|
2224
|
+
// #2915 review 5 ("a second worker over unresolved execution"): a
|
|
2225
|
+
// SEPARATE fact from `kind` threaded into every `settleRecovery` call
|
|
2226
|
+
// below — whether THIS attempt's own descendant tree is launcher-
|
|
2227
|
+
// confirmed stopped, so a settlement arriving against an already-
|
|
2228
|
+
// canary-deadline-expired reservation can resolve that reservation's
|
|
2229
|
+
// blocking marker (see `ScopedFailureBreaker.settle()`). Stays false
|
|
2230
|
+
// for native MCP (response completion is not stop proof).
|
|
2231
|
+
let confirmedStop = false;
|
|
2232
|
+
if (invocation.transport === 'shell') {
|
|
2233
|
+
// #2915 round 8 (D2, replaces the former `await debitRecovery()`
|
|
2234
|
+
// call here): the ledger debit now happens INSIDE the dispatch-
|
|
2235
|
+
// commit hook `planDirectWorkerLaunch`'s borrowed-admission branch
|
|
2236
|
+
// constructs (session-launcher.mjs), invoked by `launchIsolatedWorker`
|
|
2237
|
+
// itself immediately before the real process-start call — never
|
|
2238
|
+
// here, before the dispatch resource gate inside `launchShellWorkerImpl`
|
|
2239
|
+
// has even run. This closes the "MCP debits before queue/resource
|
|
2240
|
+
// refusals" gap (#2915 round 8 critique, D2): a resource-gate
|
|
2241
|
+
// refusal below now spends nothing.
|
|
2242
|
+
//
|
|
2243
|
+
// #2915 round 8 (D5): checked before spawn. A deadline already
|
|
2244
|
+
// passed before this bridge even attempted to spawn is a proven
|
|
2245
|
+
// zero-dispatch refusal, exactly like any other pre-spawn gate
|
|
2246
|
+
// refusal.
|
|
2247
|
+
//
|
|
2248
|
+
// #2915 round 8 review 8 (C3): cancellation itself is deliberately
|
|
2249
|
+
// NOT re-checked here (unlike the deadline) -- the authoritative
|
|
2250
|
+
// "prevent the spawn" enforcement for cancellation lives in
|
|
2251
|
+
// `dispatchGuard` below, re-checked inside the LEAF dispatch-commit
|
|
2252
|
+
// hook (D2) after every real preparation step the actual launch
|
|
2253
|
+
// performs. Adding a second, earlier check at this bridge level,
|
|
2254
|
+
// ahead of even invoking `launchShellWorkerImpl`, would race an
|
|
2255
|
+
// operator cancel arriving while an in-flight (but not yet
|
|
2256
|
+
// resolved) launch call is still legitimately starting, and
|
|
2257
|
+
// mis-refuse a launch that was already under way.
|
|
2258
|
+
if (deadlineClockNow() >= deadlineAtMs) {
|
|
2259
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2260
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['worker dispatch deadline already passed before spawn'] });
|
|
2261
|
+
}
|
|
2262
|
+
// `spawnAttempted` is still set BEFORE calling, not after inspecting
|
|
2263
|
+
// the result -- `launchShellWorkerImpl` itself may already have side
|
|
2264
|
+
// effects (dispatch gate sampling, resource checks, or a genuine
|
|
2265
|
+
// spawn) before it returns or throws, so a throw from WITHIN it can
|
|
2266
|
+
// never be proven to be pre-spawn. From here on, any exception
|
|
2267
|
+
// reaching the outer catch-all is post-spawn uncertainty.
|
|
2268
|
+
spawnAttempted = true;
|
|
2269
|
+
execution.mayHaveLaunched = true;
|
|
2270
|
+
const launch = await launchShellWorkerImpl(invocation, { allowDegraded, config: snapshot.config,
|
|
2271
|
+
name: `mcp-${entry.invocationId.slice(0, 8)}`, memoryMaxGb: options.shellWorkerMemoryMaxGb,
|
|
2272
|
+
driver: drivers[invocation.selected.cli], servedRoot: root, stdinRuntimeRoot: options.stdinRuntimeRoot, admission, invocationId: entry.invocationId, dispatchGuard, ...gateInputs });
|
|
2273
|
+
let failureHint = Buffer.alloc(0);
|
|
2274
|
+
const observeHint = chunk => {
|
|
2275
|
+
if (failureHint.length < 8192) failureHint = Buffer.concat([failureHint, Buffer.from(chunk).subarray(0, 8192 - failureHint.length)]);
|
|
2276
|
+
};
|
|
2277
|
+
launch.child?.stderr?.on('data', observeHint);
|
|
2278
|
+
isolationEvidence = { isolationMode: launch.mode ?? null, unitName: launch.unit ?? null, child: launch.child ?? null,
|
|
2279
|
+
dispatchState: launch.dispatchState === 'submission-uncertain' ? 'submission-uncertain' : (launch.launched || launch.launchAttempted) ? 'dispatched' : 'never-began' };
|
|
2280
|
+
registry.update(entry.invocationId, { isolationMode: launch.mode ?? null });
|
|
2281
|
+
if (!launch.launched && !launch.launchAttempted && launch.dispatchState !== 'submission-uncertain') {
|
|
2282
|
+
execution.mayHaveLaunched = false;
|
|
2283
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2284
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [launch.diagnostic ?? 'worker launch was refused'] });
|
|
2285
|
+
}
|
|
2286
|
+
// M2: a live stop handle for the duration of this call, so an
|
|
2287
|
+
// explicit `cancel` can actually stop the isolated unit / kill the
|
|
2288
|
+
// child, not just record that cancellation was requested.
|
|
2289
|
+
// #2915 review 4 (finding 3) / round 8 (D5): clipped to the ONE
|
|
2290
|
+
// absolute `deadlineAtMs` computed above (which already folds in the
|
|
2291
|
+
// admitted canary's own 300s window) — never re-derived from
|
|
2292
|
+
// `taskDeadlineMs` independently of it, and always measured as
|
|
2293
|
+
// "remaining from right now" so time already spent launching is
|
|
2294
|
+
// never double-counted.
|
|
2295
|
+
let deadlineMs = Math.max(0, deadlineAtMs - deadlineClockNow());
|
|
2296
|
+
const control = buildShellStopControl(launch, {
|
|
2297
|
+
waitForUnit: (unit, budgetMs, observerOptions) => (waitForUnitTerminationOverride ?? waitForUnitTermination)(unit,
|
|
2298
|
+
budgetMs, { ...observerOptions, clock: deadlineClockNow }),
|
|
2299
|
+
nowMs: deadlineClockNow,
|
|
2300
|
+
stopUnit: options.stopWorkerUnit ?? options.stopUnit,
|
|
2301
|
+
killUnit: options.killWorkerUnit,
|
|
2302
|
+
graceMs: options.shellStopGraceMs ?? (invocation.health?.terminateGraceSeconds ?? 1) * 1000,
|
|
2303
|
+
onConfirmed: async outcome => {
|
|
2304
|
+
// The shell observer proved an accepted stop plus a fresh inactive
|
|
2305
|
+
// unit that never exec'd; ExecMainCode alone cannot retain it.
|
|
2306
|
+
neverExecutedStop = outcome.neverExecuted === true;
|
|
2307
|
+
const stopEvidence = await resolveEvidence({ ...isolationEvidence, stopFirst: false });
|
|
2308
|
+
confirmedStop = stopEvidence.confirmedStop;
|
|
2309
|
+
observeStop(Promise.resolve(), isolationEvidence);
|
|
2310
|
+
if (!launch.launched) await cleanupReservation();
|
|
2311
|
+
await Promise.resolve().then(() => launch.cleanup?.()).catch(() => {});
|
|
2312
|
+
const live = registry.get(entry.invocationId);
|
|
2313
|
+
if (live?.terminalAt !== null) return;
|
|
2314
|
+
if (live.status === 'uncertain' || live.status === 'blocked') {
|
|
2315
|
+
registry.markTerminal(entry.invocationId, { control: null, termination: 'confirmed',
|
|
2316
|
+
status: live.cancelRequested ? 'cancelled' : 'failed' });
|
|
2317
|
+
} else registry.update(entry.invocationId, { control: null, termination: 'confirmed' });
|
|
2318
|
+
},
|
|
2319
|
+
});
|
|
2320
|
+
if (registry.get(entry.invocationId)?.termination !== 'confirmed') {
|
|
2321
|
+
registry.update(entry.invocationId, { control });
|
|
2322
|
+
}
|
|
2323
|
+
// Honor a cancel that arrived while the launcher was still pending.
|
|
2324
|
+
if (registry.get(entry.invocationId)?.cancelRequested) await control.stop();
|
|
2325
|
+
let outcome = launch.launched ? await control.wait(deadlineMs, { deadlineAtMs, nowMs: deadlineClockNow }) : await control.stopAndConfirm();
|
|
2326
|
+
// Preserve the control's timeout latch and refuse a late clean outcome.
|
|
2327
|
+
if (outcome.termination === 'confirmed' && !outcome.timedOut && deadlineClockNow() >= deadlineAtMs) {
|
|
2328
|
+
outcome = { ...outcome, timedOut: true };
|
|
2329
|
+
}
|
|
2330
|
+
const stopEvidence = await resolveEvidence({ ...isolationEvidence, stopFirst: false });
|
|
2331
|
+
confirmedStop = stopEvidence.confirmedStop;
|
|
2332
|
+
// #2915 review 6 (finding 1, "false recovery after timeout"): a
|
|
2333
|
+
// forced stop at the attempt's own deadline is never treated as a
|
|
2334
|
+
// candidate success, regardless of the exit code the forced stop
|
|
2335
|
+
// happened to observe (a systemd unit whose main process had
|
|
2336
|
+
// already legitimately exited 0 right before the stop escalation
|
|
2337
|
+
// fired reports exactly that exit code even though the attempt
|
|
2338
|
+
// itself was never allowed to run to its own completion). Preserved
|
|
2339
|
+
// independently of `outcome.exitCode`/`outcome.termination` through
|
|
2340
|
+
// to the candidate-success gate below (`strictlyCompleted`).
|
|
2341
|
+
observation = { transport: 'shell', status: outcome.termination === 'confirmed' && outcome.exitCode === 0 && !outcome.timedOut ? 'completed' : 'failed', termination: outcome.termination, exitCode: outcome.exitCode, timedOut: !!outcome.timedOut };
|
|
2342
|
+
// #2915 final (evidence contract): the attempt-level fact is derived
|
|
2343
|
+
// from this bridge's own launcher-observed wait status — never a
|
|
2344
|
+
// worker claim — independent of `termination`/`descendants` above.
|
|
2345
|
+
// `termination === 'confirmed'` here already reflects THIS bridge's
|
|
2346
|
+
// own trusted observation (systemd-unit query, or this process's own
|
|
2347
|
+
// `child.close` event).
|
|
2348
|
+
//
|
|
2349
|
+
// #2915 review 5 (finding 1a, "false recovery"): a definite BAD exit
|
|
2350
|
+
// is trustworthy failure evidence regardless of what happens
|
|
2351
|
+
// downstream, so it settles immediately. A confirmed CLEAN exit
|
|
2352
|
+
// (`exitCode === 0`) is only a CANDIDATE success -- settlement is
|
|
2353
|
+
// deferred past strict final-claims validation, brief-evidence
|
|
2354
|
+
// matching and the cancellation check below (common code, shared
|
|
2355
|
+
// with the MCP branch), so a worker that exits 0 but never produces
|
|
2356
|
+
// a valid, matching, uncancelled completed claim can never close a
|
|
2357
|
+
// scope through this path. Uncertain termination never implicates
|
|
2358
|
+
// anything either way.
|
|
2359
|
+
launch.child?.stderr?.removeListener('data', observeHint);
|
|
2360
|
+
if (outcome.termination !== 'confirmed') {
|
|
2361
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2362
|
+
// M2: stays LIVE — capacity is held until a stop is proven or the
|
|
2363
|
+
// conductor disposes of it explicitly, never freed on a mere
|
|
2364
|
+
// unconfirmed-termination timeout.
|
|
2365
|
+
return (confirmedStop ? registry.markTerminal : registry.update)(entry.invocationId, {
|
|
2366
|
+
status: launch.launched ? 'uncertain' : 'blocked', termination: confirmedStop ? 'confirmed' : 'uncertain',
|
|
2367
|
+
diagnostics: ['worker termination could not be confirmed before the deadline'],
|
|
2368
|
+
});
|
|
2369
|
+
}
|
|
2370
|
+
if (outcome.exitCode !== 0) {
|
|
2371
|
+
if (!failureHint.length && invocation.stderrPath) {
|
|
2372
|
+
try { failureHint = await readBoundedFile(invocation.stderrPath, 8192, root); } catch { /* no usable hint */ }
|
|
2373
|
+
}
|
|
2374
|
+
await settleObservedFailure({ ...outcome, invocationId: entry.invocationId, attemptOutcome: 'failed',
|
|
2375
|
+
cancelled: !!registry.get(entry.invocationId)?.cancelRequested, confirmedStop }, failureHint.toString('utf8'));
|
|
2376
|
+
}
|
|
2377
|
+
await Promise.resolve().then(() => launch.cleanup?.()).catch(() => {
|
|
2378
|
+
console.error('[worker-invoke] stdin cleanup failed after confirmed termination');
|
|
2379
|
+
});
|
|
2380
|
+
if (!launch.launched) {
|
|
2381
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2382
|
+
execution.mayHaveLaunched = false;
|
|
2383
|
+
return markTerminal({ control: null, status: 'failed', termination: 'confirmed',
|
|
2384
|
+
diagnostics: [launch.diagnostic ?? 'worker launch failed; termination confirmed'] });
|
|
2385
|
+
}
|
|
2386
|
+
if (outcome.timedOut || outcome.stopped) {
|
|
2387
|
+
settleRecovery({ kind: registry.get(entry.invocationId)?.cancelRequested ? 'uncertain' : 'failed', confirmedStop });
|
|
2388
|
+
return registry.markTerminal(entry.invocationId, { control: null, termination: 'confirmed',
|
|
2389
|
+
status: registry.get(entry.invocationId)?.cancelRequested ? 'cancelled' : 'failed',
|
|
2390
|
+
diagnostics: [outcome.timedOut ? 'worker deadline exceeded; termination confirmed' : 'worker stopped; termination confirmed'] });
|
|
2391
|
+
}
|
|
2392
|
+
try { rawOutput = await readBoundedFile(invocation.finalPath, invocation.maxOutputBytes, root); } catch {
|
|
2393
|
+
// #2915 review 5 (finding 1a): a confirmed clean exit that never
|
|
2394
|
+
// produced a readable final output is not a success -- settle
|
|
2395
|
+
// failed now rather than leaving it candidate-success forever
|
|
2396
|
+
// (the bad-exit branch above already settled; this is a no-op
|
|
2397
|
+
// there).
|
|
2398
|
+
settleRecovery({ kind: 'failed', confirmedStop });
|
|
2399
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['worker final output was not written'] });
|
|
2400
|
+
}
|
|
2401
|
+
} else {
|
|
2402
|
+
const { mcpRegistry, qualification } = drivers[invocation.selected.cli];
|
|
2403
|
+
const mcpServer = await mcpRegistry.get(invocation.mcp.serverRef);
|
|
2404
|
+
if (!mcpServer) {
|
|
2405
|
+
// #2915 review 4 (finding 2): another pre-spawn refusal after
|
|
2406
|
+
// admission that was never settling it.
|
|
2407
|
+
//
|
|
2408
|
+
// #2915 review 6 (finding 4): proven zero dispatch.
|
|
2409
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: true });
|
|
2410
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: ['worker MCP server is not registered'] });
|
|
2411
|
+
}
|
|
2412
|
+
// F5: pin the qualified server's definition digest NOW, at
|
|
2413
|
+
// admission, so a queued launch can detect a registry edit that
|
|
2414
|
+
// happened while it was waiting (see `serverDefinitionDigest`).
|
|
2415
|
+
const qualifiedServerDigest = serverDefinitionDigest(mcpServer);
|
|
2416
|
+
// #2915 review 5 ("a second worker over unresolved execution") /
|
|
2417
|
+
// round 8 (D5): clipped to the ONE absolute `deadlineAtMs` computed
|
|
2418
|
+
// above — never re-derived independently of it (that re-derivation,
|
|
2419
|
+
// against `admission.token.deadlineMs` alone, used to miss the
|
|
2420
|
+
// plain task-deadline case entirely and was also the source of the
|
|
2421
|
+
// "subtracts preparation time twice" defect once `defaultCallMcpWorker`
|
|
2422
|
+
// added its OWN mint-time-relative re-derivation on top — round 8
|
|
2423
|
+
// critique, D5/review 7 finding 4's last bullet). This clip exists
|
|
2424
|
+
// for callers that only ever see `mcpBinding.timeoutSeconds` (every
|
|
2425
|
+
// existing injected `callMcpWorker` test double, and the ACTUAL
|
|
2426
|
+
// enforcement inside `defaultCallMcpWorker` uses the separately
|
|
2427
|
+
// threaded `deadlineAtMs` below, never re-adding onto
|
|
2428
|
+
// `admission.token.mintedAtMs`.
|
|
2429
|
+
let mcpBinding = invocation.mcp;
|
|
2430
|
+
const remainingSeconds = Math.max(0, (deadlineAtMs - deadlineClockNow()) / 1000);
|
|
2431
|
+
if (remainingSeconds < mcpBinding.timeoutSeconds) mcpBinding = { ...mcpBinding, timeoutSeconds: remainingSeconds };
|
|
2432
|
+
// #2915 round 8 (D2, replaces the former `await debitRecovery()`
|
|
2433
|
+
// call here): same reasoning as the shell path above — the ledger
|
|
2434
|
+
// debit now happens inside the dispatch-commit hook, deep inside
|
|
2435
|
+
// `defaultLaunchMcpWorkerProcess` -> `planDirectWorkerLaunch` ->
|
|
2436
|
+
// `launchIsolatedWorker`, immediately before the real MCP worker
|
|
2437
|
+
// SERVER process actually starts.
|
|
2438
|
+
//
|
|
2439
|
+
// `spawnAttempted` is still set BEFORE calling, not after
|
|
2440
|
+
// inspecting `result.outcome` -- `callMcpWorkerImpl` itself may
|
|
2441
|
+
// already have launched a real MCP server process and sent bytes
|
|
2442
|
+
// before it returns or throws (#2892 F2's own "thrown MCP transport
|
|
2443
|
+
// error stays LIVE" case is exactly this), so a throw from WITHIN
|
|
2444
|
+
// it can never be proven to be pre-spawn.
|
|
2445
|
+
spawnAttempted = true;
|
|
2446
|
+
execution.mayHaveLaunched = true;
|
|
2447
|
+
const result = await callMcpWorkerImpl(mcpBinding, mcpServer, {
|
|
2448
|
+
sessionIsolation: invocation.mcp.sessionIsolation, mcpRegistry, allowDegraded, config: snapshot.config,
|
|
2449
|
+
// #2915 round 8 (D5): the ONE absolute deadline for this call,
|
|
2450
|
+
// computed once above.
|
|
2451
|
+
deadlineAtMs,
|
|
2452
|
+
// #2892 (CI fix): the resolved invocation's own worktree
|
|
2453
|
+
// (`invocation.cwd`, #2887's `request.workdir`) — without it the
|
|
2454
|
+
// dispatch gate's free-disk sample has no filesystem to read and
|
|
2455
|
+
// fails closed on every host, never reaching isolation at all.
|
|
2456
|
+
// See `defaultLaunchMcpWorkerProcess`'s doc comment.
|
|
2457
|
+
cwd: invocation.cwd,
|
|
2458
|
+
name: `mcp-${entry.invocationId.slice(0, 8)}`, memoryMaxGb: mcpWorkerMemoryMaxGb, mcpSchemas: qualification?.mcpSchemas,
|
|
2459
|
+
qualifiedServerDigest, registerControl: control => registry.update(entry.invocationId, { control }),
|
|
2460
|
+
// #2915 review 4 (finding 2): the SAME top-level admission this
|
|
2461
|
+
// run() already holds, borrowed through to the native-MCP launch
|
|
2462
|
+
// path — never a second, independent planner reservation. See
|
|
2463
|
+
// `defaultLaunchMcpWorkerProcess`.
|
|
2464
|
+
admission,
|
|
2465
|
+
// #2915 round 8 review 8 (C6/C3): independently resolved dispatch
|
|
2466
|
+
// identity (never derived from the borrowed admission wrapper) and
|
|
2467
|
+
// the one dispatch-commit deadline/cancellation guard, threaded
|
|
2468
|
+
// all the way to the leaf hook.
|
|
2469
|
+
invocation, invocationId: entry.invocationId, dispatchGuard, isCancelled, observeStop,
|
|
2470
|
+
...gateInputs,
|
|
2471
|
+
});
|
|
2472
|
+
isolationEvidence = { isolationMode: result.isolationMode ?? null, unitName: result.unitName ?? null,
|
|
2473
|
+
child: result.child ?? null, dispatchState: result.dispatchState ?? 'dispatched' };
|
|
2474
|
+
if (result.outcome === 'blocked') {
|
|
2475
|
+
// Never launched (dispatch gate or queue refusal): settle with no
|
|
2476
|
+
// failure evidence.
|
|
2477
|
+
//
|
|
2478
|
+
// #2915 round 8 review 8 (C2): a `blocked` outcome does NOT always
|
|
2479
|
+
// mean proven zero dispatch -- a schema-drift refusal (or any
|
|
2480
|
+
// other post-connect `blocked` return) happens AFTER a real MCP
|
|
2481
|
+
// worker server process was spawned and connected to
|
|
2482
|
+
// (`result.dispatchState === 'dispatched'`). Resolve REAL stop
|
|
2483
|
+
// evidence for that case through the shared `resolveStopEvidence`
|
|
2484
|
+
// (a server that ignores close/TERM/KILL must never be reported
|
|
2485
|
+
// confirmed-stopped) rather than hardcoding `confirmedStop: true`
|
|
2486
|
+
// for every `blocked` label (review 8, C section finding 2: "an
|
|
2487
|
+
// MCP blocked result AFTER a process was spawned is dispatched").
|
|
2488
|
+
// Every pre-spawn `blocked` path inside `callMcpWorkerImpl`
|
|
2489
|
+
// (registry requalification, admission-expiry, a thrown
|
|
2490
|
+
// pre-launch exception, or a proven pre-spawn launch refusal)
|
|
2491
|
+
// carries `dispatchState: 'never-began'` (or `'submission-
|
|
2492
|
+
// uncertain'` for a failed/timed-out systemd-run submission) and
|
|
2493
|
+
// is handled accordingly — ALWAYS proven zero dispatch only for
|
|
2494
|
+
// `'never-began'`.
|
|
2495
|
+
let blockedConfirmedStop;
|
|
2496
|
+
if (result.dispatchState === 'dispatched' || result.dispatchState === 'submission-uncertain') {
|
|
2497
|
+
const evidence = await resolveEvidence(isolationEvidence);
|
|
2498
|
+
blockedConfirmedStop = evidence.confirmedStop;
|
|
2499
|
+
} else {
|
|
2500
|
+
blockedConfirmedStop = true;
|
|
2501
|
+
execution.mayHaveLaunched = false;
|
|
2502
|
+
}
|
|
2503
|
+
settleRecovery({ kind: 'uncertain', confirmedStop: blockedConfirmedStop });
|
|
2504
|
+
return blockedConfirmedStop ? markTerminal({
|
|
2505
|
+
status: 'blocked', termination: 'confirmed',
|
|
2506
|
+
diagnostics: [result.diagnostic ?? 'worker MCP launch was refused'],
|
|
2507
|
+
}) : registry.update(entry.invocationId, {
|
|
2508
|
+
status: 'blocked', termination: blockedConfirmedStop ? 'confirmed' : 'uncertain',
|
|
2509
|
+
diagnostics: [result.diagnostic ?? 'worker MCP launch was refused'],
|
|
2510
|
+
});
|
|
2511
|
+
}
|
|
2512
|
+
confirmedStop = (await resolveEvidence({ ...isolationEvidence, stopFirst: false })).confirmedStop;
|
|
2513
|
+
if (result.outcome !== 'responded') {
|
|
2514
|
+
// Overflow holds capacity and stop control until termination is proven.
|
|
2515
|
+
if (result.diagnostic?.includes('OG_WORKER_MCP_FRAME_TOO_LARGE')) {
|
|
2516
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2517
|
+
// An explicit cancel can have confirmed termination while this run
|
|
2518
|
+
// was awaiting its initial overflow evidence.
|
|
2519
|
+
if (entry.terminalAt !== null) return entry;
|
|
2520
|
+
const patch = { status: result.termination === 'confirmed' ? 'failed' : 'uncertain',
|
|
2521
|
+
termination: result.termination ?? 'uncertain', diagnostics: [result.diagnostic] };
|
|
2522
|
+
return result.termination === 'confirmed'
|
|
2523
|
+
? registry.markTerminal(entry.invocationId, { ...patch, control: null })
|
|
2524
|
+
: registry.update(entry.invocationId, patch);
|
|
2525
|
+
}
|
|
2526
|
+
// M2: stays LIVE for the same reason as the shell timeout path above.
|
|
2527
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2528
|
+
return (confirmedStop ? registry.markTerminal : registry.update)(entry.invocationId, {
|
|
2529
|
+
status: 'uncertain', termination: confirmedStop ? 'confirmed' : 'uncertain',
|
|
2530
|
+
diagnostics: [`worker ${result.outcome}; cancellation delivered`],
|
|
2531
|
+
});
|
|
2532
|
+
}
|
|
2533
|
+
rawOutput = result.response;
|
|
2534
|
+
observation = { transport: 'mcp', status: result.response.isError ? 'failed' : 'completed', termination: 'confirmed', exitCode: result.response.isError ? 1 : null, isError: !!result.response.isError };
|
|
2535
|
+
// #2915 final (evidence contract): a transport-correlated response
|
|
2536
|
+
// that is not `isError` is a CANDIDATE successful attempt even
|
|
2537
|
+
// though `exitCode` is legitimately `null` for MCP (#2915 final
|
|
2538
|
+
// review, finding 4: "successful responses produce exitCode:null...
|
|
2539
|
+
// recorded one invocation failure and left the shared canary
|
|
2540
|
+
// active" — the former exitCode-based classification treated every
|
|
2541
|
+
// successful MCP response as a failure). Descendant-stop evidence is
|
|
2542
|
+
// a separate, independent fact this bridge never claims here.
|
|
2543
|
+
//
|
|
2544
|
+
// #2915 review 5 (finding 1a): an `isError` response IS trustworthy
|
|
2545
|
+
// failure evidence and settles immediately, exactly like a
|
|
2546
|
+
// confirmed bad exit above. A non-error response is only a
|
|
2547
|
+
// CANDIDATE -- settlement is deferred to the common code below,
|
|
2548
|
+
// past strict final-claims validation, brief-evidence matching and
|
|
2549
|
+
// the cancellation check, so a transport-correlated response alone
|
|
2550
|
+
// can never close a scope.
|
|
2551
|
+
if (result.response.isError) {
|
|
2552
|
+
await settleObservedFailure({ invocationId: entry.invocationId, attemptOutcome: 'failed',
|
|
2553
|
+
termination: 'confirmed', exitCode: 1, confirmedStop,
|
|
2554
|
+
cancelled: !!registry.get(entry.invocationId)?.cancelRequested }, JSON.stringify(result.response).slice(0, 8192));
|
|
2555
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['worker reported isError'] });
|
|
2556
|
+
}
|
|
2557
|
+
}
|
|
2558
|
+
const { ProviderAdapter } = await loadRalphExternal('lib/provider-adapter.mjs');
|
|
2559
|
+
let claims;
|
|
2560
|
+
try {
|
|
2561
|
+
claims = await new (class extends ProviderAdapter {})().parseWorkerFinal(rawOutput, {
|
|
2562
|
+
format: invocation.transport === 'shell' ? 'jsonl' : 'mcp', observation, maxOutputBytes: invocation.maxOutputBytes,
|
|
2563
|
+
});
|
|
2564
|
+
} catch (error) {
|
|
2565
|
+
// #2915 review 5 (finding 1a): the strict parse itself rejected the
|
|
2566
|
+
// claims -- the candidate success from above is settled failed
|
|
2567
|
+
// here, never left to default to an earlier premature success.
|
|
2568
|
+
//
|
|
2569
|
+
// #2915 review 6 (finding 4, "stop evidence on every settlement"):
|
|
2570
|
+
// thread the shell path's own launcher-confirmed termination fact
|
|
2571
|
+
// (`confirmedStop`) through here too — a late systemd completion
|
|
2572
|
+
// with confirmed termination but invalid claims must not retain a
|
|
2573
|
+
// permanent blocking marker merely because this rejection path
|
|
2574
|
+
// omitted it. Stays `false` for native MCP (unaffected).
|
|
2575
|
+
settleRecovery({ kind: 'failed', confirmedStop });
|
|
2576
|
+
// M5: bounded, fixed-vocabulary diagnostic only — never the raw
|
|
2577
|
+
// parser error message, which can echo attacker/worker-controlled
|
|
2578
|
+
// input text back out (a duplicate-key probe reflected a planted
|
|
2579
|
+
// marker through this exact path before the fix).
|
|
2580
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: [typeof error?.code === 'string' ? error.code : 'OG_WORKER_FINAL_INVALID'] });
|
|
2581
|
+
}
|
|
2582
|
+
const assembled = await tryAssembleDirectResult({ request, invocation, observation, claims, root,
|
|
2583
|
+
briefDigest: mcpBriefDigest, requireStructuredBrief: invocation.transport === 'mcp' });
|
|
2584
|
+
if (assembled?.conflict) {
|
|
2585
|
+
// #2915 review 5 (finding 1a): the brief changed between launch and
|
|
2586
|
+
// result assembly -- a candidate success settles failed here, never
|
|
2587
|
+
// silently assembled (or recorded recovered) against a different
|
|
2588
|
+
// brief than the one the worker was actually launched against.
|
|
2589
|
+
//
|
|
2590
|
+
// #2915 review 6 (finding 4): same `confirmedStop` threading as the
|
|
2591
|
+
// parse-failure path above.
|
|
2592
|
+
settleRecovery({ kind: 'failed', confirmedStop });
|
|
2593
|
+
return registry.markTerminal(entry.invocationId, { status: 'failed', termination: 'confirmed', diagnostics: ['OG_BRIEF_EVIDENCE_CONFLICT'] });
|
|
2594
|
+
}
|
|
2595
|
+
// M2: a cancel requested at ANY point during this run's lifetime —
|
|
2596
|
+
// even before a live `control` handle existed to deliver it to — means
|
|
2597
|
+
// this result is never trusted as `completed`, regardless of what the
|
|
2598
|
+
// worker itself claims.
|
|
2599
|
+
const liveEntry = registry.get(entry.invocationId);
|
|
2600
|
+
const cancelled = !!liveEntry?.cancelRequested;
|
|
2601
|
+
// #2915 review 6 (finding 1, "false recovery after timeout"): a
|
|
2602
|
+
// timed-out attempt (the shell path's own forced-stop-at-deadline
|
|
2603
|
+
// observation, see `observation.timedOut` above) is never treated as
|
|
2604
|
+
// strictly completed, regardless of what a worker claim says or what
|
|
2605
|
+
// exit code the forced stop happened to observe.
|
|
2606
|
+
const strictlyCompleted = claims.status === 'completed' && !observation?.timedOut;
|
|
2607
|
+
// #2915 review 5 (finding 1a): this is the ONE place a candidate
|
|
2608
|
+
// success (deferred above) is finally settled -- only after strict
|
|
2609
|
+
// completed claims, matching brief evidence (no conflict, checked
|
|
2610
|
+
// above) and the cancellation check all agree. A cancelled run never
|
|
2611
|
+
// settles 'succeeded' even with valid claims (the operator asked to
|
|
2612
|
+
// stop it); an invalid/non-completed claim settles 'failed', exactly
|
|
2613
|
+
// like every other claim-rejection path above.
|
|
2614
|
+
settleRecovery({ kind: cancelled ? 'uncertain' : strictlyCompleted ? 'succeeded' : 'failed', confirmedStop });
|
|
2615
|
+
return registry.markTerminal(entry.invocationId, {
|
|
2616
|
+
status: cancelled ? 'cancelled' : (strictlyCompleted ? 'completed' : 'failed'),
|
|
2617
|
+
termination: 'confirmed',
|
|
2618
|
+
result: assembled?.result ?? { claims },
|
|
2619
|
+
});
|
|
2620
|
+
} catch (error) {
|
|
2621
|
+
// Registry status and an error label do not establish descendant stop.
|
|
2622
|
+
if (error.workerNotLaunched) execution.mayHaveLaunched = false;
|
|
2623
|
+
const neverDispatched = !spawnAttempted || !!error.workerNotLaunched;
|
|
2624
|
+
const confirmedStop = neverDispatched || (await resolveEvidence({ ...isolationEvidence, stopFirst: false })).confirmedStop;
|
|
2625
|
+
settleRecovery({ kind: 'uncertain', confirmedStop });
|
|
2626
|
+
if (neverDispatched) {
|
|
2627
|
+
return markTerminal({ status: 'blocked', termination: 'confirmed', diagnostics: [error.message] });
|
|
2628
|
+
}
|
|
2629
|
+
if (confirmedStop) {
|
|
2630
|
+
return registry.markTerminal(entry.invocationId, { control: null,
|
|
2631
|
+
status: registry.get(entry.invocationId)?.cancelRequested ? 'cancelled' : 'failed',
|
|
2632
|
+
termination: 'confirmed', diagnostics: [error.message] });
|
|
2633
|
+
}
|
|
2634
|
+
// Keep capacity and stop control while execution remains unresolved.
|
|
2635
|
+
return registry.update(entry.invocationId, { status: 'uncertain', termination: 'uncertain', diagnostics: [error.message] });
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2638
|
+
|
|
1398
2639
|
async function tryAssembleDirectResult({ request, invocation, observation, claims, root, briefDigest, requireStructuredBrief }) {
|
|
1399
2640
|
// M5 (and F4 for mcp): `briefDigest` (fix-2892-r2) is THIS run's own
|
|
1400
2641
|
// admission-time pin for the `mcp` transport — computed and validated
|
|
@@ -1453,11 +2694,85 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1453
2694
|
}
|
|
1454
2695
|
|
|
1455
2696
|
async function status(input) {
|
|
2697
|
+
if (!input.invocation_id) {
|
|
2698
|
+
// #2915 round 8 (D4): ordinary status, read-only, no id — lists every
|
|
2699
|
+
// invocation this server currently tracks as a `dispatched` reservation
|
|
2700
|
+
// with `stop: 'unresolved'`, across every distinct breaker instance
|
|
2701
|
+
// any tracked invocation's gated admission used. Never mutates
|
|
2702
|
+
// anything (`listUnresolvedStops` is a pure read accessor on the
|
|
2703
|
+
// breaker); this is the "reachable for every long-lived owner" route
|
|
2704
|
+
// C4 requires, surfaced here rather than printed-only.
|
|
2705
|
+
const breakers = new Set(registry.recoveryOwner ? [registry.recoveryOwner] : []);
|
|
2706
|
+
for (const entry of registry.values()) if (entry.recoveryBreaker) breakers.add(entry.recoveryBreaker);
|
|
2707
|
+
const unresolvedReservations = [];
|
|
2708
|
+
for (const breaker of breakers) {
|
|
2709
|
+
if (typeof breaker.listUnresolvedStops === 'function') unresolvedReservations.push(...breaker.listUnresolvedStops());
|
|
2710
|
+
}
|
|
2711
|
+
const interventionScopes = [...breakers].flatMap(breaker => (breaker.snapshot?.().scopes ?? [])
|
|
2712
|
+
.filter(scope => scope.state === 'operator-intervention')
|
|
2713
|
+
.map(scope => ({ scopeId: scope.scopeKey, reason: scope.operatorReason,
|
|
2714
|
+
resetInstruction: { operation: 'reset_recovery_scope', scope_id: scope.scopeKey } })));
|
|
2715
|
+
return { unresolvedReservations, interventionScopes };
|
|
2716
|
+
}
|
|
1456
2717
|
const entry = registry.get(input.invocation_id);
|
|
1457
2718
|
if (!entry) throw new Error('Unknown invocation_id');
|
|
1458
2719
|
return publicStatus(entry, healthHook);
|
|
1459
2720
|
}
|
|
1460
2721
|
|
|
2722
|
+
/**
|
|
2723
|
+
* #2915 round 8 (D4, "reconciliation route"): the explicit operator-
|
|
2724
|
+
* authorized reconciliation operation on this SAME `worker-invoke`
|
|
2725
|
+
* status/cancel handler — it already runs inside the owning server and
|
|
2726
|
+
* closes over its registry. Accepted ONLY for an invocation whose stored
|
|
2727
|
+
* reservation is `dispatched` with `stop: 'unresolved'` (verified via
|
|
2728
|
+
* `ScopedFailureBreaker.hasUnresolvedStop`, a pure read accessor — never
|
|
2729
|
+
* inferred from this registry's own, separately-maintained status
|
|
2730
|
+
* fields). Applied to THAT invocation's actual breaker (`entry.
|
|
2731
|
+
* recoveryBreaker`, pinned at admission — never a fresh default breaker).
|
|
2732
|
+
* Idempotent: a second call finds nothing left to reconcile and reports
|
|
2733
|
+
* why, rather than throwing or re-applying anything. No mailbox files;
|
|
2734
|
+
* nothing is written anywhere other than the breaker's own in-memory
|
|
2735
|
+
* reservation table.
|
|
2736
|
+
*/
|
|
2737
|
+
async function reconcileStopped(input) {
|
|
2738
|
+
if (!input.invocation_id) throw new Error('reconcile_stopped requires invocation_id');
|
|
2739
|
+
const entry = registry.get(input.invocation_id);
|
|
2740
|
+
if (!entry) throw new Error('Unknown invocation_id');
|
|
2741
|
+
const breaker = entry.recoveryBreaker ?? null;
|
|
2742
|
+
const base = publicStatus(entry, healthHook);
|
|
2743
|
+
if (!breaker || typeof breaker.hasUnresolvedStop !== 'function') {
|
|
2744
|
+
return { ...base, reconciled: false, reconcileReason: 'this invocation has no tracked gated reservation to reconcile' };
|
|
2745
|
+
}
|
|
2746
|
+
if (!breaker.hasUnresolvedStop(entry.invocationId)) {
|
|
2747
|
+
return { ...base, reconciled: false, reconcileReason: 'no dispatched reservation with unresolved stop evidence exists for this invocation (already reconciled, never dispatched, or already confirmed stopped)' };
|
|
2748
|
+
}
|
|
2749
|
+
const result = breaker.reconcileStopEvidence({ invocationId: entry.invocationId, confirmedStop: true });
|
|
2750
|
+
const applied = (result?.released ?? 0) > 0;
|
|
2751
|
+
// #2915 round 8 review 8 (C7): applying a confirmed stop must ALSO
|
|
2752
|
+
// finalize this invocation's registry capacity -- a dispatched
|
|
2753
|
+
// reservation with unresolved stop kept this entry's run-slot
|
|
2754
|
+
// effectively pinned (the registry's own `hasUnresolvedStopMarker`
|
|
2755
|
+
// exemption, above, retains it past the ordinary terminal sweep for
|
|
2756
|
+
// exactly this reason). `markTerminal({})` sets `terminalAt` without
|
|
2757
|
+
// touching `status` -- a truthful terminal state that never invents
|
|
2758
|
+
// task success for an attempt whose own outcome this bridge still
|
|
2759
|
+
// never confirmed. Idempotent: a no-op if the entry was already
|
|
2760
|
+
// terminal.
|
|
2761
|
+
if (applied) registry.markTerminal(entry.invocationId, {});
|
|
2762
|
+
return {
|
|
2763
|
+
...publicStatus(entry, healthHook),
|
|
2764
|
+
reconciled: applied,
|
|
2765
|
+
...(applied ? {} : { reconcileReason: 'reservation was already reconciled or no longer exists' }),
|
|
2766
|
+
};
|
|
2767
|
+
}
|
|
2768
|
+
|
|
2769
|
+
async function resetRecoveryScope(input) {
|
|
2770
|
+
if (!input.scope_id) throw new Error('reset_recovery_scope requires scope_id');
|
|
2771
|
+
const breaker = registry.recoveryOwner;
|
|
2772
|
+
if (!breaker) throw new Error('No recovery breaker owner is bound to this registry');
|
|
2773
|
+
return { scopeId: input.scope_id, ...breaker.reset(input.scope_id) };
|
|
2774
|
+
}
|
|
2775
|
+
|
|
1461
2776
|
async function cancel(input) {
|
|
1462
2777
|
const entry = registry.get(input.invocation_id);
|
|
1463
2778
|
if (!entry) throw new Error('Unknown invocation_id');
|
|
@@ -1487,13 +2802,15 @@ export async function registerWorkerInvokeToolset(server, options = {}) {
|
|
|
1487
2802
|
|
|
1488
2803
|
server.registerTool('worker-invoke', {
|
|
1489
2804
|
title: 'Invoke or Observe a Direct-Handoff Worker',
|
|
1490
|
-
description:
|
|
2805
|
+
description: "Run, check, cancel, or reconcile stop evidence for a qualified worker without any Mission/MC session — same policy/brief/claims path as the shell direct-handoff tier. Degraded isolation is a server-side operator setting only; there is no client-facing switch. Cancel delivery is not stop confirmation. 'status' with no invocation_id lists invocations still awaiting stop-evidence reconciliation; 'reconcile_stopped' applies that reconciliation for one invocation_id once the operator has independently established its worker actually stopped. 'status' also lists interventionScopes with their reason and reset instruction; 'reset_recovery_scope' requires scope_id and resets only that owning scope after all active or unresolved executions stop, preserving cumulative allowance.",
|
|
1491
2806
|
inputSchema,
|
|
1492
2807
|
annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
|
|
1493
2808
|
}, async input => {
|
|
1494
2809
|
try {
|
|
1495
2810
|
if (input.operation === 'run') return mcpJson(publicStatus(await run(input), healthHook));
|
|
1496
2811
|
if (input.operation === 'status') return mcpJson(await status(input));
|
|
2812
|
+
if (input.operation === 'reset_recovery_scope') return mcpJson(await resetRecoveryScope(input));
|
|
2813
|
+
if (input.operation === 'reconcile_stopped') return mcpJson(await reconcileStopped(input));
|
|
1497
2814
|
return mcpJson(await cancel(input));
|
|
1498
2815
|
} catch (error) {
|
|
1499
2816
|
return mcpError(`worker-invoke: ${error.message}`);
|