@aiwg/cli 2026.10.3 → 2026.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/agentic/code/providers/model-capabilities.v1.json +96 -4
- package/bin/aiwg.mjs +50 -27
- package/dist/src/agents/grounding/knowledge-base.js +54 -6
- package/dist/src/artifacts/browser-export.js +6 -0
- package/dist/src/artifacts/checksum-manifest.js +1 -0
- package/dist/src/artifacts/citation-parser.js +173 -92
- package/dist/src/artifacts/citation-table-schema.js +59 -0
- package/dist/src/artifacts/cli.js +57 -18
- package/dist/src/artifacts/corpus-tools/snapshot.js +1 -1
- package/dist/src/artifacts/corpus-tools/vision-extract.js +2 -0
- package/dist/src/artifacts/corpus-views/ref-parser.js +3 -1
- package/dist/src/artifacts/corpus-views/renderers.js +3 -3
- package/dist/src/artifacts/discovery-eval.js +6 -4
- package/dist/src/artifacts/fortemi-core-query-adapter.js +36 -3
- package/dist/src/artifacts/fortemi-core-sync.js +166 -40
- package/dist/src/artifacts/framework-capabilities.js +171 -0
- package/dist/src/artifacts/framework-index-paths.js +73 -0
- package/dist/src/artifacts/framework-source-digest.js +75 -0
- package/dist/src/artifacts/index-builder.js +285 -131
- package/dist/src/artifacts/index-files.js +132 -18
- package/dist/src/artifacts/index-preparation.js +65 -0
- package/dist/src/artifacts/index-refresh.js +576 -0
- package/dist/src/artifacts/index-worker.mjs +9 -0
- package/dist/src/artifacts/prebuilt-build-lock.js +47 -4
- package/dist/src/artifacts/prebuilt-deployment.js +46 -0
- package/dist/src/artifacts/prebuilt-relations.js +80 -0
- package/dist/src/artifacts/query-engine.js +410 -88
- package/dist/src/artifacts/relations-builder.js +168 -0
- package/dist/src/artifacts/relations.js +148 -0
- package/dist/src/artifacts/stats.js +5 -2
- package/dist/src/artifacts/types.js +30 -10
- package/dist/src/artifacts/watcher.js +1 -1
- package/dist/src/auth/credential-store.js +2 -1
- package/dist/src/catalog/cli.d.mts +5 -0
- package/dist/src/channel/manager.d.mts +20 -0
- package/dist/src/channel/manager.mjs +19 -2
- package/dist/src/cli/command-log.js +2 -3
- package/dist/src/cli/handlers/artifacts.js +3 -0
- package/dist/src/cli/handlers/auth.js +3 -0
- package/dist/src/cli/handlers/decision.js +19 -12
- package/dist/src/cli/handlers/diagnose.js +4 -3
- package/dist/src/cli/handlers/effect.js +4 -2
- package/dist/src/cli/handlers/evidence.js +3 -0
- package/dist/src/cli/handlers/help.js +1 -0
- package/dist/src/cli/handlers/index.js +3 -1
- package/dist/src/cli/handlers/init.js +4 -1
- package/dist/src/cli/handlers/lint.js +5 -0
- package/dist/src/cli/handlers/models.js +104 -8
- package/dist/src/cli/handlers/ralph-launcher.js +2 -0
- package/dist/src/cli/handlers/refresh.js +12 -0
- package/dist/src/cli/handlers/resource-versions.js +3 -0
- package/dist/src/cli/handlers/rules.js +195 -0
- package/dist/src/cli/handlers/run.js +4 -0
- package/dist/src/cli/handlers/runtime-info.js +1 -2
- package/dist/src/cli/handlers/script-runner.js +3 -2
- package/dist/src/cli/handlers/session.js +19 -9
- package/dist/src/cli/handlers/sessions.js +98 -37
- package/dist/src/cli/handlers/setup-manifest.js +39 -44
- package/dist/src/cli/handlers/setup.js +41 -0
- package/dist/src/cli/handlers/steward.js +11 -0
- package/dist/src/cli/handlers/subcommands.js +57 -2
- package/dist/src/cli/handlers/uhp.js +3 -0
- package/dist/src/cli/handlers/use.js +294 -55
- package/dist/src/cli/handlers/utilities.js +26 -10
- package/dist/src/cli/handlers/version.js +8 -0
- package/dist/src/cli/router.js +15 -1
- package/dist/src/cli/services/deployment-verification.js +19 -9
- package/dist/src/cli/services/provider-prepare-coordinator.js +24 -0
- package/dist/src/cli/skill-usage.js +2 -3
- package/dist/src/config/aiwg-config.js +87 -37
- package/dist/src/config/cli.js +111 -5
- package/dist/src/config/decision-config.js +106 -0
- package/dist/src/config/dispatch-thresholds-runtime.d.mts +8 -0
- package/dist/src/config/gitignore.js +50 -0
- package/dist/src/config/index-graph-overrides.js +36 -0
- package/dist/src/config/project-artifacts-health.d.mts +37 -0
- package/dist/src/config/project-artifacts-runtime.d.mts +16 -0
- package/dist/src/config/user-config-dir.d.mts +9 -0
- package/dist/src/config/user-config.js +2 -1
- package/dist/src/config/user-registry.js +23 -15
- package/dist/src/config/user-state.js +7 -0
- package/dist/src/decision/ask.js +203 -43
- package/dist/src/decision/context-live-qualification.js +2 -2
- package/dist/src/decision/driver.js +13 -8
- package/dist/src/decision/ensemble/aggregate.js +23 -10
- package/dist/src/decision/ensemble-study/multifact-study.js +6 -2
- package/dist/src/decision/entry.js +2 -15
- package/dist/src/decision/graph-live-qualification.js +1 -1
- package/dist/src/decision/graph-run.js +9 -0
- package/dist/src/decision/heldout/collector.js +5 -13
- package/dist/src/decision/heldout/contract.js +30 -7
- package/dist/src/decision/jev-credentials.js +15 -1
- package/dist/src/decision/ledger.js +131 -28
- package/dist/src/decision/offload-activation.js +137 -0
- package/dist/src/decision/offload.js +550 -0
- package/dist/src/decision/qualification/load.js +4 -6
- package/dist/src/decision/savings.js +28 -0
- package/dist/src/effects/types.js +5 -1
- package/dist/src/effects/verifiers/git.js +2 -1
- package/dist/src/effects/verifiers/tracker.js +26 -14
- package/dist/src/extensions/commands/definitions.js +19 -0
- package/dist/src/extensions/project-local-remove.js +36 -1
- package/dist/src/gates/providers/loader.js +61 -17
- package/dist/src/installation/manager-command.d.mts +18 -0
- package/dist/src/installation/manager-command.mjs +14 -2
- package/dist/src/installation/manager.d.mts +49 -0
- package/dist/src/installation/manager.mjs +78 -3
- package/dist/src/issues/cli.js +4 -4
- package/dist/src/issues/local.js +24 -5
- package/dist/src/jobs/runner.js +1 -1
- package/dist/src/lint/cli.js +27 -1
- package/dist/src/lint/loader.js +2 -1
- package/dist/src/lint/reporters.js +25 -1
- package/dist/src/lint/rule-activation.js +43 -0
- package/dist/src/lint/runner.js +137 -62
- package/dist/src/mcp/adapters/codex-runtime.js +7 -1
- package/dist/src/mcp/claude-install.d.mts +4 -0
- package/dist/src/mcp/claude-install.mjs +41 -0
- package/dist/src/mcp/cli.d.mts +1 -0
- package/dist/src/mcp/cli.mjs +36 -11
- package/dist/src/mcp/config-file.d.mts +28 -0
- package/dist/src/mcp/config-layers.d.mts +36 -0
- package/dist/src/mcp/credentials.d.mts +51 -0
- package/dist/src/mcp/grok-build-config.d.mts +6 -0
- package/dist/src/mcp/grok-build-config.mjs +1 -1
- package/dist/src/mcp/helpers.mjs +3 -2
- package/dist/src/mcp/muse-native-config.d.mts +13 -0
- package/dist/src/mcp/muse-native-config.mjs +1 -0
- package/dist/src/mcp/omp-config.d.mts +4 -0
- package/dist/src/mcp/registry.js +3 -93
- package/dist/src/mcp/registry.mjs +3 -87
- package/dist/src/mcp/server-config.d.mts +3 -0
- package/dist/src/mcp/server-config.mjs +89 -0
- package/dist/src/mcp/server.mjs +37 -375
- package/dist/src/mcp/test-simple.mjs +2 -1
- package/dist/src/mcp/toml-editor.d.mts +4 -0
- package/dist/src/mcp/toml-string.d.mts +1 -0
- package/dist/src/mcp/toml-strip-mcp.d.mts +1 -0
- package/dist/src/mcp/tool-filters.d.mts +46 -0
- package/dist/src/mcp/tools/decision.mjs +12 -11
- package/dist/src/mcp/tools/discovery.mjs +62 -55
- package/dist/src/mcp/tools/orchestration.mjs +12 -2
- package/dist/src/mcp/tools/worker-invoke.mjs +1340 -23
- package/dist/src/mcp/tools/worker-shell-control.mjs +52 -21
- package/dist/src/mcp/tools/worker-stdin-runtime.mjs +3 -3
- package/dist/src/mission-protocol/codecs.js +8 -0
- package/dist/src/models/config-loader.js +7 -2
- package/dist/src/models/model-capabilities.v1.json +96 -4
- package/dist/src/models/model-discovery.js +76 -11
- package/dist/src/models/provider-policy.js +101 -7
- package/dist/src/models/qualification-digest.d.mts +2 -0
- package/dist/src/models/resolver.js +12 -10
- package/dist/src/models/worker-direct-readiness.js +65 -44
- package/dist/src/models/worker-dispatch-offer.js +3 -3
- package/dist/src/models/worker-health-readiness.js +16 -3
- package/dist/src/models/worker-installation.d.mts +39 -0
- package/dist/src/models/worker-installation.mjs +135 -28
- package/dist/src/process/bounded-child-process.d.mts +84 -0
- package/dist/src/process/bounded-child-process.mjs +43 -0
- package/dist/src/process/cli-command.d.mts +16 -0
- package/dist/src/process/signal-child.d.mts +6 -0
- package/dist/src/process/signal-child.mjs +7 -0
- package/dist/src/providers/deployed-skill-roots.js +20 -0
- package/dist/src/providers/grok-build-acp.js +3 -2
- package/dist/src/providers/grok-build-headless.js +12 -4
- package/dist/src/providers/grok-build-inspect.js +1 -0
- package/dist/src/providers/omp-agent.d.mts +8 -0
- package/dist/src/providers/omp-diagnostics.d.mts +1 -0
- package/dist/src/providers/omp-paths.d.mts +13 -0
- package/dist/src/providers/skill-ownership.d.mts +9 -0
- package/dist/src/providers/skill-ownership.mjs +88 -0
- package/dist/src/rules/activation.d.mts +21 -0
- package/dist/src/rules/activation.mjs +62 -0
- package/dist/src/rules/inline-budget.d.mts +26 -0
- package/dist/src/rules/inline-budget.mjs +66 -0
- package/dist/src/rules/inventory.js +129 -0
- package/dist/src/rules/pin-state.js +73 -0
- package/dist/src/rules/pins.d.mts +14 -0
- package/dist/src/rules/pins.mjs +112 -0
- package/dist/src/rules/rule-frontmatter.schema.json +197 -0
- package/dist/src/serve/mission-conductor.js +38 -12
- package/dist/src/serve/worker-invocation.js +30 -10
- package/dist/src/serve/worker-stdin-frame.d.mts +10 -0
- package/dist/src/sessions/adapter-registry.js +23 -19
- package/dist/src/sessions/adapters/claude.js +39 -54
- package/dist/src/sessions/adapters/codex.js +82 -67
- package/dist/src/sessions/adapters/copilot.js +4 -4
- package/dist/src/sessions/adapters/cursor.js +40 -57
- package/dist/src/sessions/adapters/deepseek-harness.js +15 -12
- package/dist/src/sessions/adapters/factory.js +31 -27
- package/dist/src/sessions/adapters/generic.js +10 -10
- package/dist/src/sessions/adapters/grok-build.js +11 -11
- package/dist/src/sessions/adapters/grokbot.js +24 -13
- package/dist/src/sessions/adapters/hermes.js +27 -29
- package/dist/src/sessions/adapters/muse-workspace-scan.js +31 -11
- package/dist/src/sessions/adapters/muse.js +150 -93
- package/dist/src/sessions/adapters/omp.js +28 -14
- package/dist/src/sessions/adapters/openclaw.js +89 -79
- package/dist/src/sessions/adapters/opencode.js +27 -24
- package/dist/src/sessions/adapters/openhuman.js +18 -60
- package/dist/src/sessions/adapters/pi.js +29 -14
- package/dist/src/sessions/adapters/warp.js +3 -3
- package/dist/src/sessions/adapters/windsurf.js +19 -48
- package/dist/src/sessions/batch-contracts.js +16 -8
- package/dist/src/sessions/batch-import.js +51 -11
- package/dist/src/sessions/contracts.js +40 -2
- package/dist/src/sessions/coverage-preflight.js +1 -1
- package/dist/src/sessions/export-plan.js +4 -3
- package/dist/src/sessions/import-exclusions.js +72 -0
- package/dist/src/sessions/import-lease.js +34 -18
- package/dist/src/sessions/import-limits.js +37 -0
- package/dist/src/sessions/importer.js +60 -38
- package/dist/src/sessions/lifecycle.js +45 -0
- package/dist/src/sessions/policy.js +133 -25
- package/dist/src/sessions/readers.js +106 -53
- package/dist/src/sessions/refresh.js +106 -0
- package/dist/src/sessions/repository.js +147 -52
- package/dist/src/sessions/workspace-discovery.js +221 -56
- package/dist/src/sessions/workspace-matching.js +107 -0
- package/dist/src/skills/importer.js +1 -0
- package/dist/src/smiths/context-pipeline/aiwg-md.js +11 -0
- package/dist/src/smiths/context-pipeline/claude-hook.js +3 -26
- package/dist/src/smiths/context-pipeline/finalization.js +6 -5
- package/dist/src/smiths/context-pipeline/generator.js +21 -4
- package/dist/src/smiths/context-pipeline/parallelism-section.js +7 -15
- package/dist/src/smiths/mcpsmith/generator.js +1 -1
- package/dist/src/smiths/toolsmith/runtime-discovery.d.mts +10 -0
- package/dist/src/smiths/toolsmith/runtime-discovery.mjs +76 -18
- package/dist/src/storage/backends/fs.js +18 -76
- package/dist/src/storage/backends/obsidian.js +1 -0
- package/dist/src/tracker/forge-http.js +2 -1
- package/dist/src/update/checker.d.mts +2 -0
- package/dist/src/update/service.d.mts +22 -0
- package/dist/src/utils/contained-filesystem.d.mts +37 -0
- package/dist/src/utils/contained-filesystem.mjs +683 -0
- package/dist/src/utils/temporary-directory.js +36 -0
- package/dist/src/writing/example-generator.js +1 -1
- package/package.json +15 -2
- package/schemas/models/model-catalog.v1.schema.json +1 -1
- package/schemas/models/provider-model-capabilities.v1.schema.json +6 -0
- package/tools/_resolve-impl.mjs +13 -15
- package/tools/agents/deploy-agents.mjs +49 -21
- package/tools/agents/providers/base.mjs +104 -134
- package/tools/agents/providers/claude.mjs +8 -24
- package/tools/agents/providers/codex.mjs +18 -11
- package/tools/agents/providers/cursor.mjs +19 -18
- package/tools/agents/providers/grok-build.mjs +3 -1
- package/tools/agents/providers/hermes.mjs +57 -47
- package/tools/agents/providers/openclaw.mjs +9 -4
- package/tools/agents/providers/pi.mjs +5 -0
- package/tools/agents/providers/provider-prepare-channel.mjs +5 -0
- package/tools/agents/providers/provider-prepare-coordinator.mjs +247 -0
- package/tools/agents/providers/provider-prepare-worker.mjs +84 -0
- package/tools/agents/providers/rule-delivery.mjs +1270 -0
- package/tools/agents/providers/warp.mjs +20 -26
- package/tools/cli/decision-offload-report.mjs +40 -0
- package/tools/cli/workspace-status.mjs +862 -0
- package/tools/plugin/package-plugins.mjs +186 -8
- package/tools/plugin/plugin-links.mjs +114 -0
- package/tools/providers/antigravity-transport.mjs +3 -1
- package/tools/providers/deepseek-harness-transport.mjs +6 -4
- package/tools/providers/omp-transport.mjs +4 -2
- package/tools/ralph-external/lib/consecutive-failure-circuit-breaker.mjs +1748 -0
- package/tools/ralph-external/lib/launcher-recovery-evidence.mjs +55 -0
- package/tools/ralph-external/lib/omp-adapter.mjs +1 -1
- package/tools/ralph-external/lib/pi-adapter.mjs +0 -4
- package/tools/ralph-external/lib/provider-adapter.mjs +59 -1
- package/tools/ralph-external/lib/recovery-probe.mjs +72 -0
- package/tools/ralph-external/lib/session-environment.mjs +28 -0
- package/tools/ralph-external/lib/systemd-client.mjs +49 -0
- package/tools/ralph-external/lib/worker-health.mjs +1 -1
- package/tools/ralph-external/lib/worker-isolation.mjs +717 -0
- package/tools/ralph-external/lib/worker-overflow.mjs +50 -0
- package/tools/ralph-external/process-monitor.mjs +1017 -0
- package/tools/ralph-external/session-launcher.mjs +5057 -0
- package/tools/scripts/resource-preflight.mjs +541 -0
- package/tools/skills/deploy-skills-codex.mjs +2 -1
- package/vscode-extension/schemas/aiwg.config.v1.json +1194 -231
|
@@ -0,0 +1,1748 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ConsecutiveFailureCircuitBreaker - Three-state circuit breaker
|
|
3
|
+
*
|
|
4
|
+
* Implements the standard circuit breaker pattern:
|
|
5
|
+
* CLOSED (normal) -> N consecutive failures -> OPEN (blocking)
|
|
6
|
+
* OPEN -> cooldown expires -> HALF_OPEN (probe)
|
|
7
|
+
* HALF_OPEN -> success -> CLOSED
|
|
8
|
+
* HALF_OPEN -> failure -> OPEN (reset cooldown)
|
|
9
|
+
*
|
|
10
|
+
* @implements #514
|
|
11
|
+
* @tests @test/unit/circuit-breaker.test.mjs
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { EventEmitter } from 'node:events';
|
|
15
|
+
|
|
16
|
+
export const CircuitState = Object.freeze({
|
|
17
|
+
CLOSED: 'closed',
|
|
18
|
+
OPEN: 'open',
|
|
19
|
+
HALF_OPEN: 'half-open',
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
export class ConsecutiveFailureCircuitBreaker extends EventEmitter {
|
|
23
|
+
/**
|
|
24
|
+
* @param {object} options
|
|
25
|
+
* @param {number} [options.failureThreshold=5] Consecutive failures to trip open
|
|
26
|
+
* @param {number} [options.cooldownMs=120000] Time in OPEN before transitioning to HALF_OPEN
|
|
27
|
+
* @param {number} [options.halfOpenMaxProbes=1] Max concurrent probes in HALF_OPEN
|
|
28
|
+
* @param {function} [options.nowMs] Clock source (defaults to Date.now). The
|
|
29
|
+
* scoped breaker always injects its own clock here so both layers share
|
|
30
|
+
* one time source (#2915: a separate Date.now() desynchronizes cooldowns
|
|
31
|
+
* under synthetic clocks).
|
|
32
|
+
*/
|
|
33
|
+
constructor(options = {}) {
|
|
34
|
+
super();
|
|
35
|
+
this.failureThreshold = options.failureThreshold ?? 5;
|
|
36
|
+
this.cooldownMs = options.cooldownMs ?? 120_000;
|
|
37
|
+
this.halfOpenMaxProbes = options.halfOpenMaxProbes ?? 1;
|
|
38
|
+
this.nowMs = options.nowMs ?? (() => Date.now());
|
|
39
|
+
|
|
40
|
+
this._state = CircuitState.CLOSED;
|
|
41
|
+
this._consecutiveFailures = 0;
|
|
42
|
+
this._lastFailureAt = null;
|
|
43
|
+
this._openedAt = null;
|
|
44
|
+
this._cooldownTimer = null;
|
|
45
|
+
this._activeProbes = 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Check whether execution is allowed in the current state.
|
|
50
|
+
* In HALF_OPEN, limits concurrent probes.
|
|
51
|
+
*/
|
|
52
|
+
canExecute() {
|
|
53
|
+
switch (this._state) {
|
|
54
|
+
case CircuitState.CLOSED:
|
|
55
|
+
return true;
|
|
56
|
+
|
|
57
|
+
case CircuitState.OPEN: {
|
|
58
|
+
// Check if cooldown has elapsed (in case timer hasn't fired yet).
|
|
59
|
+
// `!== null`, not a truthy check (#2915 final review latent-bug
|
|
60
|
+
// fix): `_openedAt` is a `nowMs()` timestamp, and a synthetic clock
|
|
61
|
+
// can legitimately open a scope at time 0, which `0 && ...` would
|
|
62
|
+
// treat as "never opened" and refuse to ever leave OPEN.
|
|
63
|
+
if (this._openedAt !== null && this.nowMs() - this._openedAt >= this.cooldownMs) {
|
|
64
|
+
this._transitionTo(CircuitState.HALF_OPEN, 'cooldown elapsed on check');
|
|
65
|
+
return this._activeProbes < this.halfOpenMaxProbes;
|
|
66
|
+
}
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
case CircuitState.HALF_OPEN:
|
|
71
|
+
return this._activeProbes < this.halfOpenMaxProbes;
|
|
72
|
+
|
|
73
|
+
default:
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Record a successful execution. Resets failure counter.
|
|
80
|
+
* In HALF_OPEN, transitions back to CLOSED.
|
|
81
|
+
*/
|
|
82
|
+
recordSuccess() {
|
|
83
|
+
this._consecutiveFailures = 0;
|
|
84
|
+
this._activeProbes = Math.max(0, this._activeProbes - 1);
|
|
85
|
+
|
|
86
|
+
if (this._state === CircuitState.HALF_OPEN) {
|
|
87
|
+
this._transitionTo(CircuitState.CLOSED, 'probe succeeded');
|
|
88
|
+
this.emit('recover');
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Record a failed execution. Increments failure counter.
|
|
94
|
+
* May trip the breaker open.
|
|
95
|
+
*/
|
|
96
|
+
recordFailure() {
|
|
97
|
+
this._consecutiveFailures++;
|
|
98
|
+
this._lastFailureAt = this.nowMs();
|
|
99
|
+
this._activeProbes = Math.max(0, this._activeProbes - 1);
|
|
100
|
+
|
|
101
|
+
if (this._state === CircuitState.HALF_OPEN) {
|
|
102
|
+
// Probe failed — back to OPEN with fresh cooldown
|
|
103
|
+
this._transitionTo(CircuitState.OPEN, 'probe failed');
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (this._state === CircuitState.CLOSED && this._consecutiveFailures >= this.failureThreshold) {
|
|
108
|
+
this._transitionTo(CircuitState.OPEN, `${this._consecutiveFailures} consecutive failures`);
|
|
109
|
+
this.emit('trip', { consecutiveFailures: this._consecutiveFailures });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Get current breaker state snapshot.
|
|
115
|
+
*/
|
|
116
|
+
getState() {
|
|
117
|
+
const now = this.nowMs();
|
|
118
|
+
let cooldownRemainingMs = 0;
|
|
119
|
+
|
|
120
|
+
if (this._state === CircuitState.OPEN && this._openedAt !== null) {
|
|
121
|
+
cooldownRemainingMs = Math.max(0, this.cooldownMs - (now - this._openedAt));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return {
|
|
125
|
+
state: this._state,
|
|
126
|
+
consecutiveFailures: this._consecutiveFailures,
|
|
127
|
+
lastFailureAt: this._lastFailureAt ? new Date(this._lastFailureAt).toISOString() : null,
|
|
128
|
+
cooldownRemainingMs,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Force-reset the breaker to CLOSED state.
|
|
134
|
+
*/
|
|
135
|
+
reset() {
|
|
136
|
+
this._clearCooldownTimer();
|
|
137
|
+
this._consecutiveFailures = 0;
|
|
138
|
+
this._lastFailureAt = null;
|
|
139
|
+
this._openedAt = null;
|
|
140
|
+
this._activeProbes = 0;
|
|
141
|
+
const from = this._state;
|
|
142
|
+
this._state = CircuitState.CLOSED;
|
|
143
|
+
if (from !== CircuitState.CLOSED) {
|
|
144
|
+
this.emit('state:change', { from, to: CircuitState.CLOSED, reason: 'manual reset' });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Clean up timers. Call when done with the breaker.
|
|
150
|
+
*/
|
|
151
|
+
destroy() {
|
|
152
|
+
this._clearCooldownTimer();
|
|
153
|
+
this.removeAllListeners();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// --- Private ---
|
|
157
|
+
|
|
158
|
+
_transitionTo(newState, reason) {
|
|
159
|
+
const from = this._state;
|
|
160
|
+
if (from === newState) return;
|
|
161
|
+
|
|
162
|
+
this._state = newState;
|
|
163
|
+
this.emit('state:change', { from, to: newState, reason });
|
|
164
|
+
|
|
165
|
+
if (newState === CircuitState.OPEN) {
|
|
166
|
+
this._openedAt = this.nowMs();
|
|
167
|
+
this._startCooldownTimer();
|
|
168
|
+
} else {
|
|
169
|
+
this._clearCooldownTimer();
|
|
170
|
+
this._openedAt = null;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (newState === CircuitState.HALF_OPEN) {
|
|
174
|
+
this._activeProbes = 0;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
_startCooldownTimer() {
|
|
179
|
+
this._clearCooldownTimer();
|
|
180
|
+
this._cooldownTimer = setTimeout(() => {
|
|
181
|
+
if (this._state === CircuitState.OPEN) {
|
|
182
|
+
this._transitionTo(CircuitState.HALF_OPEN, 'cooldown expired');
|
|
183
|
+
}
|
|
184
|
+
}, this.cooldownMs);
|
|
185
|
+
// Allow process to exit even if timer is pending
|
|
186
|
+
if (this._cooldownTimer.unref) {
|
|
187
|
+
this._cooldownTimer.unref();
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
_clearCooldownTimer() {
|
|
192
|
+
if (this._cooldownTimer) {
|
|
193
|
+
clearTimeout(this._cooldownTimer);
|
|
194
|
+
this._cooldownTimer = null;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Scoped recovery policy (03 §11.5). Every bound is an explicit number or a
|
|
201
|
+
* resolved config override — no unbounded retry, every loop has a max.
|
|
202
|
+
*
|
|
203
|
+
* - distinctThreshold/windowMs: open a scope at three distinct failures in
|
|
204
|
+
* 120s, or when every member of a frozen cohort of at least two
|
|
205
|
+
* outstanding attempts fails in that window.
|
|
206
|
+
* - cooldownMs/maxCooldownMs: earliest admission is max(120s cooldown,
|
|
207
|
+
* known 429 reset/Retry-After); a failed canary doubles the cooldown,
|
|
208
|
+
* capped at 900s. Malformed resets fall back to this backoff with
|
|
209
|
+
* bounded jitter.
|
|
210
|
+
* - probeTimeoutMs/probeMaxOutputBytes: a separate bounded relevant probe
|
|
211
|
+
* precedes the one real task canary. An echo cannot recover an upstream
|
|
212
|
+
* scope. Both dispatches consume the caller's existing allowance.
|
|
213
|
+
* - canaryTimeoutMs: bounds the single real task after a successful probe.
|
|
214
|
+
* - maxTrackedScopes/maxChargedAttempts: explicit caps on retained scope
|
|
215
|
+
* and dedup-set state, with LRU eviction of inert entries, so a
|
|
216
|
+
* long-running process's bookkeeping never grows without bound (#2915
|
|
217
|
+
* review: settled outstanding/scope maps never dropped their keys). A
|
|
218
|
+
* hard cap refuses a brand-new scope record once at maxTrackedScopes with
|
|
219
|
+
* nothing evictable, rather than silently growing past it or evicting an
|
|
220
|
+
* OPEN/outstanding scope (#2915 final review, finding 6).
|
|
221
|
+
* - maxOutstandingReservations: hard cap on live (unsettled) admission
|
|
222
|
+
* reservations, including canary reservations and uncertain blocking
|
|
223
|
+
* markers; a reservation past its own deadline is swept (treated as
|
|
224
|
+
* uncertain) rather than retained forever.
|
|
225
|
+
* - maxCoveringKeys: hard cap on distinct scope keys (including the
|
|
226
|
+
* invocation key) bound to one reservation; the invocation key is never
|
|
227
|
+
* dropped when trimming.
|
|
228
|
+
*
|
|
229
|
+
* @implements #2915
|
|
230
|
+
*/
|
|
231
|
+
export const RecoveryPolicy = Object.freeze({
|
|
232
|
+
distinctThreshold: 3,
|
|
233
|
+
windowMs: 120_000,
|
|
234
|
+
minOutstanding: 2,
|
|
235
|
+
cooldownMs: 120_000,
|
|
236
|
+
maxCooldownMs: 900_000,
|
|
237
|
+
probeTimeoutMs: 10_000,
|
|
238
|
+
probeMaxOutputBytes: 65_536,
|
|
239
|
+
canaryTimeoutMs: 300_000,
|
|
240
|
+
backoffMultiplier: 2,
|
|
241
|
+
maxJitterMs: 1_000,
|
|
242
|
+
maxRetryAfterSeconds: 86_400,
|
|
243
|
+
maxTrackedScopes: 500,
|
|
244
|
+
maxChargedAttempts: 5_000,
|
|
245
|
+
maxOutstandingReservations: 128,
|
|
246
|
+
maxCoveringKeys: 6,
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Validate numeric policy overrides. Every bound must be an explicit finite
|
|
251
|
+
* number in range — unbounded or NaN overrides are refused rather than
|
|
252
|
+
* silently widening a loop (#2915 review: policy overrides lacked validation).
|
|
253
|
+
*/
|
|
254
|
+
export function validateRecoveryPolicy(policy) {
|
|
255
|
+
const fail = (name, why) => { throw new RangeError(`Invalid recovery policy ${name}: ${why}`); };
|
|
256
|
+
const positiveInt = (name, value, min = 1, max = Infinity) => {
|
|
257
|
+
if (!Number.isInteger(value) || value < min || value > max) {
|
|
258
|
+
fail(name, `expected integer in [${min}, ${max === Infinity ? '∞' : max}], got ${value}`);
|
|
259
|
+
}
|
|
260
|
+
};
|
|
261
|
+
const positiveMs = (name, value) => {
|
|
262
|
+
if (!Number.isFinite(value) || value <= 0) fail(name, `expected finite ms > 0, got ${value}`);
|
|
263
|
+
};
|
|
264
|
+
positiveInt('distinctThreshold', policy.distinctThreshold);
|
|
265
|
+
positiveMs('windowMs', policy.windowMs);
|
|
266
|
+
positiveInt('minOutstanding', policy.minOutstanding);
|
|
267
|
+
positiveMs('cooldownMs', policy.cooldownMs);
|
|
268
|
+
positiveMs('maxCooldownMs', policy.maxCooldownMs);
|
|
269
|
+
if (policy.maxCooldownMs < policy.cooldownMs) fail('maxCooldownMs', 'must be >= cooldownMs');
|
|
270
|
+
positiveMs('probeTimeoutMs', policy.probeTimeoutMs);
|
|
271
|
+
if (policy.probeTimeoutMs > RecoveryPolicy.probeTimeoutMs) fail('probeTimeoutMs', 'cannot exceed 10s');
|
|
272
|
+
positiveInt('probeMaxOutputBytes', policy.probeMaxOutputBytes, 1, RecoveryPolicy.probeMaxOutputBytes);
|
|
273
|
+
positiveMs('canaryTimeoutMs', policy.canaryTimeoutMs);
|
|
274
|
+
if (!Number.isFinite(policy.backoffMultiplier) || policy.backoffMultiplier < 1) {
|
|
275
|
+
fail('backoffMultiplier', `expected finite number >= 1, got ${policy.backoffMultiplier}`);
|
|
276
|
+
}
|
|
277
|
+
if (!Number.isFinite(policy.maxJitterMs) || policy.maxJitterMs < 0) fail('maxJitterMs', `expected finite ms >= 0, got ${policy.maxJitterMs}`);
|
|
278
|
+
if (!Number.isFinite(policy.maxRetryAfterSeconds) || policy.maxRetryAfterSeconds < 0) {
|
|
279
|
+
fail('maxRetryAfterSeconds', `expected finite seconds >= 0, got ${policy.maxRetryAfterSeconds}`);
|
|
280
|
+
}
|
|
281
|
+
// #2915 review 4 (finding 6): these four are PROTECTIVE maxima, not mere
|
|
282
|
+
// floors — an override above the shipped default would widen the exact
|
|
283
|
+
// unbounded-growth loops this policy exists to close. Upper bound equals
|
|
284
|
+
// the shipped `RecoveryPolicy` default for each (500/128/5000/6); a
|
|
285
|
+
// caller that genuinely needs a smaller budget may still lower it.
|
|
286
|
+
positiveInt('maxTrackedScopes', policy.maxTrackedScopes, 2, RecoveryPolicy.maxTrackedScopes);
|
|
287
|
+
positiveInt('maxChargedAttempts', policy.maxChargedAttempts, 2, RecoveryPolicy.maxChargedAttempts);
|
|
288
|
+
positiveInt('maxOutstandingReservations', policy.maxOutstandingReservations, 1, RecoveryPolicy.maxOutstandingReservations);
|
|
289
|
+
positiveInt('maxCoveringKeys', policy.maxCoveringKeys, 1, RecoveryPolicy.maxCoveringKeys);
|
|
290
|
+
return policy;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** Failure scopes, narrowest first. A failure opens only its own scope. */
|
|
294
|
+
export const FailureScope = Object.freeze({
|
|
295
|
+
INVOCATION: 'invocation',
|
|
296
|
+
MODEL: 'model',
|
|
297
|
+
INSTALLATION: 'installation',
|
|
298
|
+
QUOTA: 'quota',
|
|
299
|
+
PROVIDER: 'provider',
|
|
300
|
+
HOST: 'host',
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
// Reason codes produced by classifyWorkerHealthFailure (lib/worker-health.mjs).
|
|
304
|
+
// Anything unlisted — task-local errors, unknown signatures — stays in the
|
|
305
|
+
// invocation scope so independent task failures never open a wider scope.
|
|
306
|
+
const QUOTA_CODES = new Set(['RATE_LIMIT']);
|
|
307
|
+
const HOST_CODES = new Set(['RESOURCE', 'NOT_FOUND', 'SANDBOX']);
|
|
308
|
+
const PROVIDER_CODES = new Set(['NETWORK']);
|
|
309
|
+
const INSTALLATION_CODES = new Set(['AUTH']);
|
|
310
|
+
const MODEL_CODES = new Set(['MODEL']);
|
|
311
|
+
|
|
312
|
+
export const scopeKeyFor = Object.freeze({
|
|
313
|
+
invocation: (id) => `invocation:${id ?? 'unknown'}`,
|
|
314
|
+
model: (key) => `model:${key ?? 'unknown'}`,
|
|
315
|
+
// `qualifier`, when supplied, distinguishes separately-qualified
|
|
316
|
+
// installations that share a CLI name (#2915 review: two projects
|
|
317
|
+
// resolving different installations under `codex` shared one
|
|
318
|
+
// `installation:codex` scope). Omitted/null preserves the plain
|
|
319
|
+
// `installation:<cli>` key so existing single-installation callers and
|
|
320
|
+
// tests are unaffected.
|
|
321
|
+
installation: (cli, qualifier = null) => qualifier
|
|
322
|
+
? `installation:${cli ?? 'unknown'}:${qualifier}`
|
|
323
|
+
: `installation:${cli ?? 'unknown'}`,
|
|
324
|
+
quota: (account) => `quota:${account ?? 'unknown'}`,
|
|
325
|
+
provider: (name) => `provider:${name ?? 'unknown'}`,
|
|
326
|
+
host: (id) => `host:${id ?? 'local'}`,
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
/** Scope breadth rank, narrowest first. Used to fold over-broad claims back. */
|
|
330
|
+
const SCOPE_RANK = Object.freeze({
|
|
331
|
+
[FailureScope.INVOCATION]: 0,
|
|
332
|
+
[FailureScope.MODEL]: 1,
|
|
333
|
+
[FailureScope.INSTALLATION]: 2,
|
|
334
|
+
[FailureScope.QUOTA]: 3,
|
|
335
|
+
[FailureScope.PROVIDER]: 4,
|
|
336
|
+
[FailureScope.HOST]: 5,
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
export function scopeRank(scope) {
|
|
340
|
+
return SCOPE_RANK[scope] ?? 0;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Evidence sources. Only `launcher` (launcher-observed spawn/wait/exit facts
|
|
345
|
+
* from its own child and unit status) and `resolver` (the #2887 resolved
|
|
346
|
+
* invocation and installation qualification) are trusted to widen a scope.
|
|
347
|
+
* `worker` covers everything derived from worker stderr/stdout, structured
|
|
348
|
+
* worker output, or worker-writable files — including the qualified health
|
|
349
|
+
* classifier's reason codes, which describe worker-observed text, not
|
|
350
|
+
* launcher-verified facts. Worker-sourced failures NEVER widen: they stay in
|
|
351
|
+
* the invocation scope, so one misbehaving task retried under fresh ids can
|
|
352
|
+
* poison at most its own attempts, never a provider, host, quota or
|
|
353
|
+
* installation scope (#2915 review: scope poisoning).
|
|
354
|
+
*
|
|
355
|
+
* Identity separation: the inference backend (`inferenceProvider`, e.g.
|
|
356
|
+
* `openai`) and the CLI installation (`cli`, e.g. `codex`) are distinct
|
|
357
|
+
* trusted identities. Provider scope keys use the raw backend string — never
|
|
358
|
+
* normalized through the CLI/provider-definition registry, which maps
|
|
359
|
+
* backends onto CLI ids (`openai` -> `codex`) and merges the two identities.
|
|
360
|
+
*/
|
|
361
|
+
export const RecoveryEvidenceSource = Object.freeze({
|
|
362
|
+
LAUNCHER: 'launcher',
|
|
363
|
+
RESOLVER: 'resolver',
|
|
364
|
+
WORKER: 'worker',
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
const TRUSTED_SOURCES = new Set([RecoveryEvidenceSource.LAUNCHER, RecoveryEvidenceSource.RESOLVER]);
|
|
368
|
+
|
|
369
|
+
export function isTrustedRecoveryEvidence(source) {
|
|
370
|
+
return TRUSTED_SOURCES.has(source);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Attribute a failure to the narrowest scope its evidence supports. Trusted
|
|
375
|
+
* launcher/resolver evidence with a matching trusted identity may implicate a
|
|
376
|
+
* wider scope (still `suspected` until the distinct-invocation correlation
|
|
377
|
+
* rule confirms it). Anything else — worker-derived codes, missing source,
|
|
378
|
+
* or a wider claim without its required trusted identity — folds back to the
|
|
379
|
+
* invocation scope. In particular a single task's repeated local failures
|
|
380
|
+
* (unknown codes, task errors, worker-observed signatures) can never open a
|
|
381
|
+
* provider, host, quota, installation or model scope; a genuine outage
|
|
382
|
+
* promotes beyond invocation only when N distinct invocations within the
|
|
383
|
+
* window carry the same trusted evidence (the correlation rule in
|
|
384
|
+
* recordFailure).
|
|
385
|
+
*/
|
|
386
|
+
export function attributeFailureScope({
|
|
387
|
+
reasonCode = null,
|
|
388
|
+
evidenceSource = null,
|
|
389
|
+
invocationId = null,
|
|
390
|
+
modelKey = null,
|
|
391
|
+
cli = null,
|
|
392
|
+
inferenceProvider = null,
|
|
393
|
+
provider = null,
|
|
394
|
+
quotaAccount = null,
|
|
395
|
+
hostId = null,
|
|
396
|
+
corroboratedBackends = null,
|
|
397
|
+
installationQualifier = null,
|
|
398
|
+
} = {}) {
|
|
399
|
+
const invocation = { scope: FailureScope.INVOCATION, scopeKey: scopeKeyFor.invocation(invocationId), confidence: 'suspected' };
|
|
400
|
+
if (!isTrustedRecoveryEvidence(evidenceSource)) return invocation;
|
|
401
|
+
// Legacy alias: `provider` carried either identity through the old call
|
|
402
|
+
// sites. The backend wins for provider scope; the CLI never implies it.
|
|
403
|
+
const backend = inferenceProvider ?? null;
|
|
404
|
+
if (reasonCode === 'NOT_FOUND' || reasonCode === 'SPAWN_REFUSED') {
|
|
405
|
+
// Launcher-observed failure to execute the resolved CLI binary itself
|
|
406
|
+
// (spawn ENOENT/EACCES on argv[0], never a worker-observed message —
|
|
407
|
+
// worker-sourced NOT_FOUND returns in the invocation scope above). A
|
|
408
|
+
// missing task subprocess must never implicate the host.
|
|
409
|
+
if (cli == null || cli === '') return invocation;
|
|
410
|
+
return { scope: FailureScope.INSTALLATION, scopeKey: scopeKeyFor.installation(cli, installationQualifier), confidence: 'suspected' };
|
|
411
|
+
}
|
|
412
|
+
if (QUOTA_CODES.has(reasonCode)) {
|
|
413
|
+
if (quotaAccount == null || quotaAccount === '') return invocation;
|
|
414
|
+
return { scope: FailureScope.QUOTA, scopeKey: scopeKeyFor.quota(quotaAccount), confidence: 'suspected' };
|
|
415
|
+
}
|
|
416
|
+
if (HOST_CODES.has(reasonCode)) {
|
|
417
|
+
// Host scope pauses cross-provider retries: it requires corroboration
|
|
418
|
+
// across at least two distinct trusted backends in this evidence set.
|
|
419
|
+
// Single-provider host signatures stay invocation-scoped.
|
|
420
|
+
const backends = [...new Set((corroboratedBackends ?? []).filter((b) => b != null && b !== ''))];
|
|
421
|
+
if (backends.length < 2) return invocation;
|
|
422
|
+
return { scope: FailureScope.HOST, scopeKey: scopeKeyFor.host(hostId), confidence: 'suspected' };
|
|
423
|
+
}
|
|
424
|
+
if (PROVIDER_CODES.has(reasonCode)) {
|
|
425
|
+
if (backend == null || backend === '') return invocation;
|
|
426
|
+
return { scope: FailureScope.PROVIDER, scopeKey: scopeKeyFor.provider(backend), confidence: 'suspected' };
|
|
427
|
+
}
|
|
428
|
+
if (INSTALLATION_CODES.has(reasonCode)) {
|
|
429
|
+
if (cli == null || cli === '') return invocation;
|
|
430
|
+
return { scope: FailureScope.INSTALLATION, scopeKey: scopeKeyFor.installation(cli, installationQualifier), confidence: 'suspected' };
|
|
431
|
+
}
|
|
432
|
+
if (MODEL_CODES.has(reasonCode)) {
|
|
433
|
+
if (modelKey == null || modelKey === '') return invocation;
|
|
434
|
+
return { scope: FailureScope.MODEL, scopeKey: scopeKeyFor.model(modelKey), confidence: 'suspected' };
|
|
435
|
+
}
|
|
436
|
+
return invocation;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* ScopedFailureBreaker - correlated-failure scopes over the existing
|
|
441
|
+
* ConsecutiveFailureCircuitBreaker transitions (closed → open → half-open).
|
|
442
|
+
*
|
|
443
|
+
* Each scope owns one inner breaker (failureThreshold 1: the correlation
|
|
444
|
+
* rule below is the only trip authority), sharing this instance's injected
|
|
445
|
+
* `nowMs` clock. Probes drive the inner half-open transitions; an echo
|
|
446
|
+
* probe can never close an upstream outage — only a launcher-verified
|
|
447
|
+
* relevant probe success admits one canary, and only a launcher-verified
|
|
448
|
+
* canary success recovers the scope.
|
|
449
|
+
*
|
|
450
|
+
* All window/cooldown math uses the injectable `nowMs` clock, so tests run
|
|
451
|
+
* on synthetic time (vitest fake timers cover the default Date.now clock).
|
|
452
|
+
*
|
|
453
|
+
* Restart/state (#2915D): all breaker state lives in-memory in this
|
|
454
|
+
* instance — there is deliberately no persisted store (a durable shared
|
|
455
|
+
* store would be a new subsystem; managed-tier shared ownership stays
|
|
456
|
+
* deferred to Tier 1). A restarted conductor therefore comes up with every
|
|
457
|
+
* scope CLOSED (fail-open on restart). This is the smaller safe option
|
|
458
|
+
* because two couplings survive the restart: (1) the stop-before-retry gate
|
|
459
|
+
* still requires the launcher's own confirmed-termination evidence
|
|
460
|
+
* (exit.json `termination`) before any retry, so uncertain work is never
|
|
461
|
+
* replayed blindly; (2) fresh trusted evidence re-opens a scope through the
|
|
462
|
+
* same distinct-invocation rule, so a real outage re-pauses within N
|
|
463
|
+
* failures. No uncertain probe/canary reservation survives a restart —
|
|
464
|
+
* reservations are single-use and bound to this instance.
|
|
465
|
+
*
|
|
466
|
+
* Degraded-isolation blocking markers (#2915 review 6, finding 3e): degraded
|
|
467
|
+
* isolation has no native unit query, so it can NEVER supply
|
|
468
|
+
* confirmed-termination evidence for a canary's descendants, by design (see
|
|
469
|
+
* `captureOneWorker`'s evidence contract) — a degraded canary that expires
|
|
470
|
+
* unreported leaves a `blockingMarker` that no amount of waiting will ever
|
|
471
|
+
* clear on its own. This is an accepted, documented limitation, not a bug:
|
|
472
|
+
* automatically expiring the marker after some elapsed time would silently
|
|
473
|
+
* re-admit work over a descendant tree nobody ever confirmed stopped, which
|
|
474
|
+
* is exactly the hazard this marker exists to prevent. Do not invent one.
|
|
475
|
+
* The existing, operator-reachable recovery actions are: (1)
|
|
476
|
+
* `reconcileStopEvidence({ invocationId, confirmedStop: true })`, once the
|
|
477
|
+
* operator has independently established the descendants actually stopped
|
|
478
|
+
* (e.g. by inspecting the host directly); (2) the scoped `reset(scopeKey)`
|
|
479
|
+
* (also releases that scope's own reservations, including the marker's —
|
|
480
|
+
* see its doc comment), which discards this one scope's protection instead
|
|
481
|
+
* of every scope's; (3) `resetDirectRecoveryBreaker()` / full `reset()` /
|
|
482
|
+
* process restart, which discards everything. None of these are wired to a
|
|
483
|
+
* CLI command today; they are this module's exported programmatic surface.
|
|
484
|
+
*/
|
|
485
|
+
export class ScopedFailureBreaker extends EventEmitter {
|
|
486
|
+
/**
|
|
487
|
+
* @param {object} options
|
|
488
|
+
* @param {object} [options.policy] Numeric overrides for RecoveryPolicy.
|
|
489
|
+
* @param {function} [options.nowMs] Synthetic-clock source (defaults to Date.now).
|
|
490
|
+
* @param {function} [options.jitterMs] Bounded-jitter source (defaults to Math.random-based).
|
|
491
|
+
*/
|
|
492
|
+
constructor(options = {}) {
|
|
493
|
+
super();
|
|
494
|
+
this.policy = validateRecoveryPolicy({ ...RecoveryPolicy, ...(options.policy ?? {}) });
|
|
495
|
+
// `Date.now()` is the class-level default so every existing caller that
|
|
496
|
+
// constructs this directly (including this module's own extensive
|
|
497
|
+
// synthetic-clock test suite, much of which cross-checks breaker state
|
|
498
|
+
// against `Date.now()` directly) keeps its current, epoch-based
|
|
499
|
+
// semantics unchanged. #2915 round 8 review 8 (C3 follow-up, "production
|
|
500
|
+
// deadlines use Date.now(), not a monotonic clock"): the actual
|
|
501
|
+
// production singleton (`getDirectRecoveryBreaker`, below) explicitly
|
|
502
|
+
// overrides this with a `performance.now()`-based monotonic clock
|
|
503
|
+
// instead — every comparison this class makes is relative (one `nowMs()`
|
|
504
|
+
// reading against another this SAME instance produced), so swapping the
|
|
505
|
+
// clock source at that one call site is sufficient to make production
|
|
506
|
+
// deadlines immune to a wall-clock adjustment without disturbing this
|
|
507
|
+
// constructor's own default for every other (test) caller.
|
|
508
|
+
this.nowMs = options.nowMs ?? (() => Date.now());
|
|
509
|
+
this.jitterMs = options.jitterMs
|
|
510
|
+
?? (() => Math.floor(Math.random() * (this.policy.maxJitterMs + 1)));
|
|
511
|
+
this._scopes = new Map();
|
|
512
|
+
this._outstanding = new Map();
|
|
513
|
+
this._hostFailureFacts = [];
|
|
514
|
+
this._canarySeq = 0;
|
|
515
|
+
this._observedInvocations = new Set();
|
|
516
|
+
this.canariesObserved = 0;
|
|
517
|
+
// Reservation registry (#2915 final review): every admission mints an
|
|
518
|
+
// opaque, breaker-owned token here. A reservation is removed the moment
|
|
519
|
+
// it settles (`settle`) or expires past its own deadline (swept by
|
|
520
|
+
// `_sweepReservations`, called at the top of every `admit`/`settle`), so
|
|
521
|
+
// this map's steady-state size tracks live in-flight attempts, not
|
|
522
|
+
// cumulative history.
|
|
523
|
+
this._reservations = new Map();
|
|
524
|
+
this._reservationSeq = 0;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Track a launch so the cohort frozen at the first failure sees attempts
|
|
529
|
+
* outstanding in-window. Cohort membership is immutable once frozen: later
|
|
530
|
+
* launches never join it (#2915 review: noteLaunched grew the frozen
|
|
531
|
+
* cohort, so the original members' unanimous failure could not open it).
|
|
532
|
+
* Outstanding entries older than the window are pruned here, so a leaked
|
|
533
|
+
* note (a pre-spawn failure that never settled) can pollute a cohort for
|
|
534
|
+
* at most one window, never permanently.
|
|
535
|
+
*/
|
|
536
|
+
noteLaunched(scopeKeys, invocationId) {
|
|
537
|
+
const now = this.nowMs();
|
|
538
|
+
for (const scopeKey of scopeKeys ?? []) {
|
|
539
|
+
let set = this._outstanding.get(scopeKey);
|
|
540
|
+
if (!set) { set = new Map(); this._outstanding.set(scopeKey, set); }
|
|
541
|
+
set.set(invocationId, now);
|
|
542
|
+
for (const [id, atMs] of set) {
|
|
543
|
+
if (now - atMs >= this.policy.windowMs) set.delete(id);
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Clear outstanding membership across all covering scopes once an attempt
|
|
550
|
+
* settles (used for pre-spawn failures that must leave no trace, and
|
|
551
|
+
* after recording an outcome against a single scope so sibling scopes do
|
|
552
|
+
* not retain a stale member that would veto future cohort unanimity).
|
|
553
|
+
*/
|
|
554
|
+
noteSettled({ scopeKeys = [], invocationId } = {}) {
|
|
555
|
+
for (const scopeKey of scopeKeys ?? []) this._removeOutstanding(scopeKey, invocationId);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* One atomic admission (#2915 final review, replaces the former
|
|
560
|
+
* independent `canLaunch` + `noteLaunched` pair). Returns:
|
|
561
|
+
* - `{ ok: true, token }` — admitted; `token` is an opaque,
|
|
562
|
+
* breaker-owned reservation that MUST be passed to exactly one later
|
|
563
|
+
* `settle()` call. `token.canaryKeys` names which covering scopes (if
|
|
564
|
+
* any) this admission is the half-open canary for.
|
|
565
|
+
* - `{ ok: false, blockedBy, nextEligibleAt }` — refused; no state was
|
|
566
|
+
* mutated (no reservation minted, no cohort membership recorded).
|
|
567
|
+
*
|
|
568
|
+
* `scopeKeys` plus the derived `invocation:<id>` key form the covering set
|
|
569
|
+
* for this attempt, capped at `policy.maxCoveringKeys` (the invocation key
|
|
570
|
+
* is never dropped when trimming — #2915 final review, bounded covering
|
|
571
|
+
* keys). Every covering OPEN scope must be simultaneously eligible (due,
|
|
572
|
+
* no outstanding canary, inner breaker executable) for admission to
|
|
573
|
+
* succeed; this keeps the same "all covering scopes settle together"
|
|
574
|
+
* invariant the review required (`all-covering-scopes-settle`).
|
|
575
|
+
*
|
|
576
|
+
* Reservation lifetime is bounded by `policy.maxOutstandingReservations`:
|
|
577
|
+
* once at capacity, a new admission is refused with
|
|
578
|
+
* `blockedBy: ['reservation-capacity']` rather than ever evicting a live
|
|
579
|
+
* reservation (OPEN-scope protection always wins over admitting more
|
|
580
|
+
* concurrent work).
|
|
581
|
+
*/
|
|
582
|
+
admit({ invocationId = null, scopeKeys = [], selectionDigest = null, purpose = 'task' } = {}) {
|
|
583
|
+
const now = this.nowMs();
|
|
584
|
+
this._sweepReservations(now);
|
|
585
|
+
this._sweepOutstanding(now);
|
|
586
|
+
const invocationKey = invocationId != null ? scopeKeyFor.invocation(invocationId) : null;
|
|
587
|
+
const covering = [...new Set([...(scopeKeys ?? []), ...(invocationKey ? [invocationKey] : [])])];
|
|
588
|
+
// #2915 review 5 (finding 5, "lost covering protection"): an oversized
|
|
589
|
+
// covering set is REFUSED outright, never silently trimmed. Every
|
|
590
|
+
// legitimate production caller derives covering keys from
|
|
591
|
+
// `recoveryScopeKeysForInvocation`, which never produces more than
|
|
592
|
+
// `maxCoveringKeys` (6) keys in the first place -- a set larger than
|
|
593
|
+
// that is already outside what this breaker can safely attribute, and
|
|
594
|
+
// trimming it previously dropped some of the caller's own named scopes
|
|
595
|
+
// from tracking entirely, discarding whatever protection they were
|
|
596
|
+
// meant to provide rather than refusing the admission that needed them.
|
|
597
|
+
if (covering.length > this.policy.maxCoveringKeys) {
|
|
598
|
+
return { ok: false, blockedBy: ['covering-set-too-large'], nextEligibleAt: now };
|
|
599
|
+
}
|
|
600
|
+
// #2915 review 6 (finding 2, "atomic scope tracking"): reserve every
|
|
601
|
+
// BRAND-NEW shared scope's tracking slot INSIDE this admission by
|
|
602
|
+
// actually creating its record now (`_scopeRecordBounded` performs the
|
|
603
|
+
// real capacity check/eviction) -- never a separate non-mutating dry
|
|
604
|
+
// run. Two `admit()` calls can no longer both observe "capacity exists"
|
|
605
|
+
// for the same single free slot: the first call to create a record
|
|
606
|
+
// mutates `_scopes` synchronously, so every later call's own capacity
|
|
607
|
+
// check already accounts for it (#2915 review 6: with one free slot,
|
|
608
|
+
// four concurrent admissions for distinct new scopes were all admitted
|
|
609
|
+
// and three failures later silently vanished, because the old dry run
|
|
610
|
+
// never reserved anything). Invocation-scoped keys are exempt --
|
|
611
|
+
// `maxChargedAttempts` is their own separate, much larger budget, never
|
|
612
|
+
// `maxTrackedScopes`. Any record this admission creates but does not
|
|
613
|
+
// end up needing (refused below for an unrelated reason, or still inert
|
|
614
|
+
// once this attempt settles) is released again via `_releaseIfInert`.
|
|
615
|
+
const reservedScopeKeys = [];
|
|
616
|
+
for (const key of covering) {
|
|
617
|
+
if (key === invocationKey || this._scopes.has(key)) continue;
|
|
618
|
+
const rec = this._scopeRecordBounded(key.split(':')[0], key);
|
|
619
|
+
if (!rec) {
|
|
620
|
+
for (const createdKey of reservedScopeKeys) this._releaseIfInert(createdKey);
|
|
621
|
+
return { ok: false, blockedBy: ['scope-tracking-capacity'], nextEligibleAt: now };
|
|
622
|
+
}
|
|
623
|
+
reservedScopeKeys.push(key);
|
|
624
|
+
}
|
|
625
|
+
const openKeys = covering.filter((key) => {
|
|
626
|
+
const rec = this._scopes.get(key);
|
|
627
|
+
return rec && rec.status !== 'closed';
|
|
628
|
+
});
|
|
629
|
+
const canaryKeys = [];
|
|
630
|
+
const probeRequired = [];
|
|
631
|
+
if (openKeys.length > 0) {
|
|
632
|
+
const blockedBy = [];
|
|
633
|
+
for (const scopeKey of openKeys) {
|
|
634
|
+
const rec = this._scopes.get(scopeKey);
|
|
635
|
+
// #2915 review 5 ("a second worker over unresolved execution"): a
|
|
636
|
+
// bounded blocking marker (installed by `_sweepReservations` when a
|
|
637
|
+
// canary's own deadline passed with stop still unconfirmed) refuses
|
|
638
|
+
// admission regardless of elapsed backoff -- checked BEFORE the
|
|
639
|
+
// due-floor/backoff check below, which elapsed backoff alone could
|
|
640
|
+
// otherwise satisfy. Only a settlement carrying confirmed-
|
|
641
|
+
// termination evidence for that exact reservation clears it (see
|
|
642
|
+
// `settle()`).
|
|
643
|
+
if (rec.status === 'operator-intervention') { blockedBy.push(scopeKey); continue; }
|
|
644
|
+
if (rec.blockingMarker) { blockedBy.push(scopeKey); continue; }
|
|
645
|
+
if (rec.activeCanary) { blockedBy.push(scopeKey); continue; }
|
|
646
|
+
const dueFloor = Math.max(rec.nextProbeAtMs ?? 0, rec.resetAtMs ?? 0);
|
|
647
|
+
if (now < dueFloor || !rec.breaker.canExecute()) { blockedBy.push(scopeKey); continue; }
|
|
648
|
+
if (purpose !== 'probe' && rec.status !== 'canary-ready') { probeRequired.push(scopeKey); continue; }
|
|
649
|
+
if (purpose === 'probe' && rec.status === 'canary-ready') continue;
|
|
650
|
+
canaryKeys.push(scopeKey);
|
|
651
|
+
}
|
|
652
|
+
if (blockedBy.length > 0 || probeRequired.length > 0) {
|
|
653
|
+
const nextEligibleAt = openKeys.reduce((max, key) => Math.max(max, this._scopes.get(key)?.nextProbeAtMs ?? 0), 0);
|
|
654
|
+
for (const createdKey of reservedScopeKeys) this._releaseIfInert(createdKey);
|
|
655
|
+
return { ok: false, blockedBy: [...blockedBy, ...probeRequired], nextEligibleAt,
|
|
656
|
+
probeRequired: blockedBy.length === 0 ? probeRequired : [] };
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
if (this._reservations.size >= this.policy.maxOutstandingReservations) {
|
|
660
|
+
// #2915 review 6 (finding 3f, "markers must not be able to starve
|
|
661
|
+
// unrelated work silently"): name how many of the occupied slots are
|
|
662
|
+
// unresolved blocking markers (rather than live, settleable
|
|
663
|
+
// reservations) in the refusal diagnostics, so this exact refusal is
|
|
664
|
+
// distinguishable from ordinary reservation-capacity pressure.
|
|
665
|
+
const markerCount = [...this._reservations.values()].filter((entry) => entry.stop === 'unresolved').length;
|
|
666
|
+
const blockedBy = markerCount > 0 ? ['reservation-capacity', `blocking-markers:${markerCount}`] : ['reservation-capacity'];
|
|
667
|
+
for (const createdKey of reservedScopeKeys) this._releaseIfInert(createdKey);
|
|
668
|
+
return { ok: false, blockedBy, nextEligibleAt: now };
|
|
669
|
+
}
|
|
670
|
+
this._reservationSeq += 1;
|
|
671
|
+
this._canarySeq += canaryKeys.length > 0 ? 1 : 0;
|
|
672
|
+
const id = `res-${this._reservationSeq}`;
|
|
673
|
+
// Only a canary reservation carries the bounded 300s deadline (03 §11.5:
|
|
674
|
+
// the half-open probe window) — an ordinary (non-canary) admission is
|
|
675
|
+
// not bounded by it at all, and never auto-settles via the sweep below;
|
|
676
|
+
// it occupies one slot of `maxOutstandingReservations` until its own
|
|
677
|
+
// caller settles it, however long the real attempt legitimately runs
|
|
678
|
+
// (#2915 final review risk: a canary-only deadline must never leak onto
|
|
679
|
+
// ordinary long-running attempts).
|
|
680
|
+
const deadlineMs = canaryKeys.length > 0 ? now + (purpose === 'probe' ? this.policy.probeTimeoutMs : this.policy.canaryTimeoutMs) : Infinity;
|
|
681
|
+
// `selectionDigest` (#2915 round 8, D3): stored on the FROZEN token
|
|
682
|
+
// itself — never a separate mutable field on the outer `admitDirectWorker`
|
|
683
|
+
// result — so `beginDispatch`'s identity check below has an authoritative,
|
|
684
|
+
// breaker-owned binding to validate against. A caller-side wrapper that
|
|
685
|
+
// deletes or rewrites its own copy of this value can never change what is
|
|
686
|
+
// actually compared (#2915 round 8 critique: "stored identity is only
|
|
687
|
+
// useful if compared against the actual launch").
|
|
688
|
+
const token = Object.freeze({ id, breaker: this, invocationId, selectionDigest, coveringKeys: Object.freeze(covering),
|
|
689
|
+
canaryKeys: Object.freeze(canaryKeys), purpose, maxOutputBytes: purpose === 'probe' ? this.policy.probeMaxOutputBytes : null, mintedAtMs: now, deadlineMs });
|
|
690
|
+
// `phase`/`stop` (#2915 round 8, D3): the reservation's own transition
|
|
691
|
+
// state, `admitted -> committing -> dispatched`, and its stop-evidence
|
|
692
|
+
// status, `none-needed -> unresolved -> confirmed`. Both start at their
|
|
693
|
+
// initial value for every reservation, canary or not. `revoked` (round 8):
|
|
694
|
+
// set by `_sweepReservations` when a `committing` reservation's deadline
|
|
695
|
+
// passes — it marks the committing attempt for `commitDispatch`/
|
|
696
|
+
// `abortDispatch` to observe, but (unlike `dispatched`-phase expiry) never
|
|
697
|
+
// itself releases or reopens anything, because nothing is yet known to
|
|
698
|
+
// have actually dispatched (see `_sweepReservations`, `abortDispatch`).
|
|
699
|
+
this._reservations.set(id, { token, settled: false, phase: 'admitted', stop: 'none-needed',
|
|
700
|
+
stopTouched: false, revoked: false, reservedScopeKeys });
|
|
701
|
+
this.noteLaunched(covering, invocationId);
|
|
702
|
+
for (const scopeKey of canaryKeys) {
|
|
703
|
+
const rec = this._scopes.get(scopeKey);
|
|
704
|
+
rec.activeCanary = { id, scopeKey, invocationId, reservedAtMs: now, deadlineMs };
|
|
705
|
+
if (purpose !== 'probe') { rec.canariesObserved += 1; this.canariesObserved += 1; }
|
|
706
|
+
this._transition(rec, purpose === 'probe' ? 'half-open' : 'canary', purpose === 'probe' ? 'run one bounded relevant probe' : 'verify the one task canary');
|
|
707
|
+
this._touch(rec);
|
|
708
|
+
}
|
|
709
|
+
return { ok: true, token };
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Idempotent, ownership-checked settlement of one `admit()` token (#2915
|
|
714
|
+
* final review). `outcome.kind` is `'succeeded' | 'failed' | 'uncertain'`
|
|
715
|
+
* (the launcher-owned `attemptOutcome`, never a worker claim); `failed`
|
|
716
|
+
* additionally accepts the same attribution fields `recordFailure` used
|
|
717
|
+
* to (`reasonCode`, `evidenceSource`, `cli`, `inferenceProvider`,
|
|
718
|
+
* `quotaAccount`, `hostId`, `corroboratedBackends`, `resetAtMs`,
|
|
719
|
+
* `retryAfterSeconds`, `installationQualifier`, `scope`/`scopeKey` as the
|
|
720
|
+
* caller's own claim, narrowed to what the evidence supports).
|
|
721
|
+
*
|
|
722
|
+
* A token from a different breaker instance, or one already settled
|
|
723
|
+
* (including one already swept as expired), is refused with
|
|
724
|
+
* `{ ok: false, reason }` and mutates nothing — exactly one caller ever
|
|
725
|
+
* wins the settlement of a given reservation
|
|
726
|
+
* (`old-attempt-cannot-settle-canary`).
|
|
727
|
+
*
|
|
728
|
+
* Every covering scope this reservation held settles together: canary
|
|
729
|
+
* scopes resolve per `outcome.kind` (recover / reopen+backoff / reopen
|
|
730
|
+
* uncertain), and every covering scope's outstanding cohort membership is
|
|
731
|
+
* cleared regardless of canary status (`all-covering-scopes-settle`).
|
|
732
|
+
*
|
|
733
|
+
* `outcome.confirmedStop` (#2915 review 5, "a second worker over
|
|
734
|
+
* unresolved execution") matters ONLY for a reservation whose own canary
|
|
735
|
+
* deadline already passed unresolved (`_sweepReservations` already
|
|
736
|
+
* reopened the scope and installed a blocking marker for it, below) — a
|
|
737
|
+
* settle() call against such a token is the launcher's bounded stop-
|
|
738
|
+
* escalation report arriving separately from the expiry itself, and
|
|
739
|
+
* resolves ONLY that marker, never the scope's normal recover/reopen
|
|
740
|
+
* transition (already applied exactly once, at expiry). `true` (the
|
|
741
|
+
* launcher's own confirmed termination evidence) clears the marker;
|
|
742
|
+
* anything else (including a late `kind: 'succeeded'`) leaves it in
|
|
743
|
+
* place with no further automatic resolution — late success can never
|
|
744
|
+
* close a scope this way.
|
|
745
|
+
*/
|
|
746
|
+
settle(token, outcome = { kind: 'uncertain' }) {
|
|
747
|
+
const now = this.nowMs();
|
|
748
|
+
this._sweepReservations(now);
|
|
749
|
+
this._sweepOutstanding(now);
|
|
750
|
+
// #2915 review 4 (finding 5, exact token ownership): a passed-in token
|
|
751
|
+
// is NEVER trusted for its own field values — only as a lookup key.
|
|
752
|
+
// `entry.token !== token` (reference inequality) refuses ANY copy with
|
|
753
|
+
// altered fields, including one that also copies `breaker`/`id`
|
|
754
|
+
// correctly but changes `invocationId`/`coveringKeys`/`canaryKeys`: the
|
|
755
|
+
// legitimate caller always holds the exact frozen object `admit()`
|
|
756
|
+
// returned, so only that exact object reference can ever match.
|
|
757
|
+
if (!token || typeof token.id !== 'string' || token.breaker !== this) {
|
|
758
|
+
return { ok: false, reason: 'reservation is not owned by this breaker' };
|
|
759
|
+
}
|
|
760
|
+
const entry = this._reservations.get(token.id);
|
|
761
|
+
if (!entry || entry.token !== token) {
|
|
762
|
+
return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
|
|
763
|
+
}
|
|
764
|
+
if (entry.stop === 'unresolved') {
|
|
765
|
+
// This reservation already failed its own canary deadline while
|
|
766
|
+
// `dispatched` (possibly via the sweep this very call just performed,
|
|
767
|
+
// above). Resolve the stop-evidence marker it left behind instead of
|
|
768
|
+
// re-running the normal recover/reopen transition below.
|
|
769
|
+
//
|
|
770
|
+
// #2915 review 6 (finding 3a, "blocking markers become permanently
|
|
771
|
+
// unresolvable"): a CONFIRMED stop always wins, regardless of how
|
|
772
|
+
// many unconfirmed settlements already touched this same reservation
|
|
773
|
+
// -- the former code resolved the marker on the FIRST settle() call
|
|
774
|
+
// no matter its outcome, so an earlier unconfirmed report (the common
|
|
775
|
+
// case: a degraded attempt can never supply confirmed-termination
|
|
776
|
+
// evidence) permanently rejected every later, genuinely confirmed
|
|
777
|
+
// one. An unconfirmed settlement is a no-op that resolves nothing,
|
|
778
|
+
// but each already-expired reservation still accepts exactly ONE such
|
|
779
|
+
// report (mirroring the normal "exactly once" settlement contract for
|
|
780
|
+
// every other outcome) so a caller cannot spin it indefinitely; this
|
|
781
|
+
// never forecloses a LATER confirmed report, or an explicit
|
|
782
|
+
// `reconcileStopEvidence` reconciliation, from succeeding at any time.
|
|
783
|
+
const confirmed = outcome?.confirmedStop === true;
|
|
784
|
+
if (confirmed) {
|
|
785
|
+
// A confirmed stop frees this reservation's slot; an unconfirmed
|
|
786
|
+
// stop deliberately keeps it occupying one of
|
|
787
|
+
// `maxOutstandingReservations` — the explicit bound a persistent
|
|
788
|
+
// stop-evidence marker counts against, rather than a separate marker
|
|
789
|
+
// cap or an invented auto-expiry.
|
|
790
|
+
entry.stop = 'confirmed';
|
|
791
|
+
this._release(entry);
|
|
792
|
+
return { ok: true };
|
|
793
|
+
}
|
|
794
|
+
if (entry.stopTouched) {
|
|
795
|
+
return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
|
|
796
|
+
}
|
|
797
|
+
entry.stopTouched = true;
|
|
798
|
+
return { ok: true };
|
|
799
|
+
}
|
|
800
|
+
if (entry.settled) {
|
|
801
|
+
return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
|
|
802
|
+
}
|
|
803
|
+
entry.settled = true;
|
|
804
|
+
// #2915 round 8 (C1, "outcome settlement discards unresolved execution"):
|
|
805
|
+
// captured BEFORE any of the outcome processing below, which itself
|
|
806
|
+
// never changes `entry.phase`. A reservation that reached `dispatched`
|
|
807
|
+
// genuinely started a process; whether its descendants are confirmed
|
|
808
|
+
// stopped is an INDEPENDENT fact from whether the attempt succeeded or
|
|
809
|
+
// failed, so recording the outcome below must never, by itself, be
|
|
810
|
+
// treated as proof the attempt's execution ended.
|
|
811
|
+
const wasDispatched = entry.phase === 'dispatched';
|
|
812
|
+
const confirmedStop = outcome?.confirmedStop === true;
|
|
813
|
+
const { invocationId, coveringKeys, canaryKeys } = entry.token;
|
|
814
|
+
const kind = outcome?.kind ?? 'uncertain';
|
|
815
|
+
// Enforce the canary's own deadline even at settlement time: a reported
|
|
816
|
+
// outcome that arrives after the deadline elapsed never recovers or
|
|
817
|
+
// otherwise resolves the scope through this path — it was (or will be)
|
|
818
|
+
// swept as expired instead (#2915 final review: "enforce deadlines on
|
|
819
|
+
// completion"). `now >= deadline` (#2915 round 8, D5): one consistent
|
|
820
|
+
// comparison operator for every deadline check in this module.
|
|
821
|
+
// #2915 round 8 (C1): scopes THIS token actually owned the canary for at
|
|
822
|
+
// the moment this loop ran — never the full static `canaryKeys` list,
|
|
823
|
+
// which can include a scope a LATER reservation has already taken over
|
|
824
|
+
// (the `continue` below). Used after the loop to decide which scopes, if
|
|
825
|
+
// any, get the retained-execution blocking marker: installing it on a
|
|
826
|
+
// scope this token no longer owns would wrongly block an unrelated,
|
|
827
|
+
// still-live reservation's own canary.
|
|
828
|
+
const ownedCanaryKeys = [];
|
|
829
|
+
for (const scopeKey of canaryKeys) {
|
|
830
|
+
const rec = this._scopes.get(scopeKey);
|
|
831
|
+
if (!rec || !rec.activeCanary || rec.activeCanary.id !== token.id) continue; // already reopened/expired elsewhere
|
|
832
|
+
ownedCanaryKeys.push(scopeKey);
|
|
833
|
+
if (now >= rec.activeCanary.deadlineMs) {
|
|
834
|
+
rec.activeCanary = null;
|
|
835
|
+
this._reopen(rec, now);
|
|
836
|
+
continue;
|
|
837
|
+
}
|
|
838
|
+
rec.activeCanary = null;
|
|
839
|
+
if (outcome?.callerAborted === true && confirmedStop) {
|
|
840
|
+
// Caller cancellation is no evidence of a broken provider. Keep the
|
|
841
|
+
// existing cooldown and retain a successful probe for the next canary.
|
|
842
|
+
this._transition(rec, token.purpose === 'probe' ? 'open' : 'canary-ready',
|
|
843
|
+
'caller cancelled; retry recovery without increasing backoff');
|
|
844
|
+
} else if (kind === 'succeeded' && token.purpose === 'probe') {
|
|
845
|
+
if (outcome.relevant === true && isTrustedRecoveryEvidence(outcome.evidenceSource) && confirmedStop) {
|
|
846
|
+
this._transition(rec, 'canary-ready', 'probe succeeded; admit one task canary');
|
|
847
|
+
} else { this._reopen(rec, now); }
|
|
848
|
+
} else if (kind === 'succeeded') {
|
|
849
|
+
this._recoverScope(rec, invocationId);
|
|
850
|
+
} else if (kind === 'failed') {
|
|
851
|
+
// A reset/Retry-After carried by THIS canary's own failure evidence
|
|
852
|
+
// still merges before reopening — the same "a later failure carries
|
|
853
|
+
// a further reset" rule that applies to any other failure, so a
|
|
854
|
+
// canary that fails with a longer-than-backoff reset does not get
|
|
855
|
+
// probed again before that reset elapses.
|
|
856
|
+
const reset = this._resolveResetAtMs({ resetAtMs: outcome?.resetAtMs ?? null, retryAfterSeconds: outcome?.retryAfterSeconds ?? null }, now);
|
|
857
|
+
if (reset !== null && (rec.resetAtMs === null || reset > rec.resetAtMs)) rec.resetAtMs = reset;
|
|
858
|
+
this._reopen(rec, now);
|
|
859
|
+
} else {
|
|
860
|
+
this._reopen(rec, now);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
if (kind === 'succeeded') {
|
|
864
|
+
for (const scopeKey of coveringKeys) {
|
|
865
|
+
this._recordObservedAttempt(scopeKey, invocationId);
|
|
866
|
+
if (canaryKeys.includes(scopeKey)) continue; // already resolved above
|
|
867
|
+
const rec = this._scopes.get(scopeKey);
|
|
868
|
+
if (rec && rec.status === 'closed') rec.breaker.recordSuccess();
|
|
869
|
+
}
|
|
870
|
+
if (coveringKeys.length === 0) this._recordObservedAttempt(null, invocationId);
|
|
871
|
+
} else if (kind === 'failed') {
|
|
872
|
+
this._settleFailure(token, outcome ?? {}, now);
|
|
873
|
+
}
|
|
874
|
+
// #2915 review 4 (finding 8): an 'uncertain' settlement is NEVER
|
|
875
|
+
// recorded as an observed attempt — it is the kind every pre-spawn
|
|
876
|
+
// refusal settles as (see `settleDirectWorker`), and this diagnostic
|
|
877
|
+
// counter must not read as if something dispatched when it did not.
|
|
878
|
+
// The allowance LEDGER (`debited`), not this counter, is what actually
|
|
879
|
+
// enforces attempt consumption.
|
|
880
|
+
//
|
|
881
|
+
// #2915 round 8 (D3): deletion, outstanding-membership cleanup and the
|
|
882
|
+
// reserved-scope-slot release are now ALL performed by the one private
|
|
883
|
+
// `_release` — the only code path that ever removes a reservation table
|
|
884
|
+
// entry or clears an `activeCanary`/`blockingMarker` reference naming it.
|
|
885
|
+
//
|
|
886
|
+
// #2915 round 8 (C1, "outcome settlement discards unresolved execution"):
|
|
887
|
+
// `_release` is safe here ONLY when this reservation never reached
|
|
888
|
+
// `dispatched` (nothing was ever started — identical to the `admitted`-
|
|
889
|
+
// phase branch of `_sweepReservations`) OR this very settlement carries
|
|
890
|
+
// CONFIRMED stop evidence (`outcome.confirmedStop === true`). Every other
|
|
891
|
+
// case — a dispatched reservation settling `succeeded`/`failed`/
|
|
892
|
+
// `uncertain` with no confirmed stop — records the outcome above (scope
|
|
893
|
+
// recover/reopen, failure attribution) exactly as before, but retains
|
|
894
|
+
// the reservation and installs the SAME bounded stop-evidence marker
|
|
895
|
+
// `_sweepReservations` installs on canary-deadline expiry, so a NEW
|
|
896
|
+
// canary can never be admitted over this still-unconfirmed descendant
|
|
897
|
+
// tree (#2915 round 8 review 8, finding 1: "a dispatched canary settled
|
|
898
|
+
// uncertain before its deadline... after backoff, another canary was
|
|
899
|
+
// admitted over the unresolved execution"). A later `settle()` call on
|
|
900
|
+
// this exact token lands in the `entry.stop === 'unresolved'` branch
|
|
901
|
+
// above, which already resolves the marker exactly like an expired
|
|
902
|
+
// canary's late report — the same one-more-call contract, never a
|
|
903
|
+
// separate mechanism.
|
|
904
|
+
if (!wasDispatched || confirmedStop) {
|
|
905
|
+
if (wasDispatched) entry.stop = 'confirmed';
|
|
906
|
+
this._release(entry);
|
|
907
|
+
return { ok: true };
|
|
908
|
+
}
|
|
909
|
+
entry.stop = 'unresolved';
|
|
910
|
+
for (const scopeKey of ownedCanaryKeys) {
|
|
911
|
+
const rec = this._scopes.get(scopeKey);
|
|
912
|
+
if (rec) rec.blockingMarker = { reservationId: token.id, invocationId, setAtMs: now };
|
|
913
|
+
}
|
|
914
|
+
return { ok: true };
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
/**
|
|
918
|
+
* Failure attribution + the distinct-threshold/cohort-unanimous open rule,
|
|
919
|
+
* folded into `settle()`. Scopes this token already held as a canary were
|
|
920
|
+
* resolved directly above (reopen on failure, never re-evaluated against
|
|
921
|
+
* the distinct-failure threshold, matching the former `recordFailure`
|
|
922
|
+
* canary short-circuit); every other covering scope goes through the
|
|
923
|
+
* narrowing + correlation rule unchanged from the pre-#2915-final design.
|
|
924
|
+
*/
|
|
925
|
+
_settleFailure(token, outcome, now) {
|
|
926
|
+
const { invocationId, canaryKeys } = token;
|
|
927
|
+
const supported = attributeFailureScope({
|
|
928
|
+
reasonCode: outcome.reasonCode ?? null, evidenceSource: outcome.evidenceSource ?? null, invocationId,
|
|
929
|
+
modelKey: outcome.modelKey ?? null, cli: outcome.cli ?? null, inferenceProvider: outcome.inferenceProvider ?? null,
|
|
930
|
+
quotaAccount: outcome.quotaAccount ?? null, hostId: outcome.hostId ?? null,
|
|
931
|
+
corroboratedBackends: outcome.corroboratedBackends ?? null, installationQualifier: outcome.installationQualifier ?? null,
|
|
932
|
+
});
|
|
933
|
+
// When the caller supplies an explicit `scope`/`scopeKey` claim, the
|
|
934
|
+
// recorded scope is the NARROWER of that claim and what the attached
|
|
935
|
+
// evidence supports — a wide claim on worker-derived or identity-less
|
|
936
|
+
// evidence folds back to whatever the evidence actually proves, but a
|
|
937
|
+
// claim that is already as narrow or narrower than the evidence (the
|
|
938
|
+
// normal case: the caller pre-derived this exact attribution, e.g.
|
|
939
|
+
// `classifyAttemptForSettlement`) is kept EXACTLY as given, including
|
|
940
|
+
// its specific scopeKey — a same-rank evidence key (e.g. the generic
|
|
941
|
+
// `host:local` an omitted `hostId` resolves to) must never silently
|
|
942
|
+
// replace a caller's more specific identity (e.g. `host:noodle`) at the
|
|
943
|
+
// same scope breadth. A caller that omits `scope`/`scopeKey` entirely
|
|
944
|
+
// gets the evidence-derived attribution directly.
|
|
945
|
+
let scope;
|
|
946
|
+
let scopeKey;
|
|
947
|
+
if (outcome.scope && outcome.scopeKey) {
|
|
948
|
+
scope = outcome.scope;
|
|
949
|
+
scopeKey = outcome.scopeKey;
|
|
950
|
+
if (scopeRank(supported.scope) < scopeRank(scope)) { scope = supported.scope; scopeKey = supported.scopeKey; }
|
|
951
|
+
} else {
|
|
952
|
+
scope = supported.scope;
|
|
953
|
+
scopeKey = supported.scopeKey;
|
|
954
|
+
}
|
|
955
|
+
if (canaryKeys.includes(scopeKey)) { this._recordObservedAttempt(scopeKey, invocationId); return; } // already reopened as this token's canary failure, above
|
|
956
|
+
const rec = this._scopeRecordBounded(scope, scopeKey);
|
|
957
|
+
// Charge AFTER the scope record exists (or is confirmed unattainable at
|
|
958
|
+
// the hard cap) so `rec.attemptsObserved` reflects this attempt whenever
|
|
959
|
+
// a record is created for it, rather than undercounting the very failure
|
|
960
|
+
// that first creates the scope.
|
|
961
|
+
this._recordObservedAttempt(scopeKey, invocationId);
|
|
962
|
+
if (!rec) return; // at hard capacity with nothing evictable: bookkeeping-only, never exceeds the bound
|
|
963
|
+
this._touch(rec);
|
|
964
|
+
this._removeOutstanding(scopeKey, invocationId);
|
|
965
|
+
// Genuine rolling-window retention: prune only failures older than the
|
|
966
|
+
// window. Refreezing the cohort epoch must NOT discard still-recent
|
|
967
|
+
// failures.
|
|
968
|
+
rec.failures = rec.failures.filter((f) => now - f.atMs < this.policy.windowMs);
|
|
969
|
+
if (rec.cohortFrozenAt === null || now - rec.cohortFrozenAt >= this.policy.windowMs) {
|
|
970
|
+
rec.cohort = new Set([...(this._outstanding.get(scopeKey)?.keys() ?? []), invocationId]);
|
|
971
|
+
rec.cohortFrozenAt = now;
|
|
972
|
+
}
|
|
973
|
+
if (!rec.failures.some((f) => f.invocationId === invocationId)) {
|
|
974
|
+
rec.failures.push({ invocationId, atMs: now, reasonCode: outcome.reasonCode ?? null });
|
|
975
|
+
}
|
|
976
|
+
const distinct = new Set(rec.failures.map((f) => f.invocationId));
|
|
977
|
+
const reasonLabel = outcome.reasonCode ?? 'UNKNOWN';
|
|
978
|
+
rec.evidence.set(reasonLabel, (rec.evidence.get(reasonLabel) ?? 0) + 1);
|
|
979
|
+
const reset = this._resolveResetAtMs({ resetAtMs: outcome.resetAtMs ?? null, retryAfterSeconds: outcome.retryAfterSeconds ?? null }, now);
|
|
980
|
+
if (reset !== null && (rec.resetAtMs === null || reset > rec.resetAtMs)) {
|
|
981
|
+
rec.resetAtMs = reset;
|
|
982
|
+
if (rec.status !== 'closed') rec.nextProbeAtMs = Math.max(rec.nextProbeAtMs, reset + this._boundedJitter());
|
|
983
|
+
}
|
|
984
|
+
if (rec.status !== 'closed') {
|
|
985
|
+
// Already open and this token held no canary on it (defensive: admit()
|
|
986
|
+
// never admits a non-canary reservation covering an open scope).
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
const cohort = rec.cohort ?? new Set();
|
|
990
|
+
const allOutstandingFailed = cohort.size >= this.policy.minOutstanding && [...cohort].every((id) => distinct.has(id));
|
|
991
|
+
if (distinct.size >= this.policy.distinctThreshold || allOutstandingFailed) {
|
|
992
|
+
rec.openedAtMs = now;
|
|
993
|
+
rec.nextProbeAtMs = Math.max(now + rec.episodeCooldownMs, rec.resetAtMs ?? 0) + this._boundedJitter();
|
|
994
|
+
rec.breaker.cooldownMs = rec.episodeCooldownMs;
|
|
995
|
+
rec.breaker.recordFailure();
|
|
996
|
+
this._transition(rec, 'open', 'pause covered launches; wait for reset/cooldown then probe');
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
_recoverScope(rec, invocationId) {
|
|
1001
|
+
rec.reopens = 0;
|
|
1002
|
+
rec.episodeCooldownMs = this.policy.cooldownMs;
|
|
1003
|
+
rec.resetAtMs = null;
|
|
1004
|
+
rec.nextProbeAtMs = 0;
|
|
1005
|
+
rec.failures = [];
|
|
1006
|
+
rec.cohort = null;
|
|
1007
|
+
rec.cohortFrozenAt = null;
|
|
1008
|
+
this._transition(rec, 'closed', 'canary succeeded; resume covered launches');
|
|
1009
|
+
rec.breaker.reset();
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
requireOperatorIntervention(scopeKeys, reason) {
|
|
1013
|
+
for (const key of scopeKeys) {
|
|
1014
|
+
const rec = this._scopes.get(key);
|
|
1015
|
+
if (!rec || rec.status === 'closed') continue;
|
|
1016
|
+
rec.operatorReason = `${reason}; restore a qualified scope-relevant probe, then reset this scope`;
|
|
1017
|
+
this._transition(rec, 'operator-intervention', rec.operatorReason);
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
_transition(rec, to, action) {
|
|
1022
|
+
const from = rec.status;
|
|
1023
|
+
if (from === to) return;
|
|
1024
|
+
rec.status = to;
|
|
1025
|
+
this.emit('transition', { scopeKey: rec.scopeKey, from, to, action,
|
|
1026
|
+
distinctFailures: new Set(rec.failures.map(failure => failure.invocationId)).size, reasons: [...rec.evidence.keys()],
|
|
1027
|
+
nextEligibleAtMs: rec.nextProbeAtMs, resetAtMs: rec.resetAtMs,
|
|
1028
|
+
summary: `${rec.scopeKey}: ${from} -> ${to}; ${action}; wait ${Math.max(0, rec.nextProbeAtMs - this.nowMs())}ms`,
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/**
|
|
1033
|
+
* #2915 review 4 (finding 8): DIAGNOSTIC only — `attemptsObserved`/
|
|
1034
|
+
* `canariesObserved` (renamed from `attemptsCharged`/`canariesCharged`)
|
|
1035
|
+
* report what this breaker has seen, never what it enforces. Nothing in
|
|
1036
|
+
* `admit()` reads either counter; the allowance LEDGER
|
|
1037
|
+
* (`reserveDirectWorkerAllowance`/`debitDirectWorkerAllowance` in
|
|
1038
|
+
* session-launcher.mjs) is the actual enforcement. `settle()` only calls
|
|
1039
|
+
* this for `'succeeded'`/`'failed'` kinds — an `'uncertain'` settlement
|
|
1040
|
+
* (every pre-spawn refusal settles this way) is never recorded here.
|
|
1041
|
+
*/
|
|
1042
|
+
_recordObservedAttempt(scopeKey, invocationId) {
|
|
1043
|
+
if (invocationId == null || this._observedInvocations.has(invocationId)) return false;
|
|
1044
|
+
this._observedInvocations.add(invocationId);
|
|
1045
|
+
if (this._observedInvocations.size > this.policy.maxChargedAttempts) {
|
|
1046
|
+
const oldest = this._observedInvocations.values().next().value;
|
|
1047
|
+
this._observedInvocations.delete(oldest);
|
|
1048
|
+
}
|
|
1049
|
+
const rec = scopeKey ? this._scopes.get(scopeKey) : null;
|
|
1050
|
+
if (rec) rec.attemptsObserved += 1;
|
|
1051
|
+
return true;
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
/**
|
|
1055
|
+
* Account every reservation past its own deadline — including non-canary
|
|
1056
|
+
* reservations, which share the same bounded-TTL field (though its value
|
|
1057
|
+
* is `Infinity` for them in practice) so `_reservations` cannot grow
|
|
1058
|
+
* without bound from attempts whose caller crashed before calling
|
|
1059
|
+
* `settle()`. A hung or unreported canary is treated as a failed canary:
|
|
1060
|
+
* it reopens with doubled backoff immediately (the deadline enforcement
|
|
1061
|
+
* `expireStaleReservations` used to apply only at the scope level now
|
|
1062
|
+
* applies uniformly through the reservation registry).
|
|
1063
|
+
*
|
|
1064
|
+
* #2915 review 5 ("a second worker over unresolved execution"): reopening
|
|
1065
|
+
* with backoff is NOT settlement here — the reservation itself is
|
|
1066
|
+
* deliberately retained (not deleted) and a bounded blocking marker is
|
|
1067
|
+
* installed on every canary scope it covered, because expiry alone proves
|
|
1068
|
+
* nothing about whether the attempt actually stopped. `admit()` refuses
|
|
1069
|
+
* while the marker is set, regardless of elapsed backoff; only a later
|
|
1070
|
+
* `settle()` call against this exact token, carrying confirmed-
|
|
1071
|
+
* termination evidence (`outcome.confirmedStop`), clears it — see
|
|
1072
|
+
* `settle()`. `entry.stop !== 'none-needed'` guards this block from
|
|
1073
|
+
* re-running on a later sweep of the same still-unresolved entry.
|
|
1074
|
+
*
|
|
1075
|
+
* Called at the top of every `admit`/`settle`/`beginDispatch`/
|
|
1076
|
+
* `commitDispatch`, so expiry is always observed on the very next call
|
|
1077
|
+
* even when the admitted attempt never reports back at all (process
|
|
1078
|
+
* crash, dropped callback) — the backstop that makes the design
|
|
1079
|
+
* deadlock-free once stop is eventually confirmed
|
|
1080
|
+
* (`deadline-reopens-before-readmission`).
|
|
1081
|
+
*
|
|
1082
|
+
* #2915 round 8 (D3, "expiry during dispatch commitment"): behaviour now
|
|
1083
|
+
* branches on `entry.phase`, because expiry means something different at
|
|
1084
|
+
* each phase:
|
|
1085
|
+
*
|
|
1086
|
+
* - `admitted` (never even reached `beginDispatch`): PROVEN to have never
|
|
1087
|
+
* dispatched anything, exactly like an unreported canary always was —
|
|
1088
|
+
* reopen (if this was a canary) with doubled backoff and release
|
|
1089
|
+
* outright via `_release`. No stop-evidence marker is ever installed
|
|
1090
|
+
* here: there is nothing to confirm the stop of.
|
|
1091
|
+
* - `committing` (inside the D2 hook, between `beginDispatch` and
|
|
1092
|
+
* `commitDispatch`): the reservation is marked `revoked` ONLY — never
|
|
1093
|
+
* reopened, never released. `commitDispatch` and `abortDispatch` both
|
|
1094
|
+
* observe `revoked`; only once the committing owner itself calls one of
|
|
1095
|
+
* them (acknowledging the abort) does this reservation actually resolve
|
|
1096
|
+
* — expiry alone must never classify a `committing` reservation as
|
|
1097
|
+
* unused (#2915 round 8 critique: "expiry can revoke a pending launch,
|
|
1098
|
+
* but cannot classify it as safely unused until the committing owner
|
|
1099
|
+
* acknowledges abort").
|
|
1100
|
+
* - `dispatched`: unchanged from the prior design — reopen (if a canary)
|
|
1101
|
+
* with doubled backoff and install a bounded stop-evidence marker
|
|
1102
|
+
* (`stop: 'unresolved'`), because expiry alone proves nothing about
|
|
1103
|
+
* whether the dispatched attempt actually stopped. `admit()` refuses
|
|
1104
|
+
* while the marker is set, regardless of elapsed backoff; only a later
|
|
1105
|
+
* `settle()`/`reconcileStopEvidence()` call carrying confirmed-
|
|
1106
|
+
* termination evidence clears it.
|
|
1107
|
+
*/
|
|
1108
|
+
_sweepReservations(now) {
|
|
1109
|
+
for (const [id, entry] of this._reservations) {
|
|
1110
|
+
// #2915 round 8 (C1, "the sweep directly deletes outcome-settled
|
|
1111
|
+
// entries"): a settled entry still present in the table is one
|
|
1112
|
+
// `settle()` itself deliberately retained — `entry.phase ===
|
|
1113
|
+
// 'dispatched'` with `entry.stop !== 'confirmed'` (see `settle()`) —
|
|
1114
|
+
// and is removed ONLY by `_release` (via a later `settle()` call that
|
|
1115
|
+
// now lands in the `entry.stop === 'unresolved'` branch, or
|
|
1116
|
+
// `reconcileStopEvidence`), never by this sweep. Deadline/phase expiry
|
|
1117
|
+
// below is for UNSETTLED reservations only.
|
|
1118
|
+
if (entry.settled) continue;
|
|
1119
|
+
if (now < entry.token.deadlineMs) continue;
|
|
1120
|
+
if (entry.phase === 'admitted') {
|
|
1121
|
+
const { canaryKeys } = entry.token;
|
|
1122
|
+
for (const scopeKey of canaryKeys) {
|
|
1123
|
+
const rec = this._scopes.get(scopeKey);
|
|
1124
|
+
if (rec?.activeCanary?.id === id) { rec.activeCanary = null; this._reopen(rec, now); }
|
|
1125
|
+
}
|
|
1126
|
+
this._release(entry);
|
|
1127
|
+
continue;
|
|
1128
|
+
}
|
|
1129
|
+
if (entry.phase === 'committing') {
|
|
1130
|
+
entry.revoked = true; // idempotent; commitDispatch/abortDispatch resolve it
|
|
1131
|
+
continue;
|
|
1132
|
+
}
|
|
1133
|
+
// `dispatched`.
|
|
1134
|
+
if (entry.stop !== 'none-needed') continue; // already marked unresolved/confirmed; awaiting settle()/reconcile
|
|
1135
|
+
entry.stop = 'unresolved';
|
|
1136
|
+
const { invocationId, coveringKeys, canaryKeys } = entry.token;
|
|
1137
|
+
for (const scopeKey of canaryKeys) {
|
|
1138
|
+
const rec = this._scopes.get(scopeKey);
|
|
1139
|
+
if (rec?.activeCanary?.id === id) {
|
|
1140
|
+
rec.activeCanary = null;
|
|
1141
|
+
this._reopen(rec, now);
|
|
1142
|
+
rec.blockingMarker = { reservationId: id, invocationId, setAtMs: now };
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
this.noteSettled({ scopeKeys: coveringKeys, invocationId });
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
/** Public alias retained for direct test/diagnostic use; `admit`/`settle`
|
|
1150
|
+
* already call this internally on every invocation. */
|
|
1151
|
+
expireStaleReservations() {
|
|
1152
|
+
this._sweepReservations(this.nowMs());
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
/**
|
|
1156
|
+
* Prune every outstanding cohort-membership entry older than the window,
|
|
1157
|
+
* across ALL tracked scope keys — not just the ones this call's own
|
|
1158
|
+
* covering set touches (#2915 final review, finding 6: pruning used to
|
|
1159
|
+
* touch only keys revisited by `noteLaunched`, so an invocation scope for
|
|
1160
|
+
* an attempt that never launched again, or never settled, retained its
|
|
1161
|
+
* membership forever even once the window had long since elapsed). Bounded
|
|
1162
|
+
* by the number of tracked outstanding scope keys, which is itself bounded
|
|
1163
|
+
* in steady state by `maxTrackedScopes`/`maxOutstandingReservations`.
|
|
1164
|
+
*/
|
|
1165
|
+
_sweepOutstanding(now) {
|
|
1166
|
+
for (const [key, set] of this._outstanding) {
|
|
1167
|
+
for (const [id, atMs] of set) {
|
|
1168
|
+
if (now - atMs >= this.policy.windowMs) set.delete(id);
|
|
1169
|
+
}
|
|
1170
|
+
if (set.size === 0) this._outstanding.delete(key);
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
/**
|
|
1175
|
+
* One private cleanup path (#2915 round 8, D3): the ONLY code that ever
|
|
1176
|
+
* removes a `_reservations` table entry, or clears an `activeCanary`/
|
|
1177
|
+
* `blockingMarker` scope reference naming it. Preconditions, enforced by
|
|
1178
|
+
* every caller (not re-checked here — this method trusts its callers,
|
|
1179
|
+
* all of which are other methods on this class): the reservation must
|
|
1180
|
+
* never have reached `dispatched`, OR its `stop` must already be
|
|
1181
|
+
* `'confirmed'`. A late callback — the stop-escalation observer, an
|
|
1182
|
+
* operator's `reconcileStopEvidence` — may still race this exact call; it
|
|
1183
|
+
* is idempotent only in the sense that `_reservations.delete` on an
|
|
1184
|
+
* already-absent id is a no-op, matching every other settlement path's
|
|
1185
|
+
* "exactly once" contract enforced by its own caller.
|
|
1186
|
+
*
|
|
1187
|
+
* #2915 round 8 (C1): the precondition above is now ALSO enforced here,
|
|
1188
|
+
* defensively — never only trusted of callers — so a future call site
|
|
1189
|
+
* that forgets to check it fails loudly (throws) instead of silently
|
|
1190
|
+
* releasing live, unconfirmed execution.
|
|
1191
|
+
*/
|
|
1192
|
+
_release(entry) {
|
|
1193
|
+
const { token } = entry;
|
|
1194
|
+
if (entry.phase === 'dispatched' && entry.stop !== 'confirmed') {
|
|
1195
|
+
throw new Error('ScopedFailureBreaker: refusing to release a dispatched reservation without confirmed stop');
|
|
1196
|
+
}
|
|
1197
|
+
for (const scopeKey of token.canaryKeys) {
|
|
1198
|
+
const rec = this._scopes.get(scopeKey);
|
|
1199
|
+
if (!rec) continue;
|
|
1200
|
+
if (rec.activeCanary?.id === token.id) rec.activeCanary = null;
|
|
1201
|
+
if (rec.blockingMarker?.reservationId === token.id) rec.blockingMarker = null;
|
|
1202
|
+
}
|
|
1203
|
+
this.noteSettled({ scopeKeys: token.coveringKeys, invocationId: token.invocationId });
|
|
1204
|
+
this._reservations.delete(token.id);
|
|
1205
|
+
// #2915 review 6 (finding 2): release the tracking slot THIS admission
|
|
1206
|
+
// reserved (`reservedScopeKeys`, set in `admit()`) back to the shared
|
|
1207
|
+
// pool if the scope it covers is now inert -- never a scope this
|
|
1208
|
+
// admission merely found already tracked (which may hold failure
|
|
1209
|
+
// history or outstanding members belonging to other in-flight work).
|
|
1210
|
+
for (const scopeKey of entry.reservedScopeKeys ?? []) this._releaseIfInert(scopeKey);
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
/**
|
|
1214
|
+
* Read-only phase introspection (#2915 round 8, D2/D3): lets a caller that
|
|
1215
|
+
* holds an admission (e.g. `settleDirectWorker` in session-launcher.mjs)
|
|
1216
|
+
* decide whether a reservation ever reached `dispatched` without
|
|
1217
|
+
* reconstructing authority from anything but the breaker's own stored
|
|
1218
|
+
* entry. Returns `null` for an unknown/foreign/already-released token —
|
|
1219
|
+
* never throws, never mutates.
|
|
1220
|
+
*/
|
|
1221
|
+
reservationPhase(token) {
|
|
1222
|
+
if (!token || typeof token.id !== 'string' || token.breaker !== this) return null;
|
|
1223
|
+
const entry = this._reservations.get(token.id);
|
|
1224
|
+
if (!entry || entry.token !== token) return null;
|
|
1225
|
+
return entry.phase;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
_dispatchScopeRefusal(token, now) {
|
|
1229
|
+
for (const key of token.coveringKeys) {
|
|
1230
|
+
const rec = this._scopes.get(key);
|
|
1231
|
+
if (!rec || rec.status === 'closed') continue;
|
|
1232
|
+
if (rec.blockingMarker || (rec.activeCanary?.id !== token.id
|
|
1233
|
+
&& !(token.purpose === 'probe' && rec.status === 'canary-ready' && !rec.activeCanary))
|
|
1234
|
+
|| now < Math.max(rec.resetAtMs ?? 0, rec.nextProbeAtMs ?? 0)) {
|
|
1235
|
+
return { ok: false, reason: `covering recovery scope changed before dispatch: ${key}` };
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
return null;
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
/**
|
|
1242
|
+
* Begin the actual dispatch of one admitted reservation, exactly ONCE
|
|
1243
|
+
* (#2915 round 8, D3, replaces `claimLaunch`): `admitted -> committing`.
|
|
1244
|
+
* Called once, at the leaf dispatch boundary — inside the D2
|
|
1245
|
+
* `onDispatchCommit` hook, immediately before the real process-start call.
|
|
1246
|
+
*
|
|
1247
|
+
* `invocationId` is REQUIRED and may never be `null`/`undefined`/omitted
|
|
1248
|
+
* (#2915 round 8 critique: "omitted/null bindings must not bypass checks";
|
|
1249
|
+
* round 8 review 8 finding 6 hardening: an EXPLICIT `invocationId: null`
|
|
1250
|
+
* call must refuse exactly like an omitted one, and a reservation whose
|
|
1251
|
+
* own stored `token.invocationId` is itself `null`/`undefined` — which
|
|
1252
|
+
* `admit()` never produces for any real caller, since every `invocationId`
|
|
1253
|
+
* it receives is either caller-supplied or freshly minted — can never
|
|
1254
|
+
* dispatch no matter what the caller later supplies). `selectionDigest`
|
|
1255
|
+
* is REQUIRED but MAY legitimately be `null` on both sides: `admit()`
|
|
1256
|
+
* stores `null` for it whenever the caller has no resolved selection
|
|
1257
|
+
* identity to bind (flat-option/no-`invocation` callers; see
|
|
1258
|
+
* `selectionDigestFor`), and a large share of this module's own
|
|
1259
|
+
* scope/correlation tests deliberately exercise that case — only
|
|
1260
|
+
* `undefined`/omission is refused for it, never a `null` that matches the
|
|
1261
|
+
* reservation's own stored `null`. Both are compared against the
|
|
1262
|
+
* BREAKER-OWNED, frozen `token` fields set at `admit()` time — never
|
|
1263
|
+
* against a caller's own mutable wrapper object. A caller must derive
|
|
1264
|
+
* these values from its own trusted, resolved launch (never trust them as
|
|
1265
|
+
* given by whatever object is borrowing this reservation): a reservation
|
|
1266
|
+
* minted for one invocation/selection can then never authorize dispatching
|
|
1267
|
+
* a different one, independent of what any outer wrapper object claims.
|
|
1268
|
+
*
|
|
1269
|
+
* Refuses (mutates nothing) for: an unowned/unknown/foreign token; a
|
|
1270
|
+
* second call on an already-`committing`/`dispatched` reservation; a
|
|
1271
|
+
* missing/null `invocationId` (on either side) or omitted
|
|
1272
|
+
* `selectionDigest`; a binding mismatch; or a reservation whose own
|
|
1273
|
+
* deadline has already passed (`now >= deadline`, consistent with every
|
|
1274
|
+
* other deadline check in this module).
|
|
1275
|
+
*/
|
|
1276
|
+
beginDispatch(token, { invocationId, selectionDigest } = {}) {
|
|
1277
|
+
const now = this.nowMs();
|
|
1278
|
+
this._sweepReservations(now);
|
|
1279
|
+
if (!token || typeof token.id !== 'string' || token.breaker !== this) {
|
|
1280
|
+
return { ok: false, reason: 'reservation is not owned by this breaker' };
|
|
1281
|
+
}
|
|
1282
|
+
const entry = this._reservations.get(token.id);
|
|
1283
|
+
if (!entry || entry.token !== token) {
|
|
1284
|
+
return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
|
|
1285
|
+
}
|
|
1286
|
+
if (invocationId == null || selectionDigest === undefined) {
|
|
1287
|
+
return { ok: false, reason: 'beginDispatch requires explicit invocationId and selectionDigest bindings' };
|
|
1288
|
+
}
|
|
1289
|
+
if (entry.token.invocationId == null) {
|
|
1290
|
+
return { ok: false, reason: 'reservation was admitted without an invocationId binding and can never dispatch' };
|
|
1291
|
+
}
|
|
1292
|
+
if (invocationId !== entry.token.invocationId || selectionDigest !== entry.token.selectionDigest) {
|
|
1293
|
+
return { ok: false, reason: 'reservation binding does not match the resolved dispatch' };
|
|
1294
|
+
}
|
|
1295
|
+
if (entry.phase !== 'admitted') {
|
|
1296
|
+
return { ok: false, reason: `reservation is not admitted (phase: ${entry.phase})` };
|
|
1297
|
+
}
|
|
1298
|
+
if (now >= token.deadlineMs) {
|
|
1299
|
+
return { ok: false, reason: 'reservation deadline has passed' };
|
|
1300
|
+
}
|
|
1301
|
+
const refused = this._dispatchScopeRefusal(token, now);
|
|
1302
|
+
if (refused) return refused;
|
|
1303
|
+
entry.phase = 'committing';
|
|
1304
|
+
return { ok: true };
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
/**
|
|
1308
|
+
* Commit a `committing` reservation to `dispatched`, after the hook's own
|
|
1309
|
+
* debit succeeded (#2915 round 8, D3): `committing -> dispatched`. Refuses
|
|
1310
|
+
* (mutates nothing but `revoked`, see below) for an unowned/unknown
|
|
1311
|
+
* token, one not currently `committing`, or one whose deadline passed
|
|
1312
|
+
* while committing — the latter sets `revoked` (idempotent) so the
|
|
1313
|
+
* caller's required `abortDispatch` call (below) applies the correct
|
|
1314
|
+
* "timed out while committing" handling, rather than silently
|
|
1315
|
+
* classifying a never-dispatched attempt as an ordinary clean abort.
|
|
1316
|
+
*/
|
|
1317
|
+
commitDispatch(token) {
|
|
1318
|
+
const now = this.nowMs();
|
|
1319
|
+
if (!token || typeof token.id !== 'string' || token.breaker !== this) {
|
|
1320
|
+
return { ok: false, reason: 'reservation is not owned by this breaker' };
|
|
1321
|
+
}
|
|
1322
|
+
const entry = this._reservations.get(token.id);
|
|
1323
|
+
if (!entry || entry.token !== token) {
|
|
1324
|
+
return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
|
|
1325
|
+
}
|
|
1326
|
+
if (entry.phase !== 'committing') {
|
|
1327
|
+
return { ok: false, reason: `reservation is not committing (phase: ${entry.phase})` };
|
|
1328
|
+
}
|
|
1329
|
+
if (now >= token.deadlineMs) {
|
|
1330
|
+
entry.revoked = true;
|
|
1331
|
+
return { ok: false, reason: 'reservation deadline passed while committing; call abortDispatch' };
|
|
1332
|
+
}
|
|
1333
|
+
if (entry.revoked) {
|
|
1334
|
+
return { ok: false, reason: 'reservation was revoked while committing; call abortDispatch' };
|
|
1335
|
+
}
|
|
1336
|
+
const refused = this._dispatchScopeRefusal(token, now);
|
|
1337
|
+
if (refused) return refused;
|
|
1338
|
+
entry.phase = 'dispatched';
|
|
1339
|
+
return { ok: true };
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
/**
|
|
1343
|
+
* Explicitly back out of a reservation that never reached `dispatched`
|
|
1344
|
+
* (#2915 round 8, D2/D3): the D2 hook calls this when its own work after
|
|
1345
|
+
* `beginDispatch` (the ledger debit) throws, or when `commitDispatch`
|
|
1346
|
+
* itself refused. Equivalent to `settle(token, { kind: 'uncertain',
|
|
1347
|
+
* confirmedStop: true })` — a PROVEN zero-dispatch exit, so a canary this
|
|
1348
|
+
* reservation held is treated exactly like any other unreported/aborted
|
|
1349
|
+
* probe (reopened with backoff unless explicitly caller-aborted) and is never left with a stop-evidence
|
|
1350
|
+
* marker, matching every other proven pre-spawn refusal in this module.
|
|
1351
|
+
* Refused once the reservation has actually reached `dispatched`: an
|
|
1352
|
+
* attempt that genuinely started must settle with its real outcome
|
|
1353
|
+
* through `settle()`, never a fabricated confirmed stop.
|
|
1354
|
+
*/
|
|
1355
|
+
abortDispatch(token, { callerAborted = false } = {}) {
|
|
1356
|
+
const entry = this._reservations.get(token?.id);
|
|
1357
|
+
if (entry && entry.token === token && entry.phase === 'dispatched') {
|
|
1358
|
+
return { ok: false, reason: 'reservation already dispatched; use settle() instead' };
|
|
1359
|
+
}
|
|
1360
|
+
return this.settle(token, { kind: 'uncertain', confirmedStop: true, callerAborted });
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
/**
|
|
1364
|
+
* Reconcile ONE invocation's reservation(s) from stop evidence arriving
|
|
1365
|
+
* independently of the original `admit()`/`settle()` pair (#2915 review
|
|
1366
|
+
* 6, finding 3d): a bounded stop-escalation observer, or an operator
|
|
1367
|
+
* acting on `ralph-external` diagnostics, may know only the invocation
|
|
1368
|
+
* id -- not the exact token object `admit()` minted, which may already be
|
|
1369
|
+
* out of scope by the time termination is confirmed. Only
|
|
1370
|
+
* `confirmedStop: true` releases anything; anything else is a no-op (late
|
|
1371
|
+
* or uncertain evidence can never resolve a marker or free capacity).
|
|
1372
|
+
* Clears any blocking marker naming the reservation, clears its
|
|
1373
|
+
* outstanding cohort membership, and deletes the reservation itself --
|
|
1374
|
+
* idempotent: an invocation with no matching live reservation is a no-op.
|
|
1375
|
+
*/
|
|
1376
|
+
reconcileStopEvidence({ invocationId, confirmedStop = false } = {}) {
|
|
1377
|
+
if (!confirmedStop || invocationId == null) return { released: 0 };
|
|
1378
|
+
let released = 0;
|
|
1379
|
+
for (const [, entry] of [...this._reservations]) {
|
|
1380
|
+
if (entry.token.invocationId !== invocationId) continue;
|
|
1381
|
+
if (entry.phase === 'dispatched') entry.stop = 'confirmed';
|
|
1382
|
+
this._release(entry);
|
|
1383
|
+
released += 1;
|
|
1384
|
+
}
|
|
1385
|
+
return { released };
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
/**
|
|
1389
|
+
* Read-only precondition check (#2915 round 8, D4): whether `invocationId`
|
|
1390
|
+
* currently holds a `dispatched` reservation with `stop: 'unresolved'` on
|
|
1391
|
+
* THIS breaker — exactly the condition the MCP `reconcile_stopped`
|
|
1392
|
+
* operation (src/mcp/tools/worker-invoke.mjs) must verify before calling
|
|
1393
|
+
* `reconcileStopEvidence`, so that operation only ever accepts (and only
|
|
1394
|
+
* ever reports as "applied") a reconciliation against a reservation that
|
|
1395
|
+
* genuinely needs one. Never mutates, never throws.
|
|
1396
|
+
*/
|
|
1397
|
+
hasUnresolvedStop(invocationId) {
|
|
1398
|
+
if (invocationId == null) return false;
|
|
1399
|
+
for (const entry of this._reservations.values()) {
|
|
1400
|
+
if (entry.token.invocationId === invocationId && entry.phase === 'dispatched' && entry.stop === 'unresolved') return true;
|
|
1401
|
+
}
|
|
1402
|
+
return false;
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
/**
|
|
1406
|
+
* Read-only enumeration (#2915 round 8, D4): every reservation on THIS
|
|
1407
|
+
* breaker currently sitting `dispatched` with `stop: 'unresolved'` — the
|
|
1408
|
+
* exact set an ordinary (no `invocation_id`) MCP `status` call surfaces as
|
|
1409
|
+
* "pending reconciliation", and the same set `admit()`'s own
|
|
1410
|
+
* `blocking-markers:<N>` reservation-capacity diagnostic already counts.
|
|
1411
|
+
* Never mutates, never throws.
|
|
1412
|
+
*/
|
|
1413
|
+
listUnresolvedStops(now = this.nowMs()) {
|
|
1414
|
+
const out = [];
|
|
1415
|
+
for (const entry of this._reservations.values()) {
|
|
1416
|
+
if (entry.phase === 'dispatched' && entry.stop === 'unresolved') {
|
|
1417
|
+
out.push({ invocationId: entry.token.invocationId, scopes: [...entry.token.coveringKeys], ageMs: Math.max(0, now - entry.token.mintedAtMs) });
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
return out;
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
getState(scopeKey) {
|
|
1424
|
+
const rec = this._scopes.get(scopeKey);
|
|
1425
|
+
if (!rec) return { scopeKey, state: 'closed', tracked: false };
|
|
1426
|
+
const inner = rec.breaker.getState();
|
|
1427
|
+
return {
|
|
1428
|
+
scopeKey,
|
|
1429
|
+
scope: rec.scope,
|
|
1430
|
+
tracked: true,
|
|
1431
|
+
state: rec.status,
|
|
1432
|
+
...(rec.operatorReason ? { operatorReason: rec.operatorReason } : {}),
|
|
1433
|
+
consecutiveFailures: inner.consecutiveFailures,
|
|
1434
|
+
distinctFailures: new Set(rec.failures.map((f) => f.invocationId)).size,
|
|
1435
|
+
outstanding: this._outstanding.get(scopeKey)?.size ?? 0,
|
|
1436
|
+
cohortSize: rec.cohort?.size ?? 0,
|
|
1437
|
+
activeCanary: rec.activeCanary ? { ...rec.activeCanary } : null,
|
|
1438
|
+
blockingMarker: rec.blockingMarker ? { ...rec.blockingMarker } : null,
|
|
1439
|
+
episodeCooldownMs: rec.episodeCooldownMs,
|
|
1440
|
+
nextProbeAtMs: rec.nextProbeAtMs,
|
|
1441
|
+
resetAtMs: rec.resetAtMs,
|
|
1442
|
+
attemptsObserved: rec.attemptsObserved,
|
|
1443
|
+
canariesObserved: rec.canariesObserved,
|
|
1444
|
+
cooldownRemainingMs: inner.cooldownRemainingMs,
|
|
1445
|
+
};
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
snapshot() {
|
|
1449
|
+
return {
|
|
1450
|
+
scopes: [...this._scopes.keys()].map((key) => this.getState(key)),
|
|
1451
|
+
attemptsObserved: this._observedInvocations.size,
|
|
1452
|
+
canariesObserved: this.canariesObserved,
|
|
1453
|
+
reservationsOutstanding: this._reservations.size,
|
|
1454
|
+
};
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
/** Correlate launcher-observed host faults across resolved inference backends.
|
|
1458
|
+
* Replay the first fault once corroborated, preserving the three-distinct
|
|
1459
|
+
* threshold even when the first backend alone could not establish scope.
|
|
1460
|
+
*/
|
|
1461
|
+
observeHostFailure({ invocationId, inferenceProvider, reasonCode } = {}) {
|
|
1462
|
+
if (typeof invocationId !== 'string' || !invocationId || typeof inferenceProvider !== 'string'
|
|
1463
|
+
|| !inferenceProvider || !['RESOURCE', 'SANDBOX'].includes(reasonCode)) return [];
|
|
1464
|
+
const now = this.nowMs();
|
|
1465
|
+
this._hostFailureFacts = this._hostFailureFacts.filter(fact => now - fact.atMs < this.policy.windowMs);
|
|
1466
|
+
if (!this._hostFailureFacts.some(fact => fact.invocationId === invocationId)) {
|
|
1467
|
+
this._hostFailureFacts.push({ invocationId, inferenceProvider, reasonCode, atMs: now,
|
|
1468
|
+
cohort: [...(this._outstanding.get('host:local')?.keys() ?? [])] });
|
|
1469
|
+
if (this._hostFailureFacts.length > this.policy.maxOutstandingReservations) this._hostFailureFacts.shift();
|
|
1470
|
+
}
|
|
1471
|
+
const corroboratedBackends = [...new Set(this._hostFailureFacts.map(fact => fact.inferenceProvider))];
|
|
1472
|
+
if (corroboratedBackends.length < 2) return [];
|
|
1473
|
+
const rec = this._scopeRecordBounded(FailureScope.HOST, 'host:local');
|
|
1474
|
+
const first = this._hostFailureFacts[0];
|
|
1475
|
+
if (rec && (rec.cohortFrozenAt === null || now - rec.cohortFrozenAt >= this.policy.windowMs)) {
|
|
1476
|
+
// The first fault could not yet establish host scope. Preserve who was
|
|
1477
|
+
// actually outstanding then; later sequential attempts are not a cohort.
|
|
1478
|
+
rec.cohort = new Set(first.cohort);
|
|
1479
|
+
rec.cohortFrozenAt = first.atMs;
|
|
1480
|
+
}
|
|
1481
|
+
for (const fact of this._hostFailureFacts) {
|
|
1482
|
+
if (rec?.failures.some(failure => failure.invocationId === fact.invocationId)) continue;
|
|
1483
|
+
this._settleFailure({ invocationId: fact.invocationId, canaryKeys: [] }, {
|
|
1484
|
+
reasonCode: fact.reasonCode, evidenceSource: 'launcher', hostId: 'local', corroboratedBackends,
|
|
1485
|
+
}, fact.atMs);
|
|
1486
|
+
}
|
|
1487
|
+
return corroboratedBackends;
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
/**
|
|
1491
|
+
* Reset ONE scope's failure history/backoff bookkeeping (#2915 round 8,
|
|
1492
|
+
* D3, replaces the prior whole-reservation-release behavior): resets
|
|
1493
|
+
* failure history for that scope ONLY, and never releases a reservation
|
|
1494
|
+
* with unresolved execution.
|
|
1495
|
+
*
|
|
1496
|
+
* #2915 round 8 critique ("whole-reservation reset can erase live
|
|
1497
|
+
* protection"): the prior implementation deleted EVERY reservation
|
|
1498
|
+
* covering `scopeKey` outright — including one also covering a DIFFERENT
|
|
1499
|
+
* scope as a canary, so `reset(A)` could silently drop scope B's own
|
|
1500
|
+
* still-relevant `activeCanary`/`blockingMarker` protection too. This
|
|
1501
|
+
* version never touches `_reservations` at all: if the targeted scope
|
|
1502
|
+
* currently holds a covering reservation, `activeCanary` or `blockingMarker` (unresolved
|
|
1503
|
+
* execution — someone else's covering scopes are never even inspected,
|
|
1504
|
+
* let alone released), the whole call is refused with the reconcile
|
|
1505
|
+
* instruction and NOTHING is mutated; otherwise the scope record is
|
|
1506
|
+
* destroyed outright (the same effect the prior "safe" case always had).
|
|
1507
|
+
*
|
|
1508
|
+
* #2915 round 8 review 8 (C1, "unscoped reset() also clears protection
|
|
1509
|
+
* outright"): called with no `scopeKey`, this now applies the EXACT same
|
|
1510
|
+
* per-scope rule to every tracked scope — reset failure history/backoff
|
|
1511
|
+
* for every scope that has no unresolved execution, and leave any scope
|
|
1512
|
+
* that does (an active canary or a blocking marker) completely untouched
|
|
1513
|
+
* — rather than unconditionally destroying every scope record AND
|
|
1514
|
+
* clearing `_reservations` outright. `_reservations` is never touched by
|
|
1515
|
+
* `reset()` in either form; `_release` (via `settle()`/
|
|
1516
|
+
* `reconcileStopEvidence`) is the only code that ever removes a
|
|
1517
|
+
* reservation. A full, unconditional teardown (including live execution
|
|
1518
|
+
* protection) remains available through the separate, explicitly-named
|
|
1519
|
+
* `destroy()` method, intended for test/process-exit teardown only.
|
|
1520
|
+
*/
|
|
1521
|
+
reset(scopeKey) {
|
|
1522
|
+
if (!scopeKey) this._hostFailureFacts = [];
|
|
1523
|
+
if (!scopeKey) {
|
|
1524
|
+
for (const [key, rec] of [...this._scopes]) {
|
|
1525
|
+
if (rec.activeCanary !== null || rec.blockingMarker !== null) continue; // keep execution ownership
|
|
1526
|
+
this._transition(rec, 'closed', 'operator reset; resume covered launches');
|
|
1527
|
+
rec.breaker.destroy();
|
|
1528
|
+
this._scopes.delete(key);
|
|
1529
|
+
this._outstanding.delete(key);
|
|
1530
|
+
}
|
|
1531
|
+
return { ok: true };
|
|
1532
|
+
}
|
|
1533
|
+
const rec = this._scopes.get(scopeKey);
|
|
1534
|
+
if (!rec) {
|
|
1535
|
+
if (scopeKey === 'host:local') this._hostFailureFacts = [];
|
|
1536
|
+
return { ok: true };
|
|
1537
|
+
}
|
|
1538
|
+
if (rec.activeCanary !== null || rec.blockingMarker !== null
|
|
1539
|
+
|| [...this._reservations.values()].some(entry => entry.token.coveringKeys.includes(scopeKey))) {
|
|
1540
|
+
return { ok: false, scopeKey,
|
|
1541
|
+
reason: 'scope has active or unresolved execution (a reservation, active canary or unresolved stop-evidence marker); reconcile stop evidence (reconcileStopEvidence) before resetting it' };
|
|
1542
|
+
}
|
|
1543
|
+
if (scopeKey === 'host:local') this._hostFailureFacts = [];
|
|
1544
|
+
this._transition(rec, 'closed', 'operator reset; resume covered launches');
|
|
1545
|
+
rec.breaker.destroy();
|
|
1546
|
+
this._scopes.delete(scopeKey);
|
|
1547
|
+
this._outstanding.delete(scopeKey);
|
|
1548
|
+
return { ok: true };
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
destroy() {
|
|
1552
|
+
this._hostFailureFacts = [];
|
|
1553
|
+
for (const rec of this._scopes.values()) rec.breaker.destroy();
|
|
1554
|
+
this._scopes.clear();
|
|
1555
|
+
this._outstanding.clear();
|
|
1556
|
+
this._reservations.clear();
|
|
1557
|
+
this.removeAllListeners();
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
// --- Private ---
|
|
1561
|
+
|
|
1562
|
+
/**
|
|
1563
|
+
* Create-or-fetch a tracked scope record, enforcing a true hard cap
|
|
1564
|
+
* (#2915 final review, finding 6): once at `maxTrackedScopes` with no
|
|
1565
|
+
* CLOSED/memberless scope to evict, a brand-new scope key is refused
|
|
1566
|
+
* tracking entirely — `null` — rather than growing past the bound or ever
|
|
1567
|
+
* evicting an OPEN/outstanding scope. The caller still charges the
|
|
1568
|
+
* attempt; it simply cannot attribute it to a persistently-tracked scope
|
|
1569
|
+
* record under this adversarial condition.
|
|
1570
|
+
*
|
|
1571
|
+
* Invocation-scoped records are counted and bounded SEPARATELY from every
|
|
1572
|
+
* shared scope (installation/quota/provider/host/model), against
|
|
1573
|
+
* `maxChargedAttempts` rather than `maxTrackedScopes` (#2915 review 4,
|
|
1574
|
+
* finding 6: "invocation-local records outside the shared-scope quota").
|
|
1575
|
+
* An invocation record can never legitimately open (cohort/distinct
|
|
1576
|
+
* counting is always 1 for a single invocation id), so a flood of
|
|
1577
|
+
* single-task failures can never exhaust the small shared-scope budget
|
|
1578
|
+
* that genuinely correlated installation/host/provider/quota outages
|
|
1579
|
+
* depend on, and vice versa — two OPEN shared scopes can never be forced
|
|
1580
|
+
* out to make room for one-off invocation bookkeeping.
|
|
1581
|
+
*/
|
|
1582
|
+
_scopeRecordBounded(scope, scopeKey) {
|
|
1583
|
+
let rec = this._scopes.get(scopeKey);
|
|
1584
|
+
if (rec) return rec;
|
|
1585
|
+
const isInvocation = scope === FailureScope.INVOCATION;
|
|
1586
|
+
const cap = isInvocation ? this.policy.maxChargedAttempts : this.policy.maxTrackedScopes;
|
|
1587
|
+
const count = this._scopeCount(isInvocation);
|
|
1588
|
+
if (count >= cap && !this._evictInertScopeIfPossible(isInvocation)) {
|
|
1589
|
+
return null;
|
|
1590
|
+
}
|
|
1591
|
+
rec = {
|
|
1592
|
+
scope,
|
|
1593
|
+
scopeKey,
|
|
1594
|
+
// Scope-level status is authoritative for gates and reports; the
|
|
1595
|
+
// inner breaker owns only the half-open transitions, whose
|
|
1596
|
+
// timer-driven states must never read as scope recovery. The inner
|
|
1597
|
+
// breaker shares this instance's injected clock (one clock, #2915E).
|
|
1598
|
+
status: 'closed',
|
|
1599
|
+
breaker: new ConsecutiveFailureCircuitBreaker({
|
|
1600
|
+
failureThreshold: 1,
|
|
1601
|
+
cooldownMs: this.policy.cooldownMs,
|
|
1602
|
+
halfOpenMaxProbes: 1,
|
|
1603
|
+
nowMs: this.nowMs,
|
|
1604
|
+
}),
|
|
1605
|
+
failures: [],
|
|
1606
|
+
cohort: null,
|
|
1607
|
+
cohortFrozenAt: null,
|
|
1608
|
+
evidence: new Map(),
|
|
1609
|
+
resetAtMs: null,
|
|
1610
|
+
openedAtMs: null,
|
|
1611
|
+
nextProbeAtMs: 0,
|
|
1612
|
+
episodeCooldownMs: this.policy.cooldownMs,
|
|
1613
|
+
activeCanary: null,
|
|
1614
|
+
// #2915 review 5: a bounded marker set when a canary's own deadline
|
|
1615
|
+
// passes with stop unconfirmed (see `_sweepReservations`); cleared
|
|
1616
|
+
// only by a `settle()` call carrying confirmed-termination evidence.
|
|
1617
|
+
blockingMarker: null,
|
|
1618
|
+
reopens: 0,
|
|
1619
|
+
attemptsObserved: 0,
|
|
1620
|
+
canariesObserved: 0,
|
|
1621
|
+
lastTouchedMs: this.nowMs(),
|
|
1622
|
+
};
|
|
1623
|
+
this._scopes.set(scopeKey, rec);
|
|
1624
|
+
return rec;
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
/** Last-touched bookkeeping for LRU eviction of inert (closed, memberless) scopes. */
|
|
1628
|
+
_touch(rec) {
|
|
1629
|
+
rec.lastTouchedMs = this.nowMs();
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
/** Count tracked scopes in one category (invocation-scoped vs every
|
|
1633
|
+
* shared scope) — the two independent quotas `_scopeRecordBounded`
|
|
1634
|
+
* enforces. Bounded by `_scopes.size`, itself bounded by the two maxima. */
|
|
1635
|
+
_scopeCount(isInvocation) {
|
|
1636
|
+
let count = 0;
|
|
1637
|
+
for (const rec of this._scopes.values()) {
|
|
1638
|
+
if ((rec.scope === FailureScope.INVOCATION) === isInvocation) count += 1;
|
|
1639
|
+
}
|
|
1640
|
+
return count;
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
/**
|
|
1644
|
+
* Explicit, bounded eviction (#2915 review: settled outstanding/scope maps
|
|
1645
|
+
* never dropped their own keys, so 1,000 successful attempts retained
|
|
1646
|
+
* 1,000+ map entries forever). Only CLOSED scopes with no outstanding
|
|
1647
|
+
* members AND no failure evidence still inside the rolling window are
|
|
1648
|
+
* eligible — an open scope, one with outstanding attempts, or one with
|
|
1649
|
+
* recent (in-window) failure evidence is never evicted (#2915 review 4,
|
|
1650
|
+
* finding 6: "never evict recent failure evidence or OPEN scopes" — a
|
|
1651
|
+
* two-failure CLOSED scope that has not yet reached the distinct
|
|
1652
|
+
* threshold must survive long enough for a third, still-in-window
|
|
1653
|
+
* failure to correlate against it). Only scopes matching `isInvocation`
|
|
1654
|
+
* are considered, so eviction never crosses between the two independent
|
|
1655
|
+
* quotas `_scopeRecordBounded` enforces. The scan is bounded by
|
|
1656
|
+
* `maxTrackedScopes`/`maxChargedAttempts`, never unbounded. Returns
|
|
1657
|
+
* whether an eviction actually freed a slot, so `_scopeRecordBounded` can
|
|
1658
|
+
* refuse to track a new scope at true capacity instead of growing past
|
|
1659
|
+
* the cap (#2915 final review, finding 6: the former best-effort bound
|
|
1660
|
+
* could be exceeded whenever nothing was evictable).
|
|
1661
|
+
*/
|
|
1662
|
+
_evictInertScopeIfPossible(isInvocation) {
|
|
1663
|
+
let oldestKey = null;
|
|
1664
|
+
let oldestAt = Infinity;
|
|
1665
|
+
for (const [key, rec] of this._scopes) {
|
|
1666
|
+
if ((rec.scope === FailureScope.INVOCATION) !== isInvocation) continue;
|
|
1667
|
+
if (!this._isInertScope(rec, key)) continue;
|
|
1668
|
+
if (rec.lastTouchedMs < oldestAt) { oldestAt = rec.lastTouchedMs; oldestKey = key; }
|
|
1669
|
+
}
|
|
1670
|
+
if (oldestKey === null) return false;
|
|
1671
|
+
this._scopes.get(oldestKey)?.breaker.destroy();
|
|
1672
|
+
this._scopes.delete(oldestKey);
|
|
1673
|
+
this._outstanding.delete(oldestKey);
|
|
1674
|
+
return true;
|
|
1675
|
+
}
|
|
1676
|
+
|
|
1677
|
+
/**
|
|
1678
|
+
* Shared inertness predicate (#2915 review 6, finding 2): CLOSED, no
|
|
1679
|
+
* active canary, no blocking marker, no outstanding members, and no
|
|
1680
|
+
* failure evidence still inside the rolling window. A blocking marker
|
|
1681
|
+
* (#2915 review 5) is never memberless/inert even if `status` somehow
|
|
1682
|
+
* read otherwise — never evict or release it. Used by both LRU eviction
|
|
1683
|
+
* (`_evictInertScopeIfPossible`, which also needs the oldest-touched
|
|
1684
|
+
* ordering) and targeted, single-key release (`_releaseIfInert`).
|
|
1685
|
+
*/
|
|
1686
|
+
_isInertScope(rec, scopeKey) {
|
|
1687
|
+
if (!rec || rec.status !== 'closed' || rec.activeCanary || rec.blockingMarker) return false;
|
|
1688
|
+
const outstanding = this._outstanding.get(scopeKey);
|
|
1689
|
+
if (outstanding && outstanding.size > 0) return false;
|
|
1690
|
+
const now = this.nowMs();
|
|
1691
|
+
if (rec.failures.some((f) => now - f.atMs < this.policy.windowMs)) return false;
|
|
1692
|
+
return true;
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
/**
|
|
1696
|
+
* Release ONE shared (never invocation-scoped — that quota is bounded
|
|
1697
|
+
* separately and far larger) scope record back to the tracking pool if it
|
|
1698
|
+
* is currently inert (#2915 review 6, finding 2: "release it on
|
|
1699
|
+
* settlement when the scope ends inert" / admission rollback). A no-op,
|
|
1700
|
+
* returning `false`, for an unknown, invocation-scoped, or non-inert
|
|
1701
|
+
* scope — never discards a scope still holding relevant failure history,
|
|
1702
|
+
* an active canary, a blocking marker, or outstanding members.
|
|
1703
|
+
*/
|
|
1704
|
+
_releaseIfInert(scopeKey) {
|
|
1705
|
+
const rec = this._scopes.get(scopeKey);
|
|
1706
|
+
if (!rec || rec.scope === FailureScope.INVOCATION || !this._isInertScope(rec, scopeKey)) return false;
|
|
1707
|
+
rec.breaker.destroy();
|
|
1708
|
+
this._scopes.delete(scopeKey);
|
|
1709
|
+
this._outstanding.delete(scopeKey);
|
|
1710
|
+
return true;
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
_removeOutstanding(scopeKey, invocationId) {
|
|
1714
|
+
const set = this._outstanding.get(scopeKey);
|
|
1715
|
+
if (!set || invocationId == null) return;
|
|
1716
|
+
set.delete(invocationId);
|
|
1717
|
+
if (set.size === 0) this._outstanding.delete(scopeKey);
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
_resolveResetAtMs({ resetAtMs, retryAfterSeconds }, now) {
|
|
1721
|
+
if (Number.isFinite(resetAtMs) && resetAtMs > now) return resetAtMs;
|
|
1722
|
+
if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0) {
|
|
1723
|
+
return now + Math.min(retryAfterSeconds, this.policy.maxRetryAfterSeconds) * 1000;
|
|
1724
|
+
}
|
|
1725
|
+
return null;
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
_boundedJitter() {
|
|
1729
|
+
const cap = this.policy.maxJitterMs;
|
|
1730
|
+
const value = Math.floor(Number(this.jitterMs()) || 0);
|
|
1731
|
+
return Math.max(0, Math.min(cap, value));
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
/** Reopen (or stay open) with doubled backoff, capped at maxCooldownMs. */
|
|
1735
|
+
_reopen(rec, now) {
|
|
1736
|
+
rec.reopens += 1;
|
|
1737
|
+
rec.episodeCooldownMs = Math.min(
|
|
1738
|
+
this.policy.maxCooldownMs,
|
|
1739
|
+
this.policy.cooldownMs * (this.policy.backoffMultiplier ** rec.reopens),
|
|
1740
|
+
);
|
|
1741
|
+
rec.openedAtMs = now;
|
|
1742
|
+
rec.nextProbeAtMs = Math.max(now + rec.episodeCooldownMs, rec.resetAtMs ?? 0) + this._boundedJitter();
|
|
1743
|
+
rec.breaker.cooldownMs = rec.episodeCooldownMs;
|
|
1744
|
+
rec.breaker.recordFailure();
|
|
1745
|
+
this._transition(rec, 'open', 'probe/canary failed; wait for backoff then retry one probe');
|
|
1746
|
+
this._touch(rec);
|
|
1747
|
+
}
|
|
1748
|
+
}
|