@aiwg/cli 2026.10.1 → 2026.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (361) hide show
  1. package/README.md +2 -2
  2. package/agentic/code/providers/capability-matrix.yaml +270 -0
  3. package/agentic/code/providers/model-capabilities.v1.json +3183 -24
  4. package/bin/aiwg.mjs +135 -47
  5. package/dist/src/a2a/codecs.js +9 -0
  6. package/dist/src/agents/grounding/knowledge-base.js +54 -6
  7. package/dist/src/artifacts/browser-export.js +6 -0
  8. package/dist/src/artifacts/checksum-manifest.js +1 -0
  9. package/dist/src/artifacts/citation-parser.js +173 -92
  10. package/dist/src/artifacts/citation-table-schema.js +59 -0
  11. package/dist/src/artifacts/cli.js +57 -18
  12. package/dist/src/artifacts/corpus-tools/snapshot.js +1 -1
  13. package/dist/src/artifacts/corpus-tools/vision-extract.js +107 -15
  14. package/dist/src/artifacts/corpus-views/ref-parser.js +3 -1
  15. package/dist/src/artifacts/corpus-views/renderers.js +3 -3
  16. package/dist/src/artifacts/discovery-eval.js +6 -4
  17. package/dist/src/artifacts/fortemi-core-query-adapter.js +36 -3
  18. package/dist/src/artifacts/fortemi-core-sync.js +166 -40
  19. package/dist/src/artifacts/framework-capabilities.js +171 -0
  20. package/dist/src/artifacts/framework-index-paths.js +73 -0
  21. package/dist/src/artifacts/framework-source-digest.js +75 -0
  22. package/dist/src/artifacts/index-builder.js +285 -131
  23. package/dist/src/artifacts/index-files.js +132 -18
  24. package/dist/src/artifacts/index-preparation.js +65 -0
  25. package/dist/src/artifacts/index-refresh.js +576 -0
  26. package/dist/src/artifacts/index-worker.mjs +9 -0
  27. package/dist/src/artifacts/prebuilt-build-lock.js +47 -4
  28. package/dist/src/artifacts/prebuilt-deployment.js +46 -0
  29. package/dist/src/artifacts/prebuilt-relations.js +80 -0
  30. package/dist/src/artifacts/query-engine.js +410 -88
  31. package/dist/src/artifacts/relations-builder.js +168 -0
  32. package/dist/src/artifacts/relations.js +148 -0
  33. package/dist/src/artifacts/repair.js +9 -4
  34. package/dist/src/artifacts/stats.js +5 -2
  35. package/dist/src/artifacts/types.js +30 -10
  36. package/dist/src/artifacts/watcher.js +1 -1
  37. package/dist/src/auth/credential-store.js +2 -1
  38. package/dist/src/catalog/cli.d.mts +5 -0
  39. package/dist/src/channel/manager.d.mts +20 -0
  40. package/dist/src/channel/manager.mjs +78 -12
  41. package/dist/src/cli/child-process-result.js +40 -0
  42. package/dist/src/cli/command-log.js +2 -3
  43. package/dist/src/cli/env.js +38 -16
  44. package/dist/src/cli/flag-parsing.js +236 -0
  45. package/dist/src/cli/handlers/artifacts.js +3 -0
  46. package/dist/src/cli/handlers/auth.js +3 -0
  47. package/dist/src/cli/handlers/decision.js +218 -5
  48. package/dist/src/cli/handlers/diagnose.js +13 -5
  49. package/dist/src/cli/handlers/effect.js +4 -2
  50. package/dist/src/cli/handlers/evidence.js +3 -0
  51. package/dist/src/cli/handlers/feedback.js +14 -5
  52. package/dist/src/cli/handlers/help.js +1 -0
  53. package/dist/src/cli/handlers/index.js +3 -1
  54. package/dist/src/cli/handlers/init.js +248 -30
  55. package/dist/src/cli/handlers/lint.js +5 -0
  56. package/dist/src/cli/handlers/mc.js +23 -0
  57. package/dist/src/cli/handlers/models.js +169 -10
  58. package/dist/src/cli/handlers/ralph-launcher.js +2 -0
  59. package/dist/src/cli/handlers/ralph.js +3 -0
  60. package/dist/src/cli/handlers/refresh.js +28 -0
  61. package/dist/src/cli/handlers/regenerate.js +14 -0
  62. package/dist/src/cli/handlers/resource-versions.js +3 -0
  63. package/dist/src/cli/handlers/rules.js +195 -0
  64. package/dist/src/cli/handlers/run.js +4 -0
  65. package/dist/src/cli/handlers/runtime-info.js +25 -11
  66. package/dist/src/cli/handlers/script-runner.js +19 -8
  67. package/dist/src/cli/handlers/session.js +406 -62
  68. package/dist/src/cli/handlers/sessions.js +243 -76
  69. package/dist/src/cli/handlers/setup-manifest.js +474 -79
  70. package/dist/src/cli/handlers/setup.js +180 -39
  71. package/dist/src/cli/handlers/steward.js +11 -0
  72. package/dist/src/cli/handlers/subcommands.js +60 -3
  73. package/dist/src/cli/handlers/team.js +18 -8
  74. package/dist/src/cli/handlers/uhp.js +3 -0
  75. package/dist/src/cli/handlers/use.js +307 -55
  76. package/dist/src/cli/handlers/utilities.js +125 -26
  77. package/dist/src/cli/handlers/version.js +8 -0
  78. package/dist/src/cli/prompt-utils.js +72 -0
  79. package/dist/src/cli/provider-resolution.js +11 -0
  80. package/dist/src/cli/router.js +15 -1
  81. package/dist/src/cli/services/deployment-verification.js +19 -9
  82. package/dist/src/cli/services/provider-prepare-coordinator.js +24 -0
  83. package/dist/src/cli/skill-usage.js +2 -3
  84. package/dist/src/config/aiwg-config.js +349 -35
  85. package/dist/src/config/cli.js +256 -23
  86. package/dist/src/config/decision-config.js +106 -0
  87. package/dist/src/config/dispatch-thresholds-runtime.d.mts +8 -0
  88. package/dist/src/config/dispatch-thresholds-runtime.mjs +38 -0
  89. package/dist/src/config/gitignore.js +50 -0
  90. package/dist/src/config/index-graph-overrides.js +36 -0
  91. package/dist/src/config/project-artifacts-health.d.mts +37 -0
  92. package/dist/src/config/project-artifacts-health.mjs +14 -3
  93. package/dist/src/config/project-artifacts-runtime.d.mts +16 -0
  94. package/dist/src/config/user-config-dir.d.mts +9 -0
  95. package/dist/src/config/user-config.js +38 -11
  96. package/dist/src/config/user-registry.js +23 -15
  97. package/dist/src/config/user-state.js +7 -0
  98. package/dist/src/decision/adapters/jev.js +75 -27
  99. package/dist/src/decision/ask.js +203 -43
  100. package/dist/src/decision/context-live-qualification.js +2 -2
  101. package/dist/src/decision/driver.js +14 -7
  102. package/dist/src/decision/ensemble/aggregate.js +23 -10
  103. package/dist/src/decision/ensemble-study/multifact-study.js +6 -2
  104. package/dist/src/decision/entry.js +25 -6
  105. package/dist/src/decision/graph-live-qualification.js +1 -1
  106. package/dist/src/decision/graph-run.js +9 -0
  107. package/dist/src/decision/heldout/collector.js +15 -14
  108. package/dist/src/decision/heldout/contract.js +30 -7
  109. package/dist/src/decision/index.js +1 -0
  110. package/dist/src/decision/interop.js +60 -8
  111. package/dist/src/decision/jev-credentials.js +15 -1
  112. package/dist/src/decision/ledger.js +942 -0
  113. package/dist/src/decision/offload-activation.js +137 -0
  114. package/dist/src/decision/offload.js +550 -0
  115. package/dist/src/decision/qualification/load.js +4 -6
  116. package/dist/src/decision/savings.js +28 -0
  117. package/dist/src/decision/validate.js +6 -3
  118. package/dist/src/effects/dispatch-verifier.js +252 -0
  119. package/dist/src/effects/index.js +1 -0
  120. package/dist/src/effects/types.js +5 -1
  121. package/dist/src/effects/verifiers/git.js +2 -1
  122. package/dist/src/effects/verifiers/tracker.js +26 -14
  123. package/dist/src/extensions/commands/definitions.js +19 -0
  124. package/dist/src/extensions/project-local-remove.js +36 -1
  125. package/dist/src/features/installer.js +2 -2
  126. package/dist/src/flow/graph-metadata.js +5 -0
  127. package/dist/src/gates/providers/loader.js +64 -18
  128. package/dist/src/installation/manager-command.d.mts +18 -0
  129. package/dist/src/installation/manager-command.mjs +15 -3
  130. package/dist/src/installation/manager.d.mts +49 -0
  131. package/dist/src/installation/manager.mjs +95 -5
  132. package/dist/src/issues/cli.js +4 -4
  133. package/dist/src/issues/local.js +24 -5
  134. package/dist/src/jobs/runner.js +1 -1
  135. package/dist/src/lint/cli.js +27 -1
  136. package/dist/src/lint/loader.js +2 -1
  137. package/dist/src/lint/reporters.js +25 -1
  138. package/dist/src/lint/rule-activation.js +43 -0
  139. package/dist/src/lint/runner.js +137 -62
  140. package/dist/src/mcp/adapters/codex-runtime.js +366 -56
  141. package/dist/src/mcp/claude-install.d.mts +4 -0
  142. package/dist/src/mcp/claude-install.mjs +41 -0
  143. package/dist/src/mcp/cli.d.mts +1 -0
  144. package/dist/src/mcp/cli.mjs +620 -85
  145. package/dist/src/mcp/config-file.d.mts +28 -0
  146. package/dist/src/mcp/config-file.mjs +279 -0
  147. package/dist/src/mcp/config-layers.d.mts +36 -0
  148. package/dist/src/mcp/config-layers.mjs +348 -0
  149. package/dist/src/mcp/credentials.d.mts +51 -0
  150. package/dist/src/mcp/credentials.mjs +484 -0
  151. package/dist/src/mcp/grok-build-config.d.mts +6 -0
  152. package/dist/src/mcp/grok-build-config.mjs +17 -27
  153. package/dist/src/mcp/helpers.mjs +109 -4
  154. package/dist/src/mcp/muse-native-config.d.mts +13 -0
  155. package/dist/src/mcp/muse-native-config.mjs +1 -0
  156. package/dist/src/mcp/omp-config.d.mts +4 -0
  157. package/dist/src/mcp/omp-config.mjs +50 -29
  158. package/dist/src/mcp/profiles.js +97 -23
  159. package/dist/src/mcp/profiles.mjs +99 -27
  160. package/dist/src/mcp/registry.js +248 -131
  161. package/dist/src/mcp/registry.mjs +169 -100
  162. package/dist/src/mcp/server-config.d.mts +3 -0
  163. package/dist/src/mcp/server-config.mjs +89 -0
  164. package/dist/src/mcp/server.mjs +127 -82
  165. package/dist/src/mcp/test-simple.mjs +2 -1
  166. package/dist/src/mcp/toml-editor.d.mts +4 -0
  167. package/dist/src/mcp/toml-string.d.mts +1 -0
  168. package/dist/src/mcp/toml-string.mjs +9 -0
  169. package/dist/src/mcp/toml-strip-mcp.d.mts +1 -0
  170. package/dist/src/mcp/toml-strip-mcp.mjs +135 -0
  171. package/dist/src/mcp/tool-filters.d.mts +46 -0
  172. package/dist/src/mcp/tool-filters.mjs +357 -0
  173. package/dist/src/mcp/tools/agentic-sandbox.mjs +3 -1
  174. package/dist/src/mcp/tools/decision.mjs +22 -3
  175. package/dist/src/mcp/tools/discovery.mjs +62 -55
  176. package/dist/src/mcp/tools/orchestration.mjs +50 -2
  177. package/dist/src/mcp/tools/subsystems.mjs +39 -8
  178. package/dist/src/mcp/tools/worker-invoke.mjs +2856 -0
  179. package/dist/src/mcp/tools/worker-shell-control.mjs +289 -0
  180. package/dist/src/mcp/tools/worker-stdin-runtime.mjs +79 -0
  181. package/dist/src/memory/project-registry.js +2 -2
  182. package/dist/src/mission-protocol/codecs.js +647 -1
  183. package/dist/src/mission-protocol/types.js +110 -0
  184. package/dist/src/mission-protocol/worker-status.js +2 -0
  185. package/dist/src/models/config-loader.js +76 -14
  186. package/dist/src/models/grok-build-models.js +93 -41
  187. package/dist/src/models/model-capabilities.v1.json +3183 -24
  188. package/dist/src/models/model-discovery.js +931 -67
  189. package/dist/src/models/provider-policy.js +719 -11
  190. package/dist/src/models/qualification-digest.d.mts +2 -0
  191. package/dist/src/models/qualification-digest.mjs +17 -0
  192. package/dist/src/models/resolver.js +261 -0
  193. package/dist/src/models/worker-direct-readiness.js +550 -0
  194. package/dist/src/models/worker-dispatch-offer.js +169 -0
  195. package/dist/src/models/worker-health-readiness.js +144 -0
  196. package/dist/src/models/worker-installation.d.mts +39 -0
  197. package/dist/src/models/worker-installation.mjs +986 -0
  198. package/dist/src/models/wrapper-route.js +18 -0
  199. package/dist/src/ops/registry.js +9 -14
  200. package/dist/src/plugins/standalone-packager.js +4 -3
  201. package/dist/src/process/bounded-child-process.d.mts +84 -0
  202. package/dist/src/process/bounded-child-process.mjs +430 -0
  203. package/dist/src/process/cli-command.d.mts +16 -0
  204. package/dist/src/process/cli-command.mjs +75 -0
  205. package/dist/src/process/signal-child.d.mts +6 -0
  206. package/dist/src/process/signal-child.mjs +7 -0
  207. package/dist/src/providers/capability-matrix.js +8 -0
  208. package/dist/src/providers/capability-matrix.yaml +270 -0
  209. package/dist/src/providers/deployed-skill-roots.js +20 -0
  210. package/dist/src/providers/grok-build-acp.js +3 -2
  211. package/dist/src/providers/grok-build-headless.js +53 -23
  212. package/dist/src/providers/grok-build-inspect.js +1 -0
  213. package/dist/src/providers/omp-agent.d.mts +8 -0
  214. package/dist/src/providers/omp-diagnostics.d.mts +1 -0
  215. package/dist/src/providers/omp-paths.d.mts +13 -0
  216. package/dist/src/providers/provider-definitions.js +9 -1
  217. package/dist/src/providers/provider-definitions.mjs +9 -2
  218. package/dist/src/providers/skill-ownership.d.mts +9 -0
  219. package/dist/src/providers/skill-ownership.mjs +88 -0
  220. package/dist/src/rules/activation.d.mts +21 -0
  221. package/dist/src/rules/activation.mjs +62 -0
  222. package/dist/src/rules/inline-budget.d.mts +26 -0
  223. package/dist/src/rules/inline-budget.mjs +66 -0
  224. package/dist/src/rules/inventory.js +129 -0
  225. package/dist/src/rules/pin-state.js +73 -0
  226. package/dist/src/rules/pins.d.mts +14 -0
  227. package/dist/src/rules/pins.mjs +112 -0
  228. package/dist/src/rules/rule-frontmatter.schema.json +197 -0
  229. package/dist/src/serve/executor-registry.js +16 -24
  230. package/dist/src/serve/fleet-mission-conductor.js +9 -0
  231. package/dist/src/serve/mission-conductor.js +47 -12
  232. package/dist/src/serve/shared-host-scheduler.js +2 -0
  233. package/dist/src/serve/worker-invocation.js +558 -0
  234. package/dist/src/serve/worker-stdin-frame.d.mts +10 -0
  235. package/dist/src/serve/worker-stdin-frame.mjs +107 -0
  236. package/dist/src/sessions/adapter-registry.js +49 -0
  237. package/dist/src/sessions/adapters/claude.js +39 -54
  238. package/dist/src/sessions/adapters/codex.js +82 -67
  239. package/dist/src/sessions/adapters/copilot.js +4 -4
  240. package/dist/src/sessions/adapters/cursor.js +40 -57
  241. package/dist/src/sessions/adapters/deepseek-harness.js +15 -12
  242. package/dist/src/sessions/adapters/factory.js +31 -27
  243. package/dist/src/sessions/adapters/generic.js +43 -9
  244. package/dist/src/sessions/adapters/grok-build.js +11 -11
  245. package/dist/src/sessions/adapters/grokbot-locator.js +111 -0
  246. package/dist/src/sessions/adapters/grokbot.js +434 -24
  247. package/dist/src/sessions/adapters/hermes.js +27 -29
  248. package/dist/src/sessions/adapters/muse-workspace-scan.js +128 -0
  249. package/dist/src/sessions/adapters/muse.js +200 -104
  250. package/dist/src/sessions/adapters/omp.js +28 -14
  251. package/dist/src/sessions/adapters/openclaw.js +89 -79
  252. package/dist/src/sessions/adapters/opencode.js +27 -24
  253. package/dist/src/sessions/adapters/openhuman.js +18 -60
  254. package/dist/src/sessions/adapters/pi.js +30 -14
  255. package/dist/src/sessions/adapters/warp.js +3 -3
  256. package/dist/src/sessions/adapters/windsurf.js +19 -48
  257. package/dist/src/sessions/batch-contracts.js +25 -5
  258. package/dist/src/sessions/batch-import.js +72 -51
  259. package/dist/src/sessions/candidates.js +47 -11
  260. package/dist/src/sessions/contracts.js +75 -2
  261. package/dist/src/sessions/coverage-preflight.js +134 -0
  262. package/dist/src/sessions/discovery.js +3 -0
  263. package/dist/src/sessions/export-plan.js +4 -3
  264. package/dist/src/sessions/grok-build-cli.js +67 -5
  265. package/dist/src/sessions/import-exclusions.js +72 -0
  266. package/dist/src/sessions/import-lease.js +34 -18
  267. package/dist/src/sessions/import-limits.js +37 -0
  268. package/dist/src/sessions/importer.js +84 -46
  269. package/dist/src/sessions/index.js +2 -0
  270. package/dist/src/sessions/lifecycle.js +45 -0
  271. package/dist/src/sessions/policy.js +133 -25
  272. package/dist/src/sessions/readers.js +106 -53
  273. package/dist/src/sessions/refresh.js +106 -0
  274. package/dist/src/sessions/repository.js +199 -63
  275. package/dist/src/sessions/workspace-discovery.js +315 -88
  276. package/dist/src/sessions/workspace-matching.js +107 -0
  277. package/dist/src/skills/adapters/clawhub.js +10 -8
  278. package/dist/src/skills/adapters/openclaw.js +10 -7
  279. package/dist/src/skills/importer.js +1 -0
  280. package/dist/src/smiths/context-pipeline/aiwg-md.js +11 -0
  281. package/dist/src/smiths/context-pipeline/claude-hook.js +3 -26
  282. package/dist/src/smiths/context-pipeline/finalization.js +6 -5
  283. package/dist/src/smiths/context-pipeline/generator.js +21 -4
  284. package/dist/src/smiths/context-pipeline/parallelism-section.js +195 -14
  285. package/dist/src/smiths/context-pipeline/workspace-context.js +116 -9
  286. package/dist/src/smiths/hook-bridge/codex-translator.js +6 -5
  287. package/dist/src/smiths/mcpsmith/example.js +1 -1
  288. package/dist/src/smiths/mcpsmith/generator.js +1 -1
  289. package/dist/src/smiths/toolsmith/runtime-discovery.d.mts +10 -0
  290. package/dist/src/smiths/toolsmith/runtime-discovery.mjs +85 -31
  291. package/dist/src/storage/backends/fs.js +18 -76
  292. package/dist/src/storage/backends/logseq.js +68 -7
  293. package/dist/src/storage/backends/obsidian.js +1 -0
  294. package/dist/src/storage/fortemi-qualification-receipt.js +2 -2
  295. package/dist/src/testing/generators/test-code-generator.js +57 -30
  296. package/dist/src/tracker/forge-http.js +2 -1
  297. package/dist/src/uhp/client.js +11 -1
  298. package/dist/src/uhp/mission.js +5 -0
  299. package/dist/src/update/checker.d.mts +2 -0
  300. package/dist/src/update/checker.mjs +5 -5
  301. package/dist/src/update/notifier.mjs +54 -2
  302. package/dist/src/update/service.d.mts +22 -0
  303. package/dist/src/utils/contained-filesystem.d.mts +37 -0
  304. package/dist/src/utils/contained-filesystem.mjs +683 -0
  305. package/dist/src/utils/temporary-directory.js +36 -0
  306. package/dist/src/writing/example-generator.js +1 -1
  307. package/dist/src/writing/fidelity.js +18 -1
  308. package/package.json +32 -2
  309. package/schemas/mission-v1.schema.json +2429 -0
  310. package/schemas/models/model-catalog.v1.schema.json +301 -0
  311. package/schemas/models/model-config-compatibility.v1.schema.json +281 -0
  312. package/schemas/models/model-policy.v1.schema.json +14 -0
  313. package/schemas/models/provider-model-capabilities.v1.schema.json +1072 -0
  314. package/tools/_resolve-impl.mjs +39 -2
  315. package/tools/agents/deploy-agents.mjs +64 -23
  316. package/tools/agents/providers/base.mjs +153 -166
  317. package/tools/agents/providers/claude.mjs +8 -24
  318. package/tools/agents/providers/codex.mjs +18 -11
  319. package/tools/agents/providers/cursor.mjs +19 -18
  320. package/tools/agents/providers/grok-build.mjs +3 -1
  321. package/tools/agents/providers/grokbot.mjs +11 -12
  322. package/tools/agents/providers/hermes.mjs +62 -49
  323. package/tools/agents/providers/omp.mjs +5 -2
  324. package/tools/agents/providers/openclaw-translator.mjs +19 -5
  325. package/tools/agents/providers/openclaw.mjs +9 -4
  326. package/tools/agents/providers/pi.mjs +5 -0
  327. package/tools/agents/providers/provider-prepare-channel.mjs +5 -0
  328. package/tools/agents/providers/provider-prepare-coordinator.mjs +247 -0
  329. package/tools/agents/providers/provider-prepare-worker.mjs +84 -0
  330. package/tools/agents/providers/rule-delivery.mjs +1270 -0
  331. package/tools/agents/providers/warp.mjs +20 -26
  332. package/tools/cli/decision-offload-report.mjs +40 -0
  333. package/tools/cli/workspace-status.mjs +862 -0
  334. package/tools/plugin/package-plugins.mjs +186 -8
  335. package/tools/plugin/plugin-links.mjs +114 -0
  336. package/tools/providers/antigravity-transport.mjs +3 -1
  337. package/tools/providers/deepseek-harness-transport.mjs +360 -0
  338. package/tools/providers/omp-transport.mjs +194 -0
  339. package/tools/ralph-external/lib/claude-adapter.mjs +389 -0
  340. package/tools/ralph-external/lib/codex-adapter.mjs +432 -0
  341. package/tools/ralph-external/lib/consecutive-failure-circuit-breaker.mjs +1748 -0
  342. package/tools/ralph-external/lib/deepseek-harness-adapter.mjs +397 -0
  343. package/tools/ralph-external/lib/factory-adapter.mjs +503 -0
  344. package/tools/ralph-external/lib/launcher-recovery-evidence.mjs +55 -0
  345. package/tools/ralph-external/lib/muse-adapter.mjs +364 -0
  346. package/tools/ralph-external/lib/omp-adapter.mjs +315 -0
  347. package/tools/ralph-external/lib/opencode-adapter.mjs +575 -0
  348. package/tools/ralph-external/lib/openhuman-adapter.mjs +75 -0
  349. package/tools/ralph-external/lib/pi-adapter.mjs +329 -0
  350. package/tools/ralph-external/lib/provider-adapter.mjs +748 -0
  351. package/tools/ralph-external/lib/recovery-probe.mjs +72 -0
  352. package/tools/ralph-external/lib/session-environment.mjs +28 -0
  353. package/tools/ralph-external/lib/systemd-client.mjs +49 -0
  354. package/tools/ralph-external/lib/worker-health.mjs +439 -0
  355. package/tools/ralph-external/lib/worker-isolation.mjs +717 -0
  356. package/tools/ralph-external/lib/worker-overflow.mjs +50 -0
  357. package/tools/ralph-external/process-monitor.mjs +1017 -0
  358. package/tools/ralph-external/session-launcher.mjs +5057 -0
  359. package/tools/scripts/resource-preflight.mjs +541 -0
  360. package/tools/skills/deploy-skills-codex.mjs +2 -1
  361. package/vscode-extension/schemas/aiwg.config.v1.json +2916 -0
@@ -0,0 +1,1748 @@
1
+ /**
2
+ * ConsecutiveFailureCircuitBreaker - Three-state circuit breaker
3
+ *
4
+ * Implements the standard circuit breaker pattern:
5
+ * CLOSED (normal) -> N consecutive failures -> OPEN (blocking)
6
+ * OPEN -> cooldown expires -> HALF_OPEN (probe)
7
+ * HALF_OPEN -> success -> CLOSED
8
+ * HALF_OPEN -> failure -> OPEN (reset cooldown)
9
+ *
10
+ * @implements #514
11
+ * @tests @test/unit/circuit-breaker.test.mjs
12
+ */
13
+
14
+ import { EventEmitter } from 'node:events';
15
+
16
+ export const CircuitState = Object.freeze({
17
+ CLOSED: 'closed',
18
+ OPEN: 'open',
19
+ HALF_OPEN: 'half-open',
20
+ });
21
+
22
+ export class ConsecutiveFailureCircuitBreaker extends EventEmitter {
23
+ /**
24
+ * @param {object} options
25
+ * @param {number} [options.failureThreshold=5] Consecutive failures to trip open
26
+ * @param {number} [options.cooldownMs=120000] Time in OPEN before transitioning to HALF_OPEN
27
+ * @param {number} [options.halfOpenMaxProbes=1] Max concurrent probes in HALF_OPEN
28
+ * @param {function} [options.nowMs] Clock source (defaults to Date.now). The
29
+ * scoped breaker always injects its own clock here so both layers share
30
+ * one time source (#2915: a separate Date.now() desynchronizes cooldowns
31
+ * under synthetic clocks).
32
+ */
33
+ constructor(options = {}) {
34
+ super();
35
+ this.failureThreshold = options.failureThreshold ?? 5;
36
+ this.cooldownMs = options.cooldownMs ?? 120_000;
37
+ this.halfOpenMaxProbes = options.halfOpenMaxProbes ?? 1;
38
+ this.nowMs = options.nowMs ?? (() => Date.now());
39
+
40
+ this._state = CircuitState.CLOSED;
41
+ this._consecutiveFailures = 0;
42
+ this._lastFailureAt = null;
43
+ this._openedAt = null;
44
+ this._cooldownTimer = null;
45
+ this._activeProbes = 0;
46
+ }
47
+
48
+ /**
49
+ * Check whether execution is allowed in the current state.
50
+ * In HALF_OPEN, limits concurrent probes.
51
+ */
52
+ canExecute() {
53
+ switch (this._state) {
54
+ case CircuitState.CLOSED:
55
+ return true;
56
+
57
+ case CircuitState.OPEN: {
58
+ // Check if cooldown has elapsed (in case timer hasn't fired yet).
59
+ // `!== null`, not a truthy check (#2915 final review latent-bug
60
+ // fix): `_openedAt` is a `nowMs()` timestamp, and a synthetic clock
61
+ // can legitimately open a scope at time 0, which `0 && ...` would
62
+ // treat as "never opened" and refuse to ever leave OPEN.
63
+ if (this._openedAt !== null && this.nowMs() - this._openedAt >= this.cooldownMs) {
64
+ this._transitionTo(CircuitState.HALF_OPEN, 'cooldown elapsed on check');
65
+ return this._activeProbes < this.halfOpenMaxProbes;
66
+ }
67
+ return false;
68
+ }
69
+
70
+ case CircuitState.HALF_OPEN:
71
+ return this._activeProbes < this.halfOpenMaxProbes;
72
+
73
+ default:
74
+ return false;
75
+ }
76
+ }
77
+
78
+ /**
79
+ * Record a successful execution. Resets failure counter.
80
+ * In HALF_OPEN, transitions back to CLOSED.
81
+ */
82
+ recordSuccess() {
83
+ this._consecutiveFailures = 0;
84
+ this._activeProbes = Math.max(0, this._activeProbes - 1);
85
+
86
+ if (this._state === CircuitState.HALF_OPEN) {
87
+ this._transitionTo(CircuitState.CLOSED, 'probe succeeded');
88
+ this.emit('recover');
89
+ }
90
+ }
91
+
92
+ /**
93
+ * Record a failed execution. Increments failure counter.
94
+ * May trip the breaker open.
95
+ */
96
+ recordFailure() {
97
+ this._consecutiveFailures++;
98
+ this._lastFailureAt = this.nowMs();
99
+ this._activeProbes = Math.max(0, this._activeProbes - 1);
100
+
101
+ if (this._state === CircuitState.HALF_OPEN) {
102
+ // Probe failed — back to OPEN with fresh cooldown
103
+ this._transitionTo(CircuitState.OPEN, 'probe failed');
104
+ return;
105
+ }
106
+
107
+ if (this._state === CircuitState.CLOSED && this._consecutiveFailures >= this.failureThreshold) {
108
+ this._transitionTo(CircuitState.OPEN, `${this._consecutiveFailures} consecutive failures`);
109
+ this.emit('trip', { consecutiveFailures: this._consecutiveFailures });
110
+ }
111
+ }
112
+
113
+ /**
114
+ * Get current breaker state snapshot.
115
+ */
116
+ getState() {
117
+ const now = this.nowMs();
118
+ let cooldownRemainingMs = 0;
119
+
120
+ if (this._state === CircuitState.OPEN && this._openedAt !== null) {
121
+ cooldownRemainingMs = Math.max(0, this.cooldownMs - (now - this._openedAt));
122
+ }
123
+
124
+ return {
125
+ state: this._state,
126
+ consecutiveFailures: this._consecutiveFailures,
127
+ lastFailureAt: this._lastFailureAt ? new Date(this._lastFailureAt).toISOString() : null,
128
+ cooldownRemainingMs,
129
+ };
130
+ }
131
+
132
+ /**
133
+ * Force-reset the breaker to CLOSED state.
134
+ */
135
+ reset() {
136
+ this._clearCooldownTimer();
137
+ this._consecutiveFailures = 0;
138
+ this._lastFailureAt = null;
139
+ this._openedAt = null;
140
+ this._activeProbes = 0;
141
+ const from = this._state;
142
+ this._state = CircuitState.CLOSED;
143
+ if (from !== CircuitState.CLOSED) {
144
+ this.emit('state:change', { from, to: CircuitState.CLOSED, reason: 'manual reset' });
145
+ }
146
+ }
147
+
148
+ /**
149
+ * Clean up timers. Call when done with the breaker.
150
+ */
151
+ destroy() {
152
+ this._clearCooldownTimer();
153
+ this.removeAllListeners();
154
+ }
155
+
156
+ // --- Private ---
157
+
158
+ _transitionTo(newState, reason) {
159
+ const from = this._state;
160
+ if (from === newState) return;
161
+
162
+ this._state = newState;
163
+ this.emit('state:change', { from, to: newState, reason });
164
+
165
+ if (newState === CircuitState.OPEN) {
166
+ this._openedAt = this.nowMs();
167
+ this._startCooldownTimer();
168
+ } else {
169
+ this._clearCooldownTimer();
170
+ this._openedAt = null;
171
+ }
172
+
173
+ if (newState === CircuitState.HALF_OPEN) {
174
+ this._activeProbes = 0;
175
+ }
176
+ }
177
+
178
+ _startCooldownTimer() {
179
+ this._clearCooldownTimer();
180
+ this._cooldownTimer = setTimeout(() => {
181
+ if (this._state === CircuitState.OPEN) {
182
+ this._transitionTo(CircuitState.HALF_OPEN, 'cooldown expired');
183
+ }
184
+ }, this.cooldownMs);
185
+ // Allow process to exit even if timer is pending
186
+ if (this._cooldownTimer.unref) {
187
+ this._cooldownTimer.unref();
188
+ }
189
+ }
190
+
191
+ _clearCooldownTimer() {
192
+ if (this._cooldownTimer) {
193
+ clearTimeout(this._cooldownTimer);
194
+ this._cooldownTimer = null;
195
+ }
196
+ }
197
+ }
198
+
199
+ /**
200
+ * Scoped recovery policy (03 §11.5). Every bound is an explicit number or a
201
+ * resolved config override — no unbounded retry, every loop has a max.
202
+ *
203
+ * - distinctThreshold/windowMs: open a scope at three distinct failures in
204
+ * 120s, or when every member of a frozen cohort of at least two
205
+ * outstanding attempts fails in that window.
206
+ * - cooldownMs/maxCooldownMs: earliest admission is max(120s cooldown,
207
+ * known 429 reset/Retry-After); a failed canary doubles the cooldown,
208
+ * capped at 900s. Malformed resets fall back to this backoff with
209
+ * bounded jitter.
210
+ * - probeTimeoutMs/probeMaxOutputBytes: a separate bounded relevant probe
211
+ * precedes the one real task canary. An echo cannot recover an upstream
212
+ * scope. Both dispatches consume the caller's existing allowance.
213
+ * - canaryTimeoutMs: bounds the single real task after a successful probe.
214
+ * - maxTrackedScopes/maxChargedAttempts: explicit caps on retained scope
215
+ * and dedup-set state, with LRU eviction of inert entries, so a
216
+ * long-running process's bookkeeping never grows without bound (#2915
217
+ * review: settled outstanding/scope maps never dropped their keys). A
218
+ * hard cap refuses a brand-new scope record once at maxTrackedScopes with
219
+ * nothing evictable, rather than silently growing past it or evicting an
220
+ * OPEN/outstanding scope (#2915 final review, finding 6).
221
+ * - maxOutstandingReservations: hard cap on live (unsettled) admission
222
+ * reservations, including canary reservations and uncertain blocking
223
+ * markers; a reservation past its own deadline is swept (treated as
224
+ * uncertain) rather than retained forever.
225
+ * - maxCoveringKeys: hard cap on distinct scope keys (including the
226
+ * invocation key) bound to one reservation; the invocation key is never
227
+ * dropped when trimming.
228
+ *
229
+ * @implements #2915
230
+ */
231
+ export const RecoveryPolicy = Object.freeze({
232
+ distinctThreshold: 3,
233
+ windowMs: 120_000,
234
+ minOutstanding: 2,
235
+ cooldownMs: 120_000,
236
+ maxCooldownMs: 900_000,
237
+ probeTimeoutMs: 10_000,
238
+ probeMaxOutputBytes: 65_536,
239
+ canaryTimeoutMs: 300_000,
240
+ backoffMultiplier: 2,
241
+ maxJitterMs: 1_000,
242
+ maxRetryAfterSeconds: 86_400,
243
+ maxTrackedScopes: 500,
244
+ maxChargedAttempts: 5_000,
245
+ maxOutstandingReservations: 128,
246
+ maxCoveringKeys: 6,
247
+ });
248
+
249
+ /**
250
+ * Validate numeric policy overrides. Every bound must be an explicit finite
251
+ * number in range — unbounded or NaN overrides are refused rather than
252
+ * silently widening a loop (#2915 review: policy overrides lacked validation).
253
+ */
254
+ export function validateRecoveryPolicy(policy) {
255
+ const fail = (name, why) => { throw new RangeError(`Invalid recovery policy ${name}: ${why}`); };
256
+ const positiveInt = (name, value, min = 1, max = Infinity) => {
257
+ if (!Number.isInteger(value) || value < min || value > max) {
258
+ fail(name, `expected integer in [${min}, ${max === Infinity ? '∞' : max}], got ${value}`);
259
+ }
260
+ };
261
+ const positiveMs = (name, value) => {
262
+ if (!Number.isFinite(value) || value <= 0) fail(name, `expected finite ms > 0, got ${value}`);
263
+ };
264
+ positiveInt('distinctThreshold', policy.distinctThreshold);
265
+ positiveMs('windowMs', policy.windowMs);
266
+ positiveInt('minOutstanding', policy.minOutstanding);
267
+ positiveMs('cooldownMs', policy.cooldownMs);
268
+ positiveMs('maxCooldownMs', policy.maxCooldownMs);
269
+ if (policy.maxCooldownMs < policy.cooldownMs) fail('maxCooldownMs', 'must be >= cooldownMs');
270
+ positiveMs('probeTimeoutMs', policy.probeTimeoutMs);
271
+ if (policy.probeTimeoutMs > RecoveryPolicy.probeTimeoutMs) fail('probeTimeoutMs', 'cannot exceed 10s');
272
+ positiveInt('probeMaxOutputBytes', policy.probeMaxOutputBytes, 1, RecoveryPolicy.probeMaxOutputBytes);
273
+ positiveMs('canaryTimeoutMs', policy.canaryTimeoutMs);
274
+ if (!Number.isFinite(policy.backoffMultiplier) || policy.backoffMultiplier < 1) {
275
+ fail('backoffMultiplier', `expected finite number >= 1, got ${policy.backoffMultiplier}`);
276
+ }
277
+ if (!Number.isFinite(policy.maxJitterMs) || policy.maxJitterMs < 0) fail('maxJitterMs', `expected finite ms >= 0, got ${policy.maxJitterMs}`);
278
+ if (!Number.isFinite(policy.maxRetryAfterSeconds) || policy.maxRetryAfterSeconds < 0) {
279
+ fail('maxRetryAfterSeconds', `expected finite seconds >= 0, got ${policy.maxRetryAfterSeconds}`);
280
+ }
281
+ // #2915 review 4 (finding 6): these four are PROTECTIVE maxima, not mere
282
+ // floors — an override above the shipped default would widen the exact
283
+ // unbounded-growth loops this policy exists to close. Upper bound equals
284
+ // the shipped `RecoveryPolicy` default for each (500/128/5000/6); a
285
+ // caller that genuinely needs a smaller budget may still lower it.
286
+ positiveInt('maxTrackedScopes', policy.maxTrackedScopes, 2, RecoveryPolicy.maxTrackedScopes);
287
+ positiveInt('maxChargedAttempts', policy.maxChargedAttempts, 2, RecoveryPolicy.maxChargedAttempts);
288
+ positiveInt('maxOutstandingReservations', policy.maxOutstandingReservations, 1, RecoveryPolicy.maxOutstandingReservations);
289
+ positiveInt('maxCoveringKeys', policy.maxCoveringKeys, 1, RecoveryPolicy.maxCoveringKeys);
290
+ return policy;
291
+ }
292
+
293
+ /** Failure scopes, narrowest first. A failure opens only its own scope. */
294
+ export const FailureScope = Object.freeze({
295
+ INVOCATION: 'invocation',
296
+ MODEL: 'model',
297
+ INSTALLATION: 'installation',
298
+ QUOTA: 'quota',
299
+ PROVIDER: 'provider',
300
+ HOST: 'host',
301
+ });
302
+
303
+ // Reason codes produced by classifyWorkerHealthFailure (lib/worker-health.mjs).
304
+ // Anything unlisted — task-local errors, unknown signatures — stays in the
305
+ // invocation scope so independent task failures never open a wider scope.
306
+ const QUOTA_CODES = new Set(['RATE_LIMIT']);
307
+ const HOST_CODES = new Set(['RESOURCE', 'NOT_FOUND', 'SANDBOX']);
308
+ const PROVIDER_CODES = new Set(['NETWORK']);
309
+ const INSTALLATION_CODES = new Set(['AUTH']);
310
+ const MODEL_CODES = new Set(['MODEL']);
311
+
312
+ export const scopeKeyFor = Object.freeze({
313
+ invocation: (id) => `invocation:${id ?? 'unknown'}`,
314
+ model: (key) => `model:${key ?? 'unknown'}`,
315
+ // `qualifier`, when supplied, distinguishes separately-qualified
316
+ // installations that share a CLI name (#2915 review: two projects
317
+ // resolving different installations under `codex` shared one
318
+ // `installation:codex` scope). Omitted/null preserves the plain
319
+ // `installation:<cli>` key so existing single-installation callers and
320
+ // tests are unaffected.
321
+ installation: (cli, qualifier = null) => qualifier
322
+ ? `installation:${cli ?? 'unknown'}:${qualifier}`
323
+ : `installation:${cli ?? 'unknown'}`,
324
+ quota: (account) => `quota:${account ?? 'unknown'}`,
325
+ provider: (name) => `provider:${name ?? 'unknown'}`,
326
+ host: (id) => `host:${id ?? 'local'}`,
327
+ });
328
+
329
+ /** Scope breadth rank, narrowest first. Used to fold over-broad claims back. */
330
+ const SCOPE_RANK = Object.freeze({
331
+ [FailureScope.INVOCATION]: 0,
332
+ [FailureScope.MODEL]: 1,
333
+ [FailureScope.INSTALLATION]: 2,
334
+ [FailureScope.QUOTA]: 3,
335
+ [FailureScope.PROVIDER]: 4,
336
+ [FailureScope.HOST]: 5,
337
+ });
338
+
339
+ export function scopeRank(scope) {
340
+ return SCOPE_RANK[scope] ?? 0;
341
+ }
342
+
343
+ /**
344
+ * Evidence sources. Only `launcher` (launcher-observed spawn/wait/exit facts
345
+ * from its own child and unit status) and `resolver` (the #2887 resolved
346
+ * invocation and installation qualification) are trusted to widen a scope.
347
+ * `worker` covers everything derived from worker stderr/stdout, structured
348
+ * worker output, or worker-writable files — including the qualified health
349
+ * classifier's reason codes, which describe worker-observed text, not
350
+ * launcher-verified facts. Worker-sourced failures NEVER widen: they stay in
351
+ * the invocation scope, so one misbehaving task retried under fresh ids can
352
+ * poison at most its own attempts, never a provider, host, quota or
353
+ * installation scope (#2915 review: scope poisoning).
354
+ *
355
+ * Identity separation: the inference backend (`inferenceProvider`, e.g.
356
+ * `openai`) and the CLI installation (`cli`, e.g. `codex`) are distinct
357
+ * trusted identities. Provider scope keys use the raw backend string — never
358
+ * normalized through the CLI/provider-definition registry, which maps
359
+ * backends onto CLI ids (`openai` -> `codex`) and merges the two identities.
360
+ */
361
+ export const RecoveryEvidenceSource = Object.freeze({
362
+ LAUNCHER: 'launcher',
363
+ RESOLVER: 'resolver',
364
+ WORKER: 'worker',
365
+ });
366
+
367
+ const TRUSTED_SOURCES = new Set([RecoveryEvidenceSource.LAUNCHER, RecoveryEvidenceSource.RESOLVER]);
368
+
369
+ export function isTrustedRecoveryEvidence(source) {
370
+ return TRUSTED_SOURCES.has(source);
371
+ }
372
+
373
+ /**
374
+ * Attribute a failure to the narrowest scope its evidence supports. Trusted
375
+ * launcher/resolver evidence with a matching trusted identity may implicate a
376
+ * wider scope (still `suspected` until the distinct-invocation correlation
377
+ * rule confirms it). Anything else — worker-derived codes, missing source,
378
+ * or a wider claim without its required trusted identity — folds back to the
379
+ * invocation scope. In particular a single task's repeated local failures
380
+ * (unknown codes, task errors, worker-observed signatures) can never open a
381
+ * provider, host, quota, installation or model scope; a genuine outage
382
+ * promotes beyond invocation only when N distinct invocations within the
383
+ * window carry the same trusted evidence (the correlation rule in
384
+ * recordFailure).
385
+ */
386
+ export function attributeFailureScope({
387
+ reasonCode = null,
388
+ evidenceSource = null,
389
+ invocationId = null,
390
+ modelKey = null,
391
+ cli = null,
392
+ inferenceProvider = null,
393
+ provider = null,
394
+ quotaAccount = null,
395
+ hostId = null,
396
+ corroboratedBackends = null,
397
+ installationQualifier = null,
398
+ } = {}) {
399
+ const invocation = { scope: FailureScope.INVOCATION, scopeKey: scopeKeyFor.invocation(invocationId), confidence: 'suspected' };
400
+ if (!isTrustedRecoveryEvidence(evidenceSource)) return invocation;
401
+ // Legacy alias: `provider` carried either identity through the old call
402
+ // sites. The backend wins for provider scope; the CLI never implies it.
403
+ const backend = inferenceProvider ?? null;
404
+ if (reasonCode === 'NOT_FOUND' || reasonCode === 'SPAWN_REFUSED') {
405
+ // Launcher-observed failure to execute the resolved CLI binary itself
406
+ // (spawn ENOENT/EACCES on argv[0], never a worker-observed message —
407
+ // worker-sourced NOT_FOUND returns in the invocation scope above). A
408
+ // missing task subprocess must never implicate the host.
409
+ if (cli == null || cli === '') return invocation;
410
+ return { scope: FailureScope.INSTALLATION, scopeKey: scopeKeyFor.installation(cli, installationQualifier), confidence: 'suspected' };
411
+ }
412
+ if (QUOTA_CODES.has(reasonCode)) {
413
+ if (quotaAccount == null || quotaAccount === '') return invocation;
414
+ return { scope: FailureScope.QUOTA, scopeKey: scopeKeyFor.quota(quotaAccount), confidence: 'suspected' };
415
+ }
416
+ if (HOST_CODES.has(reasonCode)) {
417
+ // Host scope pauses cross-provider retries: it requires corroboration
418
+ // across at least two distinct trusted backends in this evidence set.
419
+ // Single-provider host signatures stay invocation-scoped.
420
+ const backends = [...new Set((corroboratedBackends ?? []).filter((b) => b != null && b !== ''))];
421
+ if (backends.length < 2) return invocation;
422
+ return { scope: FailureScope.HOST, scopeKey: scopeKeyFor.host(hostId), confidence: 'suspected' };
423
+ }
424
+ if (PROVIDER_CODES.has(reasonCode)) {
425
+ if (backend == null || backend === '') return invocation;
426
+ return { scope: FailureScope.PROVIDER, scopeKey: scopeKeyFor.provider(backend), confidence: 'suspected' };
427
+ }
428
+ if (INSTALLATION_CODES.has(reasonCode)) {
429
+ if (cli == null || cli === '') return invocation;
430
+ return { scope: FailureScope.INSTALLATION, scopeKey: scopeKeyFor.installation(cli, installationQualifier), confidence: 'suspected' };
431
+ }
432
+ if (MODEL_CODES.has(reasonCode)) {
433
+ if (modelKey == null || modelKey === '') return invocation;
434
+ return { scope: FailureScope.MODEL, scopeKey: scopeKeyFor.model(modelKey), confidence: 'suspected' };
435
+ }
436
+ return invocation;
437
+ }
438
+
439
+ /**
440
+ * ScopedFailureBreaker - correlated-failure scopes over the existing
441
+ * ConsecutiveFailureCircuitBreaker transitions (closed → open → half-open).
442
+ *
443
+ * Each scope owns one inner breaker (failureThreshold 1: the correlation
444
+ * rule below is the only trip authority), sharing this instance's injected
445
+ * `nowMs` clock. Probes drive the inner half-open transitions; an echo
446
+ * probe can never close an upstream outage — only a launcher-verified
447
+ * relevant probe success admits one canary, and only a launcher-verified
448
+ * canary success recovers the scope.
449
+ *
450
+ * All window/cooldown math uses the injectable `nowMs` clock, so tests run
451
+ * on synthetic time (vitest fake timers cover the default Date.now clock).
452
+ *
453
+ * Restart/state (#2915D): all breaker state lives in-memory in this
454
+ * instance — there is deliberately no persisted store (a durable shared
455
+ * store would be a new subsystem; managed-tier shared ownership stays
456
+ * deferred to Tier 1). A restarted conductor therefore comes up with every
457
+ * scope CLOSED (fail-open on restart). This is the smaller safe option
458
+ * because two couplings survive the restart: (1) the stop-before-retry gate
459
+ * still requires the launcher's own confirmed-termination evidence
460
+ * (exit.json `termination`) before any retry, so uncertain work is never
461
+ * replayed blindly; (2) fresh trusted evidence re-opens a scope through the
462
+ * same distinct-invocation rule, so a real outage re-pauses within N
463
+ * failures. No uncertain probe/canary reservation survives a restart —
464
+ * reservations are single-use and bound to this instance.
465
+ *
466
+ * Degraded-isolation blocking markers (#2915 review 6, finding 3e): degraded
467
+ * isolation has no native unit query, so it can NEVER supply
468
+ * confirmed-termination evidence for a canary's descendants, by design (see
469
+ * `captureOneWorker`'s evidence contract) — a degraded canary that expires
470
+ * unreported leaves a `blockingMarker` that no amount of waiting will ever
471
+ * clear on its own. This is an accepted, documented limitation, not a bug:
472
+ * automatically expiring the marker after some elapsed time would silently
473
+ * re-admit work over a descendant tree nobody ever confirmed stopped, which
474
+ * is exactly the hazard this marker exists to prevent. Do not invent one.
475
+ * The existing, operator-reachable recovery actions are: (1)
476
+ * `reconcileStopEvidence({ invocationId, confirmedStop: true })`, once the
477
+ * operator has independently established the descendants actually stopped
478
+ * (e.g. by inspecting the host directly); (2) the scoped `reset(scopeKey)`
479
+ * (also releases that scope's own reservations, including the marker's —
480
+ * see its doc comment), which discards this one scope's protection instead
481
+ * of every scope's; (3) `resetDirectRecoveryBreaker()` / full `reset()` /
482
+ * process restart, which discards everything. None of these are wired to a
483
+ * CLI command today; they are this module's exported programmatic surface.
484
+ */
485
+ export class ScopedFailureBreaker extends EventEmitter {
486
+ /**
487
+ * @param {object} options
488
+ * @param {object} [options.policy] Numeric overrides for RecoveryPolicy.
489
+ * @param {function} [options.nowMs] Synthetic-clock source (defaults to Date.now).
490
+ * @param {function} [options.jitterMs] Bounded-jitter source (defaults to Math.random-based).
491
+ */
492
+ constructor(options = {}) {
493
+ super();
494
+ this.policy = validateRecoveryPolicy({ ...RecoveryPolicy, ...(options.policy ?? {}) });
495
+ // `Date.now()` is the class-level default so every existing caller that
496
+ // constructs this directly (including this module's own extensive
497
+ // synthetic-clock test suite, much of which cross-checks breaker state
498
+ // against `Date.now()` directly) keeps its current, epoch-based
499
+ // semantics unchanged. #2915 round 8 review 8 (C3 follow-up, "production
500
+ // deadlines use Date.now(), not a monotonic clock"): the actual
501
+ // production singleton (`getDirectRecoveryBreaker`, below) explicitly
502
+ // overrides this with a `performance.now()`-based monotonic clock
503
+ // instead — every comparison this class makes is relative (one `nowMs()`
504
+ // reading against another this SAME instance produced), so swapping the
505
+ // clock source at that one call site is sufficient to make production
506
+ // deadlines immune to a wall-clock adjustment without disturbing this
507
+ // constructor's own default for every other (test) caller.
508
+ this.nowMs = options.nowMs ?? (() => Date.now());
509
+ this.jitterMs = options.jitterMs
510
+ ?? (() => Math.floor(Math.random() * (this.policy.maxJitterMs + 1)));
511
+ this._scopes = new Map();
512
+ this._outstanding = new Map();
513
+ this._hostFailureFacts = [];
514
+ this._canarySeq = 0;
515
+ this._observedInvocations = new Set();
516
+ this.canariesObserved = 0;
517
+ // Reservation registry (#2915 final review): every admission mints an
518
+ // opaque, breaker-owned token here. A reservation is removed the moment
519
+ // it settles (`settle`) or expires past its own deadline (swept by
520
+ // `_sweepReservations`, called at the top of every `admit`/`settle`), so
521
+ // this map's steady-state size tracks live in-flight attempts, not
522
+ // cumulative history.
523
+ this._reservations = new Map();
524
+ this._reservationSeq = 0;
525
+ }
526
+
527
+ /**
528
+ * Track a launch so the cohort frozen at the first failure sees attempts
529
+ * outstanding in-window. Cohort membership is immutable once frozen: later
530
+ * launches never join it (#2915 review: noteLaunched grew the frozen
531
+ * cohort, so the original members' unanimous failure could not open it).
532
+ * Outstanding entries older than the window are pruned here, so a leaked
533
+ * note (a pre-spawn failure that never settled) can pollute a cohort for
534
+ * at most one window, never permanently.
535
+ */
536
+ noteLaunched(scopeKeys, invocationId) {
537
+ const now = this.nowMs();
538
+ for (const scopeKey of scopeKeys ?? []) {
539
+ let set = this._outstanding.get(scopeKey);
540
+ if (!set) { set = new Map(); this._outstanding.set(scopeKey, set); }
541
+ set.set(invocationId, now);
542
+ for (const [id, atMs] of set) {
543
+ if (now - atMs >= this.policy.windowMs) set.delete(id);
544
+ }
545
+ }
546
+ }
547
+
548
+ /**
549
+ * Clear outstanding membership across all covering scopes once an attempt
550
+ * settles (used for pre-spawn failures that must leave no trace, and
551
+ * after recording an outcome against a single scope so sibling scopes do
552
+ * not retain a stale member that would veto future cohort unanimity).
553
+ */
554
+ noteSettled({ scopeKeys = [], invocationId } = {}) {
555
+ for (const scopeKey of scopeKeys ?? []) this._removeOutstanding(scopeKey, invocationId);
556
+ }
557
+
558
+ /**
559
+ * One atomic admission (#2915 final review, replaces the former
560
+ * independent `canLaunch` + `noteLaunched` pair). Returns:
561
+ * - `{ ok: true, token }` — admitted; `token` is an opaque,
562
+ * breaker-owned reservation that MUST be passed to exactly one later
563
+ * `settle()` call. `token.canaryKeys` names which covering scopes (if
564
+ * any) this admission is the half-open canary for.
565
+ * - `{ ok: false, blockedBy, nextEligibleAt }` — refused; no state was
566
+ * mutated (no reservation minted, no cohort membership recorded).
567
+ *
568
+ * `scopeKeys` plus the derived `invocation:<id>` key form the covering set
569
+ * for this attempt, capped at `policy.maxCoveringKeys` (the invocation key
570
+ * is never dropped when trimming — #2915 final review, bounded covering
571
+ * keys). Every covering OPEN scope must be simultaneously eligible (due,
572
+ * no outstanding canary, inner breaker executable) for admission to
573
+ * succeed; this keeps the same "all covering scopes settle together"
574
+ * invariant the review required (`all-covering-scopes-settle`).
575
+ *
576
+ * Reservation lifetime is bounded by `policy.maxOutstandingReservations`:
577
+ * once at capacity, a new admission is refused with
578
+ * `blockedBy: ['reservation-capacity']` rather than ever evicting a live
579
+ * reservation (OPEN-scope protection always wins over admitting more
580
+ * concurrent work).
581
+ */
582
+ admit({ invocationId = null, scopeKeys = [], selectionDigest = null, purpose = 'task' } = {}) {
583
+ const now = this.nowMs();
584
+ this._sweepReservations(now);
585
+ this._sweepOutstanding(now);
586
+ const invocationKey = invocationId != null ? scopeKeyFor.invocation(invocationId) : null;
587
+ const covering = [...new Set([...(scopeKeys ?? []), ...(invocationKey ? [invocationKey] : [])])];
588
+ // #2915 review 5 (finding 5, "lost covering protection"): an oversized
589
+ // covering set is REFUSED outright, never silently trimmed. Every
590
+ // legitimate production caller derives covering keys from
591
+ // `recoveryScopeKeysForInvocation`, which never produces more than
592
+ // `maxCoveringKeys` (6) keys in the first place -- a set larger than
593
+ // that is already outside what this breaker can safely attribute, and
594
+ // trimming it previously dropped some of the caller's own named scopes
595
+ // from tracking entirely, discarding whatever protection they were
596
+ // meant to provide rather than refusing the admission that needed them.
597
+ if (covering.length > this.policy.maxCoveringKeys) {
598
+ return { ok: false, blockedBy: ['covering-set-too-large'], nextEligibleAt: now };
599
+ }
600
+ // #2915 review 6 (finding 2, "atomic scope tracking"): reserve every
601
+ // BRAND-NEW shared scope's tracking slot INSIDE this admission by
602
+ // actually creating its record now (`_scopeRecordBounded` performs the
603
+ // real capacity check/eviction) -- never a separate non-mutating dry
604
+ // run. Two `admit()` calls can no longer both observe "capacity exists"
605
+ // for the same single free slot: the first call to create a record
606
+ // mutates `_scopes` synchronously, so every later call's own capacity
607
+ // check already accounts for it (#2915 review 6: with one free slot,
608
+ // four concurrent admissions for distinct new scopes were all admitted
609
+ // and three failures later silently vanished, because the old dry run
610
+ // never reserved anything). Invocation-scoped keys are exempt --
611
+ // `maxChargedAttempts` is their own separate, much larger budget, never
612
+ // `maxTrackedScopes`. Any record this admission creates but does not
613
+ // end up needing (refused below for an unrelated reason, or still inert
614
+ // once this attempt settles) is released again via `_releaseIfInert`.
615
+ const reservedScopeKeys = [];
616
+ for (const key of covering) {
617
+ if (key === invocationKey || this._scopes.has(key)) continue;
618
+ const rec = this._scopeRecordBounded(key.split(':')[0], key);
619
+ if (!rec) {
620
+ for (const createdKey of reservedScopeKeys) this._releaseIfInert(createdKey);
621
+ return { ok: false, blockedBy: ['scope-tracking-capacity'], nextEligibleAt: now };
622
+ }
623
+ reservedScopeKeys.push(key);
624
+ }
625
+ const openKeys = covering.filter((key) => {
626
+ const rec = this._scopes.get(key);
627
+ return rec && rec.status !== 'closed';
628
+ });
629
+ const canaryKeys = [];
630
+ const probeRequired = [];
631
+ if (openKeys.length > 0) {
632
+ const blockedBy = [];
633
+ for (const scopeKey of openKeys) {
634
+ const rec = this._scopes.get(scopeKey);
635
+ // #2915 review 5 ("a second worker over unresolved execution"): a
636
+ // bounded blocking marker (installed by `_sweepReservations` when a
637
+ // canary's own deadline passed with stop still unconfirmed) refuses
638
+ // admission regardless of elapsed backoff -- checked BEFORE the
639
+ // due-floor/backoff check below, which elapsed backoff alone could
640
+ // otherwise satisfy. Only a settlement carrying confirmed-
641
+ // termination evidence for that exact reservation clears it (see
642
+ // `settle()`).
643
+ if (rec.status === 'operator-intervention') { blockedBy.push(scopeKey); continue; }
644
+ if (rec.blockingMarker) { blockedBy.push(scopeKey); continue; }
645
+ if (rec.activeCanary) { blockedBy.push(scopeKey); continue; }
646
+ const dueFloor = Math.max(rec.nextProbeAtMs ?? 0, rec.resetAtMs ?? 0);
647
+ if (now < dueFloor || !rec.breaker.canExecute()) { blockedBy.push(scopeKey); continue; }
648
+ if (purpose !== 'probe' && rec.status !== 'canary-ready') { probeRequired.push(scopeKey); continue; }
649
+ if (purpose === 'probe' && rec.status === 'canary-ready') continue;
650
+ canaryKeys.push(scopeKey);
651
+ }
652
+ if (blockedBy.length > 0 || probeRequired.length > 0) {
653
+ const nextEligibleAt = openKeys.reduce((max, key) => Math.max(max, this._scopes.get(key)?.nextProbeAtMs ?? 0), 0);
654
+ for (const createdKey of reservedScopeKeys) this._releaseIfInert(createdKey);
655
+ return { ok: false, blockedBy: [...blockedBy, ...probeRequired], nextEligibleAt,
656
+ probeRequired: blockedBy.length === 0 ? probeRequired : [] };
657
+ }
658
+ }
659
+ if (this._reservations.size >= this.policy.maxOutstandingReservations) {
660
+ // #2915 review 6 (finding 3f, "markers must not be able to starve
661
+ // unrelated work silently"): name how many of the occupied slots are
662
+ // unresolved blocking markers (rather than live, settleable
663
+ // reservations) in the refusal diagnostics, so this exact refusal is
664
+ // distinguishable from ordinary reservation-capacity pressure.
665
+ const markerCount = [...this._reservations.values()].filter((entry) => entry.stop === 'unresolved').length;
666
+ const blockedBy = markerCount > 0 ? ['reservation-capacity', `blocking-markers:${markerCount}`] : ['reservation-capacity'];
667
+ for (const createdKey of reservedScopeKeys) this._releaseIfInert(createdKey);
668
+ return { ok: false, blockedBy, nextEligibleAt: now };
669
+ }
670
+ this._reservationSeq += 1;
671
+ this._canarySeq += canaryKeys.length > 0 ? 1 : 0;
672
+ const id = `res-${this._reservationSeq}`;
673
+ // Only a canary reservation carries the bounded 300s deadline (03 §11.5:
674
+ // the half-open probe window) — an ordinary (non-canary) admission is
675
+ // not bounded by it at all, and never auto-settles via the sweep below;
676
+ // it occupies one slot of `maxOutstandingReservations` until its own
677
+ // caller settles it, however long the real attempt legitimately runs
678
+ // (#2915 final review risk: a canary-only deadline must never leak onto
679
+ // ordinary long-running attempts).
680
+ const deadlineMs = canaryKeys.length > 0 ? now + (purpose === 'probe' ? this.policy.probeTimeoutMs : this.policy.canaryTimeoutMs) : Infinity;
681
+ // `selectionDigest` (#2915 round 8, D3): stored on the FROZEN token
682
+ // itself — never a separate mutable field on the outer `admitDirectWorker`
683
+ // result — so `beginDispatch`'s identity check below has an authoritative,
684
+ // breaker-owned binding to validate against. A caller-side wrapper that
685
+ // deletes or rewrites its own copy of this value can never change what is
686
+ // actually compared (#2915 round 8 critique: "stored identity is only
687
+ // useful if compared against the actual launch").
688
+ const token = Object.freeze({ id, breaker: this, invocationId, selectionDigest, coveringKeys: Object.freeze(covering),
689
+ canaryKeys: Object.freeze(canaryKeys), purpose, maxOutputBytes: purpose === 'probe' ? this.policy.probeMaxOutputBytes : null, mintedAtMs: now, deadlineMs });
690
+ // `phase`/`stop` (#2915 round 8, D3): the reservation's own transition
691
+ // state, `admitted -> committing -> dispatched`, and its stop-evidence
692
+ // status, `none-needed -> unresolved -> confirmed`. Both start at their
693
+ // initial value for every reservation, canary or not. `revoked` (round 8):
694
+ // set by `_sweepReservations` when a `committing` reservation's deadline
695
+ // passes — it marks the committing attempt for `commitDispatch`/
696
+ // `abortDispatch` to observe, but (unlike `dispatched`-phase expiry) never
697
+ // itself releases or reopens anything, because nothing is yet known to
698
+ // have actually dispatched (see `_sweepReservations`, `abortDispatch`).
699
+ this._reservations.set(id, { token, settled: false, phase: 'admitted', stop: 'none-needed',
700
+ stopTouched: false, revoked: false, reservedScopeKeys });
701
+ this.noteLaunched(covering, invocationId);
702
+ for (const scopeKey of canaryKeys) {
703
+ const rec = this._scopes.get(scopeKey);
704
+ rec.activeCanary = { id, scopeKey, invocationId, reservedAtMs: now, deadlineMs };
705
+ if (purpose !== 'probe') { rec.canariesObserved += 1; this.canariesObserved += 1; }
706
+ this._transition(rec, purpose === 'probe' ? 'half-open' : 'canary', purpose === 'probe' ? 'run one bounded relevant probe' : 'verify the one task canary');
707
+ this._touch(rec);
708
+ }
709
+ return { ok: true, token };
710
+ }
711
+
712
+ /**
713
+ * Idempotent, ownership-checked settlement of one `admit()` token (#2915
714
+ * final review). `outcome.kind` is `'succeeded' | 'failed' | 'uncertain'`
715
+ * (the launcher-owned `attemptOutcome`, never a worker claim); `failed`
716
+ * additionally accepts the same attribution fields `recordFailure` used
717
+ * to (`reasonCode`, `evidenceSource`, `cli`, `inferenceProvider`,
718
+ * `quotaAccount`, `hostId`, `corroboratedBackends`, `resetAtMs`,
719
+ * `retryAfterSeconds`, `installationQualifier`, `scope`/`scopeKey` as the
720
+ * caller's own claim, narrowed to what the evidence supports).
721
+ *
722
+ * A token from a different breaker instance, or one already settled
723
+ * (including one already swept as expired), is refused with
724
+ * `{ ok: false, reason }` and mutates nothing — exactly one caller ever
725
+ * wins the settlement of a given reservation
726
+ * (`old-attempt-cannot-settle-canary`).
727
+ *
728
+ * Every covering scope this reservation held settles together: canary
729
+ * scopes resolve per `outcome.kind` (recover / reopen+backoff / reopen
730
+ * uncertain), and every covering scope's outstanding cohort membership is
731
+ * cleared regardless of canary status (`all-covering-scopes-settle`).
732
+ *
733
+ * `outcome.confirmedStop` (#2915 review 5, "a second worker over
734
+ * unresolved execution") matters ONLY for a reservation whose own canary
735
+ * deadline already passed unresolved (`_sweepReservations` already
736
+ * reopened the scope and installed a blocking marker for it, below) — a
737
+ * settle() call against such a token is the launcher's bounded stop-
738
+ * escalation report arriving separately from the expiry itself, and
739
+ * resolves ONLY that marker, never the scope's normal recover/reopen
740
+ * transition (already applied exactly once, at expiry). `true` (the
741
+ * launcher's own confirmed termination evidence) clears the marker;
742
+ * anything else (including a late `kind: 'succeeded'`) leaves it in
743
+ * place with no further automatic resolution — late success can never
744
+ * close a scope this way.
745
+ */
746
+ settle(token, outcome = { kind: 'uncertain' }) {
747
+ const now = this.nowMs();
748
+ this._sweepReservations(now);
749
+ this._sweepOutstanding(now);
750
+ // #2915 review 4 (finding 5, exact token ownership): a passed-in token
751
+ // is NEVER trusted for its own field values — only as a lookup key.
752
+ // `entry.token !== token` (reference inequality) refuses ANY copy with
753
+ // altered fields, including one that also copies `breaker`/`id`
754
+ // correctly but changes `invocationId`/`coveringKeys`/`canaryKeys`: the
755
+ // legitimate caller always holds the exact frozen object `admit()`
756
+ // returned, so only that exact object reference can ever match.
757
+ if (!token || typeof token.id !== 'string' || token.breaker !== this) {
758
+ return { ok: false, reason: 'reservation is not owned by this breaker' };
759
+ }
760
+ const entry = this._reservations.get(token.id);
761
+ if (!entry || entry.token !== token) {
762
+ return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
763
+ }
764
+ if (entry.stop === 'unresolved') {
765
+ // This reservation already failed its own canary deadline while
766
+ // `dispatched` (possibly via the sweep this very call just performed,
767
+ // above). Resolve the stop-evidence marker it left behind instead of
768
+ // re-running the normal recover/reopen transition below.
769
+ //
770
+ // #2915 review 6 (finding 3a, "blocking markers become permanently
771
+ // unresolvable"): a CONFIRMED stop always wins, regardless of how
772
+ // many unconfirmed settlements already touched this same reservation
773
+ // -- the former code resolved the marker on the FIRST settle() call
774
+ // no matter its outcome, so an earlier unconfirmed report (the common
775
+ // case: a degraded attempt can never supply confirmed-termination
776
+ // evidence) permanently rejected every later, genuinely confirmed
777
+ // one. An unconfirmed settlement is a no-op that resolves nothing,
778
+ // but each already-expired reservation still accepts exactly ONE such
779
+ // report (mirroring the normal "exactly once" settlement contract for
780
+ // every other outcome) so a caller cannot spin it indefinitely; this
781
+ // never forecloses a LATER confirmed report, or an explicit
782
+ // `reconcileStopEvidence` reconciliation, from succeeding at any time.
783
+ const confirmed = outcome?.confirmedStop === true;
784
+ if (confirmed) {
785
+ // A confirmed stop frees this reservation's slot; an unconfirmed
786
+ // stop deliberately keeps it occupying one of
787
+ // `maxOutstandingReservations` — the explicit bound a persistent
788
+ // stop-evidence marker counts against, rather than a separate marker
789
+ // cap or an invented auto-expiry.
790
+ entry.stop = 'confirmed';
791
+ this._release(entry);
792
+ return { ok: true };
793
+ }
794
+ if (entry.stopTouched) {
795
+ return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
796
+ }
797
+ entry.stopTouched = true;
798
+ return { ok: true };
799
+ }
800
+ if (entry.settled) {
801
+ return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
802
+ }
803
+ entry.settled = true;
804
+ // #2915 round 8 (C1, "outcome settlement discards unresolved execution"):
805
+ // captured BEFORE any of the outcome processing below, which itself
806
+ // never changes `entry.phase`. A reservation that reached `dispatched`
807
+ // genuinely started a process; whether its descendants are confirmed
808
+ // stopped is an INDEPENDENT fact from whether the attempt succeeded or
809
+ // failed, so recording the outcome below must never, by itself, be
810
+ // treated as proof the attempt's execution ended.
811
+ const wasDispatched = entry.phase === 'dispatched';
812
+ const confirmedStop = outcome?.confirmedStop === true;
813
+ const { invocationId, coveringKeys, canaryKeys } = entry.token;
814
+ const kind = outcome?.kind ?? 'uncertain';
815
+ // Enforce the canary's own deadline even at settlement time: a reported
816
+ // outcome that arrives after the deadline elapsed never recovers or
817
+ // otherwise resolves the scope through this path — it was (or will be)
818
+ // swept as expired instead (#2915 final review: "enforce deadlines on
819
+ // completion"). `now >= deadline` (#2915 round 8, D5): one consistent
820
+ // comparison operator for every deadline check in this module.
821
+ // #2915 round 8 (C1): scopes THIS token actually owned the canary for at
822
+ // the moment this loop ran — never the full static `canaryKeys` list,
823
+ // which can include a scope a LATER reservation has already taken over
824
+ // (the `continue` below). Used after the loop to decide which scopes, if
825
+ // any, get the retained-execution blocking marker: installing it on a
826
+ // scope this token no longer owns would wrongly block an unrelated,
827
+ // still-live reservation's own canary.
828
+ const ownedCanaryKeys = [];
829
+ for (const scopeKey of canaryKeys) {
830
+ const rec = this._scopes.get(scopeKey);
831
+ if (!rec || !rec.activeCanary || rec.activeCanary.id !== token.id) continue; // already reopened/expired elsewhere
832
+ ownedCanaryKeys.push(scopeKey);
833
+ if (now >= rec.activeCanary.deadlineMs) {
834
+ rec.activeCanary = null;
835
+ this._reopen(rec, now);
836
+ continue;
837
+ }
838
+ rec.activeCanary = null;
839
+ if (outcome?.callerAborted === true && confirmedStop) {
840
+ // Caller cancellation is no evidence of a broken provider. Keep the
841
+ // existing cooldown and retain a successful probe for the next canary.
842
+ this._transition(rec, token.purpose === 'probe' ? 'open' : 'canary-ready',
843
+ 'caller cancelled; retry recovery without increasing backoff');
844
+ } else if (kind === 'succeeded' && token.purpose === 'probe') {
845
+ if (outcome.relevant === true && isTrustedRecoveryEvidence(outcome.evidenceSource) && confirmedStop) {
846
+ this._transition(rec, 'canary-ready', 'probe succeeded; admit one task canary');
847
+ } else { this._reopen(rec, now); }
848
+ } else if (kind === 'succeeded') {
849
+ this._recoverScope(rec, invocationId);
850
+ } else if (kind === 'failed') {
851
+ // A reset/Retry-After carried by THIS canary's own failure evidence
852
+ // still merges before reopening — the same "a later failure carries
853
+ // a further reset" rule that applies to any other failure, so a
854
+ // canary that fails with a longer-than-backoff reset does not get
855
+ // probed again before that reset elapses.
856
+ const reset = this._resolveResetAtMs({ resetAtMs: outcome?.resetAtMs ?? null, retryAfterSeconds: outcome?.retryAfterSeconds ?? null }, now);
857
+ if (reset !== null && (rec.resetAtMs === null || reset > rec.resetAtMs)) rec.resetAtMs = reset;
858
+ this._reopen(rec, now);
859
+ } else {
860
+ this._reopen(rec, now);
861
+ }
862
+ }
863
+ if (kind === 'succeeded') {
864
+ for (const scopeKey of coveringKeys) {
865
+ this._recordObservedAttempt(scopeKey, invocationId);
866
+ if (canaryKeys.includes(scopeKey)) continue; // already resolved above
867
+ const rec = this._scopes.get(scopeKey);
868
+ if (rec && rec.status === 'closed') rec.breaker.recordSuccess();
869
+ }
870
+ if (coveringKeys.length === 0) this._recordObservedAttempt(null, invocationId);
871
+ } else if (kind === 'failed') {
872
+ this._settleFailure(token, outcome ?? {}, now);
873
+ }
874
+ // #2915 review 4 (finding 8): an 'uncertain' settlement is NEVER
875
+ // recorded as an observed attempt — it is the kind every pre-spawn
876
+ // refusal settles as (see `settleDirectWorker`), and this diagnostic
877
+ // counter must not read as if something dispatched when it did not.
878
+ // The allowance LEDGER (`debited`), not this counter, is what actually
879
+ // enforces attempt consumption.
880
+ //
881
+ // #2915 round 8 (D3): deletion, outstanding-membership cleanup and the
882
+ // reserved-scope-slot release are now ALL performed by the one private
883
+ // `_release` — the only code path that ever removes a reservation table
884
+ // entry or clears an `activeCanary`/`blockingMarker` reference naming it.
885
+ //
886
+ // #2915 round 8 (C1, "outcome settlement discards unresolved execution"):
887
+ // `_release` is safe here ONLY when this reservation never reached
888
+ // `dispatched` (nothing was ever started — identical to the `admitted`-
889
+ // phase branch of `_sweepReservations`) OR this very settlement carries
890
+ // CONFIRMED stop evidence (`outcome.confirmedStop === true`). Every other
891
+ // case — a dispatched reservation settling `succeeded`/`failed`/
892
+ // `uncertain` with no confirmed stop — records the outcome above (scope
893
+ // recover/reopen, failure attribution) exactly as before, but retains
894
+ // the reservation and installs the SAME bounded stop-evidence marker
895
+ // `_sweepReservations` installs on canary-deadline expiry, so a NEW
896
+ // canary can never be admitted over this still-unconfirmed descendant
897
+ // tree (#2915 round 8 review 8, finding 1: "a dispatched canary settled
898
+ // uncertain before its deadline... after backoff, another canary was
899
+ // admitted over the unresolved execution"). A later `settle()` call on
900
+ // this exact token lands in the `entry.stop === 'unresolved'` branch
901
+ // above, which already resolves the marker exactly like an expired
902
+ // canary's late report — the same one-more-call contract, never a
903
+ // separate mechanism.
904
+ if (!wasDispatched || confirmedStop) {
905
+ if (wasDispatched) entry.stop = 'confirmed';
906
+ this._release(entry);
907
+ return { ok: true };
908
+ }
909
+ entry.stop = 'unresolved';
910
+ for (const scopeKey of ownedCanaryKeys) {
911
+ const rec = this._scopes.get(scopeKey);
912
+ if (rec) rec.blockingMarker = { reservationId: token.id, invocationId, setAtMs: now };
913
+ }
914
+ return { ok: true };
915
+ }
916
+
917
+ /**
918
+ * Failure attribution + the distinct-threshold/cohort-unanimous open rule,
919
+ * folded into `settle()`. Scopes this token already held as a canary were
920
+ * resolved directly above (reopen on failure, never re-evaluated against
921
+ * the distinct-failure threshold, matching the former `recordFailure`
922
+ * canary short-circuit); every other covering scope goes through the
923
+ * narrowing + correlation rule unchanged from the pre-#2915-final design.
924
+ */
925
+ _settleFailure(token, outcome, now) {
926
+ const { invocationId, canaryKeys } = token;
927
+ const supported = attributeFailureScope({
928
+ reasonCode: outcome.reasonCode ?? null, evidenceSource: outcome.evidenceSource ?? null, invocationId,
929
+ modelKey: outcome.modelKey ?? null, cli: outcome.cli ?? null, inferenceProvider: outcome.inferenceProvider ?? null,
930
+ quotaAccount: outcome.quotaAccount ?? null, hostId: outcome.hostId ?? null,
931
+ corroboratedBackends: outcome.corroboratedBackends ?? null, installationQualifier: outcome.installationQualifier ?? null,
932
+ });
933
+ // When the caller supplies an explicit `scope`/`scopeKey` claim, the
934
+ // recorded scope is the NARROWER of that claim and what the attached
935
+ // evidence supports — a wide claim on worker-derived or identity-less
936
+ // evidence folds back to whatever the evidence actually proves, but a
937
+ // claim that is already as narrow or narrower than the evidence (the
938
+ // normal case: the caller pre-derived this exact attribution, e.g.
939
+ // `classifyAttemptForSettlement`) is kept EXACTLY as given, including
940
+ // its specific scopeKey — a same-rank evidence key (e.g. the generic
941
+ // `host:local` an omitted `hostId` resolves to) must never silently
942
+ // replace a caller's more specific identity (e.g. `host:noodle`) at the
943
+ // same scope breadth. A caller that omits `scope`/`scopeKey` entirely
944
+ // gets the evidence-derived attribution directly.
945
+ let scope;
946
+ let scopeKey;
947
+ if (outcome.scope && outcome.scopeKey) {
948
+ scope = outcome.scope;
949
+ scopeKey = outcome.scopeKey;
950
+ if (scopeRank(supported.scope) < scopeRank(scope)) { scope = supported.scope; scopeKey = supported.scopeKey; }
951
+ } else {
952
+ scope = supported.scope;
953
+ scopeKey = supported.scopeKey;
954
+ }
955
+ if (canaryKeys.includes(scopeKey)) { this._recordObservedAttempt(scopeKey, invocationId); return; } // already reopened as this token's canary failure, above
956
+ const rec = this._scopeRecordBounded(scope, scopeKey);
957
+ // Charge AFTER the scope record exists (or is confirmed unattainable at
958
+ // the hard cap) so `rec.attemptsObserved` reflects this attempt whenever
959
+ // a record is created for it, rather than undercounting the very failure
960
+ // that first creates the scope.
961
+ this._recordObservedAttempt(scopeKey, invocationId);
962
+ if (!rec) return; // at hard capacity with nothing evictable: bookkeeping-only, never exceeds the bound
963
+ this._touch(rec);
964
+ this._removeOutstanding(scopeKey, invocationId);
965
+ // Genuine rolling-window retention: prune only failures older than the
966
+ // window. Refreezing the cohort epoch must NOT discard still-recent
967
+ // failures.
968
+ rec.failures = rec.failures.filter((f) => now - f.atMs < this.policy.windowMs);
969
+ if (rec.cohortFrozenAt === null || now - rec.cohortFrozenAt >= this.policy.windowMs) {
970
+ rec.cohort = new Set([...(this._outstanding.get(scopeKey)?.keys() ?? []), invocationId]);
971
+ rec.cohortFrozenAt = now;
972
+ }
973
+ if (!rec.failures.some((f) => f.invocationId === invocationId)) {
974
+ rec.failures.push({ invocationId, atMs: now, reasonCode: outcome.reasonCode ?? null });
975
+ }
976
+ const distinct = new Set(rec.failures.map((f) => f.invocationId));
977
+ const reasonLabel = outcome.reasonCode ?? 'UNKNOWN';
978
+ rec.evidence.set(reasonLabel, (rec.evidence.get(reasonLabel) ?? 0) + 1);
979
+ const reset = this._resolveResetAtMs({ resetAtMs: outcome.resetAtMs ?? null, retryAfterSeconds: outcome.retryAfterSeconds ?? null }, now);
980
+ if (reset !== null && (rec.resetAtMs === null || reset > rec.resetAtMs)) {
981
+ rec.resetAtMs = reset;
982
+ if (rec.status !== 'closed') rec.nextProbeAtMs = Math.max(rec.nextProbeAtMs, reset + this._boundedJitter());
983
+ }
984
+ if (rec.status !== 'closed') {
985
+ // Already open and this token held no canary on it (defensive: admit()
986
+ // never admits a non-canary reservation covering an open scope).
987
+ return;
988
+ }
989
+ const cohort = rec.cohort ?? new Set();
990
+ const allOutstandingFailed = cohort.size >= this.policy.minOutstanding && [...cohort].every((id) => distinct.has(id));
991
+ if (distinct.size >= this.policy.distinctThreshold || allOutstandingFailed) {
992
+ rec.openedAtMs = now;
993
+ rec.nextProbeAtMs = Math.max(now + rec.episodeCooldownMs, rec.resetAtMs ?? 0) + this._boundedJitter();
994
+ rec.breaker.cooldownMs = rec.episodeCooldownMs;
995
+ rec.breaker.recordFailure();
996
+ this._transition(rec, 'open', 'pause covered launches; wait for reset/cooldown then probe');
997
+ }
998
+ }
999
+
1000
+ _recoverScope(rec, invocationId) {
1001
+ rec.reopens = 0;
1002
+ rec.episodeCooldownMs = this.policy.cooldownMs;
1003
+ rec.resetAtMs = null;
1004
+ rec.nextProbeAtMs = 0;
1005
+ rec.failures = [];
1006
+ rec.cohort = null;
1007
+ rec.cohortFrozenAt = null;
1008
+ this._transition(rec, 'closed', 'canary succeeded; resume covered launches');
1009
+ rec.breaker.reset();
1010
+ }
1011
+
1012
+ requireOperatorIntervention(scopeKeys, reason) {
1013
+ for (const key of scopeKeys) {
1014
+ const rec = this._scopes.get(key);
1015
+ if (!rec || rec.status === 'closed') continue;
1016
+ rec.operatorReason = `${reason}; restore a qualified scope-relevant probe, then reset this scope`;
1017
+ this._transition(rec, 'operator-intervention', rec.operatorReason);
1018
+ }
1019
+ }
1020
+
1021
+ _transition(rec, to, action) {
1022
+ const from = rec.status;
1023
+ if (from === to) return;
1024
+ rec.status = to;
1025
+ this.emit('transition', { scopeKey: rec.scopeKey, from, to, action,
1026
+ distinctFailures: new Set(rec.failures.map(failure => failure.invocationId)).size, reasons: [...rec.evidence.keys()],
1027
+ nextEligibleAtMs: rec.nextProbeAtMs, resetAtMs: rec.resetAtMs,
1028
+ summary: `${rec.scopeKey}: ${from} -> ${to}; ${action}; wait ${Math.max(0, rec.nextProbeAtMs - this.nowMs())}ms`,
1029
+ });
1030
+ }
1031
+
1032
+ /**
1033
+ * #2915 review 4 (finding 8): DIAGNOSTIC only — `attemptsObserved`/
1034
+ * `canariesObserved` (renamed from `attemptsCharged`/`canariesCharged`)
1035
+ * report what this breaker has seen, never what it enforces. Nothing in
1036
+ * `admit()` reads either counter; the allowance LEDGER
1037
+ * (`reserveDirectWorkerAllowance`/`debitDirectWorkerAllowance` in
1038
+ * session-launcher.mjs) is the actual enforcement. `settle()` only calls
1039
+ * this for `'succeeded'`/`'failed'` kinds — an `'uncertain'` settlement
1040
+ * (every pre-spawn refusal settles this way) is never recorded here.
1041
+ */
1042
+ _recordObservedAttempt(scopeKey, invocationId) {
1043
+ if (invocationId == null || this._observedInvocations.has(invocationId)) return false;
1044
+ this._observedInvocations.add(invocationId);
1045
+ if (this._observedInvocations.size > this.policy.maxChargedAttempts) {
1046
+ const oldest = this._observedInvocations.values().next().value;
1047
+ this._observedInvocations.delete(oldest);
1048
+ }
1049
+ const rec = scopeKey ? this._scopes.get(scopeKey) : null;
1050
+ if (rec) rec.attemptsObserved += 1;
1051
+ return true;
1052
+ }
1053
+
1054
+ /**
1055
+ * Account every reservation past its own deadline — including non-canary
1056
+ * reservations, which share the same bounded-TTL field (though its value
1057
+ * is `Infinity` for them in practice) so `_reservations` cannot grow
1058
+ * without bound from attempts whose caller crashed before calling
1059
+ * `settle()`. A hung or unreported canary is treated as a failed canary:
1060
+ * it reopens with doubled backoff immediately (the deadline enforcement
1061
+ * `expireStaleReservations` used to apply only at the scope level now
1062
+ * applies uniformly through the reservation registry).
1063
+ *
1064
+ * #2915 review 5 ("a second worker over unresolved execution"): reopening
1065
+ * with backoff is NOT settlement here — the reservation itself is
1066
+ * deliberately retained (not deleted) and a bounded blocking marker is
1067
+ * installed on every canary scope it covered, because expiry alone proves
1068
+ * nothing about whether the attempt actually stopped. `admit()` refuses
1069
+ * while the marker is set, regardless of elapsed backoff; only a later
1070
+ * `settle()` call against this exact token, carrying confirmed-
1071
+ * termination evidence (`outcome.confirmedStop`), clears it — see
1072
+ * `settle()`. `entry.stop !== 'none-needed'` guards this block from
1073
+ * re-running on a later sweep of the same still-unresolved entry.
1074
+ *
1075
+ * Called at the top of every `admit`/`settle`/`beginDispatch`/
1076
+ * `commitDispatch`, so expiry is always observed on the very next call
1077
+ * even when the admitted attempt never reports back at all (process
1078
+ * crash, dropped callback) — the backstop that makes the design
1079
+ * deadlock-free once stop is eventually confirmed
1080
+ * (`deadline-reopens-before-readmission`).
1081
+ *
1082
+ * #2915 round 8 (D3, "expiry during dispatch commitment"): behaviour now
1083
+ * branches on `entry.phase`, because expiry means something different at
1084
+ * each phase:
1085
+ *
1086
+ * - `admitted` (never even reached `beginDispatch`): PROVEN to have never
1087
+ * dispatched anything, exactly like an unreported canary always was —
1088
+ * reopen (if this was a canary) with doubled backoff and release
1089
+ * outright via `_release`. No stop-evidence marker is ever installed
1090
+ * here: there is nothing to confirm the stop of.
1091
+ * - `committing` (inside the D2 hook, between `beginDispatch` and
1092
+ * `commitDispatch`): the reservation is marked `revoked` ONLY — never
1093
+ * reopened, never released. `commitDispatch` and `abortDispatch` both
1094
+ * observe `revoked`; only once the committing owner itself calls one of
1095
+ * them (acknowledging the abort) does this reservation actually resolve
1096
+ * — expiry alone must never classify a `committing` reservation as
1097
+ * unused (#2915 round 8 critique: "expiry can revoke a pending launch,
1098
+ * but cannot classify it as safely unused until the committing owner
1099
+ * acknowledges abort").
1100
+ * - `dispatched`: unchanged from the prior design — reopen (if a canary)
1101
+ * with doubled backoff and install a bounded stop-evidence marker
1102
+ * (`stop: 'unresolved'`), because expiry alone proves nothing about
1103
+ * whether the dispatched attempt actually stopped. `admit()` refuses
1104
+ * while the marker is set, regardless of elapsed backoff; only a later
1105
+ * `settle()`/`reconcileStopEvidence()` call carrying confirmed-
1106
+ * termination evidence clears it.
1107
+ */
1108
+ _sweepReservations(now) {
1109
+ for (const [id, entry] of this._reservations) {
1110
+ // #2915 round 8 (C1, "the sweep directly deletes outcome-settled
1111
+ // entries"): a settled entry still present in the table is one
1112
+ // `settle()` itself deliberately retained — `entry.phase ===
1113
+ // 'dispatched'` with `entry.stop !== 'confirmed'` (see `settle()`) —
1114
+ // and is removed ONLY by `_release` (via a later `settle()` call that
1115
+ // now lands in the `entry.stop === 'unresolved'` branch, or
1116
+ // `reconcileStopEvidence`), never by this sweep. Deadline/phase expiry
1117
+ // below is for UNSETTLED reservations only.
1118
+ if (entry.settled) continue;
1119
+ if (now < entry.token.deadlineMs) continue;
1120
+ if (entry.phase === 'admitted') {
1121
+ const { canaryKeys } = entry.token;
1122
+ for (const scopeKey of canaryKeys) {
1123
+ const rec = this._scopes.get(scopeKey);
1124
+ if (rec?.activeCanary?.id === id) { rec.activeCanary = null; this._reopen(rec, now); }
1125
+ }
1126
+ this._release(entry);
1127
+ continue;
1128
+ }
1129
+ if (entry.phase === 'committing') {
1130
+ entry.revoked = true; // idempotent; commitDispatch/abortDispatch resolve it
1131
+ continue;
1132
+ }
1133
+ // `dispatched`.
1134
+ if (entry.stop !== 'none-needed') continue; // already marked unresolved/confirmed; awaiting settle()/reconcile
1135
+ entry.stop = 'unresolved';
1136
+ const { invocationId, coveringKeys, canaryKeys } = entry.token;
1137
+ for (const scopeKey of canaryKeys) {
1138
+ const rec = this._scopes.get(scopeKey);
1139
+ if (rec?.activeCanary?.id === id) {
1140
+ rec.activeCanary = null;
1141
+ this._reopen(rec, now);
1142
+ rec.blockingMarker = { reservationId: id, invocationId, setAtMs: now };
1143
+ }
1144
+ }
1145
+ this.noteSettled({ scopeKeys: coveringKeys, invocationId });
1146
+ }
1147
+ }
1148
+
1149
+ /** Public alias retained for direct test/diagnostic use; `admit`/`settle`
1150
+ * already call this internally on every invocation. */
1151
+ expireStaleReservations() {
1152
+ this._sweepReservations(this.nowMs());
1153
+ }
1154
+
1155
+ /**
1156
+ * Prune every outstanding cohort-membership entry older than the window,
1157
+ * across ALL tracked scope keys — not just the ones this call's own
1158
+ * covering set touches (#2915 final review, finding 6: pruning used to
1159
+ * touch only keys revisited by `noteLaunched`, so an invocation scope for
1160
+ * an attempt that never launched again, or never settled, retained its
1161
+ * membership forever even once the window had long since elapsed). Bounded
1162
+ * by the number of tracked outstanding scope keys, which is itself bounded
1163
+ * in steady state by `maxTrackedScopes`/`maxOutstandingReservations`.
1164
+ */
1165
+ _sweepOutstanding(now) {
1166
+ for (const [key, set] of this._outstanding) {
1167
+ for (const [id, atMs] of set) {
1168
+ if (now - atMs >= this.policy.windowMs) set.delete(id);
1169
+ }
1170
+ if (set.size === 0) this._outstanding.delete(key);
1171
+ }
1172
+ }
1173
+
1174
+ /**
1175
+ * One private cleanup path (#2915 round 8, D3): the ONLY code that ever
1176
+ * removes a `_reservations` table entry, or clears an `activeCanary`/
1177
+ * `blockingMarker` scope reference naming it. Preconditions, enforced by
1178
+ * every caller (not re-checked here — this method trusts its callers,
1179
+ * all of which are other methods on this class): the reservation must
1180
+ * never have reached `dispatched`, OR its `stop` must already be
1181
+ * `'confirmed'`. A late callback — the stop-escalation observer, an
1182
+ * operator's `reconcileStopEvidence` — may still race this exact call; it
1183
+ * is idempotent only in the sense that `_reservations.delete` on an
1184
+ * already-absent id is a no-op, matching every other settlement path's
1185
+ * "exactly once" contract enforced by its own caller.
1186
+ *
1187
+ * #2915 round 8 (C1): the precondition above is now ALSO enforced here,
1188
+ * defensively — never only trusted of callers — so a future call site
1189
+ * that forgets to check it fails loudly (throws) instead of silently
1190
+ * releasing live, unconfirmed execution.
1191
+ */
1192
+ _release(entry) {
1193
+ const { token } = entry;
1194
+ if (entry.phase === 'dispatched' && entry.stop !== 'confirmed') {
1195
+ throw new Error('ScopedFailureBreaker: refusing to release a dispatched reservation without confirmed stop');
1196
+ }
1197
+ for (const scopeKey of token.canaryKeys) {
1198
+ const rec = this._scopes.get(scopeKey);
1199
+ if (!rec) continue;
1200
+ if (rec.activeCanary?.id === token.id) rec.activeCanary = null;
1201
+ if (rec.blockingMarker?.reservationId === token.id) rec.blockingMarker = null;
1202
+ }
1203
+ this.noteSettled({ scopeKeys: token.coveringKeys, invocationId: token.invocationId });
1204
+ this._reservations.delete(token.id);
1205
+ // #2915 review 6 (finding 2): release the tracking slot THIS admission
1206
+ // reserved (`reservedScopeKeys`, set in `admit()`) back to the shared
1207
+ // pool if the scope it covers is now inert -- never a scope this
1208
+ // admission merely found already tracked (which may hold failure
1209
+ // history or outstanding members belonging to other in-flight work).
1210
+ for (const scopeKey of entry.reservedScopeKeys ?? []) this._releaseIfInert(scopeKey);
1211
+ }
1212
+
1213
+ /**
1214
+ * Read-only phase introspection (#2915 round 8, D2/D3): lets a caller that
1215
+ * holds an admission (e.g. `settleDirectWorker` in session-launcher.mjs)
1216
+ * decide whether a reservation ever reached `dispatched` without
1217
+ * reconstructing authority from anything but the breaker's own stored
1218
+ * entry. Returns `null` for an unknown/foreign/already-released token —
1219
+ * never throws, never mutates.
1220
+ */
1221
+ reservationPhase(token) {
1222
+ if (!token || typeof token.id !== 'string' || token.breaker !== this) return null;
1223
+ const entry = this._reservations.get(token.id);
1224
+ if (!entry || entry.token !== token) return null;
1225
+ return entry.phase;
1226
+ }
1227
+
1228
+ _dispatchScopeRefusal(token, now) {
1229
+ for (const key of token.coveringKeys) {
1230
+ const rec = this._scopes.get(key);
1231
+ if (!rec || rec.status === 'closed') continue;
1232
+ if (rec.blockingMarker || (rec.activeCanary?.id !== token.id
1233
+ && !(token.purpose === 'probe' && rec.status === 'canary-ready' && !rec.activeCanary))
1234
+ || now < Math.max(rec.resetAtMs ?? 0, rec.nextProbeAtMs ?? 0)) {
1235
+ return { ok: false, reason: `covering recovery scope changed before dispatch: ${key}` };
1236
+ }
1237
+ }
1238
+ return null;
1239
+ }
1240
+
1241
+ /**
1242
+ * Begin the actual dispatch of one admitted reservation, exactly ONCE
1243
+ * (#2915 round 8, D3, replaces `claimLaunch`): `admitted -> committing`.
1244
+ * Called once, at the leaf dispatch boundary — inside the D2
1245
+ * `onDispatchCommit` hook, immediately before the real process-start call.
1246
+ *
1247
+ * `invocationId` is REQUIRED and may never be `null`/`undefined`/omitted
1248
+ * (#2915 round 8 critique: "omitted/null bindings must not bypass checks";
1249
+ * round 8 review 8 finding 6 hardening: an EXPLICIT `invocationId: null`
1250
+ * call must refuse exactly like an omitted one, and a reservation whose
1251
+ * own stored `token.invocationId` is itself `null`/`undefined` — which
1252
+ * `admit()` never produces for any real caller, since every `invocationId`
1253
+ * it receives is either caller-supplied or freshly minted — can never
1254
+ * dispatch no matter what the caller later supplies). `selectionDigest`
1255
+ * is REQUIRED but MAY legitimately be `null` on both sides: `admit()`
1256
+ * stores `null` for it whenever the caller has no resolved selection
1257
+ * identity to bind (flat-option/no-`invocation` callers; see
1258
+ * `selectionDigestFor`), and a large share of this module's own
1259
+ * scope/correlation tests deliberately exercise that case — only
1260
+ * `undefined`/omission is refused for it, never a `null` that matches the
1261
+ * reservation's own stored `null`. Both are compared against the
1262
+ * BREAKER-OWNED, frozen `token` fields set at `admit()` time — never
1263
+ * against a caller's own mutable wrapper object. A caller must derive
1264
+ * these values from its own trusted, resolved launch (never trust them as
1265
+ * given by whatever object is borrowing this reservation): a reservation
1266
+ * minted for one invocation/selection can then never authorize dispatching
1267
+ * a different one, independent of what any outer wrapper object claims.
1268
+ *
1269
+ * Refuses (mutates nothing) for: an unowned/unknown/foreign token; a
1270
+ * second call on an already-`committing`/`dispatched` reservation; a
1271
+ * missing/null `invocationId` (on either side) or omitted
1272
+ * `selectionDigest`; a binding mismatch; or a reservation whose own
1273
+ * deadline has already passed (`now >= deadline`, consistent with every
1274
+ * other deadline check in this module).
1275
+ */
1276
+ beginDispatch(token, { invocationId, selectionDigest } = {}) {
1277
+ const now = this.nowMs();
1278
+ this._sweepReservations(now);
1279
+ if (!token || typeof token.id !== 'string' || token.breaker !== this) {
1280
+ return { ok: false, reason: 'reservation is not owned by this breaker' };
1281
+ }
1282
+ const entry = this._reservations.get(token.id);
1283
+ if (!entry || entry.token !== token) {
1284
+ return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
1285
+ }
1286
+ if (invocationId == null || selectionDigest === undefined) {
1287
+ return { ok: false, reason: 'beginDispatch requires explicit invocationId and selectionDigest bindings' };
1288
+ }
1289
+ if (entry.token.invocationId == null) {
1290
+ return { ok: false, reason: 'reservation was admitted without an invocationId binding and can never dispatch' };
1291
+ }
1292
+ if (invocationId !== entry.token.invocationId || selectionDigest !== entry.token.selectionDigest) {
1293
+ return { ok: false, reason: 'reservation binding does not match the resolved dispatch' };
1294
+ }
1295
+ if (entry.phase !== 'admitted') {
1296
+ return { ok: false, reason: `reservation is not admitted (phase: ${entry.phase})` };
1297
+ }
1298
+ if (now >= token.deadlineMs) {
1299
+ return { ok: false, reason: 'reservation deadline has passed' };
1300
+ }
1301
+ const refused = this._dispatchScopeRefusal(token, now);
1302
+ if (refused) return refused;
1303
+ entry.phase = 'committing';
1304
+ return { ok: true };
1305
+ }
1306
+
1307
+ /**
1308
+ * Commit a `committing` reservation to `dispatched`, after the hook's own
1309
+ * debit succeeded (#2915 round 8, D3): `committing -> dispatched`. Refuses
1310
+ * (mutates nothing but `revoked`, see below) for an unowned/unknown
1311
+ * token, one not currently `committing`, or one whose deadline passed
1312
+ * while committing — the latter sets `revoked` (idempotent) so the
1313
+ * caller's required `abortDispatch` call (below) applies the correct
1314
+ * "timed out while committing" handling, rather than silently
1315
+ * classifying a never-dispatched attempt as an ordinary clean abort.
1316
+ */
1317
+ commitDispatch(token) {
1318
+ const now = this.nowMs();
1319
+ if (!token || typeof token.id !== 'string' || token.breaker !== this) {
1320
+ return { ok: false, reason: 'reservation is not owned by this breaker' };
1321
+ }
1322
+ const entry = this._reservations.get(token.id);
1323
+ if (!entry || entry.token !== token) {
1324
+ return { ok: false, reason: 'reservation is unknown, already settled, expired, or does not match the stored token' };
1325
+ }
1326
+ if (entry.phase !== 'committing') {
1327
+ return { ok: false, reason: `reservation is not committing (phase: ${entry.phase})` };
1328
+ }
1329
+ if (now >= token.deadlineMs) {
1330
+ entry.revoked = true;
1331
+ return { ok: false, reason: 'reservation deadline passed while committing; call abortDispatch' };
1332
+ }
1333
+ if (entry.revoked) {
1334
+ return { ok: false, reason: 'reservation was revoked while committing; call abortDispatch' };
1335
+ }
1336
+ const refused = this._dispatchScopeRefusal(token, now);
1337
+ if (refused) return refused;
1338
+ entry.phase = 'dispatched';
1339
+ return { ok: true };
1340
+ }
1341
+
1342
+ /**
1343
+ * Explicitly back out of a reservation that never reached `dispatched`
1344
+ * (#2915 round 8, D2/D3): the D2 hook calls this when its own work after
1345
+ * `beginDispatch` (the ledger debit) throws, or when `commitDispatch`
1346
+ * itself refused. Equivalent to `settle(token, { kind: 'uncertain',
1347
+ * confirmedStop: true })` — a PROVEN zero-dispatch exit, so a canary this
1348
+ * reservation held is treated exactly like any other unreported/aborted
1349
+ * probe (reopened with backoff unless explicitly caller-aborted) and is never left with a stop-evidence
1350
+ * marker, matching every other proven pre-spawn refusal in this module.
1351
+ * Refused once the reservation has actually reached `dispatched`: an
1352
+ * attempt that genuinely started must settle with its real outcome
1353
+ * through `settle()`, never a fabricated confirmed stop.
1354
+ */
1355
+ abortDispatch(token, { callerAborted = false } = {}) {
1356
+ const entry = this._reservations.get(token?.id);
1357
+ if (entry && entry.token === token && entry.phase === 'dispatched') {
1358
+ return { ok: false, reason: 'reservation already dispatched; use settle() instead' };
1359
+ }
1360
+ return this.settle(token, { kind: 'uncertain', confirmedStop: true, callerAborted });
1361
+ }
1362
+
1363
+ /**
1364
+ * Reconcile ONE invocation's reservation(s) from stop evidence arriving
1365
+ * independently of the original `admit()`/`settle()` pair (#2915 review
1366
+ * 6, finding 3d): a bounded stop-escalation observer, or an operator
1367
+ * acting on `ralph-external` diagnostics, may know only the invocation
1368
+ * id -- not the exact token object `admit()` minted, which may already be
1369
+ * out of scope by the time termination is confirmed. Only
1370
+ * `confirmedStop: true` releases anything; anything else is a no-op (late
1371
+ * or uncertain evidence can never resolve a marker or free capacity).
1372
+ * Clears any blocking marker naming the reservation, clears its
1373
+ * outstanding cohort membership, and deletes the reservation itself --
1374
+ * idempotent: an invocation with no matching live reservation is a no-op.
1375
+ */
1376
+ reconcileStopEvidence({ invocationId, confirmedStop = false } = {}) {
1377
+ if (!confirmedStop || invocationId == null) return { released: 0 };
1378
+ let released = 0;
1379
+ for (const [, entry] of [...this._reservations]) {
1380
+ if (entry.token.invocationId !== invocationId) continue;
1381
+ if (entry.phase === 'dispatched') entry.stop = 'confirmed';
1382
+ this._release(entry);
1383
+ released += 1;
1384
+ }
1385
+ return { released };
1386
+ }
1387
+
1388
+ /**
1389
+ * Read-only precondition check (#2915 round 8, D4): whether `invocationId`
1390
+ * currently holds a `dispatched` reservation with `stop: 'unresolved'` on
1391
+ * THIS breaker — exactly the condition the MCP `reconcile_stopped`
1392
+ * operation (src/mcp/tools/worker-invoke.mjs) must verify before calling
1393
+ * `reconcileStopEvidence`, so that operation only ever accepts (and only
1394
+ * ever reports as "applied") a reconciliation against a reservation that
1395
+ * genuinely needs one. Never mutates, never throws.
1396
+ */
1397
+ hasUnresolvedStop(invocationId) {
1398
+ if (invocationId == null) return false;
1399
+ for (const entry of this._reservations.values()) {
1400
+ if (entry.token.invocationId === invocationId && entry.phase === 'dispatched' && entry.stop === 'unresolved') return true;
1401
+ }
1402
+ return false;
1403
+ }
1404
+
1405
+ /**
1406
+ * Read-only enumeration (#2915 round 8, D4): every reservation on THIS
1407
+ * breaker currently sitting `dispatched` with `stop: 'unresolved'` — the
1408
+ * exact set an ordinary (no `invocation_id`) MCP `status` call surfaces as
1409
+ * "pending reconciliation", and the same set `admit()`'s own
1410
+ * `blocking-markers:<N>` reservation-capacity diagnostic already counts.
1411
+ * Never mutates, never throws.
1412
+ */
1413
+ listUnresolvedStops(now = this.nowMs()) {
1414
+ const out = [];
1415
+ for (const entry of this._reservations.values()) {
1416
+ if (entry.phase === 'dispatched' && entry.stop === 'unresolved') {
1417
+ out.push({ invocationId: entry.token.invocationId, scopes: [...entry.token.coveringKeys], ageMs: Math.max(0, now - entry.token.mintedAtMs) });
1418
+ }
1419
+ }
1420
+ return out;
1421
+ }
1422
+
1423
+ getState(scopeKey) {
1424
+ const rec = this._scopes.get(scopeKey);
1425
+ if (!rec) return { scopeKey, state: 'closed', tracked: false };
1426
+ const inner = rec.breaker.getState();
1427
+ return {
1428
+ scopeKey,
1429
+ scope: rec.scope,
1430
+ tracked: true,
1431
+ state: rec.status,
1432
+ ...(rec.operatorReason ? { operatorReason: rec.operatorReason } : {}),
1433
+ consecutiveFailures: inner.consecutiveFailures,
1434
+ distinctFailures: new Set(rec.failures.map((f) => f.invocationId)).size,
1435
+ outstanding: this._outstanding.get(scopeKey)?.size ?? 0,
1436
+ cohortSize: rec.cohort?.size ?? 0,
1437
+ activeCanary: rec.activeCanary ? { ...rec.activeCanary } : null,
1438
+ blockingMarker: rec.blockingMarker ? { ...rec.blockingMarker } : null,
1439
+ episodeCooldownMs: rec.episodeCooldownMs,
1440
+ nextProbeAtMs: rec.nextProbeAtMs,
1441
+ resetAtMs: rec.resetAtMs,
1442
+ attemptsObserved: rec.attemptsObserved,
1443
+ canariesObserved: rec.canariesObserved,
1444
+ cooldownRemainingMs: inner.cooldownRemainingMs,
1445
+ };
1446
+ }
1447
+
1448
+ snapshot() {
1449
+ return {
1450
+ scopes: [...this._scopes.keys()].map((key) => this.getState(key)),
1451
+ attemptsObserved: this._observedInvocations.size,
1452
+ canariesObserved: this.canariesObserved,
1453
+ reservationsOutstanding: this._reservations.size,
1454
+ };
1455
+ }
1456
+
1457
+ /** Correlate launcher-observed host faults across resolved inference backends.
1458
+ * Replay the first fault once corroborated, preserving the three-distinct
1459
+ * threshold even when the first backend alone could not establish scope.
1460
+ */
1461
+ observeHostFailure({ invocationId, inferenceProvider, reasonCode } = {}) {
1462
+ if (typeof invocationId !== 'string' || !invocationId || typeof inferenceProvider !== 'string'
1463
+ || !inferenceProvider || !['RESOURCE', 'SANDBOX'].includes(reasonCode)) return [];
1464
+ const now = this.nowMs();
1465
+ this._hostFailureFacts = this._hostFailureFacts.filter(fact => now - fact.atMs < this.policy.windowMs);
1466
+ if (!this._hostFailureFacts.some(fact => fact.invocationId === invocationId)) {
1467
+ this._hostFailureFacts.push({ invocationId, inferenceProvider, reasonCode, atMs: now,
1468
+ cohort: [...(this._outstanding.get('host:local')?.keys() ?? [])] });
1469
+ if (this._hostFailureFacts.length > this.policy.maxOutstandingReservations) this._hostFailureFacts.shift();
1470
+ }
1471
+ const corroboratedBackends = [...new Set(this._hostFailureFacts.map(fact => fact.inferenceProvider))];
1472
+ if (corroboratedBackends.length < 2) return [];
1473
+ const rec = this._scopeRecordBounded(FailureScope.HOST, 'host:local');
1474
+ const first = this._hostFailureFacts[0];
1475
+ if (rec && (rec.cohortFrozenAt === null || now - rec.cohortFrozenAt >= this.policy.windowMs)) {
1476
+ // The first fault could not yet establish host scope. Preserve who was
1477
+ // actually outstanding then; later sequential attempts are not a cohort.
1478
+ rec.cohort = new Set(first.cohort);
1479
+ rec.cohortFrozenAt = first.atMs;
1480
+ }
1481
+ for (const fact of this._hostFailureFacts) {
1482
+ if (rec?.failures.some(failure => failure.invocationId === fact.invocationId)) continue;
1483
+ this._settleFailure({ invocationId: fact.invocationId, canaryKeys: [] }, {
1484
+ reasonCode: fact.reasonCode, evidenceSource: 'launcher', hostId: 'local', corroboratedBackends,
1485
+ }, fact.atMs);
1486
+ }
1487
+ return corroboratedBackends;
1488
+ }
1489
+
1490
+ /**
1491
+ * Reset ONE scope's failure history/backoff bookkeeping (#2915 round 8,
1492
+ * D3, replaces the prior whole-reservation-release behavior): resets
1493
+ * failure history for that scope ONLY, and never releases a reservation
1494
+ * with unresolved execution.
1495
+ *
1496
+ * #2915 round 8 critique ("whole-reservation reset can erase live
1497
+ * protection"): the prior implementation deleted EVERY reservation
1498
+ * covering `scopeKey` outright — including one also covering a DIFFERENT
1499
+ * scope as a canary, so `reset(A)` could silently drop scope B's own
1500
+ * still-relevant `activeCanary`/`blockingMarker` protection too. This
1501
+ * version never touches `_reservations` at all: if the targeted scope
1502
+ * currently holds a covering reservation, `activeCanary` or `blockingMarker` (unresolved
1503
+ * execution — someone else's covering scopes are never even inspected,
1504
+ * let alone released), the whole call is refused with the reconcile
1505
+ * instruction and NOTHING is mutated; otherwise the scope record is
1506
+ * destroyed outright (the same effect the prior "safe" case always had).
1507
+ *
1508
+ * #2915 round 8 review 8 (C1, "unscoped reset() also clears protection
1509
+ * outright"): called with no `scopeKey`, this now applies the EXACT same
1510
+ * per-scope rule to every tracked scope — reset failure history/backoff
1511
+ * for every scope that has no unresolved execution, and leave any scope
1512
+ * that does (an active canary or a blocking marker) completely untouched
1513
+ * — rather than unconditionally destroying every scope record AND
1514
+ * clearing `_reservations` outright. `_reservations` is never touched by
1515
+ * `reset()` in either form; `_release` (via `settle()`/
1516
+ * `reconcileStopEvidence`) is the only code that ever removes a
1517
+ * reservation. A full, unconditional teardown (including live execution
1518
+ * protection) remains available through the separate, explicitly-named
1519
+ * `destroy()` method, intended for test/process-exit teardown only.
1520
+ */
1521
+ reset(scopeKey) {
1522
+ if (!scopeKey) this._hostFailureFacts = [];
1523
+ if (!scopeKey) {
1524
+ for (const [key, rec] of [...this._scopes]) {
1525
+ if (rec.activeCanary !== null || rec.blockingMarker !== null) continue; // keep execution ownership
1526
+ this._transition(rec, 'closed', 'operator reset; resume covered launches');
1527
+ rec.breaker.destroy();
1528
+ this._scopes.delete(key);
1529
+ this._outstanding.delete(key);
1530
+ }
1531
+ return { ok: true };
1532
+ }
1533
+ const rec = this._scopes.get(scopeKey);
1534
+ if (!rec) {
1535
+ if (scopeKey === 'host:local') this._hostFailureFacts = [];
1536
+ return { ok: true };
1537
+ }
1538
+ if (rec.activeCanary !== null || rec.blockingMarker !== null
1539
+ || [...this._reservations.values()].some(entry => entry.token.coveringKeys.includes(scopeKey))) {
1540
+ return { ok: false, scopeKey,
1541
+ reason: 'scope has active or unresolved execution (a reservation, active canary or unresolved stop-evidence marker); reconcile stop evidence (reconcileStopEvidence) before resetting it' };
1542
+ }
1543
+ if (scopeKey === 'host:local') this._hostFailureFacts = [];
1544
+ this._transition(rec, 'closed', 'operator reset; resume covered launches');
1545
+ rec.breaker.destroy();
1546
+ this._scopes.delete(scopeKey);
1547
+ this._outstanding.delete(scopeKey);
1548
+ return { ok: true };
1549
+ }
1550
+
1551
+ destroy() {
1552
+ this._hostFailureFacts = [];
1553
+ for (const rec of this._scopes.values()) rec.breaker.destroy();
1554
+ this._scopes.clear();
1555
+ this._outstanding.clear();
1556
+ this._reservations.clear();
1557
+ this.removeAllListeners();
1558
+ }
1559
+
1560
+ // --- Private ---
1561
+
1562
+ /**
1563
+ * Create-or-fetch a tracked scope record, enforcing a true hard cap
1564
+ * (#2915 final review, finding 6): once at `maxTrackedScopes` with no
1565
+ * CLOSED/memberless scope to evict, a brand-new scope key is refused
1566
+ * tracking entirely — `null` — rather than growing past the bound or ever
1567
+ * evicting an OPEN/outstanding scope. The caller still charges the
1568
+ * attempt; it simply cannot attribute it to a persistently-tracked scope
1569
+ * record under this adversarial condition.
1570
+ *
1571
+ * Invocation-scoped records are counted and bounded SEPARATELY from every
1572
+ * shared scope (installation/quota/provider/host/model), against
1573
+ * `maxChargedAttempts` rather than `maxTrackedScopes` (#2915 review 4,
1574
+ * finding 6: "invocation-local records outside the shared-scope quota").
1575
+ * An invocation record can never legitimately open (cohort/distinct
1576
+ * counting is always 1 for a single invocation id), so a flood of
1577
+ * single-task failures can never exhaust the small shared-scope budget
1578
+ * that genuinely correlated installation/host/provider/quota outages
1579
+ * depend on, and vice versa — two OPEN shared scopes can never be forced
1580
+ * out to make room for one-off invocation bookkeeping.
1581
+ */
1582
+ _scopeRecordBounded(scope, scopeKey) {
1583
+ let rec = this._scopes.get(scopeKey);
1584
+ if (rec) return rec;
1585
+ const isInvocation = scope === FailureScope.INVOCATION;
1586
+ const cap = isInvocation ? this.policy.maxChargedAttempts : this.policy.maxTrackedScopes;
1587
+ const count = this._scopeCount(isInvocation);
1588
+ if (count >= cap && !this._evictInertScopeIfPossible(isInvocation)) {
1589
+ return null;
1590
+ }
1591
+ rec = {
1592
+ scope,
1593
+ scopeKey,
1594
+ // Scope-level status is authoritative for gates and reports; the
1595
+ // inner breaker owns only the half-open transitions, whose
1596
+ // timer-driven states must never read as scope recovery. The inner
1597
+ // breaker shares this instance's injected clock (one clock, #2915E).
1598
+ status: 'closed',
1599
+ breaker: new ConsecutiveFailureCircuitBreaker({
1600
+ failureThreshold: 1,
1601
+ cooldownMs: this.policy.cooldownMs,
1602
+ halfOpenMaxProbes: 1,
1603
+ nowMs: this.nowMs,
1604
+ }),
1605
+ failures: [],
1606
+ cohort: null,
1607
+ cohortFrozenAt: null,
1608
+ evidence: new Map(),
1609
+ resetAtMs: null,
1610
+ openedAtMs: null,
1611
+ nextProbeAtMs: 0,
1612
+ episodeCooldownMs: this.policy.cooldownMs,
1613
+ activeCanary: null,
1614
+ // #2915 review 5: a bounded marker set when a canary's own deadline
1615
+ // passes with stop unconfirmed (see `_sweepReservations`); cleared
1616
+ // only by a `settle()` call carrying confirmed-termination evidence.
1617
+ blockingMarker: null,
1618
+ reopens: 0,
1619
+ attemptsObserved: 0,
1620
+ canariesObserved: 0,
1621
+ lastTouchedMs: this.nowMs(),
1622
+ };
1623
+ this._scopes.set(scopeKey, rec);
1624
+ return rec;
1625
+ }
1626
+
1627
+ /** Last-touched bookkeeping for LRU eviction of inert (closed, memberless) scopes. */
1628
+ _touch(rec) {
1629
+ rec.lastTouchedMs = this.nowMs();
1630
+ }
1631
+
1632
+ /** Count tracked scopes in one category (invocation-scoped vs every
1633
+ * shared scope) — the two independent quotas `_scopeRecordBounded`
1634
+ * enforces. Bounded by `_scopes.size`, itself bounded by the two maxima. */
1635
+ _scopeCount(isInvocation) {
1636
+ let count = 0;
1637
+ for (const rec of this._scopes.values()) {
1638
+ if ((rec.scope === FailureScope.INVOCATION) === isInvocation) count += 1;
1639
+ }
1640
+ return count;
1641
+ }
1642
+
1643
+ /**
1644
+ * Explicit, bounded eviction (#2915 review: settled outstanding/scope maps
1645
+ * never dropped their own keys, so 1,000 successful attempts retained
1646
+ * 1,000+ map entries forever). Only CLOSED scopes with no outstanding
1647
+ * members AND no failure evidence still inside the rolling window are
1648
+ * eligible — an open scope, one with outstanding attempts, or one with
1649
+ * recent (in-window) failure evidence is never evicted (#2915 review 4,
1650
+ * finding 6: "never evict recent failure evidence or OPEN scopes" — a
1651
+ * two-failure CLOSED scope that has not yet reached the distinct
1652
+ * threshold must survive long enough for a third, still-in-window
1653
+ * failure to correlate against it). Only scopes matching `isInvocation`
1654
+ * are considered, so eviction never crosses between the two independent
1655
+ * quotas `_scopeRecordBounded` enforces. The scan is bounded by
1656
+ * `maxTrackedScopes`/`maxChargedAttempts`, never unbounded. Returns
1657
+ * whether an eviction actually freed a slot, so `_scopeRecordBounded` can
1658
+ * refuse to track a new scope at true capacity instead of growing past
1659
+ * the cap (#2915 final review, finding 6: the former best-effort bound
1660
+ * could be exceeded whenever nothing was evictable).
1661
+ */
1662
+ _evictInertScopeIfPossible(isInvocation) {
1663
+ let oldestKey = null;
1664
+ let oldestAt = Infinity;
1665
+ for (const [key, rec] of this._scopes) {
1666
+ if ((rec.scope === FailureScope.INVOCATION) !== isInvocation) continue;
1667
+ if (!this._isInertScope(rec, key)) continue;
1668
+ if (rec.lastTouchedMs < oldestAt) { oldestAt = rec.lastTouchedMs; oldestKey = key; }
1669
+ }
1670
+ if (oldestKey === null) return false;
1671
+ this._scopes.get(oldestKey)?.breaker.destroy();
1672
+ this._scopes.delete(oldestKey);
1673
+ this._outstanding.delete(oldestKey);
1674
+ return true;
1675
+ }
1676
+
1677
+ /**
1678
+ * Shared inertness predicate (#2915 review 6, finding 2): CLOSED, no
1679
+ * active canary, no blocking marker, no outstanding members, and no
1680
+ * failure evidence still inside the rolling window. A blocking marker
1681
+ * (#2915 review 5) is never memberless/inert even if `status` somehow
1682
+ * read otherwise — never evict or release it. Used by both LRU eviction
1683
+ * (`_evictInertScopeIfPossible`, which also needs the oldest-touched
1684
+ * ordering) and targeted, single-key release (`_releaseIfInert`).
1685
+ */
1686
+ _isInertScope(rec, scopeKey) {
1687
+ if (!rec || rec.status !== 'closed' || rec.activeCanary || rec.blockingMarker) return false;
1688
+ const outstanding = this._outstanding.get(scopeKey);
1689
+ if (outstanding && outstanding.size > 0) return false;
1690
+ const now = this.nowMs();
1691
+ if (rec.failures.some((f) => now - f.atMs < this.policy.windowMs)) return false;
1692
+ return true;
1693
+ }
1694
+
1695
+ /**
1696
+ * Release ONE shared (never invocation-scoped — that quota is bounded
1697
+ * separately and far larger) scope record back to the tracking pool if it
1698
+ * is currently inert (#2915 review 6, finding 2: "release it on
1699
+ * settlement when the scope ends inert" / admission rollback). A no-op,
1700
+ * returning `false`, for an unknown, invocation-scoped, or non-inert
1701
+ * scope — never discards a scope still holding relevant failure history,
1702
+ * an active canary, a blocking marker, or outstanding members.
1703
+ */
1704
+ _releaseIfInert(scopeKey) {
1705
+ const rec = this._scopes.get(scopeKey);
1706
+ if (!rec || rec.scope === FailureScope.INVOCATION || !this._isInertScope(rec, scopeKey)) return false;
1707
+ rec.breaker.destroy();
1708
+ this._scopes.delete(scopeKey);
1709
+ this._outstanding.delete(scopeKey);
1710
+ return true;
1711
+ }
1712
+
1713
+ _removeOutstanding(scopeKey, invocationId) {
1714
+ const set = this._outstanding.get(scopeKey);
1715
+ if (!set || invocationId == null) return;
1716
+ set.delete(invocationId);
1717
+ if (set.size === 0) this._outstanding.delete(scopeKey);
1718
+ }
1719
+
1720
+ _resolveResetAtMs({ resetAtMs, retryAfterSeconds }, now) {
1721
+ if (Number.isFinite(resetAtMs) && resetAtMs > now) return resetAtMs;
1722
+ if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0) {
1723
+ return now + Math.min(retryAfterSeconds, this.policy.maxRetryAfterSeconds) * 1000;
1724
+ }
1725
+ return null;
1726
+ }
1727
+
1728
+ _boundedJitter() {
1729
+ const cap = this.policy.maxJitterMs;
1730
+ const value = Math.floor(Number(this.jitterMs()) || 0);
1731
+ return Math.max(0, Math.min(cap, value));
1732
+ }
1733
+
1734
+ /** Reopen (or stay open) with doubled backoff, capped at maxCooldownMs. */
1735
+ _reopen(rec, now) {
1736
+ rec.reopens += 1;
1737
+ rec.episodeCooldownMs = Math.min(
1738
+ this.policy.maxCooldownMs,
1739
+ this.policy.cooldownMs * (this.policy.backoffMultiplier ** rec.reopens),
1740
+ );
1741
+ rec.openedAtMs = now;
1742
+ rec.nextProbeAtMs = Math.max(now + rec.episodeCooldownMs, rec.resetAtMs ?? 0) + this._boundedJitter();
1743
+ rec.breaker.cooldownMs = rec.episodeCooldownMs;
1744
+ rec.breaker.recordFailure();
1745
+ this._transition(rec, 'open', 'probe/canary failed; wait for backoff then retry one probe');
1746
+ this._touch(rec);
1747
+ }
1748
+ }