@edgehero/pi-dispatch 4.3.1 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/.env.example +84 -3
  2. package/README.md +20 -7
  3. package/deploy/console.service +52 -0
  4. package/deploy/docker-compose.yml +3 -1
  5. package/deploy/pi-dispatch-valkey.container +4 -2
  6. package/deploy/sandbox-gateway.service +23 -0
  7. package/deploy/worker-env-wrapper.cmd +6 -0
  8. package/deploy/worker-env-wrapper.sh +10 -1
  9. package/package.json +117 -19
  10. package/src/agent-snapshot.mjs +49 -0
  11. package/src/answerer-runtime.mjs +194 -0
  12. package/src/answerer.mjs +178 -0
  13. package/src/approval-policy.mjs +76 -0
  14. package/src/approval-runtime.mjs +188 -0
  15. package/src/approval-session.mjs +137 -0
  16. package/src/approval-state.mjs +170 -0
  17. package/src/auth-cli.mjs +86 -0
  18. package/src/auth-connections.mjs +145 -0
  19. package/src/auth-policy.mjs +32 -0
  20. package/src/auth-store.mjs +97 -0
  21. package/src/azure-host.mjs +66 -4
  22. package/src/azure-prompt.mjs +5 -4
  23. package/src/backend-conformance.mjs +34 -2
  24. package/src/backend-local.mjs +103 -34
  25. package/src/backend-openshell.mjs +1540 -0
  26. package/src/backend-podman.mjs +63 -2
  27. package/src/backend-registry.mjs +61 -4
  28. package/src/backends.mjs +395 -6
  29. package/src/bullmq-internals.mjs +208 -0
  30. package/src/canary-cli.mjs +28 -0
  31. package/src/canary-lock.mjs +41 -0
  32. package/src/canary-runtime.mjs +237 -0
  33. package/src/canary.mjs +53 -0
  34. package/src/cancel-cli.mjs +47 -34
  35. package/src/cancel-state.mjs +213 -15
  36. package/src/capabilities.mjs +36 -1
  37. package/src/capacity-cli.mjs +28 -8
  38. package/src/capacity-records.mjs +75 -16
  39. package/src/capacity.mjs +65 -0
  40. package/src/chat-cli.mjs +288 -0
  41. package/src/chat-fork.mjs +547 -0
  42. package/src/chat-mode.mjs +156 -0
  43. package/src/chat-view.mjs +720 -0
  44. package/src/checkpoint-paths.mjs +113 -0
  45. package/src/cli.mjs +128 -41
  46. package/src/comment-tables.mjs +322 -0
  47. package/src/community-auth-adapters.mjs +87 -0
  48. package/src/community-auth-helper.mjs +76 -0
  49. package/src/community-auth-metadata.mjs +6 -0
  50. package/src/community-auth-output-pins.mjs +38 -0
  51. package/src/community-auth-patches.mjs +168 -0
  52. package/src/community-auth-pins.mjs +38 -0
  53. package/src/config.mjs +190 -29
  54. package/src/connection.mjs +83 -7
  55. package/src/console-answer.mjs +113 -0
  56. package/src/console-audit.mjs +604 -0
  57. package/src/console-bootstrap.mjs +159 -0
  58. package/src/console-client.mjs +848 -0
  59. package/src/console-probe.mjs +106 -0
  60. package/src/console-tokens.mjs +459 -0
  61. package/src/container-names.mjs +51 -0
  62. package/src/container-spec.mjs +20 -1
  63. package/src/cpu-reserve.mjs +4 -1
  64. package/src/defer-reasons.mjs +200 -0
  65. package/src/defer-write.mjs +208 -0
  66. package/src/deployment-venue.mjs +23 -6
  67. package/src/docker-run.mjs +11 -0
  68. package/src/doctor-bundles.mjs +130 -0
  69. package/src/doctor-model-proposals.mjs +109 -0
  70. package/src/doctor-openshell.mjs +238 -0
  71. package/src/doctor-results.mjs +602 -0
  72. package/src/doctor-spawner.mjs +215 -0
  73. package/src/doctor.mjs +1639 -196
  74. package/src/dollar-budget.mjs +212 -3
  75. package/src/endpoint-status.mjs +100 -0
  76. package/src/env-allowlist.mjs +96 -6
  77. package/src/env-file.mjs +6 -4
  78. package/src/file-expect.mjs +47 -0
  79. package/src/forgejo-host.mjs +42 -5
  80. package/src/forgejo-prompt.mjs +5 -4
  81. package/src/forges.mjs +33 -3
  82. package/src/fork-job.mjs +58 -0
  83. package/src/fork-policy.mjs +23 -0
  84. package/src/fork-store.mjs +338 -0
  85. package/src/get-token.mjs +1 -1
  86. package/src/git-dirty.mjs +119 -14
  87. package/src/git-hardening.mjs +85 -0
  88. package/src/github-host.mjs +31 -6
  89. package/src/github-prompt.mjs +7 -16
  90. package/src/gitlab-host.mjs +42 -3
  91. package/src/gitlab-prompt.mjs +5 -4
  92. package/src/held-document.mjs +23 -0
  93. package/src/held-replies.mjs +325 -0
  94. package/src/held-settle.mjs +266 -0
  95. package/src/hook-runs.mjs +188 -0
  96. package/src/host-budget.mjs +8 -5
  97. package/src/host-clone.mjs +72 -0
  98. package/src/host-loads.mjs +557 -0
  99. package/src/host-registry.mjs +66 -6
  100. package/src/host-telemetry.mjs +92 -0
  101. package/src/http-body.mjs +78 -0
  102. package/src/image-preflight.mjs +156 -13
  103. package/src/import-pi.mjs +60 -172
  104. package/src/inbox-relay.mjs +482 -0
  105. package/src/inbox-settle.mjs +196 -0
  106. package/src/index.mjs +734 -129
  107. package/src/init.mjs +35 -5
  108. package/src/job-data-version.mjs +75 -0
  109. package/src/job-id.mjs +30 -15
  110. package/src/job-layout.mjs +22 -0
  111. package/src/job-size.mjs +16 -0
  112. package/src/live-jobs.mjs +14 -6
  113. package/src/live-probes.mjs +6 -2
  114. package/src/live-transcript.mjs +872 -0
  115. package/src/loaded-record.mjs +609 -0
  116. package/src/mcp-config.mjs +1 -1
  117. package/src/mcp-preflight.mjs +36 -5
  118. package/src/model-attempts.mjs +84 -0
  119. package/src/model-catalog.mjs +83 -0
  120. package/src/model-cooldowns.mjs +22 -0
  121. package/src/model-failover-protocol.mjs +11 -0
  122. package/src/model-failover.mjs +134 -0
  123. package/src/model-observation.mjs +50 -0
  124. package/src/model-proposal-protocol.mjs +21 -0
  125. package/src/model-proposals.mjs +75 -0
  126. package/src/model-ref.mjs +67 -10
  127. package/src/model-roles-file.mjs +24 -0
  128. package/src/model-roles.mjs +73 -0
  129. package/src/model-status.mjs +34 -0
  130. package/src/models-json.mjs +4 -1
  131. package/src/money.mjs +1 -1
  132. package/src/oauth-openai.mjs +121 -0
  133. package/src/older-state.mjs +249 -0
  134. package/src/on-failure.mjs +17 -13
  135. package/src/open-browser.mjs +31 -2
  136. package/src/opener-cli.mjs +135 -0
  137. package/src/opener-protocol.mjs +27 -0
  138. package/src/opener-terminal.mjs +128 -0
  139. package/src/openshell-cli.mjs +155 -0
  140. package/src/openshell-policy.mjs +163 -0
  141. package/src/outbox.mjs +14 -0
  142. package/src/output-cap.mjs +1 -0
  143. package/src/package-canary-record.mjs +26 -0
  144. package/src/package-canary.mjs +135 -0
  145. package/src/package-names.mjs +9 -0
  146. package/src/package-stage.mjs +418 -0
  147. package/src/packages.mjs +24 -16
  148. package/src/parked.mjs +2 -1
  149. package/src/pause-windows.mjs +136 -17
  150. package/src/podman-stack.mjs +40 -14
  151. package/src/portfolio-snapshot.mjs +26 -7
  152. package/src/prepare-github.mjs +41 -77
  153. package/src/prepare-local.mjs +32 -2
  154. package/src/prepare.mjs +45 -11
  155. package/src/private-store-boundary.mjs +70 -0
  156. package/src/processor.mjs +1550 -173
  157. package/src/projects.mjs +23 -0
  158. package/src/prompt-fence.mjs +18 -0
  159. package/src/provider-key.mjs +7 -0
  160. package/src/provider-steering.mjs +1 -0
  161. package/src/question-context.mjs +35 -0
  162. package/src/question-group-runtime.mjs +91 -0
  163. package/src/question-groups.mjs +171 -0
  164. package/src/queue.mjs +264 -55
  165. package/src/reason-catalog.mjs +448 -0
  166. package/src/receiver-status.mjs +230 -0
  167. package/src/record-reason.mjs +37 -0
  168. package/src/recovery-queue.mjs +566 -0
  169. package/src/redact-secrets.mjs +219 -0
  170. package/src/reserve-estimate.mjs +101 -0
  171. package/src/reserved-env.mjs +36 -1
  172. package/src/retention.mjs +67 -0
  173. package/src/run-budget-ledger.mjs +283 -0
  174. package/src/run-checkpoint.mjs +627 -0
  175. package/src/run-container.mjs +749 -33
  176. package/src/run-history.mjs +791 -45
  177. package/src/run-mirror.mjs +75 -7
  178. package/src/run-progress.mjs +751 -0
  179. package/src/runtime-observations.mjs +3 -1
  180. package/src/runtime-settings.mjs +272 -37
  181. package/src/sandbox-cli.mjs +78 -2
  182. package/src/sandbox-deployment.mjs +51 -0
  183. package/src/sandbox-gateway-protocol.mjs +378 -0
  184. package/src/sandbox-hosts.mjs +100 -0
  185. package/src/sandbox-inventory.mjs +83 -0
  186. package/src/sandbox-policy.mjs +225 -0
  187. package/src/sandbox-prepare-requests.mjs +145 -0
  188. package/src/sandbox-prepare.mjs +285 -0
  189. package/src/sandbox-refs.mjs +70 -0
  190. package/src/sandbox-session-monitor.mjs +4 -0
  191. package/src/sandbox-session-watch.mjs +124 -0
  192. package/src/sandbox-sessions.mjs +161 -0
  193. package/src/sandbox-store.mjs +63 -6
  194. package/src/sandbox-workspace-guard.mjs +39 -0
  195. package/src/sandbox.mjs +27 -200
  196. package/src/scheduler-stall-guard.mjs +14 -8
  197. package/src/schedules.mjs +19 -6
  198. package/src/secret-bundles.mjs +245 -0
  199. package/src/secret-profiles.mjs +103 -3
  200. package/src/secrets.mjs +19 -9
  201. package/src/service-env.mjs +38 -5
  202. package/src/service.mjs +638 -106
  203. package/src/session-key.mjs +60 -0
  204. package/src/session-store.mjs +168 -10
  205. package/src/setup-contract.mjs +40 -0
  206. package/src/setup-egress.mjs +113 -0
  207. package/src/setup-host-entry.mjs +33 -0
  208. package/src/setup-host.mjs +242 -0
  209. package/src/setup-lock.mjs +41 -0
  210. package/src/size-records.mjs +15 -3
  211. package/src/start.mjs +1353 -135
  212. package/src/status-comment.mjs +415 -0
  213. package/src/stderr-transcript.mjs +318 -0
  214. package/src/steer-ack.mjs +122 -0
  215. package/src/steer-cli.mjs +213 -0
  216. package/src/steer-framing.mjs +185 -0
  217. package/src/steer-inbox.mjs +693 -0
  218. package/src/store-version.mjs +139 -0
  219. package/src/subscription-config.mjs +56 -0
  220. package/src/subscription-limits.mjs +3 -0
  221. package/src/subscription-quota.mjs +259 -0
  222. package/src/subscription-runtime.mjs +123 -0
  223. package/src/subscription-status.mjs +42 -0
  224. package/src/subscription-wait.mjs +11 -0
  225. package/src/subscriptions.mjs +7 -1
  226. package/src/thinking-level.mjs +19 -0
  227. package/src/touch-outbox.mjs +61 -0
  228. package/src/touch-state.mjs +133 -0
  229. package/src/triggers-file.mjs +299 -39
  230. package/src/triggers.mjs +780 -38
  231. package/src/trust-paths.mjs +171 -0
  232. package/src/up.mjs +398 -51
  233. package/src/updates-cli.mjs +61 -0
  234. package/src/updates-look.mjs +127 -0
  235. package/src/updates-reconcile.mjs +182 -0
  236. package/src/updates-subscription-adapters.mjs +8 -0
  237. package/src/updates.mjs +252 -0
  238. package/src/valkey-auth.mjs +32 -9
  239. package/src/valkey-endpoint.mjs +14 -4
  240. package/src/wait-for.mjs +1 -1
  241. package/src/wait-state.mjs +62 -3
  242. package/src/watch-board.mjs +38 -0
  243. package/src/watch-closer.mjs +1 -1
  244. package/src/watch-contract.mjs +102 -0
  245. package/src/watch-decision.mjs +78 -0
  246. package/src/watch-drainer.mjs +14 -0
  247. package/src/watch-issue-adapter.mjs +44 -0
  248. package/src/watch-issue.mjs +111 -0
  249. package/src/watch-record.mjs +10 -0
  250. package/src/watch-runtime.mjs +167 -0
  251. package/src/watch-store.mjs +183 -0
  252. package/src/watch-trace.mjs +34 -0
  253. package/src/workspace-checkpoint.mjs +1750 -0
  254. package/src/zero-touch-fixes.mjs +99 -0
  255. package/src/zero-touch.mjs +225 -0
  256. package/src/repeat-slot.mjs +0 -13
package/src/processor.mjs CHANGED
@@ -1,4 +1,12 @@
1
- import { DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, DOCKER_NEVER_STARTED_EXITS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_CONF_WIDENS_JOB, PODMAN_SERVICE_LOCAL, RUNTIME_ADDS_NO_MOUNTS } from "./backends.mjs";
1
+ import { cleanTouch, touchScopeOf } from "./touch-state.mjs";
2
+ import { SubscriptionWait } from "./subscription-wait.mjs";
3
+ import { readQuestionContext } from "./question-context.mjs";
4
+ import { stageModelRoleInputs } from "./agent-snapshot.mjs";
5
+ import { effectiveCostCapMicros } from "./money.mjs";
6
+ import { validateFork, forkBranch, forkArchiveFact, FORK_THINKING_LEVELS } from "./fork-job.mjs";
7
+ import { sessionKeyFor } from "./session-key.mjs";
8
+ import { join } from "node:path";
9
+ import { ABSENT, OPENSHELL_BIND_LINGERING, backendFor, DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, DOCKER_NEVER_STARTED_EXITS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_CONF_WIDENS_JOB, OPENSHELL_GATEWAY_LOCAL, PODMAN_SERVICE_LOCAL, RUNTIME_ADDS_NO_MOUNTS } from "./backends.mjs";
2
10
  import { resolveBackendName } from "./backend-registry.mjs";
3
11
  import { lstatSync } from "node:fs";
4
12
  import { checkTokenCap, recordTokenSpend, releaseLedgers, reserveLedgers } from "./budget.mjs";
@@ -6,12 +14,18 @@ import { configError } from "./config.mjs";
6
14
  import { unqualifiedScope } from "./pause-windows.mjs";
7
15
  import { PROJECT_CAP_REASON } from "./scoped-limits.mjs";
8
16
  import { DEFAULT_SECRETS_PROFILE, secretsArmed } from "./secrets.mjs";
17
+ import { mergeBundleSecrets } from "./secret-bundles.mjs";
18
+ import { providerKeyCandidates } from "./env-allowlist.mjs";
9
19
  import { RESERVED_ENV_NAMES } from "./triggers.mjs";
10
- import { MCP_REFUSAL_COMMENT } from "./mcp-preflight.mjs";
20
+ import { imageFactsOf, loadedWithBase, loadedWithImage } from "./loaded-record.mjs";
21
+ import { mcpRefusalComment } from "./mcp-preflight.mjs";
11
22
  import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
12
- import { COST_CAP_WHYS, EXIT_OOM_KILLED, RUNNER_POLICY_REASONS } from "./run-history.mjs";
23
+ import { CONTAINER_EXIT_UNKNOWN, CONTAINER_INFRA_EXIT } from "./record-reason.mjs";
24
+ import { COST_CAP_WHYS, EXIT_OOM_KILLED, RECORD_CLOCK_SKEW_MS, RUNNER_POLICY_REASONS, recoveryCounts, recoveryCountsFromProgress } from "./run-history.mjs";
25
+ import { baseHasExtensions as baseHasExtensionsDefault, resetTrustPaths as resetTrustPathsDefault } from "./trust-paths.mjs";
26
+ import { normalizeHost, removeCheckpointPointer as removePointerDefault } from "./run-checkpoint.mjs";
13
27
  import { DEFAULT_EGRESS_PROXY } from "./egress.mjs";
14
- import { CAPABILITY_GATES, EXIT_AUTH_CAPABILITY } from "./image-preflight.mjs";
28
+ import { CAPABILITY_GATES, EXIT_AUTH_CAPABILITY, STEER_STDIN_CAPABILITY, TRANSCRIPT_STDERR_CAPABILITY } from "./image-preflight.mjs";
15
29
  import { modelListProblem, modelOnList, splitModelEntry } from "./model-ref.mjs";
16
30
  import { ALLOCATION_CAP_REASON, ENVELOPE_MISMATCH_REASON } from "./allocation.mjs";
17
31
 
@@ -19,8 +33,15 @@ import { ALLOCATION_CAP_REASON, ENVELOPE_MISMATCH_REASON } from "./allocation.mj
19
33
  export const LOCAL_FOLDER_PROJECT_CHANGED = "local-folder-project-changed";
20
34
  // Issue #505: a portfolio cron job refused before it spends, because this host could not apply the plan it would write.
21
35
  export const PORTFOLIO_NO_ENVELOPE = "portfolio-no-envelope";
22
- import { DOLLAR_CAP_REASON, DOLLAR_KEY_PREFIX, dollarLedgers, dollarSettlement, dollarsRecord, holdPart, meteredMicros, modelDollarSettlement, releaseDollars, reserveDollars, settleDollars } from "./dollar-budget.mjs";
23
- import { zeroRatedVerdict } from "./model-endpoints.mjs";
36
+ import { DOLLAR_CAP_REASON, DOLLAR_KEY_PREFIX, deploymentWindowsOf, dollarLedgers, dollarSettlement, dollarsRecord, holdPart, meteredMicros, modelDollarSettlement, releaseDollars, reserveDollars, settleDollars } from "./dollar-budget.mjs";
37
+ import { callableModelRefs, zeroRatedFor } from "./reserve-estimate.mjs";
38
+ import { outputCapView } from "./output-cap.mjs";
39
+ import { answerTrailComment, consoleAnswerBy, consoleDraftOf } from "./console-answer.mjs";
40
+ import { forgeDeliveryJobId } from "./job-id.mjs";
41
+ import { FORGE_JOB_RETENTION_MS } from "./retention.mjs";
42
+ // The forge-comment tables (issue #710): every fixed sentence this file posts, in one leaf the reason catalog can import
43
+ // without the job path. Re-exported below under the names they always had here.
44
+ import { MODEL_AGENTS_INVALID_COMMENT, APPROVAL_COMMENTS, APPROVAL_PARK_NEWER_COMMENT, BUNDLE_COMMENTS, ENVELOPE_MISMATCH_COMMENTS, HELD_TAKEN_COMMENT, JOB_USER_COMMENTS, MODEL_NOT_ALLOWED_COMMENTS, MODEL_UNKNOWN_COMMENT, modelUnknownComment, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, OVERLAY_REFUSED_COMMENT, PARKED_COMMENT, PARKED_COMMENT_PR, PODMAN_CONF_COMMENTS, RECOVERY_COMMENTS, REFUSAL_COMMENTS, TERMINAL_COMMENTS, VENUE_BUNDLE_REFUSED_COMMENT, VENUE_REFUSED_COMMENT, parkedComment } from "./comment-tables.mjs";
24
45
 
25
46
  /**
26
47
  * The forge comment's reason for each observation a floor refusal missed (issues #278 and #345), keyed like
@@ -38,6 +59,9 @@ export const OBSERVATION_COMMENT = Object.freeze({
38
59
  [PODMAN_BOUNDS_DELEGATED]: "the worker's rootless Podman is not observed applying a container's pid, memory and cpu bounds",
39
60
  [PODMAN_ADDS_NO_MOUNTS]: "the worker's rootless Podman is not observed adding no mounts of its own to a job container",
40
61
  [PODMAN_SERVICE_LOCAL]: "the podman CLI is not observed running containers on this host rather than through a remote service, so the job's credentials could cross a network the deployment does not own",
62
+ // Issue #683. Never reached through a floor alone: the openshell venue refuses a gateway it does not observe local
63
+ // whatever the floor says (`venueRefused`), and this sentence is the floor's half for a deployment that also asks.
64
+ [OPENSHELL_GATEWAY_LOCAL]: "the OpenShell gateway is not observed listening on this host, so the job's credentials could cross a network the deployment does not own",
41
65
  });
42
66
 
43
67
  /**
@@ -107,72 +131,9 @@ export function peakReachedLimit(memPeak, limit) {
107
131
  return memPeak >= Math.ceil((limit * 9) / 10);
108
132
  }
109
133
 
110
- // The post-spend terminal comments (issue #288). Every FREE refusal above the container already comments;
111
- // these are the paths where money was spent and the run still ended without the agent's own status step,
112
- // which used to tell the issue nothing (REQ-JOB-STATUS-COMMENTS' acceptance -- "exactly one completion or
113
- // failure comment" -- held only below the spend line). Keyed by the reason token so a new abort
114
- // classification adds a ROW here, never a re-plumb; sentences are FIXED and path-free, because for a
115
- // local job the adapter logs the full text into a persistent service log (the "this folder" discipline
116
- // at the scope-cap refusal). The completed path stays silent on purpose: exit 0 is where the AGENT'S own
117
- // status comment lives (the prompt contract instructs it, including for "I cannot fix this"), and exit 2
118
- // by construction means the agent was cut off before that step.
119
- // EXPORTED only so a test can hold every RUNNER_POLICY_REASONS member to a row here (issue #437 review).
120
- export const TERMINAL_COMMENTS = {
121
- "worker-abort": "Stopped: the worker ended this run before it finished (the 30-minute job limit, or a worker shutdown). Partial work may exist. Not retried.",
122
- "operator-cancel": "Stopped: the operator cancelled this run. Partial work may exist. Not retried.",
123
- "runner-policy": "Stopped: the run ended inside the container before finishing (a turn or token budget, or an in-container configuration refusal). Partial work may exist. Not retried.",
124
- // Issue #437. Names the cause but never the provider's own message, which may echo a key fragment. "Or
125
- // access" because a 403 is as often a key that works but may not use this model or route as a bad key.
126
- "provider-auth-refused": "Stopped: the AI provider refused this worker's credentials or access (an authentication or permission error). The operator needs to check the provider key and what it is allowed to use. Not retried.",
127
- // Issues #501, #502. The two stops name the policy, never the amount or the model: both are operator
128
- // configuration, and the comment's reader may be an issue author who can act on neither.
129
- "cost-cap": "Stopped: the next AI call could have taken this run past its cost limit, so it was not made. Partial work may exist. Not retried.",
130
- "model-not-allowed": "Stopped: the run tried to call an AI model this trigger does not allow, or to change an AI request in a way it does not allow, so the call was not made. Partial work may exist. Not retried.",
131
- "cost-cap-unenforceable": "Stopped: this run has a cost limit, and the job image could not enforce it before each AI call, so nothing was sent to the AI provider. The operator needs to update the job image. Not retried.",
132
- "model-policy-unenforceable": "Stopped: this run is limited to certain AI models, and the job image could not enforce that before each AI call, so nothing was sent to the AI provider. The operator needs to update the job image. Not retried.",
133
- // The runner's refusals of the job's MCP servers, each before the prompt. Never the server's name, its host or a
134
- // variable: the exit line carries the name and the cause in its message, which the worker never parses (it reaches
135
- // the worker's log as the container's output). Each says what the operator can check; the issue's author can act on
136
- // none of it.
137
- "mcp-unavailable": "Stopped: an MCP server this trigger names could not be used (it did not start, failed its handshake or refused the connection, an authentication error among them), so the run ended before the agent started. The operator needs to check that server and the job image. Not retried.",
138
- "mcp-config-invalid": "Stopped: the list of MCP servers the worker wrote for this job is not one the job image accepts, so the run ended before the agent started. The operator needs to run a worker and a job image of the same release. Not retried.",
139
- "mcp-config-mismatch": "Stopped: the MCP servers the worker wrote for this job are not the ones it named for it, so the run ended before the agent started. The operator needs to run a worker and a job image of the same release. Not retried.",
140
- "mcp-secret-missing": "Stopped: a credential an MCP server this trigger names needs was not set inside the job, so the run ended before the agent started. The operator needs to check the secrets this trigger passes. Not retried.",
141
- "mcp-command-missing": "Stopped: the job image does not have the program an MCP server this trigger names runs, so the run ended before the agent started. The operator needs to install that server in the job image or the overlay. Not retried.",
142
- "mcp-shadowed": "Stopped: another extension in this job registers a tool or command that only the job's MCP servers may, so the run ended before the agent started. The operator needs to remove that extension or this trigger's MCP servers. Not retried.",
143
- // Issue #625. The runner's two refusals of a job armed with run.awaitInput, each before the prompt.
144
- "await-input-shadowed": "Stopped: another extension in this job registers the tool a run uses to stop and wait for an answer, so the run ended before the agent started. The operator needs to remove that extension or this trigger's run.awaitInput. Not retried.",
145
- "await-input-without-session": "Stopped: this run may stop to wait for an answer but had nowhere to keep its work, so it ended before the agent started. The operator needs to run a worker and a job image of the same release. Not retried.",
146
- // Issue #596. Never the size or the project: both are operator configuration, and the reader may be an issue author
147
- // who can act on neither. The worker log and the run record carry them.
148
- [EXIT_OOM_KILLED]: "Stopped: the job's container ran out of memory and was stopped. Partial work may exist. Not retried, because the same size would stop the same way. The operator can raise this job's memory size.",
149
- };
150
-
151
- // Issue #625: the one comment the COMPLETED path posts, on a run that ended awaiting-input. The agent posted its own
152
- // question (the tool tells it to, before it ends); this says how the run continues. Fixed and path-free, the
153
- // TERMINAL_COMMENTS rule, and it carries no model text: the question never crosses the exit line.
154
- // On an issue any reply continues it (the receiver routes it on the marker); a pull request's reply keeps the phrase,
155
- // since its session key needs a head branch that costs a token to read (parked.mjs).
156
- export const PARKED_COMMENT = "This run stopped to wait for an answer to the question above. Reply here, and the next run continues from here.";
157
- export const PARKED_COMMENT_PR = "This run stopped to wait for an answer to the question above. Reply in a comment that starts with this repository's pi-dispatch comment phrase, and the next run continues from here.";
158
- export function parkedComment(target) {
159
- return target?.type === "pull_request" ? PARKED_COMMENT_PR : PARKED_COMMENT;
160
- }
161
-
162
- // Issue #502: the `model-unknown` refusal's comment. Names no model: the reader may be an issue author.
163
- export const MODEL_UNKNOWN_COMMENT = "Refused: this job names an AI model that this deployment does not know (it is in neither pi's model catalog nor the overlay models.json), so no container was started and nothing was spent. Ask the operator to check the trigger's model settings. Not run.";
164
-
165
- // The `model-unknown` refusal whose `why` is `overlay-*` (PR #558, the end-of-round check of #501 and #502): the deployment's overlay
166
- // models.json is the problem, not the job's model, so the generic text above would send the author to the wrong
167
- // place. Fixed text naming no path and no model; the operator finds which file and why in the worker log.
168
- export const OVERLAY_REFUSED_COMMENT = "Refused before starting: the deployment's model settings file (models.json in the overlay) cannot be used for this job, so no container was started and nothing was spent. The operator needs to fix that file. Not run.";
169
-
170
- // Issue #341: the forge comments for a `job-user-unmappable` refusal, keyed by cause. Shorter than the operator
171
- // texts in job-user.mjs on purpose: a comment's reader may be an issue author, who can act on none of it.
172
- const JOB_USER_COMMENTS = Object.freeze({
173
- default: "Refused: the worker host's container runtime cannot give this job a non-root user that can read and write its own files. Not run.",
174
- "runtime-unreadable": "Refused: the worker host's container runtime answered in a form the worker cannot read, so which user this job may run as is unknown. Not run.",
175
- });
134
+ // The post-spend terminal comments (TERMINAL_COMMENTS, issue #288), the park's, the model and job-user refusals', the
135
+ // durable-run refusals' and the bundle refusals' live in comment-tables.mjs since issue #710, with the rule each follows.
136
+ export { HELD_TAKEN_COMMENT, MODEL_UNKNOWN_COMMENT, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, OVERLAY_REFUSED_COMMENT, PARKED_COMMENT, PARKED_COMMENT_PR, RECOVERY_COMMENTS, TERMINAL_COMMENTS, parkedComment };
176
137
 
177
138
  /**
178
139
  * The runtime a venue's own preflights talk to, named in a retry's words (issue #354). The message is the job_failed log
@@ -225,20 +186,287 @@ function modelPrefixesOf(modelDollars) {
225
186
  }
226
187
 
227
188
  /**
228
- * The models a job may call (issue #501, #503 part 7), as `{ provider, id }` refs: its main model, and every entry of
229
- * its effective allowed-model list when it has one. A list entry that does not split is kept as `null`, which the
230
- * zero-rated check reads as "not zero-rated" (fail closed).
189
+ * How long a console answer's trail claim lives: a forge job's retained data (`FORGE_JOB_RETENTION_MS`, retention.mjs),
190
+ * the longest the same id can return. Derived, so a change to the retention moves the claim with it.
231
191
  */
232
- function callableModelRefs(job) {
233
- const refs = [{ provider: job.provider, id: job.model }];
234
- for (const entry of Array.isArray(job.models) ? job.models : []) {
235
- const ref = splitModelEntry(entry);
236
- refs.push(ref === null ? null : { provider: ref.provider, id: ref.model });
192
+ export const CONSOLE_TRAIL_CLAIM_MS = FORGE_JOB_RETENTION_MS;
193
+
194
+ /**
195
+ * Claim the one trail comment of a console answer (issue #642): true when this attempt set `console:trail:<jobId>` and
196
+ * so must post, false when an earlier attempt of the same job already did, when the job's id cannot be derived, or when
197
+ * Valkey did not answer (each logged by its kind, never the text). The id is the queue's own (`forgeDeliveryJobId` over
198
+ * the job's kind, delivery id and replica), so a retry of the same job, which keeps its data, reaches the same key.
199
+ * Never throws.
200
+ */
201
+ async function claimConsoleTrail(redis, job, log) {
202
+ let jobId;
203
+ try {
204
+ jobId = forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica);
205
+ } catch {
206
+ log("console_trail_skipped", { kind: job.kind ?? null, reason: "no-job-id" });
207
+ return false;
208
+ }
209
+ try {
210
+ const set = await redis.set(`console:trail:${jobId}`, "1", "PX", CONSOLE_TRAIL_CLAIM_MS, "NX");
211
+ if (set === "OK") return true;
212
+ log("console_trail_skipped", { kind: job.kind ?? null, reason: "already-posted" });
213
+ return false;
214
+ } catch (error) {
215
+ log("console_trail_skipped", { kind: job.kind ?? null, reason: "valkey-error", code: typeof error?.code === "string" ? error.code : "error" });
216
+ return false;
237
217
  }
238
- return refs;
218
+ }
219
+
220
+ // `callableModelRefs` (the models a job may call) moved to reserve-estimate.mjs (issue #642) with the zero-rated test
221
+ // it feeds, so the console's reserve estimate and this processor's reserve judge one rule (`zeroRatedFor`).
222
+
223
+ // ── Durable runs (issue #645) ───────────────────────────────────────────────────────────────────────────────────────
224
+
225
+ /** How many times one run may be continued from its checkpoint: a recovery job numbered past this is `recovery-cap`. */
226
+ export const RECOVERY_CAP = 2;
227
+ /** The reason an armed run's attempt RETURNS with when its continuation was queued instead of a retry thrown. */
228
+ export const RECOVERY_ENQUEUED = "recovery-enqueued";
229
+ /** A recovery job whose run already reached its final outcome elsewhere: refused silently, never commented. */
230
+ export const RECOVERY_SUPERSEDED = "recovery-superseded";
231
+ /** How long a recovery job waits before asking again whether its base run's container is still running here. */
232
+ export const RECOVERY_WAIT_MS = 30_000;
233
+ /** How long a recovery job waits before trying again after a never-started fault (it has no attempt to retry with). */
234
+ export const RECOVERY_RETRY_WAIT_MS = 60_000;
235
+
236
+ /**
237
+ * The run this attempt belongs to, decided from the job data alone (issue #645): `{ armed, runId, isRecovery, recovery,
238
+ * recoverSkipped, seq, recovered, checkpoints }`. The last three are filled in as the attempt goes, so the record can say
239
+ * them on every exit.
240
+ *
241
+ * `runId` is `job.recoveryOf` for a recovery job, else the queue's own id, DERIVED like the console trail's
242
+ * (`forgeDeliveryJobId` over kind, delivery id and replica: queue.mjs `enqueueForgeJob` names every forge job so). The
243
+ * processor is handed the effective job, which has no `.id`, and a derivation from data the queue itself keyed on is one
244
+ * answer for every attempt, retries and recoveries included; a new wiring argument would be a second. A recovery job's
245
+ * id (`<runId>.r<n>`) is never parsed: `recoveryOf` and `recovery` carry it.
246
+ *
247
+ * Not armed, with the reason the record names (fail open and say which): a local job (`local`, v1 recovers forge jobs
248
+ * only, so it runs exactly as it did before #645) and a forge job whose id cannot be derived (`no-run-id`).
249
+ */
250
+ function durableOf(job) {
251
+ const d = { armed: false, runId: null, isRecovery: false, recovery: null, recoverSkipped: null, seq: null, recovered: null, checkpoints: null };
252
+ if (job?.recover !== true) return d;
253
+ if (job.kind === "local") {
254
+ d.recoverSkipped = "local";
255
+ return d;
256
+ }
257
+ if (typeof job.recoveryOf === "string" && job.recoveryOf !== "") {
258
+ d.runId = job.recoveryOf;
259
+ d.isRecovery = true;
260
+ d.recovery = job.recovery;
261
+ } else {
262
+ try {
263
+ d.runId = forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica);
264
+ } catch {
265
+ d.recoverSkipped = "no-run-id";
266
+ return d;
267
+ }
268
+ }
269
+ d.armed = true;
270
+ return d;
271
+ }
272
+
273
+ /** The durable-run facts the record carries, added to a result only when there are any (so every other is unchanged). */
274
+ function withDurable(result, d) {
275
+ if (result === null || typeof result !== "object") return result;
276
+ if (d.recoverSkipped === null && d.recovered === null && d.checkpoints === null) return result;
277
+ return { ...result, ...(d.recoverSkipped !== null ? { recoverSkipped: d.recoverSkipped } : {}), ...(d.recovered !== null ? { recovered: d.recovered } : {}), ...(d.checkpoints !== null ? { checkpoints: d.checkpoints } : {}) };
278
+ }
279
+
280
+ /** The same on a throw, so a retried or failed attempt's record says them too (buildRecord reads the error). */
281
+ function tagDurable(e, d) {
282
+ if (e === null || typeof e !== "object") return;
283
+ try {
284
+ if (d.recoverSkipped !== null && e.recoverSkipped === undefined) e.recoverSkipped = d.recoverSkipped;
285
+ if (d.recovered !== null && e.recovered === undefined) e.recovered = d.recovered;
286
+ if (d.checkpoints !== null && e.checkpoints === undefined) e.checkpoints = d.checkpoints;
287
+ } catch {
288
+ // a frozen error keeps its own fields
289
+ }
290
+ }
291
+
292
+ /**
293
+ * The run reached its final outcome (a completed or policy result): remove its checkpoint store entry, its Valkey
294
+ * pointer and its budget ledger. Each best effort and logged by name, never thrown: every one has a TTL or a sweep
295
+ * behind it, and a cleanup fault must not turn a finished run into a failure.
296
+ */
297
+ async function endDurableRun(d, deps) {
298
+ const log = typeof deps.log === "function" ? deps.log : () => {};
299
+ const quietly = async (what, fn) => {
300
+ try {
301
+ const r = await fn();
302
+ if (r && typeof r === "object" && (r.error || r.skipped)) log("durable_cleanup_failed", { what, code: String(r.error ?? r.skipped).slice(0, 40) });
303
+ } catch (error) {
304
+ log("durable_cleanup_failed", { what, code: typeof error?.code === "string" ? error.code : "error" });
305
+ }
306
+ };
307
+ if (deps.workspaceCheckpointer) await quietly("store", () => deps.workspaceCheckpointer.remove(d.runId));
308
+ // The wiring's seam when it hands one over; else run-checkpoint.mjs's own delete on this processor's client, so a bare
309
+ // wiring cannot leave a pointer that makes a later recovery think the run is still live. It answers false, never a
310
+ // throw, when Valkey did not answer: logged, and the pointer's 7-day expiry is the backstop.
311
+ await quietly("pointer", async () => {
312
+ const done = typeof deps.removeCheckpointPointer === "function" ? await deps.removeCheckpointPointer({ runId: d.runId }) : await removePointerDefault({ redis: deps.redis, runId: d.runId });
313
+ return done === false ? { error: "unanswered" } : null;
314
+ });
315
+ if (deps.runBudgetLedger) await quietly("ledger", () => deps.runBudgetLedger.remove(d.runId));
316
+ }
317
+
318
+ /** A confirmed draft still belongs to the forge/account where it was captured. Check before even a
319
+ * refusal/approval comment can mint credentials; a restarted worker may serve another configured instance. */
320
+ export async function checkDraftAnswerSource(job, deps) {
321
+ const provenance = consoleDraftOf(job?.console);
322
+ if (!provenance) return null;
323
+ const refused = reason => ({ outcome: "policy", reason, budgetReserved: false });
324
+ if (provenance.invalid || job.parkedReply !== true || !deps?.answerDraftStore || typeof deps.answerDraftRuntime?.checkSource !== "function") return refused("answer-draft-refused");
325
+ let meta, input;
326
+ try {
327
+ meta = await deps.answerDraftStore.readMeta(provenance.draftSourceRunId);
328
+ input = await deps.answerDraftStore.privateInput(provenance.draftSourceRunId);
329
+ } catch { throw new InfraRetry("answer draft provenance could not be read", { reason: "container-never-started", budgetReserved: false }); }
330
+ if (!meta || !["drafted", "sent"].includes(meta.status) || meta.draftRunId !== provenance.draftRunId || meta.draftDigest !== provenance.draftDigest || input?.source?.kind !== job.kind || String(input.source.repo).toLowerCase() !== String(job.repo).toLowerCase() || input.source.target?.type !== job.target?.type || input.source.target?.number !== job.target?.number || (input.source.projectId ?? null) !== (job.projectId ?? null) || (input.source.azure?.projectId ?? null) !== (job.azure?.projectId ?? null)) return refused("answer-draft-refused");
331
+ try { return await deps.answerDraftRuntime.checkSource(input.source) === true ? null : refused("answer-draft-config-changed"); }
332
+ catch { throw new InfraRetry("answer draft current forge identity could not be read", { reason: "container-never-started", budgetReserved: false }); }
239
333
  }
240
334
 
241
335
  export async function runJob(job, deps) {
336
+ const sourceRefusal = await checkDraftAnswerSource(job, deps);
337
+ if (sourceRefusal) return sourceRefusal;
338
+ const touches = [];
339
+ let touchLineageSkipped = null;
340
+ // Issue #796: what the runner said of a canary's candidate, carried on every way out, since a job can load its candidate and still end any way at all.
341
+ // Issue #800: what the job loaded, the same way and for the same reason, a retried attempt's record included: a server
342
+ // that timed out exits 1 and retries (`mcp-unreachable`), and that record is the one a person opens to find out why.
343
+ const canaryFacts = { load: null, loaded: null };
344
+ const recordTouch = async (value) => {
345
+ const event = cleanTouch(value);
346
+ if (!event) return;
347
+ touches.push(event);
348
+ try {
349
+ if (deps.touchOutbox) { const intent = await deps.touchOutbox.begin(event); await deps.touchOutbox.applied(intent); }
350
+ else if (deps.touchStore) await deps.touchStore.write(event);
351
+ else await deps.recordTouch?.(event);
352
+ } catch { deps.log?.("touch_record_pending", {}); }
353
+ };
354
+ try {
355
+ const result = withCanaryLoad(await runJobInner(job, { ...deps, recordTouch, recordTouchGap: () => { touchLineageSkipped = "source-unavailable"; }, canaryFacts }), canaryFacts);
356
+ return result && typeof result === "object" && (touches.length || touchLineageSkipped) ? { ...result, ...(touches.length ? { touches } : {}), ...(touchLineageSkipped ? { touchLineageSkipped } : {}) } : result;
357
+ } catch (error) {
358
+ if (canaryFacts.load !== null && error && typeof error === "object") error.canaryLoad = canaryFacts.load;
359
+ if (error && typeof error === "object") {
360
+ const ran = error.loaded ?? null;
361
+ // A throw that already holds what the container said (`InfraRetry`'s `loaded`) came out of the run's own wrapper before
362
+ // the processor could note it, so what is noted here is the worker's own facts alone. The line stands, with them added.
363
+ if (ran !== null && canaryFacts.loaded !== null) error.loaded = loadedWithBase({ ...ran, ...imageFactsOf(canaryFacts.loaded) }, canaryFacts.loaded.baseSha);
364
+ else if (ran === null && canaryFacts.loaded !== null) error.loaded = canaryFacts.loaded;
365
+ }
366
+ if (touches.length && error && typeof error === "object") error.touches = touches;
367
+ if (touchLineageSkipped && error && typeof error === "object") error.touchLineageSkipped = touchLineageSkipped;
368
+ throw error;
369
+ }
370
+ }
371
+
372
+ function withCanaryLoad(result, facts) {
373
+ if (result === null || typeof result !== "object" || (facts.load === null && facts.loaded === null)) return result;
374
+ return { ...result, ...(facts.load !== null ? { canaryLoad: facts.load } : {}), ...(facts.loaded !== null ? { loaded: facts.loaded } : {}) };
375
+ }
376
+
377
+ async function runJobInner(job, deps) {
378
+ // Queue fields never confer approval authority. Resolve the host-owned decision before the mint or any reserve.
379
+ job = { ...job };
380
+ delete job.approvalDecision;
381
+ let approval = null;
382
+ if (job.gates !== undefined || job.approvalRequestId !== undefined || job.resume === true && deps.approvalRuntime) {
383
+ const id = deps.queueJobId ?? forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica);
384
+ if (!deps.approvalRuntime) return { outcome: "policy", reason: "approval-store-unavailable", budgetReserved: false, exitCode: null, turns: null, tokens: null };
385
+ approval = await deps.approvalRuntime.before({ ...job, id });
386
+ if (approval.error) throw new InfraRetry("approval state could not be read", { reason: "container-never-started", budgetReserved: false });
387
+ // Issue #761: the live triggers file was written by a newer pi-dispatch, so this build cannot judge the approval
388
+ // declaration. Before the mint and every reserve, and before any request is decided or claimed: the job waits for
389
+ // a worker that reads the file (`newer-release`), never refused, so neither the delivery nor the decision is lost.
390
+ if (approval.newer === true) throw new NewerReleaseHold("the triggers file");
391
+ if (!approval.ok) {
392
+ await deps.comment?.(job, `Refused: ${approval.refused}. Check the pending approval and current trigger declaration. Not run.`);
393
+ return { outcome: "policy", reason: approval.refused, ...(approval.audit ? { approval: approval.audit } : {}), budgetReserved: false, exitCode: null, turns: null, tokens: null };
394
+ }
395
+ if (approval.decision) {
396
+ job.approvalDecision = approval.decision;
397
+ // A phrase-prefixed /approve is a decision, not a new flow override. Continue the requesting flow.
398
+ if (typeof approval.request?.resumeJob?.flow === "string") job.flow = approval.request.resumeJob.flow;
399
+ if (deps.approvalSessionCheck) {
400
+ const session = deps.approvalSessionCheck(approval.request);
401
+ if (!session.ok) {
402
+ await deps.approvalRuntime.finish({ requestId: approval.request.requestId, jobId: id });
403
+ return { outcome: "policy", reason: "approval-session-unavailable", approval: approval.audit, budgetReserved: false, exitCode: null, turns: null, tokens: null };
404
+ }
405
+ }
406
+ }
407
+ if (approval.budgetCapUsd !== undefined) job.maxCostMicros = effectiveCostCapMicros(approval.budgetCapUsd, deps.deploymentMaxCostUsd ?? null);
408
+ deps = { ...deps, approvalRequest: approval.request ?? null, approvalJobId: id };
409
+ }
410
+
411
+ const durable = durableOf(job);
412
+ // Issue #704: what this attempt did with the project's secret bundle, for the record (`bundle` or `bundleSkipped`).
413
+ const bundleFacts = { bundle: null, bundleSkipped: null };
414
+ const archiveFacts = { value: null };
415
+ let result;
416
+ try {
417
+ result = await runAttempt(job, deps, durable, bundleFacts, archiveFacts);
418
+ if (approval?.audit && !result?.approval) result = { ...result, approval: approval.audit };
419
+ if (approval?.request && result?.outcome === "policy") {
420
+ const closed = await deps.approvalRuntime.finish({ requestId: approval.request.requestId, jobId: deps.approvalJobId });
421
+ if (!closed.ok) result = { ...result, reason: "approval-finish-unavailable" };
422
+ }
423
+ } catch (e) {
424
+ try { if (archiveFacts.value && e && typeof e === "object") e.forkSource = archiveFacts.value; } catch {}
425
+ if (approval?.audit && e && typeof e === "object") e.approval = approval.audit;
426
+ tagDurable(e, durable);
427
+ tagBundle(e, bundleFacts);
428
+ // A RECOVERY job has one attempt (recovery-queue.mjs), so a never-started retry would fail it for good over a
429
+ // blip (a pointer read, a ledger write, a restore that met a lock). Every such throw becomes a wait hold instead:
430
+ // `makeProcessor` delays it without spending the attempt, and the run's recovery deadline (`recoverBy`, the
431
+ // host-lost sweep) bounds how long it can wait. Nothing was started, so nothing is lost by asking again.
432
+ if (durable.isRecovery && e instanceof InfraRetry && e.reason === "container-never-started" && !(e instanceof RecoveryWaitHold)) {
433
+ const hold = new RecoveryWaitHold(e.message, { cause: e, reason: e.reason, provider: e.provider, model: e.model, budgetReserved: e.budgetReserved, dollars: e.dollars, resources: e.resources, exitCode: e.exitCode, turns: e.turns, tokens: e.tokens, session: e.session, usage: e.usage, waitMs: RECOVERY_RETRY_WAIT_MS });
434
+ tagDurable(hold, durable);
435
+ tagBundle(hold, bundleFacts);
436
+ deps?.log?.("recovery_retry_held", { delayMs: RECOVERY_RETRY_WAIT_MS });
437
+ throw hold;
438
+ }
439
+ throw e;
440
+ }
441
+ // The run's FINAL outcome cleans up after it. Not a queued recovery (its checkpoint is what the recovery restores),
442
+ // and not `recovery-superseded`: that one is decided on a pointer read that came back empty, and an empty read must
443
+ // never be the reason another run's store is deleted; the run that settled cleaned up after itself.
444
+ if (durable.armed && result?.reason !== RECOVERY_ENQUEUED && result?.reason !== RECOVERY_SUPERSEDED) await endDurableRun(durable, deps);
445
+ return withBundle(withDurable(archiveFacts.value ? { ...result, forkSource: archiveFacts.value } : result, durable), bundleFacts);
446
+ }
447
+
448
+ /**
449
+ * The secret-bundle facts the record carries (issue #704), added to a result only when there are any, so every other
450
+ * result is unchanged. NAMES only: `bundle` is `{ id, variables, shadowed }`, never a value.
451
+ */
452
+ function withBundle(result, b) {
453
+ if (result === null || typeof result !== "object") return result;
454
+ if (b.bundle === null && b.bundleSkipped === null) return result;
455
+ return { ...result, ...(b.bundle !== null ? { bundle: b.bundle } : {}), ...(b.bundleSkipped !== null ? { bundleSkipped: b.bundleSkipped } : {}) };
456
+ }
457
+
458
+ /** The same on a throw, so a retried or failed attempt's record says what it held (buildRecord reads the error). */
459
+ function tagBundle(e, b) {
460
+ if (e === null || typeof e !== "object") return;
461
+ try {
462
+ if (b.bundle !== null && e.bundle === undefined) e.bundle = b.bundle;
463
+ if (b.bundleSkipped !== null && e.bundleSkipped === undefined) e.bundleSkipped = b.bundleSkipped;
464
+ } catch {
465
+ // a frozen error keeps its own fields
466
+ }
467
+ }
468
+
469
+ async function runAttempt(job, deps, durable, bundleFacts = { bundle: null, bundleSkipped: null }, archiveFacts = { value: null }) {
242
470
  const {
243
471
  redis,
244
472
  caps, // { day, week, month }; week/month null when that window is disabled (REQ-SPEND-CAPS-MULTI-WINDOW)
@@ -256,7 +484,8 @@ export async function runJob(job, deps) {
256
484
  // this job names is on this host (image-preflight.mjs). Default admits everything, so a wiring that
257
485
  // omits it behaves exactly as before -- the container's own failure stays the backstop.
258
486
  imagePreflight = async () => ({ ok: true }),
259
- // (job) => { ok } | { refused, at, jobId }. The one-shot pre-spend check (issue #231,
487
+ subscriptionAuthRequired = async () => false,
488
+ // (job) => { ok } | { refused, at, jobId } | { newer } (issue #761). The one-shot pre-spend check (issue #231,
260
489
  // DES-ONE-SHOT-DISARM-IN-THE-FILE). Default admits everything -- an unwired processor behaves
261
490
  // exactly as before, and the gate below only calls it for a job whose matched rule was a
262
491
  // one-shot, so the default is never a probe running on every delivery.
@@ -271,6 +500,8 @@ export async function runJob(job, deps) {
271
500
  // the real read happens where it always did, because threading a live credential through the processor
272
501
  // would put it in scope for every log line and record between here and the container.
273
502
  checkProviderCredential = () => ({ ok: true }),
503
+ prepareSubscriptionAuth = null,
504
+ validateSubscriptionAuth = null,
274
505
  // (refs) => { ok } | { unknown: { provider, id }, why } | { unavailable: code } (issue #502). Is each of this
275
506
  // job's models, its main one and every listed one, a model pi knows (model-catalog.mjs `checkModelsKnown`,
276
507
  // the builtin catalog plus the overlay models.json)? Admit-everything by default, like the credential probe
@@ -306,6 +537,11 @@ export async function runJob(job, deps) {
306
537
  // so the answer is not swallowed against it. Takes no job: the key belongs to the REAL BullMQ job, which only the
307
538
  // wiring holds (index.mjs binds queue.mjs's `releaseSemanticWindow` to it). Never throws. The default does nothing.
308
539
  releaseParkedWindow = async () => {},
540
+ // () => Promise<{ resumed } | null> (issue #638, DES-AWAIT-INPUT-ENDS-THE-JOB): take the replies held for this run
541
+ // while it was parking, after the window is released: the first recent one is enqueued as the resumed job, the
542
+ // rest answered once (held-settle.mjs `takeHeld`). Bound by the wiring to the REAL job, like the release. Never
543
+ // throws. The default takes nothing, so a wiring without it parks exactly as before.
544
+ takeHeldReplies = async () => null,
309
545
  // The session store's root, or null when the feature is unavailable (REQ-RESUMABLE-SESSION). Read
310
546
  // ONLY to answer the fail-closed gate below -- nothing here opens it, and it never reaches the
311
547
  // container env (docker-run.mjs mounts a per-job COPY, never the store).
@@ -342,6 +578,29 @@ export async function runJob(job, deps) {
342
578
  // would leave an INJECTED resolver running on every job, which is how a probe nobody wanted starts
343
579
  // spawning a subprocess per delivery to learn nothing.
344
580
  resolveSecrets = async (job) => ({ profileUnknown: job.secretsProfile ?? DEFAULT_SECRETS_PROFILE }),
581
+ // Issue #704, project secret bundles. `findSecretBundle(job, { project })` is the worker's own read of the triggers
582
+ // file's `secretBundles` (triggers-file.mjs `makeFindSecretBundle`); the default finds none, so a bare processor
583
+ // and every deployment without the key run exactly as before. `secretBundles` is the bundle seam
584
+ // (secret-bundles.mjs `makeBundleResolver`, its overlay and signal bound by the wiring): `lookup(profile)` with
585
+ // no spawn, `resolve(job, binding)` with one. Its default FAILS CLOSED, `resolveSecrets`' posture and reason:
586
+ // a wiring that finds a bundle and cannot resolve one must refuse, never start the job without the environment.
587
+ findSecretBundle = () => ({ none: true }),
588
+ secretBundles = { lookup: (profile) => ({ profileUnknown: profile }), resolve: async (_job, binding) => ({ profileUnknown: binding?.profile ?? null }) },
589
+ // `(job, token) => { headRef, headRepo }` for a pull-request target, the forge host's own (prepare-github.mjs reads
590
+ // the same seam for run.resume): whether the head lives in the base repository, answered by the FORGE, never the
591
+ // payload. Asked only for a job whose project has a bundle without `forkReviews`. Null on a bare wiring, which
592
+ // can confirm no head and so grants no bundle to a pull request (`bundleSkipped: "fork"`, the direction that
593
+ // grants nothing).
594
+ resolvePullRequestHead = null,
595
+ // Issue #704 review: whether a profile this host lacks DEFERS the job rather than refusing it: true only for a job
596
+ // on a multi-host worker's shared queue (index.mjs, the never-fits gate's predicate), where another host may serve
597
+ // it. False by default, so a single host refuses, as it always did.
598
+ bundleProfileDeferrable = false,
599
+ // Issue #704 part 3: the job's venue's judgement of its project secret bundle (the registry's `bundleGate`,
600
+ // backend-openshell.mjs's for both OpenShell venues), `(job, { endpoints, vars, shadowed })` => `{ ok, placeholders?,
601
+ // plain? }`, `{ venueRefused: { reason, comment } }` or `{ unavailable }`. The default admits, which is every venue
602
+ // that holds its env as values: there a bundle's variables simply join the env.
603
+ bundleGate = async () => ({ ok: true }),
345
604
  // (job) => { ok, document } | { refused, name, why } | { unavailable: code } (REQ-MCP-OPERATOR-SERVERS,
346
605
  // mcp-preflight.mjs). The default passes a job that names no server and refuses one that does, so a wiring that
347
606
  // forgot the check can never run a job without the servers its trigger names.
@@ -381,7 +640,7 @@ export async function runJob(job, deps) {
381
640
  // never sets it classifies every abort as worker-abort, exactly as before.
382
641
  runContainer,
383
642
  cleanup, // (dirs) => void
384
- comment, // (job, text) => void (issue status; no-op for local jobs)
643
+ comment: rawComment, // (job, text) => void (issue status; no-op for local jobs)
385
644
  log = () => {},
386
645
  // The outbox chain collector (INT-OUTBOX-CONTRACT). No-op default so a job whose wiring omits it --
387
646
  // or a github job with no /outbox -- chains nothing. It NEVER throws (outbox.mjs), so its counts are
@@ -392,6 +651,7 @@ export async function runJob(job, deps) {
392
651
  // NEVER throws, so a refused plan is a recorded outcome of a completed job and never a retry. The default collects
393
652
  // nothing (null: no plan), so a wiring that omits it records `plan: null`.
394
653
  collectPlan = async () => null,
654
+ collectWatch = async () => {},
395
655
  // Issue #501: the deployment's dollar windows `{ day, week, month }` in micro-dollars (each null when unset), or
396
656
  // null when no window is set. Null is the default and the off switch: nothing is reserved or settled and no
397
657
  // `budget:usd:*` key is written, so a deployment with no dollar setting is byte-identical.
@@ -414,6 +674,10 @@ export async function runJob(job, deps) {
414
674
  otherDollars = null,
415
675
  dollarCapSource = null,
416
676
  envelopeMismatch = false,
677
+ // Issue #761: the endpoint tag of the Valkey this worker's dollar counters live in (`valkeyEndpointTag`), which the
678
+ // record's `dollars.reservedIn` carries so doctor sums only the records of the counter it reads. Null (an unwired
679
+ // processor) writes no `reservedIn`, so such a record keeps the shape it had.
680
+ valkeyTag = null,
417
681
  // Issue #505: whether the LIVE triggers file still flags this job's cron trigger `run.portfolio: true`, as
418
682
  // `(job) => boolean`, and this host's envelope `delegation` block (`{ enabled, writers }`), null with no envelope.
419
683
  // The flag on the job data is what the trigger said when the job was queued; the file is what the operator says
@@ -438,8 +702,63 @@ export async function runJob(job, deps) {
438
702
  // `builtinModel`). Read only by the zero-rated check; the default knows no builtin model, so an unwired
439
703
  // processor judges overlay models alone and reserves for every other.
440
704
  builtinModel = () => null,
705
+ // Issue #645, durable runs. Every one defaults to inert, and none is read unless the job armed `run.recover`, so an
706
+ // unwired processor runs every other job exactly as before. `workspaceCheckpointer` is workspace-checkpoint.mjs's
707
+ // (null when PI_CHECKPOINTS_DIR is unset), `runBudgetLedger` run-budget-ledger.mjs's, `readCheckpointPointer`
708
+ // `({ runId }) => pointer | null` and `enqueueRecovery` `({ job, cause, pointer }) => { enqueued, jobId, n } |
709
+ // { enqueued: false, reason }` (recovery-queue.mjs, bound to the real BullMQ job by the wiring). `stageRecovered`
710
+ // is the session store's (session-store.mjs). `resetTrustPaths` is trust-paths.mjs's. `runningJobs` is the
711
+ // processor's `{ list() }` of the jobs running on this host, and `hostName` this worker's name, as the checkpoint
712
+ // manifest records it.
713
+ workspaceCheckpointer = null,
714
+ forkStore = null,
715
+ checkForkPolicy = async () => ({ refused: "fork-policy-unavailable" }),
716
+ checkThinking = async () => false,
717
+ runBudgetLedger = null,
718
+ readCheckpointPointer = null,
719
+ enqueueRecovery = null,
720
+ stageRecovered = null,
721
+ resetTrustPaths = resetTrustPathsDefault,
722
+ runningJobs = null,
723
+ onDollarHold = () => {},
724
+ hostName = null,
725
+ // `({ runId, pointer, attempt }) => { seq } | { skipped }` (start.mjs): a salvage checkpoint from a crashed attempt's
726
+ // job directory still on this host, asked for a recovery job before its manifest is read. Null takes none.
727
+ salvageCheckpoint = null,
728
+ // `(jobId, { keep }) => record | null` (start.mjs `readRunRecord`): the settled-run check a recovery asks first, this
729
+ // host's file and then the fleet's copy, the first that passes `keep` when one is given. Null skips it, and the
730
+ // pointer and store logic below still decide.
731
+ readRecord = null,
732
+ // `({ workspace, baseSha }) => boolean` (trust-paths.mjs): whether the base commit holds anything under
733
+ // `.pi/extensions/`, which makes a run unrecoverable (see its use).
734
+ baseHasExtensions = baseHasExtensionsDefault,
441
735
  now = new Date(),
442
736
  } = deps;
737
+ // `armed` can still drop to false for a fresh attempt whose base holds `.pi/extensions/` (below), before anything durable.
738
+ let { armed } = durable;
739
+ const { runId, isRecovery } = durable;
740
+ // The checkpoint a recovery job restores, read and judged among the free gates below.
741
+ let manifest = null;
742
+ let forkSource = null;
743
+ let sourcePrimed = false;
744
+ let sourceRunId = deps.queueJobId ?? null;
745
+ if (sourceRunId === null) { try { sourceRunId = forgeDeliveryJobId(job.kind, job.trigger?.deliveryId, job.replica); } catch {} }
746
+ const forkRefusal = async (reason) => {
747
+ await comment(job, `Refused: ${reason}. No fork container was started. Check the source run, current trigger policy and retained workspace points.`);
748
+ return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
749
+ };
750
+ let transcriptPath = null;
751
+ // Whether this attempt's dollar hold has a durable record (`recordHold`): only then do its settle and release go through
752
+ // the ledger's compare-and-set. A hold that failed to be recorded is given back directly, since no one else can see it.
753
+ let holdRecorded = false;
754
+ // Issue #645: the job-count slot an armed run carries from an earlier attempt (`claimSlot` answered "carried"). It is
755
+ // still counted, by the attempt that reserved it, so this attempt reserves none and refunds none.
756
+ let slotCarried = false;
757
+ // Whether an armed run's slot belongs to the RUN rather than to this attempt (any successful claim). Such a slot is not
758
+ // refunded by a never-started retry: the retry carries it (`claimSlot` answers "carried"), so refunding it here would
759
+ // let the retry run on a slot nothing counts. Only the run's final outcome gives a slot back (a refusal or
760
+ // config-refused, exactly as before).
761
+ let slotOwnedByRun = false;
443
762
 
444
763
  // "Forge-backed" is the negation of local, not an enumeration of forges: a job that is not editing a
445
764
  // folder on this host is working against a remote, and every gate below applies for the same reason
@@ -452,6 +771,11 @@ export async function runJob(job, deps) {
452
771
  const wantsForgeToken = isForgeBacked || job.github === true;
453
772
  let token = null;
454
773
  let prepared = null;
774
+ const comment = job.answerDraft ? async () => {} : rawComment;
775
+ let watchCompleted = false;
776
+ let answerUsage = null;
777
+ let watchUsage = null;
778
+ let watchPolicyRefused = false;
455
779
  // The job-count reservations still standing, in reserve order (issue #499 part B): what `reserveLedgers` took and
456
780
  // no refund has given back yet. EVERY refund below is `releaseLedgers` over this one list, last first, so no path
457
781
  // can give back one ledger and forget another, and none can give one back twice: a released ledger leaves the list.
@@ -461,7 +785,8 @@ export async function runJob(job, deps) {
461
785
  // refund, global-only as INT-RUN-HISTORY-FILE-CONTRACT has it. A scoped or project slot a failed refund left behind
462
786
  // is in the `budget_release_failed` log line, not in this field.
463
787
  let globalLedger = null;
464
- const globalHeld = () => globalLedger !== null && held.includes(globalLedger);
788
+ // A carried slot (issue #645) is held too: by the run, under the attempt that reserved it.
789
+ const globalHeld = () => (globalLedger !== null && held.includes(globalLedger)) || slotCarried;
465
790
  // Give back every held job-count slot, NEVER throwing: a refund that did not land must not replace the caller's
466
791
  // classification with a Redis message. Logged with `at`; returns whether the list is empty afterwards.
467
792
  const refundLedgers = async (at) => {
@@ -489,9 +814,14 @@ export async function runJob(job, deps) {
489
814
  // taken or once it has been settled or given back, so no path can settle or refund it twice. `dollars` is what
490
815
  // the record says about it (INT-RUN-HISTORY-FILE-CONTRACT), null until the reservation step ran.
491
816
  let dollarHold = null;
817
+ // A reporting failure must never change admission, settlement or the result of a paid run.
818
+ const publishDollarHold = (hold) => { try { onDollarHold(hold); } catch {} };
492
819
  let dollars = null;
493
820
  // Issue #596: what the container used (`resources` off its exit line), null until a container ran and reported it.
494
821
  let resources = null;
822
+ // Issue #644: the run's progress summary (run-progress.mjs `summary`), null until a container ran with the reducer on.
823
+ // Display only: carried to the record like `resources`, and never read by any decision below.
824
+ let progress = null;
495
825
 
496
826
  try {
497
827
  // The one-shot pre-spend check (issue #231), FIRST on the ladder: one file read, cheaper than
@@ -504,6 +834,11 @@ export async function runJob(job, deps) {
504
834
  // there, not optional hardening.
505
835
  if (job.trigger?.matched?.once === true) {
506
836
  const spent = await checkOnceSpent(job);
837
+ // Issue #761: the one answer that is not fail-open. A triggers file written by a newer pi-dispatch may hold this
838
+ // one-shot's disarm in a shape this build cannot read, and this build's own disarm of it would be refused, so
839
+ // running would fire the one-shot again on every delivery. Deferred (`newer-release`), never refused: a refusal
840
+ // would use up the delivery. Thrown here, first on the ladder, so nothing is minted, cloned or reserved.
841
+ if (spent.newer === true) throw new NewerReleaseHold("the triggers file");
507
842
  if (spent.refused) {
508
843
  // Commented like every sibling policy refusal: explainability is this refusal's whole
509
844
  // purpose, and only a DISTINCT re-close reaches it past the GUID dedup, so the noise
@@ -523,8 +858,12 @@ export async function runJob(job, deps) {
523
858
  // standing between a stale receiver and a paid job that ran when the operator wrote "wait".
524
859
  {
525
860
  const skew = await checkWaitSkew(job);
526
- // Issue #502: an authored NARROWING field the job arrived without (`AUTHORED_NARROWING_FIELDS`, triggers-file.mjs),
527
- // today `run.models` and `run.maxCostUsd`. The same two causes as a dropped wait, and the same refusal shape; without it the job would
861
+ // Issue #761: a triggers file written by a newer pi-dispatch may author a narrowing field this build has no row
862
+ // for, which no job field can show missing, so neither "skewed" nor "fine" is an answer this build can give. The
863
+ // job waits for a worker that reads the file (`newer-release`), before any spend, the once check's posture.
864
+ if (skew.newer === true) throw new NewerReleaseHold("the triggers file");
865
+ // Issue #502: an authored NARROWING field the job arrived without, one of the rows of `AUTHORED_NARROWING_FIELDS`
866
+ // (triggers-file.mjs). The same two causes as a dropped wait, and the same refusal shape; without it the job would
528
867
  // run on the deployment's list, or on none, while every record reads like a correct run. The FIELD is named,
529
868
  // never its value: the comment's reader may be an issue author.
530
869
  if (skew.skewed && typeof skew.field === "string") {
@@ -567,6 +906,16 @@ export async function runJob(job, deps) {
567
906
  log("refused_backend_unblessed", { backend: job.backend, blessed: blessedBackends });
568
907
  return { outcome: "policy", reason: "backend-unblessed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
569
908
  }
909
+ // Issue #683: a folder job (a local run, a cron trigger's folder) on a venue that declares `localFolders` absent.
910
+ // The loader refuses a trigger that NAMES such a venue (`refusesLocalWorkspace`); this is the one it could not
911
+ // see, a trigger with no `run.backend` on a deployment whose DEFAULT is such a venue. A copy is edited elsewhere
912
+ // and written back afterwards, racing every edit the operator makes meanwhile (`DES-OPENSHELL-VENUE`).
913
+ const venueName = resolveBackendName(job, blessedBackends[0]);
914
+ if (job.kind === "local" && backendFor(venueName)?.declares?.localFolders === ABSENT) {
915
+ await comment(job, `Refused: this job runs against a folder on this machine, and its venue "${venueName}" copies a job's files instead of editing a folder in place. Run it on a venue that mounts folders (local or podman). Not run.`);
916
+ log("refused_venue_local_folders", { backend: venueName });
917
+ return { outcome: "policy", reason: "venue-local-folders-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
918
+ }
570
919
 
571
920
  // Issue #278. PI_BACKEND_FLOOR can ask for a guarantee a backend declares but EARNS only while something
572
921
  // about this host is observed -- `local`'s credentialTransit, which holds only while the docker CLI sends
@@ -614,6 +963,16 @@ export async function runJob(job, deps) {
614
963
  budgetReserved: false,
615
964
  };
616
965
  }
966
+ // Issue #683: a venue's own refusal, made before spend whatever the floor says: the openshell venue's gateway is not
967
+ // on this host, has a gateway-wide policy in force, or the host has no Landlock (`CONST-VENUE-EGRESS-LAYER`'s third
968
+ // and fourth conditions). Determinate, so a RETURN (CONST-RETRY-INFRA-ONLY), ahead of the image preflight and the
969
+ // mint. The comment is the venue's fixed sentence for its reason, never anything the runtime said.
970
+ if (observed?.venueRefused && typeof observed.venueRefused.reason === "string") {
971
+ const { reason, comment: text } = observed.venueRefused;
972
+ await comment(job, typeof text === "string" && text !== "" ? text : VENUE_REFUSED_COMMENT);
973
+ log("refused_venue", { reason });
974
+ return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
975
+ }
617
976
  if (observed?.unavailable) {
618
977
  // The local venue's words stay what they were (they are its log line and BullMQ's failedReason); another venue
619
978
  // is not docker's to name.
@@ -662,15 +1021,15 @@ export async function runJob(job, deps) {
662
1021
  job,
663
1022
  rootful
664
1023
  ? key
665
- ? "Refused: the worker host's Podman configuration adds to every job's container what this venue does not allow (variables, the Podman service's groups, looser limits and filters, or the programs that run it), so the operator must change it before local jobs run. Not run."
666
- : "Refused: the worker host's Podman configuration could not be read in full, or is written in a form the worker does not decode, so whether it adds to a job's container what this venue does not allow is not known, and the operator must fix that before local jobs run. Not run."
1024
+ ? PODMAN_CONF_COMMENTS["rootful-widens"]
1025
+ : PODMAN_CONF_COMMENTS["rootful-unread"]
667
1026
  : live
668
1027
  ? key
669
- ? "Refused: the worker host's running Podman network still lets a job's container reach the host's own services, with an option from a configuration since changed, so the operator must restart that network before podman jobs run. Not run."
670
- : "Refused: the worker host's running Podman network could not be read, so whether it lets a job's container reach more than this venue allows is not known, and the operator must fix that before podman jobs run. Not run."
1028
+ ? PODMAN_CONF_COMMENTS["live-widens"]
1029
+ : PODMAN_CONF_COMMENTS["live-unread"]
671
1030
  : key
672
- ? "Refused: the worker host's Podman configuration lets a job's container reach more than this venue allows (the host's own services, the worker account's groups, or looser limits and filters than the worker sets), so the operator must change it before podman jobs run. Not run."
673
- : "Refused: the worker host's Podman configuration could not be read in full, or is written in a form the worker does not decode, so whether it lets a job's container reach more than this venue allows is not known, and the operator must fix that before podman jobs run. Not run.",
1031
+ ? PODMAN_CONF_COMMENTS.widens
1032
+ : PODMAN_CONF_COMMENTS.unread,
674
1033
  );
675
1034
  log("refused_podman_conf_widens_job", { key: key ?? null, ...(live ? { live: true } : {}), ...(rootful ? { rootful: true } : {}), message: observed.podmanConfRefused.message });
676
1035
  return { outcome: "policy", reason: PODMAN_CONF_WIDENS_JOB, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
@@ -681,12 +1040,22 @@ export async function runJob(job, deps) {
681
1040
  // image refuses without minting a credential it will not use, cloning a repo it will not read, or
682
1041
  // burning a cap slot. Jobs run with --pull=never (docker-run.mjs), so an absent image is never
683
1042
  // fetched -- this refusal IS the whole diagnosis, not a race with a background pull.
684
- const img = await imagePreflight(job);
1043
+ const needsSubscriptionAuth = await subscriptionAuthRequired(job);
1044
+ if (typeof needsSubscriptionAuth !== "boolean") throw new Error("the subscription image requirement is unavailable");
1045
+ const checkedImage = await imagePreflight(job, { subscriptionAuthRequired: needsSubscriptionAuth });
1046
+ // Enforce the same host-only requirement at the DI seam too: a preflight returning capabilities
1047
+ // cannot accidentally admit an older image before access preparation, mint or reservations.
1048
+ const img = checkedImage.ok && needsSubscriptionAuth && !(checkedImage.capabilities ?? []).includes("subscriptionAuth")
1049
+ ? { subscriptionAuthUnsupported: checkedImage.image ?? job.image ?? "configured job image", declared: checkedImage.capabilities ?? [] }
1050
+ : checkedImage;
685
1051
  // The image's declared pi version, read on the inspect the preflight already ran. Needed BEFORE the
686
1052
  // container starts, because a transcript written by a different pi may hold tool-call arguments the
687
1053
  // current schema no longer accepts -- so the resume has to be refused, not repaired mid-run. Null
688
1054
  // when the image declares none, which downstream means "never resume": the safe direction.
689
1055
  const piVersion = img.piVersion ?? null;
1056
+ // Issue #800: from here a refusal can say which image it would have run on. Before this, and on a preflight that found
1057
+ // no usable image, no value is set, so such a record is byte-identical to what it was.
1058
+ if (deps.canaryFacts && img.ok === true) deps.canaryFacts.loaded = loadedWithImage(null, img);
690
1059
  if (img.missing) {
691
1060
  await comment(job, `Refused: the job image "${img.missing}" is not present on the worker host. Not run.`);
692
1061
  log("refused_image_missing", { image: img.missing });
@@ -716,6 +1085,22 @@ export async function runJob(job, deps) {
716
1085
  log("refused_image_forge_unsupported", { image: img.forgeUnsupported, kind: img.kind, declared: img.declared });
717
1086
  return { outcome: "policy", reason: "job-image-forge-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
718
1087
  }
1088
+ if (deps.approvalRequest && deps.approvalSessionCheck) {
1089
+ const ready = deps.approvalSessionCheck(deps.approvalRequest, { piVersion: img.piVersion, venue: resolveBackendName(job, blessedBackends[0]), provider: job.provider, modelId: job.model });
1090
+ if (!ready.ok) return { outcome: "policy", reason: "approval-session-unavailable", exitCode: null, turns: null, tokens: null, budgetReserved: false };
1091
+ }
1092
+
1093
+ if (job.fallbackModels !== undefined && !(img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY)) {
1094
+ const required = CAPABILITY_GATES.find((row) => row.token === "modelFailover");
1095
+ await comment(job, required.comment(img.image ?? job.image ?? "selected image", "lacks authenticated exit support"));
1096
+ return { outcome: "policy", reason: "job-image-model-failover-unsupported", exitCode: null, turns: null, tokens: null, budgetReserved: false };
1097
+ }
1098
+ if (job.answerDraft && !(img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY)) return { outcome: "policy", reason: "job-image-answer-draft-unsupported", budgetReserved: false };
1099
+ if (job.gates && !(img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY)) {
1100
+ await comment(job, APPROVAL_COMMENTS["job-image-approval-auth-unsupported"]);
1101
+ return { outcome: "policy", reason: "job-image-approval-auth-unsupported", exitCode: null, turns: null, tokens: null, budgetReserved: false };
1102
+ }
1103
+
719
1104
  // The image capability gates (one table, CAPABILITY_GATES in image-preflight.mjs): the image is present and
720
1105
  // does not declare a feature this job carries, so its runner would silently ignore it. Determinate, so a
721
1106
  // refusal rather than a retry, and pre-spend, because no version of this gets better by running. Like the
@@ -790,7 +1175,7 @@ export async function runJob(job, deps) {
790
1175
  if (known?.unknown) {
791
1176
  // An `overlay-*` why is the deployment's file, not the job's model: its own comment.
792
1177
  const why = typeof known.why === "string" ? known.why : null;
793
- await comment(job, why?.startsWith("overlay-") ? OVERLAY_REFUSED_COMMENT : MODEL_UNKNOWN_COMMENT);
1178
+ await comment(job, modelUnknownComment(why));
794
1179
  // `why` is a fixed token: `overlay-unparseable` tells the operator the file is the problem, not the id.
795
1180
  log("refused_model_unknown", { provider: known.unknown.provider ?? null, model: known.unknown.id ?? null, why: known.why ?? null });
796
1181
  return { outcome: "policy", reason: "model-unknown", why, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
@@ -800,7 +1185,7 @@ export async function runJob(job, deps) {
800
1185
  // does not list `PI_MODEL`, or a `dispatch_set model` that moved the default off a trigger's list. Both
801
1186
  // halves are compared, exact (`modelOnList`), the runner guard's rule.
802
1187
  if (Array.isArray(job.models) && !modelOnList(job.models, job.provider, job.model)) {
803
- await comment(job, "Refused: the AI model this job would run on is not on the list of models it is allowed to use, so no container was started and nothing was spent. Ask the operator to check the trigger's model settings. Not run.");
1188
+ await comment(job, MODEL_NOT_ALLOWED_COMMENTS.main);
804
1189
  log("refused_model_not_allowed", { provider: job.provider ?? null, model: job.model ?? null });
805
1190
  return { outcome: "policy", reason: "model-not-allowed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
806
1191
  }
@@ -809,12 +1194,15 @@ export async function runJob(job, deps) {
809
1194
  // the container started; on any other api the worker is stricter than the runner, by decision. Refused here, free. The log names the listed model, the operator's own configuration; the
810
1195
  // comment names none.
811
1196
  if (known?.fallbackUnlisted) {
812
- await comment(job, "Refused: a model this job is allowed to use declares fallback models that are not on the job's list, so no container was started and nothing was spent. Ask the operator to list those models too, or remove that model. Not run.");
1197
+ await comment(job, MODEL_NOT_ALLOWED_COMMENTS["fallback-unlisted"]);
813
1198
  log("refused_model_not_allowed", { provider: known.fallbackUnlisted.provider ?? null, model: known.fallbackUnlisted.id ?? null, why: "fallback-unlisted" });
814
1199
  return { outcome: "policy", reason: "model-not-allowed", why: "fallback-unlisted", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
815
1200
  }
816
1201
  }
817
1202
 
1203
+ if (job.watch?.project && job.watch.project !== pickupProject) return { outcome: "policy", reason: "watch-project-mismatch", budgetReserved: false };
1204
+ if (job.answerDraft?.project && job.answerDraft.project !== pickupProject) return { outcome: "policy", reason: "answer-draft-config-changed", budgetReserved: false };
1205
+
818
1206
  // Is there a credential to run this job with at all? FREE, determinate and I/O-light: a pure function
819
1207
  // of the job's provider, the worker env, and (only when the env has no key) one small readFileSync of
820
1208
  // pi's auth.json. So it goes here, with the other free gates, which is further than issue #310 asked
@@ -853,8 +1241,8 @@ export async function runJob(job, deps) {
853
1241
  await comment(
854
1242
  job,
855
1243
  absent
856
- ? "Refused: this worker has no budget envelope while the other workers share an applied budget split, so no container was started and nothing was spent. Ask the operator to install the envelope on this worker, or to turn delegated allocation off for every worker (`pi-dispatch doctor` says how). Not run."
857
- : "Refused: this worker's budget envelope differs from the one the current budget split was made for, so no container was started and nothing was spent. Ask the operator to run `pi-dispatch doctor`. Not run.",
1244
+ ? ENVELOPE_MISMATCH_COMMENTS["no-envelope"]
1245
+ : ENVELOPE_MISMATCH_COMMENTS.mismatch,
858
1246
  );
859
1247
  log("refused_envelope_mismatch", absent ? { envelope: "none" } : {});
860
1248
  return { outcome: "policy", reason: ENVELOPE_MISMATCH_REASON, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
@@ -884,28 +1272,46 @@ export async function runJob(job, deps) {
884
1272
  }
885
1273
  }
886
1274
 
887
- const credential = await checkProviderCredential(job, { modelEndpoints });
888
- if (credential.unavailable) {
889
- // Issue #503: the overlay models.json could not be read at this pickup for a transient reason, so the gate has no
890
- // verdict for a provider that may be keyless. Retried, never refused: a refusal is permanent and public, and the
891
- // next attempt may read the file. The code is a fixed errno token, never file text.
892
- log("provider_credential_unavailable", { provider: job.provider ?? null, reason: credential.unavailable });
893
- throw new InfraRetry("the provider credential could not be decided", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
894
- }
895
- if (!credential.ok) {
896
- // The PROVIDER is named and the message is not. The provider is operator-authored config, already
897
- // on the record and the mirror, and named freely by the sibling refusals (`backend-unblessed` names
898
- // the backend, `job-image-missing` names the image), so withholding it would tell the operator less
899
- // than is safe. The message is withheld from BOTH surfaces: `credentialFromPiAuth`'s refusals carry
900
- // `auth.json`'s absolute path, which is an OS account name, and the scoped-budget refusal below
901
- // keeps a host path out of its own log line citing no-pii-in-logs. `doctor` prints the variable and
902
- // the path, on the operator's terminal, which is where that belongs.
903
- await comment(job, `Refused: this deployment has no usable credential for the "${job.provider}" provider, so no container was started and nothing was spent. Ask the operator to run \`pi-dispatch doctor\`. Not run.`);
904
- log("refused_provider_unconfigured", { provider: job.provider ?? null });
905
- return { outcome: "policy", reason: "provider-unconfigured", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1275
+ const credentialJobs = [job];
1276
+ if (job.watch && job.watchDecisionModel && job.watchDecisionModel.provider !== job.provider) credentialJobs.push({ ...job, provider: job.watchDecisionModel.provider, model: job.watchDecisionModel.id });
1277
+ if (job.questionGrouping && job.questionGroupModel && job.questionGroupModel.provider !== job.provider) credentialJobs.push({ ...job, provider: job.questionGroupModel.provider, model: job.questionGroupModel.id });
1278
+ if (job.answerDraft && job.answerDecisionModel && job.answerDecisionModel.provider !== job.provider) credentialJobs.push({ ...job, provider: job.answerDecisionModel.provider, model: job.answerDecisionModel.id });
1279
+ for (const credentialJob of credentialJobs) {
1280
+ const credential = await checkProviderCredential(credentialJob, { modelEndpoints });
1281
+ if (credential.unavailable) {
1282
+ // Issue #503: the overlay models.json could not be read at this pickup for a transient reason, so the gate has no
1283
+ // verdict for a provider that may be keyless. Retried, never refused: a refusal is permanent and public, and the
1284
+ // next attempt may read the file. The code is a fixed errno token, never file text.
1285
+ log("provider_credential_unavailable", { provider: credentialJob.provider ?? null, reason: credential.unavailable });
1286
+ throw new InfraRetry("the provider credential could not be decided", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
1287
+ }
1288
+ if (!credential.ok) {
1289
+ // The PROVIDER is named and the message is not. The provider is operator-authored config, already
1290
+ // on the record and the mirror, and named freely by the sibling refusals (`backend-unblessed` names
1291
+ // the backend, `job-image-missing` names the image), so withholding it would tell the operator less
1292
+ // than is safe. The message is withheld from BOTH surfaces: `credentialFromPiAuth`'s refusals carry
1293
+ // `auth.json`'s absolute path, which is an OS account name, and the scoped-budget refusal below
1294
+ // keeps a host path out of its own log line citing no-pii-in-logs. `doctor` prints the variable and
1295
+ // the path, on the operator's terminal, which is where that belongs.
1296
+ await comment(job, `Refused: this deployment has no usable credential for the "${credentialJob.provider}" provider, so no container was started and nothing was spent. Ask the operator to run \`pi-dispatch doctor\`. Not run.`);
1297
+ log("refused_provider_unconfigured", { provider: credentialJob.provider ?? null });
1298
+ return { outcome: "policy", reason: "provider-unconfigured", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1299
+ }
906
1300
  }
907
1301
 
908
1302
 
1303
+ // Access-only handoff is prepared after local determinate gates and before any paid work.
1304
+ let subscriptionAuth = [];
1305
+ if (prepareSubscriptionAuth) {
1306
+ const preparedAuth = await prepareSubscriptionAuth(job);
1307
+ if (!preparedAuth.ok) {
1308
+ if (preparedAuth.wait) throw new SubscriptionWait(preparedAuth);
1309
+ if (preparedAuth.unavailable) throw new InfraRetry("subscription authentication unavailable", { reason: "container-never-started", budgetReserved: false });
1310
+ return { outcome: "policy", reason: preparedAuth.reason ?? "subscription-quota", exitCode: null, budgetReserved: false };
1311
+ }
1312
+ subscriptionAuth = preparedAuth.snapshots;
1313
+ }
1314
+
909
1315
  // REQ-EGRESS-ALLOWLIST. The egress policy this deployment claims must be able to serve this job
910
1316
  // BEFORE the job costs anything. It is one `docker inspect` when the policy is armed and ZERO spawns
911
1317
  // when it is not, so a deployment without one pays nothing at all.
@@ -990,13 +1396,39 @@ export async function runJob(job, deps) {
990
1396
  // mark beside its transcript, and it is free to read, so it is asked here, before the mint and before any spend.
991
1397
  // Without it the reply is ordinary conversation on an issue: refused, silently (no comment, since the person was
992
1398
  // talking to people), and the stale marker is cleared so the next comment is not routed here again.
1399
+ if (job.fork !== undefined) {
1400
+ try { validateFork(job.fork); } catch { return await forkRefusal("fork-request-invalid"); }
1401
+ if (job.kind === "local" || job.target?.type !== "issue" || job.command || job.replica !== undefined || job.replicas !== undefined) return await forkRefusal("fork-target-unsupported");
1402
+ const venue = resolveBackendName(job, blessedBackends[0]);
1403
+ if (venue === "openshell") return await forkRefusal("openshell-fork-unsupported");
1404
+ if (job.awaitInput === true) return await forkRefusal("fork-await-input-unsupported");
1405
+ if (!isRecovery && !forkStore) return await forkRefusal("fork-source-store-unset");
1406
+ const policy = await checkForkPolicy(job);
1407
+ if (policy?.ok !== true) return await forkRefusal(policy?.refused ?? "fork-policy-unavailable");
1408
+ const expected = job.forkReserveMicros;
1409
+ if (expected !== null && (!Number.isSafeInteger(expected) || expected < 0)) return await forkRefusal("fork-request-invalid");
1410
+ if (expected !== null && !zeroRatedFor(job, { modelEndpoints, builtinModel }).zeroRated && (expected === 0 || !Number.isSafeInteger(job.maxCostMicros) || job.maxCostMicros > expected)) return await forkRefusal("fork-reservation-changed");
1411
+ const sourceKey = sessionKeyFor({ ...job, fork: undefined });
1412
+ if (sourceKey !== job.fork.rootKey) return await forkRefusal("fork-source-target-mismatch");
1413
+ // A recovery continues THIS fork's checkpoint; selecting the parent's original point again loses its work.
1414
+ if (!isRecovery) {
1415
+ forkSource = await forkStore.readPoint({ runId: job.fork.fromJob, sourceDigest: job.fork.sourceDigest, entryId: job.fork.fromEntry, position: job.fork.position });
1416
+ if (forkSource?.unavailable) throw new InfraRetry("the fork source could not be read just now", { reason: "container-never-started", budgetReserved: false });
1417
+ if (!forkSource?.ok) return await forkRefusal(forkSource?.refused ?? "fork-source-unavailable");
1418
+ if (forkSource.manifest.sessionKey !== sourceKey) return await forkRefusal("fork-source-target-mismatch");
1419
+ if (forkSource.manifest.piVersion !== piVersion) return await forkRefusal("fork-source-pi-version-mismatch");
1420
+ if (forkSource.manifest.venue !== venue) return await forkRefusal("fork-source-venue-mismatch");
1421
+ if (hostName && forkSource.manifest.host !== normalizeHost(hostName)) return await forkRefusal("fork-source-host-mismatch");
1422
+ }
1423
+ }
1424
+ if (job.thinkingLevel !== undefined && (!FORK_THINKING_LEVELS.includes(job.thinkingLevel) || !(await checkThinking(job)))) return await forkRefusal("thinking-level-unsupported");
993
1425
  if (job.parkedReply === true && !isParked(job)) {
994
1426
  log("refused_not_parked", { kind: job.kind ?? null });
995
1427
  await parkedLabel(job, false);
996
1428
  return { outcome: "policy", reason: "not-parked", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
997
1429
  }
998
1430
  if (job.resume === true && !sessionsDir) {
999
- await comment(job, "Refused: this trigger set `run.resume` but PI_SESSIONS_DIR is unset, so there is nowhere to persist the transcript -- the job would run with no session and still report success. Set PI_SESSIONS_DIR to a private directory outside every repo, or drop `run.resume` from this trigger. Not run.");
1431
+ await comment(job, REFUSAL_COMMENTS["sessions-dir-unset"]);
1000
1432
  // The variable NAME, never a value: there is no path to print here (its absence IS the refusal),
1001
1433
  // and the store's path is the one setting SECURITY.md calls a PII store. `kind` is host-assigned,
1002
1434
  // the same PII class as the `repo` on the branch refusal below.
@@ -1007,6 +1439,144 @@ export async function runJob(job, deps) {
1007
1439
  return { outcome: "policy", reason: "sessions-dir-unset", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1008
1440
  }
1009
1441
 
1442
+ // Issue #645, durable runs: `run.recover`'s free refusals, here beside run.resume's and for its reasons. Free (a
1443
+ // value in hand, a Valkey GET, a manifest read off this host's disk), determinate, and strictly before the mint,
1444
+ // the clone, the token-cap read and every reserve (CONST-BUDGET-BEFORE-TOKENS).
1445
+ //
1446
+ // The store's absence is run.resume's one fail-CLOSED case again: without a checkpoint store nothing is kept, a
1447
+ // lost run is retried from scratch, and the job still looks protected. A wiring that has a store but not the
1448
+ // rest of the machinery (a ledger, the pointer read, the enqueuer) is a defect of this build, not of the operator's
1449
+ // settings, so it is a config refusal: refunded, `doctor`-pointed, never run half-armed.
1450
+ if (armed && !workspaceCheckpointer) {
1451
+ await comment(job, RECOVERY_COMMENTS["recover-store-unset"]);
1452
+ log("refused_checkpoints_dir_unset", { kind: job.kind ?? null });
1453
+ return { outcome: "policy", reason: "recover-store-unset", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1454
+ }
1455
+ if (armed && (!runBudgetLedger || typeof readCheckpointPointer !== "function" || typeof enqueueRecovery !== "function")) {
1456
+ throw configError("run.recover is armed, and this worker was built without the run budget ledger, the checkpoint pointer read or the recovery enqueuer");
1457
+ }
1458
+ if (isRecovery) {
1459
+ const n = durable.recovery;
1460
+ // SETTLED, first of all: the run (its base attempt or an earlier recovery) already wrote a completed or policy
1461
+ // record, so it reached its final outcome and there is nothing to continue. Silent, and cleans nothing (that
1462
+ // outcome cleaned up after itself). Asked BEFORE the pointer and the store, because those can disagree after a
1463
+ // cleanup that half failed (a store removal that failed beside a pointer delete that landed), and that
1464
+ // disagreement must never put a failure comment under a run that completed.
1465
+ // Only a record of THIS run settles it (issue #811). Ids come back (a lost queue, a hand redelivery) while records
1466
+ // stay for the retention window, so a `completed` record an older job left under the run's id would end this
1467
+ // recovery as superseded, silently, and the lost run would be neither continued nor reported. The recovery's data
1468
+ // carries the base run's creation time (`recoveryOfCreatedAt`, recovery-queue.mjs `recoveryDataOf`), and a record
1469
+ // whose time (`startedAt`, or `endedAt` with no start) is older than that, less `RECORD_CLOCK_SKEW_MS`, is
1470
+ // another job's and settles nothing. Only a time that PARSES is judged so: `whyNotThisJob` would take a record
1471
+ // with no readable time for another job's, but here that would run a run that may have completed, and paying
1472
+ // twice for a settled run is the worse mistake, so such a record still settles, as before. A recovery without
1473
+ // the field (queued by a build from before it) keeps the check without an age. Both sides are read, this host's
1474
+ // file and the fleet's copy, each held to the rule (`keep`), so a stale record on this host cannot hide the
1475
+ // settled one in the mirror; the answer is judged again, so a reader that ignores `keep` cannot settle the run
1476
+ // on a record that fails it. Every field read is in the record's stable core, so a newer release's record is
1477
+ // judged the same way.
1478
+ const since = Number.isSafeInteger(job.recoveryOfCreatedAt) && job.recoveryOfCreatedAt >= 0 ? job.recoveryOfCreatedAt : null;
1479
+ const settlesRun = (rec, id) => {
1480
+ if (!rec || typeof rec !== "object" || rec.jobId !== id || (rec.outcome !== "completed" && rec.outcome !== "policy")) return false;
1481
+ if (since === null) return true;
1482
+ const at = Date.parse(rec.startedAt ?? rec.endedAt ?? "");
1483
+ return !(Number.isFinite(at) && at < since - RECORD_CLOCK_SKEW_MS);
1484
+ };
1485
+ if (typeof readRecord === "function") {
1486
+ const ids = [runId, ...Array.from({ length: Number.isSafeInteger(n) && n > 1 ? Math.min(n, RECOVERY_CAP + 1) - 1 : 0 }, (_, k) => `${runId}.r${k + 1}`)];
1487
+ for (const id of ids) {
1488
+ let rec = null;
1489
+ try {
1490
+ rec = await readRecord(id, { keep: (record) => settlesRun(record, id) });
1491
+ } catch {
1492
+ rec = null;
1493
+ }
1494
+ if (settlesRun(rec, id)) {
1495
+ log("recovery_superseded", { n: Number.isSafeInteger(n) ? n : null, by: "record" });
1496
+ return { outcome: "policy", reason: RECOVERY_SUPERSEDED, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1497
+ }
1498
+ }
1499
+ }
1500
+ // THE CAP: a run continued twice is not continued a third time, whatever its checkpoint holds.
1501
+ if (!Number.isSafeInteger(n) || n < 1 || n > RECOVERY_CAP) {
1502
+ await comment(job, RECOVERY_COMMENTS["recovery-cap"]);
1503
+ log("refused_recovery_cap", { n: Number.isSafeInteger(n) ? n : null, cap: RECOVERY_CAP });
1504
+ return { outcome: "policy", reason: "recovery-cap", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1505
+ }
1506
+ // SUPERSEDED: the base run reached its final outcome after this recovery was queued (a completed or policy result
1507
+ // removes the pointer), so there is nothing left to continue. Silent: that outcome already said what it had to.
1508
+ // A read that THROWS is not an answer, so it is retried, never taken for "settled".
1509
+ let pointer;
1510
+ try {
1511
+ pointer = await readCheckpointPointer({ runId });
1512
+ } catch {
1513
+ throw new InfraRetry("the run's checkpoint pointer could not be read", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
1514
+ }
1515
+ // The pointer read answers null for "no pointer" AND for "Valkey did not answer in time" (run-checkpoint.mjs never
1516
+ // throws), so null alone must not decide: a silent refusal would drop a live run's continuation over a blip. A
1517
+ // run's final outcome removes its STORE ENTRY before its pointer (`endDurableRun`), so superseded is a missing
1518
+ // pointer AND a missing store. A missing pointer beside a store that is still there is ambiguous, and is
1519
+ // retried loudly rather than guessed either way (continuing could re-run a run that completed).
1520
+ if (pointer === null || pointer === undefined) {
1521
+ const still = await workspaceCheckpointer.readManifest(runId);
1522
+ if (still?.absent === true) {
1523
+ log("recovery_superseded", { n });
1524
+ return { outcome: "policy", reason: RECOVERY_SUPERSEDED, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1525
+ }
1526
+ // A store a newer pi-dispatch wrote is one this build cannot judge either way: waited on like the read below.
1527
+ if (still?.newer === true) throw new NewerReleaseHold("the run's checkpoint");
1528
+ log("recovery_pointer_unread", { n });
1529
+ throw new InfraRetry("the run's checkpoint pointer is missing while its checkpoint is still there", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
1530
+ }
1531
+ // THE BASE STILL RUNNING HERE: its container may still be alive and spending, so this recovery must neither
1532
+ // settle its hold as crashed nor start a second agent on the same branch. It waits, without an attempt
1533
+ // (`RecoveryWaitHold`), until that container is gone. Asked of this host only: the recovery is queued to the
1534
+ // host the checkpoint is on, which is the host the base ran on.
1535
+ // THIS job is in that list too (the processor admits a job before it runs it), under the id the recovery queue
1536
+ // gave it (`<runId>.r<n>`, recovery-queue.mjs), so its own entry is excluded: without that every recovery would
1537
+ // find itself running and wait for ever.
1538
+ const ownId = `${runId}.r${n}`;
1539
+ const live = typeof runningJobs?.list === "function" ? runningJobs.list() : [];
1540
+ if (live.some((e) => typeof e?.id === "string" && e.id !== ownId && (e.id === runId || e.id.startsWith(`${runId}.r`)))) {
1541
+ log("recovery_waits_for_base", { n, delayMs: RECOVERY_WAIT_MS });
1542
+ throw new RecoveryWaitHold("the run this recovery continues is still running on this host", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false, waitMs: RECOVERY_WAIT_MS });
1543
+ }
1544
+ // SALVAGE, before the manifest is read: a crashed attempt whose job directory is still on this host (the worker
1545
+ // died, the directory did not) gets one last checkpoint taken from it (`kind: "salvage"`), so the recovery
1546
+ // continues from where the run really got to rather than from its last live checkpoint. The wiring's
1547
+ // (start.mjs); it never throws by contract, and is guarded anyway: a failed salvage leaves the last committed
1548
+ // checkpoint, which is still a correct place to continue from.
1549
+ if (typeof salvageCheckpoint === "function") {
1550
+ const salvaged = await Promise.resolve()
1551
+ .then(() => salvageCheckpoint({ runId, pointer, attempt: pointer.attempt }))
1552
+ .catch(() => ({ skipped: "error" }));
1553
+ log("recovery_salvage", Number.isSafeInteger(salvaged?.seq) ? { seq: salvaged.seq } : { skipped: typeof salvaged?.skipped === "string" ? salvaged.skipped.slice(0, 40) : "unknown" });
1554
+ }
1555
+ // THE CHECKPOINT, judged before anything spends: valid (readManifest hashes the transcript against its
1556
+ // manifest), taken on THIS host, by THIS image's pi version, in THIS job's venue. A transcript from another pi
1557
+ // may hold tool-call arguments the current schema refuses (the run.resume rule), and a checkpoint from another
1558
+ // venue was quiesced under another runtime's pause.
1559
+ const read = await workspaceCheckpointer.readManifest(runId);
1560
+ // Issue #761: a checkpoint a newer pi-dispatch wrote is not an absent one. Refused here, the run's final outcome
1561
+ // would remove its store and its pointer (`endDurableRun`) and a host upgraded again could never continue it; so
1562
+ // the recovery WAITS (`newer-release`, index.mjs) for a worker that reads the checkpoint, without spending its one
1563
+ // attempt (the never-started conversion in `runJobInner` takes only `container-never-started`, which this is
1564
+ // not), with no comment and no record, and with the store, the pointer and the job directory left in place. What
1565
+ // bounds the wait is the host-lost sweep's hard maximum (recovery-queue.mjs, `RECOVERY_MAX_WAIT_MS`).
1566
+ if (read?.newer === true) throw new NewerReleaseHold("the run's checkpoint");
1567
+ const m = read?.ok === true ? read.manifest : null;
1568
+ const venue = resolveBackendName(job, blessedBackends[0]);
1569
+ const why = m === null ? (read?.absent ? "absent" : "invalid") : typeof hostName === "string" && hostName !== "" && m.host !== normalizeHost(hostName) ? "host" : piVersion === null || m.piVersion !== piVersion ? "pi-version" : m.venue !== venue ? "venue" : (m.provider !== job.provider || m.model !== job.model) ? "model" : null;
1570
+ if (why !== null) {
1571
+ await comment(job, RECOVERY_COMMENTS["recovery-checkpoint-absent"]);
1572
+ // `why` is a fixed token, and `invalid`'s detail is the store's own fixed reason (`field:seq`, `transcript-mismatch`).
1573
+ log("refused_recovery_checkpoint_absent", { n, why, ...(why === "invalid" && typeof read?.reason === "string" ? { detail: read.reason.slice(0, 40) } : {}) });
1574
+ return { outcome: "policy", reason: "recovery-checkpoint-absent", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1575
+ }
1576
+ manifest = m;
1577
+ transcriptPath = read.transcriptPath;
1578
+ }
1579
+
1010
1580
  // REQ-PER-TRIGGER-SKILLS. A trigger that named a skills directory the worker cannot see would run
1011
1581
  // its flow WITHOUT the skills it was written against, produce a plausible report, and exit 0. Free
1012
1582
  // and determinate -- one lstat, no credential needed to know the answer -- so it belongs among the
@@ -1016,7 +1586,7 @@ export async function runJob(job, deps) {
1016
1586
  // image blocks every job on this host, an unset sessions dir blocks every armed trigger, a bad
1017
1587
  // skillsDir blocks one trigger.
1018
1588
  if (job.skillsDir && !isReadableDir(job.skillsDir)) {
1019
- await comment(job, "Refused: this trigger set `run.skillsDir`, and that path is absent or is not a directory on the worker host. The job would have run without the skills the flow was written against. Not run.");
1589
+ await comment(job, REFUSAL_COMMENTS["skills-dir-missing"]);
1020
1590
  // The FIELD name, never its value. `comment` posts publicly on the issue, so a host path here
1021
1591
  // would publish the operator's filesystem layout to anyone reading the thread; the log line is
1022
1592
  // the same restraint refused_sessions_dir_unset keeps.
@@ -1024,6 +1594,22 @@ export async function runJob(job, deps) {
1024
1594
  return { outcome: "policy", reason: "skills-dir-missing", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1025
1595
  }
1026
1596
 
1597
+ // Issue #796: a canary's refusals, free and determinate like the one above, so before the mint and everything after it. The registry read is free too; one that cannot
1598
+ // be asked is infrastructure and retries (CONST-RETRY-INFRA-ONLY), since the same job can succeed. The ladder's narrowest gate, so it asks the network last.
1599
+ let canaryPlan = null;
1600
+ if (job.packageCanary !== undefined) {
1601
+ if (!deps.packageCanary) return { outcome: "policy", reason: "package-canary-invalid", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
1602
+ try {
1603
+ canaryPlan = await deps.packageCanary.check(job);
1604
+ } catch {
1605
+ throw new InfraRetry("the registry could not be asked about the canary's version", { reason: "container-never-started", budgetReserved: false, provider: job.provider ?? null, model: job.model ?? null });
1606
+ }
1607
+ if (canaryPlan.refused) {
1608
+ log("refused_package_canary", { reason: canaryPlan.refused });
1609
+ return { outcome: "policy", reason: canaryPlan.refused, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
1610
+ }
1611
+ }
1612
+
1027
1613
  // REQ-MCP-OPERATOR-SERVERS. A trigger that named MCP servers its job cannot use would run its flow without them,
1028
1614
  // or hand a server the job's own credentials, or open a connection the egress proxy refuses at its first call:
1029
1615
  // each a plausible report and a clean exit. One read of the overlay's mcp.json, no credential and no network, so
@@ -1038,11 +1624,84 @@ export async function runJob(job, deps) {
1038
1624
  // The server NAME is the operator's own configuration, the same class as the image ref on the image refusal.
1039
1625
  // `why` can name an internal host or a variable (never a value, a url's path or its query), so it goes to the
1040
1626
  // worker's log and the public comment says only which rule refused.
1041
- await comment(job, `Refused: this trigger names MCP server${mcpChecked.name ? ` "${mcpChecked.name}"` : "s"} in \`run.mcp\`, and ${MCP_REFUSAL_COMMENT[mcpChecked.refused] ?? "it cannot be used"}. The job would have run without the servers its flow was written against; the worker's log says why (\`refused_mcp\`). Not run.`);
1627
+ await comment(job, mcpRefusalComment(mcpChecked.refused, mcpChecked.name));
1042
1628
  log("refused_mcp", { kind: job.kind ?? null, reason: mcpChecked.refused, server: mcpChecked.name ?? null, why: mcpChecked.why });
1043
1629
  return { outcome: "policy", reason: mcpChecked.refused, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1044
1630
  }
1045
1631
 
1632
+ // PROJECT SECRET BUNDLES, the free half (issue #704, REQ-TRIGGER-SECRETS, DES-PROJECT-SECRET-BUNDLES). WHICH jobs
1633
+ // get their project's bundle is decided here, before the mint, the clone and every reserve, from one cached read
1634
+ // of the triggers file and the host's declarations: no spawn, no network (CONST-BUDGET-BEFORE-TOKENS). The read
1635
+ // itself comes after the run.secrets resolver and, like it, before the mint, except for a pull request that needs
1636
+ // the fork check: that one needs the forge and the job's token, so its read follows the mint (below).
1637
+ //
1638
+ // TRIGGER-MADE JOBS ONLY, and the predicate is the portfolio gate's: a chained `/outbox` child and every manual
1639
+ // run (`pi-dispatch run`, `/dispatch run`, `dispatch_run`) hold none, each recorded as `bundleSkipped`. The
1640
+ // operator's grant was to the triggers they reviewed, not to what a job queues next or what a model asks for
1641
+ // (outbox.mjs keeps `run.secrets` off children for the same reason). Chained is asked first: a child may carry
1642
+ // its parent's trigger context, and it must never read as manual. A trigger's `run.secretBundle: false` is
1643
+ // `opted-out`, read from the live file rather than the job data (triggers-file.mjs says why).
1644
+ //
1645
+ // Every refusal RETURNS (CONST-RETRY-INFRA-ONLY): an undeclared or doubly declared profile, a resumed job
1646
+ // (`run.resume` would replay a transcript that may hold an echoed value into every later job on its key, the
1647
+ // rule that refuses `run.secrets` beside it at load, which cannot be applied at load here because the file
1648
+ // cannot know a job's project), and a trigger the file no longer holds. A file that has never been read and
1649
+ // validated in this process RETRIES a job with a project (the finder answers from its last good read otherwise),
1650
+ // and a profile this host lacks is DEFERRED on a multi-host worker's shared queue, the never-fits gate's rule,
1651
+ // since another host may serve it. A venue's refusal of the bundle itself comes with the read (`readBundle`).
1652
+ const bundleChained = job.parentJobId !== undefined || job.chainDepth !== undefined;
1653
+ const bundleManual = job.trigger === undefined || job.trigger === null;
1654
+ let bundleBinding = null;
1655
+ const bundleFound = await Promise.resolve()
1656
+ .then(() => findSecretBundle(job, { project: pickupProject }))
1657
+ .catch(() => ({ unreadable: "read-error", transient: true }));
1658
+ if (bundleFound?.unreadable !== undefined) {
1659
+ // Never a good read in this process, and the job has a project: which bundle it would hold is unknown. RETRIED,
1660
+ // never refused (a refused webhook delivery is gone, and the fix is the operator's next save) and never run
1661
+ // without (the silent no-op). A chained or manual job could hold none whatever the file says, so it runs.
1662
+ if (!bundleChained && !bundleManual) {
1663
+ // The errno or our own word (`invalid`), never a path or file text.
1664
+ log("secret_bundles_unreadable", { kind: job.kind ?? null, why: String(bundleFound.unreadable).slice(0, 40) });
1665
+ throw new InfraRetry("the triggers file's secretBundles could not be read, and has not been in this process", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
1666
+ }
1667
+ } else if (bundleFound && bundleFound.none !== true) {
1668
+ if (bundleChained) bundleFacts.bundleSkipped = "chained";
1669
+ else if (bundleManual) bundleFacts.bundleSkipped = "manual";
1670
+ else if (bundleFound.triggerChanged === true) {
1671
+ await comment(job, BUNDLE_COMMENTS["secret-bundle-trigger-changed"]);
1672
+ log("refused_secret_bundle_trigger_changed", { kind: job.kind ?? null, bundle: bundleFound.id });
1673
+ return { outcome: "policy", reason: "secret-bundle-trigger-changed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1674
+ } else if (bundleFound.optedOut === true) bundleFacts.bundleSkipped = "opted-out";
1675
+ else {
1676
+ if (job.resume === true) {
1677
+ await comment(job, BUNDLE_COMMENTS["secret-bundle-resume"]);
1678
+ log("refused_secret_bundle_resume", { kind: job.kind ?? null, bundle: bundleFound.id });
1679
+ return { outcome: "policy", reason: "secret-bundle-resume", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1680
+ }
1681
+ const looked = secretBundles.lookup(bundleFound.bundle.profile);
1682
+ if (looked?.ok !== true && bundleProfileDeferrable === true) {
1683
+ // A multi-host worker's SHARED queue: another host may declare the profile, and routing does not steer a
1684
+ // bundled job there (DES-PROJECT-SECRET-BUNDLES says why). So this host does not refuse a job it merely
1685
+ // cannot serve: it hands it back, the never-fits gate's deferral, without spending an attempt.
1686
+ log("secret_bundle_profile_deferred", { kind: job.kind ?? null, bundle: bundleFound.id, profile: bundleFound.bundle.profile, ambiguous: looked?.ambiguous !== undefined });
1687
+ throw new SecretBundleProfileHold("this host cannot serve the job's secret bundle profile; another host may", { reason: looked?.ambiguous !== undefined ? "secret-bundle-profile-ambiguous" : "secret-bundle-profile-unknown", provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false });
1688
+ }
1689
+ if (looked?.ok !== true) {
1690
+ // The profile LABEL in the log only, and the bundle id: both operator config. The comment names neither,
1691
+ // the run.secrets refusals' restraint (the declared names would publish the operator's vault topology).
1692
+ const reason = looked?.ambiguous !== undefined ? "secret-bundle-profile-ambiguous" : "secret-bundle-profile-unknown";
1693
+ await comment(job, BUNDLE_COMMENTS[reason]);
1694
+ log(reason === "secret-bundle-profile-ambiguous" ? "refused_secret_bundle_profile_ambiguous" : "refused_secret_bundle_profile_unknown", { kind: job.kind ?? null, bundle: bundleFound.id, profile: bundleFound.bundle.profile });
1695
+ return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1696
+ }
1697
+ // Issue #704 part 3: no venue refusal here any more. Part 2 refused a bundled job on both OpenShell venues at
1698
+ // this point, which was AHEAD of the fork check, so a fork's job that would have run without the bundle was
1699
+ // refused for it. The venue's own judgement of the bundle (`bundleGate`) now runs inside the read below,
1700
+ // which a fork's job never reaches.
1701
+ bundleBinding = { id: bundleFound.id, ...bundleFound.bundle };
1702
+ }
1703
+ }
1704
+
1046
1705
  // REQ-TRIGGER-SECRETS. A trigger that named secret references the worker cannot resolve would run its
1047
1706
  // flow with those variables UNSET, get a 401 from whatever it was meant to reach, write a plausible
1048
1707
  // report about why the integration is down, and exit 0.
@@ -1070,10 +1729,11 @@ export async function runJob(job, deps) {
1070
1729
  // carry `secrets` the current set would refuse, and buildContainerEnv would write every one of them.
1071
1730
  // The same set the loader uses, imported, so the two cannot drift, and checked before the resolver so
1072
1731
  // no wiring of it can skip the check.
1073
- const loadReserved = secretsArmed(job) ? Object.keys(job.secrets ?? {}).find((name) => RESERVED_ENV_NAMES.has(name)) : undefined;
1074
- const resolved = loadReserved !== undefined ? { reserved: loadReserved, atLoad: true } : secretsArmed(job) ? await resolveSecrets(job) : { ok: true, secrets: {} };
1732
+ const secondaryCredentialNames = new Set(job.answerDraft && job.answerDecisionModel ? providerKeyCandidates(job.answerDecisionModel.provider) : job.questionGrouping && job.questionGroupModel ? providerKeyCandidates(job.questionGroupModel.provider) : job.watch && job.watchDecisionModel ? providerKeyCandidates(job.watchDecisionModel.provider) : []);
1733
+ const loadReserved = secretsArmed(job) ? Object.keys(job.secrets ?? {}).find((name) => RESERVED_ENV_NAMES.has(name) || secondaryCredentialNames.has(name)) : undefined;
1734
+ const resolved = loadReserved !== undefined ? { reserved: loadReserved, atLoad: RESERVED_ENV_NAMES.has(loadReserved) } : secretsArmed(job) ? await resolveSecrets(job) : { ok: true, secrets: {} };
1075
1735
  if (resolved.profileUnknown) {
1076
- await comment(job, "Refused: this trigger set `run.secrets`, and the resolver profile it names is not usable on this worker host. No profile of that name is declared, or its resolver is absent or not executable. The job would have started with those variables unset, and an agent that gets a 401 writes a plausible report and exits 0. Run `pi-dispatch doctor` on the worker to see which profiles it has. Not run.");
1736
+ await comment(job, REFUSAL_COMMENTS["secret-profile-unknown"]);
1077
1737
  // The operator's own profile LABEL, and never a path, a reference, or a byte the resolver printed.
1078
1738
  // `comment` posts publicly on the issue: a resolver path there publishes the operator's filesystem
1079
1739
  // layout, and the DECLARED profile names would publish their vault topology -- which is why the
@@ -1090,7 +1750,7 @@ export async function runJob(job, deps) {
1090
1750
  // anywhere -- redirect a profile the operator wrote in .env. This project already refuses ambiguity
1091
1751
  // rather than resolving it (PI_EGRESS: "a typo must never leave you believing you have a policy you
1092
1752
  // do not"), and an operator who sees this fixes it in seconds.
1093
- await comment(job, "Refused: this trigger set `run.secrets`, and the resolver profile it names is declared twice on this worker host, once in the environment and once in the settings overlay. Neither wins, on purpose: the job would otherwise run against whichever one happened to be picked. Remove one of the two. Not run.");
1753
+ await comment(job, REFUSAL_COMMENTS["secret-profile-ambiguous"]);
1094
1754
  log("refused_secret_profile_ambiguous", { kind: job.kind ?? null, profile: resolved.ambiguous });
1095
1755
  return { outcome: "policy", reason: "secret-profile-ambiguous", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1096
1756
  }
@@ -1147,7 +1807,136 @@ export async function runJob(job, deps) {
1147
1807
  log("secret_resolver_unreachable", { kind: job.kind ?? null, name: resolved.unreachable, failure: resolved.failure ?? null, code: resolved.code ?? null, stderrBytes: resolved.stderrBytes ?? 0 });
1148
1808
  throw new InfraRetry(`secret resolver could not answer for ${resolved.unreachable}`, { reason: "secret-resolver-unreachable", provider: job.provider ?? null, model: job.model ?? null });
1149
1809
  }
1150
- const secrets = resolved.secrets ?? {};
1810
+ // `let`: a project bundle's variables join it after the mint (below), under the trigger's own values.
1811
+ let secrets = resolved.secrets ?? {};
1812
+ // Issue #704 part 3: the bundle's `{ endpoints, plain }` split, for the venue (null: no bundle reached this job).
1813
+ let bundleKeys = null;
1814
+
1815
+ // Issue #704 part 3: the venue's answer, the same handling for both of its asks below. A refusal RETURNS with the
1816
+ // venue's fixed sentence (its log names the variable and host, never the comment); a source it could not read
1817
+ // RETRIES, as the observation preflight's does.
1818
+ // The hosts this job's OWN credentials are sent to beyond the forge's (issue #704 part 3, review round 1, H2), which a
1819
+ // bundle endpoint may not claim on a venue that binds one provider per host: the composed `baseUrl` of every model
1820
+ // the job may call (the overlay's models.json over pi's catalog, `outputCapView`, the composition the output cap and
1821
+ // the zero-rated check read), every declared model endpoint's host (keyed or not: the simple rule), and every
1822
+ // HTTP MCP server's host (its headers may carry a run.secrets value). A model whose baseUrl cannot be composed adds
1823
+ // nothing, and is said so in DES-OPENSHELL-VENUE.
1824
+ const credentialHosts = () => {
1825
+ const hosts = new Set();
1826
+ const add = (url) => {
1827
+ try {
1828
+ if (typeof url === "string" && url !== "") hosts.add(new URL(url).hostname.toLowerCase());
1829
+ } catch {}
1830
+ };
1831
+ for (const ref of callableModelRefs(job)) {
1832
+ if (ref === null) continue;
1833
+ try {
1834
+ add(outputCapView({ models: modelEndpoints?.models ?? null, provider: ref.provider, modelId: ref.id, builtinModel })?.baseUrl);
1835
+ } catch {}
1836
+ }
1837
+ for (const ep of modelEndpoints?.endpoints ?? []) if (typeof ep?.host === "string") hosts.add(ep.host.toLowerCase());
1838
+ for (const server of Object.values(mcpChecked?.document?.servers ?? {})) add(server?.url);
1839
+ return [...hosts].sort();
1840
+ };
1841
+ const askBundleGate = async (ask) => {
1842
+ const judged = await bundleGate(job, { ...ask, credentialHosts: credentialHosts(), capabilities: img.capabilities ?? [] });
1843
+ if (judged?.venueRefused && typeof judged.venueRefused.reason === "string") {
1844
+ const { reason, comment: text } = judged.venueRefused;
1845
+ await comment(job, typeof text === "string" && text !== "" ? text : VENUE_BUNDLE_REFUSED_COMMENT);
1846
+ log("refused_venue", { reason, bundle: bundleBinding.id });
1847
+ return { refused: { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false } }; // return => not retried
1848
+ }
1849
+ if (judged?.unavailable) throw new InfraRetry("the job's venue could not judge its secret bundle", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
1850
+ return { judged };
1851
+ };
1852
+ // PROJECT SECRET BUNDLES, the read (issue #704). ONE resolver call, then the strict parse, the reserved names and
1853
+ // the merge. BEFORE the mint for every job that needs no fork check (CONST-BUDGET-BEFORE-TOKENS: its determinate
1854
+ // refusals spend nothing, not even a token), and right after the mint for a pull request whose head must be asked
1855
+ // of the forge first; either way before branch protection, the clone, the token-cap read and every reserve.
1856
+ // Returns null, or the result to return.
1857
+ const readBundle = async () => {
1858
+ if (bundleBinding === null) return null;
1859
+ // Issue #704 part 3: the hosts first, before the resolver runs, so a file naming a host this job's venue cannot
1860
+ // bind costs no read of the operator's manager. Here rather than among the free gates above (where part 2
1861
+ // refused every OpenShell bundle), because only here is the fork check behind it: a fork's job runs without
1862
+ // the bundle, so nothing about the bundle may refuse it.
1863
+ const endpoints = bundleBinding.endpoints ?? {};
1864
+ if (Object.keys(endpoints).length > 0) {
1865
+ const pre = await askBundleGate({ endpoints, vars: null });
1866
+ if (pre.refused) return pre.refused;
1867
+ }
1868
+ const bundleRead = await secretBundles.resolve(job, bundleBinding);
1869
+ if (bundleRead?.ok !== true) {
1870
+ if (bundleRead?.unreachable) {
1871
+ // INDETERMINATE (exit 1, an unknown exit, a spawn fault, a timeout): retried, the run.secrets rule. A
1872
+ // rate-limited manager read lands here, which is why its documented resolvers exit 1 for one.
1873
+ log("secret_bundle_resolver_unreachable", { kind: job.kind ?? null, bundle: bundleBinding.id, failure: bundleRead.failure ?? null, code: bundleRead.code ?? null, stderrBytes: bundleRead.stderrBytes ?? 0 });
1874
+ throw new InfraRetry(`the secret bundle's resolver could not answer for bundle ${bundleBinding.id}`, { reason: "secret-resolver-unreachable", provider: job.provider ?? null, model: job.model ?? null });
1875
+ }
1876
+ if (bundleRead?.reserved !== undefined) {
1877
+ // The variable NAME, in the log, so the operator can rename it in their manager. Not in the public
1878
+ // comment: a bundle's names are the contents of the operator's manager, not of the reviewed file.
1879
+ await comment(job, BUNDLE_COMMENTS["secret-name-reserved"]);
1880
+ log("refused_secret_bundle_name_reserved", { kind: job.kind ?? null, bundle: bundleBinding.id, name: bundleRead.reserved });
1881
+ return { outcome: "policy", reason: "secret-name-reserved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1882
+ }
1883
+ if (bundleRead?.profileUnknown !== undefined || bundleRead?.ambiguous !== undefined) {
1884
+ // Only if the declarations changed between the lookup above and this read: refused as there.
1885
+ const reason = bundleRead.ambiguous !== undefined ? "secret-bundle-profile-ambiguous" : "secret-bundle-profile-unknown";
1886
+ await comment(job, BUNDLE_COMMENTS[reason]);
1887
+ log(reason === "secret-bundle-profile-ambiguous" ? "refused_secret_bundle_profile_ambiguous" : "refused_secret_bundle_profile_unknown", { kind: job.kind ?? null, bundle: bundleBinding.id, profile: bundleBinding.profile });
1888
+ return { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1889
+ }
1890
+ // DETERMINATE: exit 2, an empty, oversized or malformed output, a bad name or value. `name` and `line`
1891
+ // ride the log only, for the reason above; the comment says which rule, in our words.
1892
+ const why = BUNDLE_FAILURES[bundleRead?.failure] ?? "did not return a usable environment";
1893
+ await comment(job, `Refused: this job's project has a secret bundle, and its resolver ${why}. The job would have started without the project's environment, and an agent that gets a 401 writes a plausible report and exits 0. Run the bundle's resolver by hand against its reference to see why: this comment carries neither the reference, nor the resolver's path, nor a byte of what it printed. Not run.`);
1894
+ log("refused_secret_bundle_unresolved", { kind: job.kind ?? null, bundle: bundleBinding.id, failure: bundleRead?.failure ?? null, name: bundleRead?.name ?? null, line: bundleRead?.line ?? null, code: bundleRead?.code ?? null, stderrBytes: bundleRead?.stderrBytes ?? 0 });
1895
+ return { outcome: "policy", reason: "secret-unresolved", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1896
+ }
1897
+ // THE MERGE: the trigger's own run.secrets win a clash (secret-bundles.mjs `mergeBundleSecrets`), and the
1898
+ // shadowed names are recorded. The values join `secrets`, so every path that already treats a run.secrets
1899
+ // value as a secret treats these alike: buildContainerEnv injects them, the redaction covers every
1900
+ // non-NON_SECRET_ENV_KEYS name (run-container.mjs), and a checkpoint refuses to store any of them.
1901
+ const merged = mergeBundleSecrets(bundleRead.vars, secrets);
1902
+ // Issue #704 part 3: which received variables the bundle binds to a host (`endpoints`) and which go in plain; a
1903
+ // variable a run.secrets value shadowed is in neither unless `endpoints` names it (below). Said to every
1904
+ // venue (`bundleKeys` on the call bag); only a venue whose layer holds credentials reads it, and it judges the
1905
+ // split first, now with the values (`bundleGate`): a value its gateway would refuse is a refusal here, before
1906
+ // spend, never a create that fails as infrastructure and is retried into the same answer.
1907
+ const received = Object.fromEntries(Object.entries(bundleRead.vars).filter(([name]) => !merged.shadowed.includes(name)));
1908
+ const post = await askBundleGate({ endpoints, vars: received, shadowed: merged.shadowed });
1909
+ if (post.refused) return post.refused;
1910
+ // Review round 1, M2: a declared endpoint name a run.secrets value shadowed stays on that host's provider, now with
1911
+ // the trigger's value (the one that wins), so it resolves at the host the reviewed file binds the name to.
1912
+ bundleKeys = {
1913
+ endpoints: Object.fromEntries([...Object.keys(received), ...merged.shadowed].filter((name) => Object.hasOwn(endpoints, name)).map((name) => [name, endpoints[name]])),
1914
+ plain: Object.keys(received).filter((name) => !Object.hasOwn(endpoints, name)).sort(),
1915
+ };
1916
+ secrets = merged.secrets;
1917
+ bundleFacts.bundle = { id: bundleBinding.id, variables: Object.keys(bundleRead.vars).sort(), shadowed: merged.shadowed };
1918
+ // On a venue whose own layer holds the credentials, the record says how many reached the agent as placeholders
1919
+ // and how many as values (INT-RUN-HISTORY-FILE-CONTRACT), the venue's own counts.
1920
+ if (backendFor(resolveBackendName(job, blessedBackends[0]))?.credentialCustody === "layer" && Number.isSafeInteger(post.judged?.placeholders) && Number.isSafeInteger(post.judged?.plain)) {
1921
+ bundleFacts.bundle.placeholders = post.judged.placeholders;
1922
+ bundleFacts.bundle.plain = post.judged.plain;
1923
+ bundleFacts.bundle.placeholderNames = post.judged.placeholderNames;
1924
+ }
1925
+ log("secret_bundle_applied", { kind: job.kind ?? null, bundle: bundleBinding.id, count: bundleFacts.bundle.variables.length, shadowed: merged.shadowed.length });
1926
+ return null;
1927
+ };
1928
+ const bundleHeadCheck = bundleBinding !== null && job.target?.type === "pull_request" && bundleBinding.forkReviews !== true;
1929
+ if (bundleBinding !== null && !bundleHeadCheck) {
1930
+ const refused = await readBundle();
1931
+ if (refused !== null) return refused;
1932
+ }
1933
+
1934
+
1935
+ const secondaryCollision = Object.keys(secrets ?? {}).find(name => secondaryCredentialNames.has(name));
1936
+ if (secondaryCollision !== undefined) {
1937
+ log("refused_watch_provider_credential_override", { name: secondaryCollision });
1938
+ return { outcome: "policy", reason: "secret-name-reserved", budgetReserved: false, provider: job.provider ?? null, model: job.model ?? null };
1939
+ }
1151
1940
 
1152
1941
  if (wantsForgeToken) {
1153
1942
  token = await mintToken(job);
@@ -1161,12 +1950,59 @@ export async function runJob(job, deps) {
1161
1950
  }
1162
1951
  }
1163
1952
 
1953
+ // PROJECT SECRET BUNDLES, the fork check (issue #704). AFTER the mint, because it needs the job's own token, and
1954
+ // BEFORE branch protection, the clone, the token-cap read and every reserve, so a refusal of the read that follows
1955
+ // still spends nothing but the minted token (CONST-BUDGET-BEFORE-TOKENS: every free refusal stays above the mint,
1956
+ // and so does the read of every job that needs no fork check). A refusal past the mint leaves that token to
1957
+ // expire, exactly as the unprotected-branch refusal below does: it is scoped to this job's repository and
1958
+ // short-lived (CONST-TOKEN-SCOPED-PER-JOB), and no path in this project revokes one, so this adds no second rule.
1959
+ //
1960
+ // A job on a pull request whose head lives in another repository reads a stranger's text (CONST-TRIGGER-AUTHOR-GATE
1961
+ // admits a collaborator's review of a fork), and must not hold every project secret while it does, so it runs
1962
+ // WITHOUT the bundle (`bundleSkipped: "fork"`) unless the bundle opted in with `forkReviews: true`. Answered by the
1963
+ // forge's own `resolvePullRequestHead`, the seam run.resume's fork gate already uses: the head repository must
1964
+ // EQUAL the job's, case-insensitively on GitHub and Forgejo, whose repository names are (GitLab and Azure answer by
1965
+ // project or repository id and report the job's own name back, so their comparison is exact). A webhook's
1966
+ // `head.repo` is attacker-supplied, and an `issue_comment` on a pull request carries no head at all, so the
1967
+ // payload cannot answer. A DETERMINATE forge answer (401, 403, 404: the token cannot see the pull request, or it
1968
+ // is gone) cannot become a yes by retrying, so the job runs without the bundle (`bundleSkipped:
1969
+ // "head-unverified"`); a network fault, a 5xx or a timeout RETRIES (InfraRetry), so a blip neither grants a bundle
1970
+ // nor drops one.
1971
+ if (bundleHeadCheck) {
1972
+ let head = null;
1973
+ let unverified = false;
1974
+ if (typeof resolvePullRequestHead === "function") {
1975
+ try {
1976
+ head = (await resolvePullRequestHead(job, token)) ?? {};
1977
+ } catch (error) {
1978
+ const status = headReadStatus(error);
1979
+ // A GitHub rate limit answers 403 (or 429) with `x-ratelimit-remaining: 0` or a `retry-after`: not a
1980
+ // determinate "you cannot see this pull request", so it RETRIES like any other transient answer.
1981
+ const rateLimited = headReadRateLimited(error);
1982
+ log("secret_bundle_head_unreadable", { kind: job.kind ?? null, status, rateLimited });
1983
+ if (rateLimited || (status !== 401 && status !== 403 && status !== 404)) throw new InfraRetry("the pull request's head could not be read to decide whether it is a fork", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
1984
+ unverified = true;
1985
+ }
1986
+ }
1987
+ const fold = job.kind === "github" || job.kind === "forgejo" ? (v) => v.toLowerCase() : (v) => v;
1988
+ const same = !unverified && typeof head?.headRepo === "string" && typeof job.repo === "string" && fold(head.headRepo) === fold(job.repo);
1989
+ if (!same) {
1990
+ bundleFacts.bundleSkipped = unverified ? "head-unverified" : "fork";
1991
+ log("secret_bundle_skipped", { kind: job.kind ?? null, bundle: bundleBinding.id, reason: bundleFacts.bundleSkipped });
1992
+ bundleBinding = null;
1993
+ }
1994
+ }
1995
+ if (bundleHeadCheck) {
1996
+ const refused = await readBundle();
1997
+ if (refused !== null) return refused;
1998
+ }
1999
+
1164
2000
  if (isForgeBacked) {
1165
2001
  // REQ-BRANCH-PROTECTION-PRECONDITION. The agent's token can merge, so branch protection is the
1166
2002
  // only technical barrier to a self-merge. Refuse before spending anything. Forge-backed jobs
1167
2003
  // only: a local job has no remote branch to protect.
1168
2004
  if (!(await isDefaultBranchProtected(job, token))) {
1169
- await comment(job, "Refused: the default branch is not protected. See SECURITY.md.");
2005
+ await comment(job, REFUSAL_COMMENTS["unprotected-branch"]);
1170
2006
  log("refused_unprotected", { repo: job.repo });
1171
2007
  // exitCode/turns/tokens null: refused pre-container, so no container exit, turn, or token count exists.
1172
2008
  return { outcome: "policy", reason: "unprotected-branch", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
@@ -1176,7 +2012,10 @@ export async function runJob(job, deps) {
1176
2012
  // `podmanStore` (issue #429) only where the venue's job user carried one: the podman store the container ran in.
1177
2013
  // `portfolio` (issue #505) only for a job the gate above confirmed: prepare then writes /job/portfolio.json, after
1178
2014
  // asking the live file once more. Absent otherwise, so every other job's prepare call is unchanged.
1179
- prepared = await prepareWorkspace(job, token, { piVersion, jobUser: { user: jobUser?.user ?? null, home: jobUser?.home ?? null }, ...(typeof jobUser?.store === "string" ? { podmanStore: jobUser.store } : {}), ...(portfolio ? { portfolio: true } : {}), ...(jobSize ? { size: jobSize } : {}), ...(mcpChecked.document ? { mcp: mcpChecked.document } : {}) }); // resolves SHA, clones, materialises .pi/, writes prompt
2015
+ // The existing read-only event file carries accepted human text. Environment/exec metadata stays id-only,
2016
+ // including on gateways that log their exec argv. Keep the queued command for the operator's forge trail.
2017
+ const stagedJob = job.approvalDecision ? { ...job, trigger: { ...job.trigger, comment: { ...job.trigger?.comment, body: job.approvalDecision.text ?? "" } } } : job;
2018
+ prepared = await prepareWorkspace(stagedJob, token, { piVersion, jobUser: { user: jobUser?.user ?? null, home: jobUser?.home ?? null }, ...(typeof jobUser?.store === "string" ? { podmanStore: jobUser.store } : {}), ...(portfolio ? { portfolio: true } : {}), ...(jobSize ? { size: jobSize } : {}), ...(mcpChecked.document ? { mcp: mcpChecked.document } : {}), ...(job.watch ? { watchSecrets: secrets } : {}), ...(isRecovery ? { pin: { sha: manifest.workspace.baseSha, from: workspaceCheckpointer.baseSourceUrl(runId) } } : forkSource ? { pin: { sha: forkSource.manifest.baseSha, from: forkStore.baseSourceUrl(job.fork.fromJob) } } : {}) }); // resolves SHA, clones, materialises .pi/, writes prompt
1180
2019
 
1181
2020
  // A determinate prepare refusal -- sha-gone (the default branch advanced past the resolved tip),
1182
2021
  // or a `pi-*` materialiser cap breach (the repo's .pi/ is too large to place in /job, issue #60)
@@ -1189,6 +2028,27 @@ export async function runJob(job, deps) {
1189
2028
  if (prepared?.outcome === "policy") {
1190
2029
  return { ...prepared, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
1191
2030
  }
2031
+ // Issue #796: the candidate goes under this job's own directory, before any reservation: a version that cannot be installed spends nothing. After the prepare, because that
2032
+ // is what makes the directory, and on a forge job that is after the mint. The `finally` below removes the directory whichever way this goes.
2033
+ if (canaryPlan) {
2034
+ let installed;
2035
+ try {
2036
+ installed = await deps.packageCanary.install(job, prepared.jobDir, canaryPlan);
2037
+ } catch {
2038
+ throw new InfraRetry("the canary's candidate could not be installed", { reason: "container-never-started", budgetReserved: false, provider: job.provider ?? null, model: job.model ?? null });
2039
+ }
2040
+ if (installed.refused) {
2041
+ log("refused_package_canary", { reason: installed.refused });
2042
+ return { outcome: "policy", reason: installed.refused, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
2043
+ }
2044
+ prepared = { ...prepared, packageCanary: installed };
2045
+ }
2046
+
2047
+
2048
+ if (deps.approvalRequest && (prepared.session?.key !== deps.approvalRequest.sessionKey || prepared.session?.resume !== true)) {
2049
+ await comment(job, APPROVAL_COMMENTS["approval-session-unavailable"]);
2050
+ return { outcome: "policy", reason: "approval-session-unavailable", exitCode: null, turns: null, tokens: null, budgetReserved: false };
2051
+ }
1192
2052
 
1193
2053
  // THE RESOLVED FOLDER'S PROJECT (issue #504 part B). The pickup decided the project, and so the share a job reserves
1194
2054
  // against, from the folder AS NAMED; the container mounts the folder prepare RESOLVED. A link or a case variant
@@ -1196,15 +2056,115 @@ export async function runJob(job, deps) {
1196
2056
  // Refused here, after prepare and before the token-cap read and every reserve: free, determinate, never retried. A
1197
2057
  // resolved folder in no project is not refused, because the named spelling is the operator's own membership
1198
2058
  // (a symlinked member is listed by the path the triggers use, docs/projects.md).
1199
- if (job.kind === "local" && typeof folderProject === "function" && typeof prepared?.workspace === "string") {
2059
+ if (job.kind === "local" && !job.questionGrouping && typeof folderProject === "function" && typeof prepared?.workspace === "string") {
1200
2060
  const resolvedProject = folderProject(prepared.workspace);
1201
2061
  if (resolvedProject !== null && resolvedProject !== pickupProject) {
1202
- await comment(job, "Refused: this job's folder now leads into another project's folder, so no container was started and nothing was spent. Not run.");
2062
+ await comment(job, REFUSAL_COMMENTS["local-folder-project-changed"]);
1203
2063
  log("refused_local_folder_project_changed", { pickup: pickupProject, resolved: resolvedProject });
1204
2064
  return { outcome: "policy", reason: LOCAL_FOLDER_PROJECT_CHANGED, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1205
2065
  }
1206
2066
  }
1207
2067
 
2068
+ // The archive's base is copied before agent-written git state exists. Never mount the archive itself.
2069
+ if (forkStore && sourceRunId && (job.resume === true || job.fork !== undefined) && sessionKeyFor(job) && prepared.sha && !isRecovery) {
2070
+ let primed;
2071
+ try { primed = await forkStore.primeBase({ runId: sourceRunId, workspace: prepared.workspace, baseSha: prepared.sha }); } catch { primed = { refused: "fork-source-unavailable" }; }
2072
+ sourcePrimed = primed?.ok === true;
2073
+ archiveFacts.value = forkArchiveFact({ reason: sourcePrimed ? "not-completed" : primed?.refused ?? primed?.skipped });
2074
+ if (!sourcePrimed) log("fork_source_skipped", { reason: primed?.refused ?? primed?.skipped ?? "base-unavailable" });
2075
+ }
2076
+ // Issue #645: A BASE WITH `.pi/extensions/` IS NOT RECOVERABLE. A merge-gated extension may import a file outside
2077
+ // `.pi/` (`../lib/x.js`) or a bare package resolved from the workspace's `node_modules`, and in a recovered run
2078
+ // those come from the restored tree, which the agent wrote. The trust reset puts `.pi/` back but cannot enumerate
2079
+ // what an extension imports, so the only sound answer is not to recover such a run at all. Decided here, at arm
2080
+ // time, from the fresh clone's own `.git` (the worker's, before any container), so the run is simply not armed:
2081
+ // no ledger claim, no checkpoints, no pointer, and it runs exactly as it did before #645, the record saying
2082
+ // `recoverSkipped: "workspace-extensions"`. A base that could not be read is not armed either, fail closed, and
2083
+ // says `base-unprimed`. The recovery side asks again before it restores (defence in depth).
2084
+ if (armed && !isRecovery) {
2085
+ let unsafe;
2086
+ try {
2087
+ unsafe = await baseHasExtensions({ workspace: prepared.workspace, baseSha: prepared.sha });
2088
+ } catch {
2089
+ unsafe = null;
2090
+ }
2091
+ if (unsafe !== false) {
2092
+ armed = false;
2093
+ durable.armed = false;
2094
+ durable.recoverSkipped = unsafe === true ? "workspace-extensions" : "base-unprimed";
2095
+ log("recover_not_armed", { reason: durable.recoverSkipped });
2096
+ }
2097
+ }
2098
+
2099
+ // Issue #645: A RECOVERY JOB'S WORKSPACE. The preparer cloned the checkpoint's base commit from the checkpoint store
2100
+ // (`pin`), so the checkout is exactly the tree the checkpoint is a diff against; the restore applies the crashed
2101
+ // run's work on top. All of it local and free, before the token-cap read and every reserve.
2102
+ if (isRecovery) {
2103
+ const baseSha = manifest.workspace.baseSha;
2104
+ // The arm-time rule, again, on the base the store just handed over: a checkpoint of such a base can only exist
2105
+ // from a worker that armed it before this rule, and is refused rather than restored. Unreadable: retried.
2106
+ let unsafe;
2107
+ try {
2108
+ unsafe = await baseHasExtensions({ workspace: prepared.workspace, baseSha });
2109
+ } catch {
2110
+ unsafe = null;
2111
+ }
2112
+ if (unsafe === true) {
2113
+ await comment(job, RECOVERY_COMMENTS["recovery-unsafe-extensions"]);
2114
+ log("refused_recovery_unsafe_extensions", { n: durable.recovery });
2115
+ return { outcome: "policy", reason: "recovery-unsafe-extensions", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
2116
+ }
2117
+ if (unsafe !== false) throw new InfraRetry("whether the recovered base holds .pi/extensions could not be read", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
2118
+ const restored = await workspaceCheckpointer.restore({ runId, manifest, intoWorkspace: prepared.workspace });
2119
+ if (restored?.refused) {
2120
+ // Determinate: the store refused its own checkpoint (tampered, another run's, a base that does not match).
2121
+ await comment(job, RECOVERY_COMMENTS["recovery-checkpoint-absent"]);
2122
+ log("refused_recovery_checkpoint_absent", { n: durable.recovery, why: "restore-refused", detail: String(restored.refused).slice(0, 40) });
2123
+ return { outcome: "policy", reason: "recovery-checkpoint-absent", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
2124
+ }
2125
+ if (!restored || restored.skipped || !Number.isSafeInteger(restored.written)) {
2126
+ log("recovery_restore_skipped", { reason: typeof restored?.skipped === "string" ? restored.skipped.slice(0, 40) : "error" });
2127
+ throw new InfraRetry("the checkpoint could not be restored", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
2128
+ }
2129
+ // THE TRUST PATHS: what pi loads as the repository's own code and instructions (`.pi/`, `.agents/`, the root
2130
+ // AGENTS/CLAUDE files) goes back to the base commit's content, since the restored tree was written by the agent.
2131
+ // A reset that failed means the recovery would start on the agent's configuration, so it does not start.
2132
+ try {
2133
+ await resetTrustPaths({ workspace: prepared.workspace, baseSha });
2134
+ } catch (error) {
2135
+ log("recovery_trust_reset_failed", { code: typeof error?.code === "string" ? error.code : "error" });
2136
+ throw new InfraRetry("the recovered workspace's trust paths could not be reset", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
2137
+ }
2138
+ // THE CONVERSATION: the checkpoint's transcript, staged where every job's transcript file is. A resume job's is a
2139
+ // session with its key (promoted on completion as before); any other job's is its live file.
2140
+ const staged = typeof stageRecovered === "function" ? await stageRecovered({ jobDir: prepared.jobDir, transcriptPath, piVersion, job, resolved: prepared.recoveryResolved ?? {} }) : null;
2141
+ if (!staged || typeof staged.hostDir !== "string") {
2142
+ throw new InfraRetry("the checkpoint's transcript could not be staged", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
2143
+ }
2144
+ prepared = staged.key ? { ...prepared, session: staged } : { ...prepared, live: { hostDir: staged.hostDir } };
2145
+ durable.recovered = { from: manifest.kind, n: durable.recovery, fromAttempt: manifest.attempt, checkpointSeq: manifest.seq, orphans: null, replayed: null, interrupted: null, notStarted: null };
2146
+ }
2147
+
2148
+ if (forkSource) {
2149
+ const baseSha = forkSource.manifest.baseSha;
2150
+ const unsafe = await baseHasExtensions({ workspace: prepared.workspace, baseSha });
2151
+ if (unsafe === true) return await forkRefusal("fork-unsafe-extensions");
2152
+ if (unsafe !== false) throw new InfraRetry("the fork base could not be inspected", { reason: "container-never-started", budgetReserved: false });
2153
+ const restored = await forkStore.restorePoint({ runId: job.fork.fromJob, sourceDigest: job.fork.sourceDigest, entryId: job.fork.fromEntry, position: job.fork.position, intoWorkspace: prepared.workspace });
2154
+ if (restored?.unavailable || restored?.skipped) throw new InfraRetry("the fork workspace could not be restored just now", { reason: "container-never-started", budgetReserved: false });
2155
+ if (!restored?.ok) return await forkRefusal(restored?.refused ?? "fork-workspace-unavailable");
2156
+ await resetTrustPaths({ workspace: prepared.workspace, baseSha });
2157
+ const staged = await stageRecovered?.({ jobDir: prepared.jobDir, transcriptPath: forkSource.transcriptPath, piVersion, job: { ...job, resume: true } });
2158
+ if (!staged?.key) throw new InfraRetry("the fork transcript could not be staged", { reason: "container-never-started", budgetReserved: false });
2159
+ prepared = { ...prepared, session: staged };
2160
+ await comment(job, `Fork of run \`${job.fork.fromJob}\`, ${job.fork.position} entry \`${job.fork.fromEntry}\`, using \`${job.provider}/${job.model}\` on \`${forkBranch(job)}\`. The original attempt is preserved.`);
2161
+ }
2162
+ try { await (deps.stageModelRoleInputs ?? stageModelRoleInputs)(job, prepared); } catch (error) {
2163
+ if (!error?.piDispatchConfig) throw new InfraRetry("the trusted agent snapshot could not be staged", { reason: "container-never-started", provider: job.provider, model: job.model, budgetReserved: false });
2164
+ log("refused_model_agents_invalid", {});
2165
+ await comment(job, MODEL_AGENTS_INVALID_COMMENT);
2166
+ return { outcome: "policy", reason: "model-agents-invalid", exitCode: null, turns: null, tokens: null, provider: job.provider, model: job.model, budgetReserved: false };
2167
+ }
1208
2168
  // Daily TOKEN cap (issue #25): the deliberate check-AFTER control. Token cost is only known
1209
2169
  // post-run, so this cannot check-and-increment before the spend the way the job-count cap does
1210
2170
  // (CONST-BUDGET-BEFORE-TOKENS). It is a read-only GET of prior jobs' recorded spend -- it consumes
@@ -1235,10 +2195,23 @@ export async function runJob(job, deps) {
1235
2195
  // ledger's day counted, an EXPIRE that faults leaves its key without a TTL): `reserveBudget` does not say which of
1236
2196
  // its windows landed, so giving them back could DECR a window that never rose. That is the pre-existing
1237
2197
  // mid-reserve posture of one ledger, unchanged.
2198
+ // Issue #645: an armed run's job-count slot is the RUN's, claimed once in its budget ledger. "new": this attempt
2199
+ // reserves the ledgers below, exactly as every job did before. "carried": an earlier attempt (a retry's first try,
2200
+ // or the attempt a recovery continues) reserved it and is still counted in it, so nothing is reserved again. A
2201
+ // ledger that did not answer leaves the question open, so nothing is reserved and the attempt is retried.
2202
+ if (armed) {
2203
+ const claim = await runBudgetLedger.claimSlot(runId);
2204
+ if (claim !== "new" && claim !== "carried") {
2205
+ log("run_budget_claim_failed", { code: typeof claim?.error === "string" ? claim.error : "unknown" });
2206
+ throw new InfraRetry("the run's budget ledger could not be read", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
2207
+ }
2208
+ slotCarried = claim === "carried";
2209
+ slotOwnedByRun = true;
2210
+ }
1238
2211
  globalLedger = { scope: null, keyPrefix: null, caps, softHoldPct, reason: null };
1239
2212
  let counted;
1240
2213
  try {
1241
- counted = await reserveLedgers(redis, [...(scopedLedgers ?? []), globalLedger], held, { now });
2214
+ counted = slotCarried ? { allowed: true } : await reserveLedgers(redis, [...(scopedLedgers ?? []), globalLedger], held, { now });
1242
2215
  } catch (error) {
1243
2216
  // Valkey failed mid-walk. No container can have started, and `held` lists exactly the reservations that
1244
2217
  // landed, so they go back (last first, never throwing) before the error escapes; otherwise a repo and a
@@ -1283,6 +2256,29 @@ export async function runJob(job, deps) {
1283
2256
  return { outcome: "policy", reason: counted.refusedBy.reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
1284
2257
  }
1285
2258
 
2259
+ // Issue #645: THIS CONTAINER'S NUMBER in the run (`nextSeq`: attempt 1 is 1, its retry 2, a recovery's container the
2260
+ // next), which addresses its dollar hold in the run's ledger and names its checkpoints' attempt. Before the dollar
2261
+ // reserve, because a hold is recorded under it: without a number there is no record, and without a record there is
2262
+ // no hold. A recovery then settles AT THE FLOOR every hold of the run's earlier containers still `held`: those
2263
+ // containers are dead (the base-still-running gate above holds this job until none of the run's is alive here),
2264
+ // and their real spend is unknown, so the reservation already in every window is what they are charged
2265
+ // (`settleCrashed`, bookkeeping only, no window key moves). A settle that failed is logged and the run goes on: the
2266
+ // holds stay `held` and in the windows at the floor, which is the safe side, and nothing releases an earlier
2267
+ // container's hold but that container itself.
2268
+ if (armed) {
2269
+ const seq = await runBudgetLedger.nextSeq(runId);
2270
+ if (!Number.isSafeInteger(seq) || seq < 1) {
2271
+ log("run_budget_seq_failed", { code: typeof seq?.error === "string" ? seq.error : "unknown" });
2272
+ throw new InfraRetry("the run's container number could not be taken", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
2273
+ }
2274
+ durable.seq = seq;
2275
+ if (isRecovery) {
2276
+ const crashed = await runBudgetLedger.settleCrashed(runId, seq);
2277
+ if (Array.isArray(crashed?.settled)) log("run_budget_settled_crashed", { holds: crashed.settled.length });
2278
+ else log("run_budget_settle_crashed_failed", { code: typeof crashed?.error === "string" ? crashed.error : "unknown" });
2279
+ }
2280
+ }
2281
+
1286
2282
  // DOLLAR windows (issue #501, part 3), after EVERY job-count reserve and before the container: the last gate
1287
2283
  // before the spend line (CONST-BUDGET-BEFORE-TOKENS). Every free gate and every job-count ledger come first, so
1288
2284
  // a refusal anywhere above never touches a dollar counter. The amount is the job's per-job cost cap, which the
@@ -1310,7 +2306,7 @@ export async function runJob(job, deps) {
1310
2306
  // job carried a non-null cap), so an image that would ignore the 0 never gets here. A cap that is ALREADY 0
1311
2307
  // (a malformed queued value reads as 0, `effectiveCostCapMicros`) has nothing to reserve either.
1312
2308
  const refs = callableModelRefs(job);
1313
- const zero = job.maxCostMicros === 0 ? { zeroRated: true } : zeroRatedVerdict({ models: modelEndpoints?.models ?? null, endpoints: modelEndpoints?.endpoints ?? [], refs, builtinModel });
2309
+ const zero = zeroRatedFor(job, { modelEndpoints, builtinModel });
1314
2310
  if (zero.zeroRated) {
1315
2311
  containerJob = { ...job, maxCostMicros: 0 };
1316
2312
  dollars = dollarsRecord({ reservedMicros: 0, settledMicros: 0, basis: "unreserved" });
@@ -1370,20 +2366,134 @@ export async function runJob(job, deps) {
1370
2366
  return { outcome: "policy", reason: refusalReason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: globalHeld(), dollars: stranded ? dollarsRecord({ reservedMicros: job.maxCostMicros, settledMicros: job.maxCostMicros, basis: "floor", modelBasis }) : dollarsRecord({ reservedMicros: job.maxCostMicros, settledMicros: 0, basis: "refunded", modelBasis }) }; // return => not retried
1371
2367
  }
1372
2368
  dollarHold = reservation.hold;
1373
- dollars = dollarsRecord({ reservedMicros: job.maxCostMicros, settledMicros: job.maxCostMicros, basis: "floor", modelBasis: modelPrefixes.size > 0 ? "floor" : null });
2369
+ publishDollarHold({ amountMicros: dollarHold.amountMicros, at: new Date(now).getTime() });
2370
+ // Issue #761: every key holds the whole reservation until a settle or a release moves it, so every deployment
2371
+ // window the hold reserved in holds what this record states (`deploymentWindowsOf`). Each later path below
2372
+ // replaces `dollars`, with the windows that hold what IT states.
2373
+ dollars = dollarsRecord({ reservedMicros: job.maxCostMicros, settledMicros: job.maxCostMicros, basis: "floor", modelBasis: modelPrefixes.size > 0 ? "floor" : null, reservedIn: deploymentWindowsOf(dollarHold, { applied: 0, target: job.maxCostMicros, recorded: job.maxCostMicros, valkey: valkeyTag }) });
2374
+ // Issue #645: an armed run never runs a container without a DURABLE record of its hold (run-budget-ledger.mjs's
2375
+ // contract), so a recovery after a crash can see it and settle it exactly once. A record that did not land
2376
+ // throws a never-started retry before any container; the catch gives the hold back directly (`holdRecorded`
2377
+ // false: no one else can see it to settle or release it).
2378
+ if (armed) {
2379
+ const rec = await runBudgetLedger.recordHold(runId, durable.seq, dollarHold);
2380
+ if (rec?.ok !== true) {
2381
+ log("run_budget_hold_unrecorded", { code: typeof rec?.error === "string" ? rec.error : "unknown" });
2382
+ throw new InfraRetry("the run's dollar hold could not be recorded", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
2383
+ }
2384
+ holdRecorded = true;
2385
+ }
1374
2386
  }
1375
2387
  }
1376
2388
 
2389
+ // Issue #642: ONE trail comment says who answered from the operator console and quotes what (console-answer.mjs:
2390
+ // mentions neutralised, no control byte). Posted HERE, after every pre-spend gate and every reserve has passed and
2391
+ // just before the container starts, so the issue never reads "answered" above a refusal that then dropped the
2392
+ // answer (a `not-parked`, a missing session store, a cap); and still before the provider is called, so it spends
2393
+ // nothing (CONST-BUDGET-BEFORE-TOKENS). The `comment` adapter posts as the deployment's own identity, which the
2394
+ // receiver's self-guard drops as `self`, so the trail never starts a job. A `console` field whose name is not a
2395
+ // console token's (a hand-built queue entry; enqueueForgeJob refuses one) posts nothing.
2396
+ //
2397
+ // ONCE PER JOB, not once per attempt: a forge job has `attempts: 2`, so an InfraRetry after this point re-runs every
2398
+ // gate and would reach here again, and a console retry of a failed answer job is the same job id again. The claim
2399
+ // is `SET console:trail:<jobId> 1 NX PX CONSOLE_TRAIL_CLAIM_MS` (the retention of a forge job's data, so the claim
2400
+ // outlives every way the same id can come back), and only the attempt that set it posts. On a Valkey error the trail
2401
+ // is SKIPPED and logged, never posted blind: at most once is the promise, because a duplicate trail reads on the forge
2402
+ // as a second answer, while a missing one leaves the answer in the job and the audit log. Never throws: the trail is
2403
+ // provenance, and a failed claim is no reason to refuse a job every gate has admitted.
2404
+ if (job.parkedReply === true && job.console !== undefined) {
2405
+ const by = consoleAnswerBy(job.console);
2406
+ if (by === null) log("console_answer_by_invalid", { kind: job.kind ?? null });
2407
+ else if (await claimConsoleTrail(redis, job, log)) await comment(job, answerTrailComment(by, job.trigger?.comment?.body, job.console?.draftRunId));
2408
+ }
2409
+
1377
2410
  // The user the gate above decided is the user that runs: one answer, never two call sites that agree.
1378
2411
  // Issue #545: an image that declares `exitAuth` is handed a per-job key on stdin and signs its exit line with it, and
1379
2412
  // then only a signed line is read (run-container.mjs, run-history.mjs `authenticExitLines`). An image that does not
1380
2413
  // declare it is read as before, under the #542 trust rule below alone.
1381
2414
  const exitAuth = (img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY);
1382
- const { code, aborted, abortReason, turns, tokens, session, usage, context, detached, exitReason, exitWhy = null, exitLineCode = null, exitAuth: exitAuthResult = null, exitOomKilled = false, memoryLimit = null, resources: ranResources = null } = await runContainer({ job: containerJob, token, prepared, secrets, user: jobUser?.user ?? null, home: jobUser?.home ?? null, relabel: jobUser?.relabel === true, ...(modelEndpoints?.endpoints?.length > 0 ? { modelEndpoints } : {}), ...(exitAuth ? { exitAuth: true } : {}), ...(jobSize ? { size: jobSize } : {}), ...(Number.isSafeInteger(jobUser?.hostCpus) ? { hostCpus: jobUser.hostCpus } : {}), ...(Number.isSafeInteger(cpuBudgetCenti) ? { cpuBudgetCenti } : {}), ...(Array.isArray(jobUser?.unenforced) && jobUser.unenforced.length > 0 ? { unenforced: jobUser.unenforced } : {}) });
2415
+ // Issue #686: an image whose runner can mirror its transcript over stderr. Handed as a fact; runContainer arms it only
2416
+ // on a venue that copies (`transcript: "stdio"`) and only beside a key, so every other run is called as before.
2417
+ const transcriptStderr = (img.capabilities ?? []).includes(TRANSCRIPT_STDERR_CAPABILITY);
2418
+ // Issue #686, part 2: an image whose runner reads steer frames on stdin. Handed as a fact, like the one above;
2419
+ // runContainer arms it only for a steer job on a venue that copies, so every other run is called as before.
2420
+ const steerStdin = (img.capabilities ?? []).includes(STEER_STDIN_CAPABILITY);
2421
+ // Issue #645: a FRESH armed run seeds the checkpoint store from its clone, now, before the container: the one moment
2422
+ // the workspace's `.git` is still the worker's own (workspace-checkpoint.mjs `primeBase`). After every reserve, so a
2423
+ // refused job never writes to the store. A recovery's base is already there. A base that could not be primed means
2424
+ // no checkpoint of this run can ever be restored: the run goes on unprotected and the record says so.
2425
+ let checkpointArgs = null;
2426
+ const baseSha = isRecovery ? manifest.workspace.baseSha : prepared.sha;
2427
+ if (armed) {
2428
+ if (!isRecovery) {
2429
+ const objectFormat = typeof baseSha === "string" && baseSha.length === 64 ? "sha256" : "sha1";
2430
+ let primed = null;
2431
+ try {
2432
+ primed = await workspaceCheckpointer.primeBase({ runId, workspace: prepared.workspace, baseSha, objectFormat });
2433
+ } catch {
2434
+ primed = null;
2435
+ }
2436
+ if (primed?.ok !== true) {
2437
+ durable.recoverSkipped = "base-unprimed";
2438
+ log("recover_base_unprimed", { reason: typeof primed?.skipped === "string" ? primed.skipped.slice(0, 40) : "error" });
2439
+ }
2440
+ }
2441
+ // The live checkpointer's open arguments (run-checkpoint.mjs): the run, this container's number, the base, and
2442
+ // the values a snapshot must refuse to store (the job's token and its resolved secrets). `attempt` IS the
2443
+ // container number: unique across retries and recoveries, which a BullMQ attempt count is not. `meta` is what the
2444
+ // manifest records and a recovery judges above: never a string the job chose.
2445
+ const secretValues = [token, ...Object.values(secrets ?? {})].filter((v) => typeof v === "string" && v !== "");
2446
+ checkpointArgs = { runId, attempt: durable.seq, baseSha, branch: null, secretValues, meta: { piVersion, imageDigest: img.imageDigest ?? null, venue: resolveBackendName(job, blessedBackends[0]), provider: job.provider ?? null, model: job.model ?? null } };
2447
+ }
2448
+ if (job.watch?.stage === "decision") await deps.watchRuntime?.started(job);
2449
+ if (subscriptionAuth.length > 0 && !(await validateSubscriptionAuth(subscriptionAuth))) throw new InfraRetry("subscription binding changed before container start", { reason: "container-never-started", budgetReserved: true });
2450
+ // Issue #800: the commit the repository's part of this run came from, beside the image's facts. Noted here, the last
2451
+ // thing before the container, and not at the prepare above: a refusal between the two loaded nothing, so it has nothing
2452
+ // to point at, and its record stays byte-identical to what it was. Every way out from here keeps it, a retried attempt's too.
2453
+ if (deps.canaryFacts) deps.canaryFacts.loaded = loadedWithBase(deps.canaryFacts.loaded, baseSha);
2454
+ const { canaryLoad: canaryLoadOfRun = null, loaded: loadedOfRun = null, code, aborted, abortReason, turns, tokens, session, usage, context, detached, exitReason, approval: approvalMetadata = null, exitWhy = null, exitLineCode = null, exitAuth: exitAuthResult = null, exitOomKilled = false, memoryLimit = null, resources: ranResources = null, progress: ranProgress = null, checkpoint: ranCheckpoint = null, recovery: ranRecovery = null, lingering = false } = await runContainer({ job: containerJob, token, prepared, secrets, ...(subscriptionAuth.length ? { subscriptionAuth } : {}), ...(bundleKeys !== null ? { bundleKeys } : {}), user: jobUser?.user ?? null, home: jobUser?.home ?? null, relabel: jobUser?.relabel === true, ...(modelEndpoints?.endpoints?.length > 0 ? { modelEndpoints } : {}), ...(exitAuth ? { exitAuth: true } : {}), ...(transcriptStderr ? { transcriptStderr: true } : {}), ...(steerStdin ? { steerStdin: true } : {}), ...(jobSize ? { size: jobSize } : {}), ...(Number.isSafeInteger(jobUser?.hostCpus) ? { hostCpus: jobUser.hostCpus } : {}), ...(Number.isSafeInteger(cpuBudgetCenti) ? { cpuBudgetCenti } : {}), ...(Array.isArray(jobUser?.unenforced) && jobUser.unenforced.length > 0 ? { unenforced: jobUser.unenforced } : {}), ...(checkpointArgs !== null ? { recover: true, recovering: isRecovery, checkpoint: checkpointArgs } : {}) });
2455
+ const answeredPark = prepared.session?.parkedSource;
2456
+ // A phrase-prefixed comment uses ordinary routing, but resuming this exact parked transcript still
2457
+ // answers its question. A held receipt also identifies an answer whose legacy source cannot be proved.
2458
+ if (!deps.approvalRequest && session?.resumed === true && (job.parkedReply === true || typeof job.trigger?.comment?.body === "string" && (answeredPark || Number.isSafeInteger(job.replyReceivedAt)))) {
2459
+ if (answeredPark) {
2460
+ const receivedAt = Number.isSafeInteger(deps.replyReceivedAt) ? deps.replyReceivedAt : now.getTime();
2461
+ await deps.recordTouch({ runId: answeredPark.runId, cause: "question-answer", at: Math.max(answeredPark.at, receivedAt), receivedAt: Math.max(answeredPark.at, receivedAt), answerSource: job.console?.draftRunId ? "draft" : job.console?.by === "panel" ? "panel" : job.console ? "console" : "forge", ...(job.console?.draftRunId ? { draftRunId: job.console.draftRunId } : {}), parkedAt: answeredPark.at, scopeId: touchScopeOf(job, hostName), project: pickupProject ?? undefined, nonce: sourceRunId });
2462
+ } else {
2463
+ deps.recordTouchGap();
2464
+ log("touch_lineage_unavailable", { reason: "source-unavailable" });
2465
+ }
2466
+ }
2467
+
1383
2468
  containerRan = true;
2469
+ watchPolicyRefused = code === EXIT_POLICY;
2470
+ watchUsage = Number.isFinite(tokens?.cost) && tokens.cost >= 0 ? { costUsd: tokens.cost } : null;
2471
+ answerUsage = { tokens };
2472
+ if (sourcePrimed && !lingering && !detached && !aborted && code === EXIT_COMPLETED) {
2473
+ const rootKey = sessionKeyFor(job).split(".")[0];
2474
+ const transcriptPath = join(prepared.session?.hostDir ?? prepared.live?.hostDir ?? join(prepared.jobDir, "session"), "current.jsonl");
2475
+ let pointSkipped = null;
2476
+ // Dead-container snapshots also make the completed boundary available without live recovery.
2477
+ if (typeof forkStore.capturePoint === "function") {
2478
+ try {
2479
+ const captured = await forkStore.capturePoint({ runId: sourceRunId, workspace: prepared.workspace, baseSha: prepared.sha, quiesced: "dead", kind: "exit", attempt: durable.seq ?? 0, transcriptPath, offset: lstatSync(transcriptPath).size, secretValues: [token, ...Object.values(secrets ?? {})].filter((v) => typeof v === "string" && v !== ""), meta: { piVersion, venue: resolveBackendName(job, blessedBackends[0]), provider: job.provider ?? null, model: job.model ?? null } });
2480
+ if (!captured?.ok) {
2481
+ pointSkipped = captured?.refused ?? captured?.skipped ?? "checkpoint-unavailable";
2482
+ log("fork_checkpoint_skipped", { reason: pointSkipped });
2483
+ }
2484
+ } catch { pointSkipped = "checkpoint-unavailable"; log("fork_checkpoint_skipped", { reason: "unavailable" }); }
2485
+ }
2486
+ let saved;
2487
+ try { saved = forkStore.publish({ runId: sourceRunId, sessionKey: rootKey, transcriptPath: join(prepared.session?.hostDir ?? prepared.live?.hostDir ?? join(prepared.jobDir, "session"), "current.jsonl"), baseSha: prepared.sha, venue: resolveBackendName(job, blessedBackends[0]), piVersion, attempt: durable.seq ?? 0 }); } catch { saved = { refused: "fork-source-unavailable" }; }
2488
+ archiveFacts.value = forkArchiveFact({ saved: saved?.ok === true, reason: saved?.refused, pointSkipped });
2489
+ log("fork_source_saved", saved?.ok ? { saved: true } : { saved: false, reason: saved?.refused ?? "unavailable" });
2490
+ }
1384
2491
  // Issue #596: what the container used, off its exit line, rebuilt by the sink (null from a runContainer that predates
1385
2492
  // the field). Every result and every throw below carries it, so a retried attempt's record says what it used too.
1386
2493
  resources = ranResources ?? null;
2494
+ if (deps.canaryFacts && job.packageCanary !== undefined) deps.canaryFacts.load = canaryLoadOfRun ?? null;
2495
+ if (deps.canaryFacts && loadedOfRun !== null) deps.canaryFacts.loaded = loadedWithBase(loadedWithImage(loadedOfRun, img), baseSha);
2496
+ progress = ranProgress ?? null;
1387
2497
  // Issue #596: CONFIRMED killed for memory. The image's supervisor (image/runner/src/supervise.mjs) outlives the runner,
1388
2498
  // whose process tree it gives the highest OOM score, and when the runner dies of SIGKILL with the cgroup's
1389
2499
  // `oom_kill` above 0 it writes the signed line `code: 137, reason: "oom-killed"` (parseExitOomKilled). All three
@@ -1394,6 +2504,79 @@ export async function runJob(job, deps) {
1394
2504
  // `oom_kill` counts the HOST's OOM killer too (`peakReachedLimit`). A report below it stays infrastructure.
1395
2505
  const oomReported = exitOomKilled === true && exitAuthResult === "verified" && code === EXIT_SIGKILL;
1396
2506
  const oomKilled = oomReported && peakReachedLimit(resources?.memPeak ?? null, memoryLimit);
2507
+ // Issue #645: what the runner says it did with the crashed attempt's orphaned tool calls (the exit line's `recovery`).
2508
+ // Issue #662: a recovery whose own container was killed writes no exit line, so its counts come from the transcript
2509
+ // instead (the reducer's recovery marks, this attempt's lines only), labelled `countsFrom: "transcript"`. The marks
2510
+ // are in an agent-writable file, so the exit line's counts win whenever the exit line CARRIES a `recovery` block
2511
+ // (`recoveryCounts` not null); an exit line without one (a verified line whose recovery step threw, say) falls back
2512
+ // to the transcript like a missing line. Neither feeds any decision.
2513
+ if (durable.recovered !== null) {
2514
+ const counts = recoveryCounts(ranRecovery);
2515
+ const fromTranscript = counts === null ? recoveryCountsFromProgress(progress) : null;
2516
+ if (counts !== null) durable.recovered = { ...durable.recovered, ...counts, countsFrom: "exit-line" };
2517
+ else if (fromTranscript !== null) durable.recovered = { ...durable.recovered, ...fromTranscript, countsFrom: "transcript" };
2518
+ }
2519
+ // A container the checkpointer stopped because it could not unpause it (`checkpointStopped`, code 137, not aborted)
2520
+ // is an infrastructure exit like any other here: not the worker's abort, not an OOM (no supervisor report), so it
2521
+ // reaches the unknown-exit arm below and recovers from the checkpoint that was committed before the pause.
2522
+ // Issue #645: THE EXIT CHECKPOINT, then the checkpointer's close, before anything below can return or throw. Taken
2523
+ // only on an exit this attempt may recover from: infrastructure (exit 1, an unknown code, a detached container),
2524
+ // never a worker's abort, an OOM kill, a completed or policy exit (none of those recovers) nor a never-started one
2525
+ // (nothing ran, and today's retry is cheaper). The container is dead, so nothing is paused. B1's handle is read
2526
+ // defensively: absent, or without `exit`, means no exit checkpoint, never a failure. Neither call may throw out of here.
2527
+ const neverStartedExit = isNeverStartedExit({ code, aborted, detached }, neverStartedExits(job));
2528
+ const recoverable = armed && !aborted && !oomKilled && code !== EXIT_COMPLETED && code !== EXIT_POLICY && !neverStartedExit;
2529
+ // Issue #686 part 3, review round 2 of #714: a venue whose box was not confirmed gone after the run (`lingering`,
2530
+ // `openshell-bind`) takes NO exit checkpoint: the host tree may still be written, so a snapshot of it is not a state
2531
+ // the run reached. A recovery then continues from the last committed checkpoint.
2532
+ if (ranCheckpoint !== null && typeof ranCheckpoint === "object") {
2533
+ if (recoverable && lingering !== true && typeof ranCheckpoint.exit === "function") {
2534
+ const taken = await Promise.resolve().then(() => ranCheckpoint.exit({ kind: "exit" })).catch(() => ({ skipped: "error" }));
2535
+ log("checkpoint_exit", taken && Number.isSafeInteger(taken.seq) ? { seq: taken.seq } : { skipped: typeof taken?.skipped === "string" ? taken.skipped.slice(0, 40) : "unknown" });
2536
+ }
2537
+ const closed = typeof ranCheckpoint.close === "function" ? await Promise.resolve().then(() => ranCheckpoint.close()).catch(() => null) : null;
2538
+ // `close()` answers the counts; `stats()` (a function on B1's handle) is the fallback when close did not.
2539
+ const stats = closed ?? (typeof ranCheckpoint.stats === "function" ? await Promise.resolve().then(() => ranCheckpoint.stats()).catch(() => null) : null);
2540
+ const count = (v) => (Number.isSafeInteger(v) && v >= 0 ? v : null);
2541
+ if (stats && count(stats.written) !== null && count(stats.failed) !== null && count(stats.skipped) !== null) durable.checkpoints = { written: stats.written, failed: stats.failed, skipped: stats.skipped };
2542
+ }
2543
+ // An infrastructure exit of an armed run: queue its continuation from the checkpoint instead of throwing a retry
2544
+ // (CONST-RETRY-INFRA-ONLY holds: nothing is returned as success, and the queued recovery IS the retry, one that
2545
+ // keeps the work). RETURNED, so BullMQ spends no second attempt that would start from scratch beside it. Not
2546
+ // queued (no checkpoint, the cap, a claim already taken, Valkey) -> today's InfraRetry, unchanged.
2547
+ // A RECOVERY job has one attempt, so a throw here would fail it with the generic failure comment and nothing else:
2548
+ // when its container ran, ended infra and no further recovery was queued, it RETURNS `recovery-failed` with its own
2549
+ // comment instead (the run's final outcome, so its checkpoint, pointer and ledger are cleaned up).
2550
+ const notQueued = async (err) => {
2551
+ if (!isRecovery) throw err;
2552
+ await comment(job, RECOVERY_COMMENTS["recovery-failed"]);
2553
+ return { outcome: "failed", reason: "recovery-failed", exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
2554
+ };
2555
+ const recoverOrThrow = async (err) => {
2556
+ if (!recoverable) return notQueued(err);
2557
+ let pointer = null;
2558
+ try {
2559
+ pointer = await readCheckpointPointer({ runId });
2560
+ } catch {
2561
+ pointer = null;
2562
+ }
2563
+ if (pointer === null || pointer === undefined) {
2564
+ log("recovery_not_enqueued", { cause: "infra", reason: "no-checkpoint" });
2565
+ return notQueued(err);
2566
+ }
2567
+ let queued;
2568
+ try {
2569
+ queued = await enqueueRecovery({ job, cause: "infra", pointer });
2570
+ } catch {
2571
+ queued = { enqueued: false, reason: "error" };
2572
+ }
2573
+ if (queued?.enqueued !== true) {
2574
+ log("recovery_not_enqueued", { cause: "infra", reason: typeof queued?.reason === "string" && /^[a-z0-9-]{1,40}$/.test(queued.reason) ? queued.reason : "unknown" });
2575
+ return notQueued(err);
2576
+ }
2577
+ log("recovery_enqueued_after_exit", { cause: "infra", n: Number.isSafeInteger(queued.n) ? queued.n : null, exitCode: code });
2578
+ return { outcome: "failed", reason: RECOVERY_ENQUEUED, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
2579
+ };
1397
2580
  // Numbers only (bytes), never a path or a project: the operator's trace of a kill read as the host's. Never on a run
1398
2581
  // the worker stopped itself (a timeout, a cancel, a shutdown): that 137 is the worker's own, the abort decides it
1399
2582
  // below, and a line naming the host's OOM killer would send the operator after a kill that never happened.
@@ -1418,7 +2601,7 @@ export async function runJob(job, deps) {
1418
2601
  // completed, runner policy, a worker abort or cancel, exit 1, an unknown exit, and a detached container. A
1419
2602
  // never-started exit is not settled: the catch refunds it whole, and `isNeverStartedExit` is the one answer both
1420
2603
  // ask. Before any classification below, so every return and every throw carries the same `dollars`.
1421
- const neverStarted = isNeverStartedExit({ code, aborted, detached }, neverStartedExits(job));
2604
+ const neverStarted = neverStartedExit;
1422
2605
  if (dollarHold !== null && !neverStarted) {
1423
2606
  // The exit line is trusted only when the container exited ON ITS OWN (the worker did not abort, cancel, time
1424
2607
  // out or detach it, so the runner had the chance to write its genuine last line) AND that line's own `code` is
@@ -1428,10 +2611,19 @@ export async function runJob(job, deps) {
1428
2611
  const { settledMicros, basis } = dollarSettlement({ tokens, usage: usage ?? null, reservedMicros, trusted });
1429
2612
  // The deployment, the repo or folder and the project windows settle to the job's cost; each model window to its
1430
2613
  // own row of the usage ledger (`modelDollarSettlement`), never to the job's total.
2614
+ // Issue #645: a recorded hold is settled through the run's ledger first, `held` -> `settled`, and only the call
2615
+ // that MOVED it adjusts a window key. Anything else ("not-held": someone already finished it; a ledger error)
2616
+ // leaves the reservation in every window, the floor, which is the safe side and what the record then says.
2617
+ let ledgerMoved = true;
2618
+ if (holdRecorded) {
2619
+ const moved = await runBudgetLedger.settle(runId, durable.seq);
2620
+ ledgerMoved = moved === "moved";
2621
+ if (!ledgerMoved) log("run_budget_settle_skipped", { result: typeof moved === "string" ? moved : typeof moved?.error === "string" ? moved.error : "unknown" });
2622
+ }
1431
2623
  const jobPart = holdPart(dollarHold, (prefix) => jobCostPrefixes.has(prefix));
1432
- const { applied, of } = await settleDollars(redis, jobPart, settledMicros, { log });
2624
+ const { applied, of } = ledgerMoved ? await settleDollars(redis, jobPart, settledMicros, { log }) : { applied: 0, of: 1 };
1433
2625
  let modelBasis = null;
1434
- for (const part of dollarHold.ledgers ?? []) {
2626
+ for (const part of ledgerMoved ? (dollarHold.ledgers ?? []) : []) {
1435
2627
  const ref = modelPrefixes.get(part.keyPrefix);
1436
2628
  if (ref === undefined) continue;
1437
2629
  const m = modelDollarSettlement({ ref, basis, tokens, usage: usage ?? null, reservedMicros, trusted });
@@ -1445,8 +2637,14 @@ export async function runJob(job, deps) {
1445
2637
  // A fault that adjusted no key leaves the whole reservation in every window, which is a floor whatever the
1446
2638
  // basis would have been, and the record says so. A partial one is logged (dollar_settle_error) and recorded
1447
2639
  // as computed: the keys that were adjusted hold the settled amount.
1448
- dollars = applied === 0 && of > 0 && settledMicros !== reservedMicros ? dollarsRecord({ reservedMicros, settledMicros: reservedMicros, basis: "floor", modelBasis }) : dollarsRecord({ reservedMicros, settledMicros, basis, modelBasis });
2640
+ if (!ledgerMoved && modelPrefixes.size > 0) modelBasis = "floor";
2641
+ const kept = applied === 0 && of > 0 && (settledMicros !== reservedMicros || !ledgerMoved);
2642
+ // Issue #761: the deployment windows whose keys hold what this record states, along the keys the settle walked
2643
+ // (`jobPart`, the deployment's first): a key it moved holds the settled amount, one it did not the reservation.
2644
+ const reservedIn = deploymentWindowsOf(jobPart, { applied, target: settledMicros, recorded: kept ? reservedMicros : settledMicros, valkey: valkeyTag });
2645
+ dollars = kept ? dollarsRecord({ reservedMicros, settledMicros: reservedMicros, basis: "floor", modelBasis, reservedIn }) : dollarsRecord({ reservedMicros, settledMicros, basis, modelBasis, reservedIn });
1449
2646
  dollarHold = null;
2647
+ publishDollarHold(null);
1450
2648
  log("dollar_settled", { reservedMicros: dollars.reservedMicros, settledMicros: dollars.settledMicros, basis: dollars.basis, ...(modelBasis !== null ? { modelBasis } : {}) });
1451
2649
  } else if (dollars?.basis === "unreserved") {
1452
2650
  // Nothing was held, so nothing is written. A metered cost above 0 here would mean the runner's cap of 0 let a
@@ -1460,7 +2658,7 @@ export async function runJob(job, deps) {
1460
2658
  // mid-job). It DID start, so this is never refunded as never-started: it keeps its slot and retries as infrastructure,
1461
2659
  // BEFORE the exit-code switch, where the same code would read as a free never-started exit.
1462
2660
  if (detached === true) {
1463
- throw new InfraRetry(`the container outlived its docker run, exit ${code}`, { reason: "container-detached", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources });
2661
+ return await recoverOrThrow(new InfraRetry(`the container outlived its docker run, exit ${code}`, { reason: "container-detached", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources }));
1464
2662
  }
1465
2663
 
1466
2664
  // A WORKER-initiated stop (30-min timeout via cancelJob, graceful-shutdown docker stop, or an
@@ -1475,11 +2673,11 @@ export async function runJob(job, deps) {
1475
2673
  // `abortReason` itself never reaches the record -- buildRecord copies named fields only.
1476
2674
  // exitCode/turns/tokens carry the container's own exit, turn count, and usage totals; budgetReserved true post-reserve.
1477
2675
  if (aborted) {
1478
- const reason = abortReason === "operator-cancel" ? "operator-cancel" : "worker-abort";
2676
+ const reason = ["operator-cancel", "agent-cancel"].includes(abortReason) ? abortReason : "worker-abort";
1479
2677
  // Awaited bare like every determinate refusal above: the adapter never throws by contract, and
1480
2678
  // the one swallowed comment in this file (the catch's) justifies itself by its position.
1481
2679
  await comment(job, TERMINAL_COMMENTS[reason]);
1482
- return { outcome: "policy", reason, exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}) };
2680
+ return { outcome: "policy", reason, exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
1483
2681
  }
1484
2682
 
1485
2683
  // Issue #596: a runner the kernel killed for memory, CONFIRMED (`oomKilled` above). After the abort branch, so a
@@ -1495,21 +2693,34 @@ export async function runJob(job, deps) {
1495
2693
  // The project and the size go to the log and the record, never to the comment (the TERMINAL_COMMENTS rule).
1496
2694
  log("oom_killed", { jobId: job.id ?? null });
1497
2695
  await comment(job, TERMINAL_COMMENTS[EXIT_OOM_KILLED]);
1498
- return { outcome: "policy", reason: EXIT_OOM_KILLED, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}) };
2696
+ return { outcome: "policy", reason: EXIT_OOM_KILLED, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
2697
+ }
2698
+ // Issue #686 part 3, review round 2 of #714 (CONST-RETRY-INFRA-ONLY): a box not confirmed gone after the run. Exit 2
2699
+ // keeps its own reason below, and a worker's abort stays the abort. A COMPLETED run is not thrown away as a retry,
2700
+ // and a FOLDER job is not retried while the first sandbox may still write the folder: both RETURN the venue's named
2701
+ // result, never retried, spend kept and settled above. The completed run's work stays where the agent put it (a
2702
+ // pushed branch, a folder's edits), but nothing more is done with its result: no session promotion, no chained
2703
+ // child, no completion comment, since the tree it would read may still change. Any other exit of a forge job is
2704
+ // infrastructure as before (its retry gets a fresh clone; its recovery the last committed checkpoint).
2705
+ if (lingering === true && !aborted && code !== EXIT_POLICY && (code === EXIT_COMPLETED || job.kind === "local")) {
2706
+ log("venue_box_lingering", { exitCode: code, kind: job.kind ?? null });
2707
+ await comment(job, OPENSHELL_BIND_LINGERING.comment);
2708
+ return { outcome: "policy", reason: OPENSHELL_BIND_LINGERING.reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
1499
2709
  }
1500
2710
 
1501
2711
  switch (code) {
1502
2712
  case EXIT_COMPLETED: {
2713
+ watchCompleted = true;
1503
2714
  // The SOLE chain-collection point. Read the completed parent's /outbox and enqueue children
1504
2715
  // BEFORE the `finally` deletes jobDir -- the await resolves inside this case, so the read
1505
2716
  // finishes before control leaves to cleanup. NOT reached on any other branch (policy, abort,
1506
2717
  // over-budget, infra): an InfraRetry job is retried, so chaining there would double-enqueue.
1507
2718
  // collectChain never throws; chainEnqueued/chainRefused are additive telemetry only.
1508
- const chain = await collectChain({ job, prepared });
2719
+ const chain = job.answerDraft ? { enqueued: 0, refused: 0 } : await collectChain({ job, prepared });
1509
2720
  // Issue #505: the plan, after the chain and on this branch only, for the reason the chain is here: a policy or
1510
2721
  // infra exit collects nothing, and a retried job must not apply what its failed attempt wrote. `portfolio` is
1511
2722
  // the pickup's decision; the collector asks the live file again, and both must agree. Never throws.
1512
- const plan = await collectPlan({ job, prepared, portfolio });
2723
+ const plan = job.answerDraft ? null : await collectPlan({ job, prepared, portfolio });
1513
2724
  // COMPLETED-ONLY PROMOTION, and the exclusivity is the point rather than an optimisation.
1514
2725
  // A policy or infra exit leaves the canonical transcript byte-identical to what it was
1515
2726
  // before this run, so a retry starts from exactly what the first attempt did -- promote on
@@ -1522,22 +2733,69 @@ export async function runJob(job, deps) {
1522
2733
  // exitReason is RUNNER_COMPLETED_REASONS-filtered and read only here, on code 0, so it cannot move a job
1523
2734
  // between retry classes. Only a job armed with run.awaitInput may park: a line that says so on any other
1524
2735
  // job is ignored, since that runner was never offered the tool.
1525
- const parked = exitReason === "awaiting-input" && job.awaitInput === true;
1526
- const promoted = prepared.session ? promoteSession(prepared.session, { piVersion, context, parked }) : null;
1527
- // The window closes AFTER the promotion (an answer that gets in must find the `parked` mark and the transcript)
1528
- // and BEFORE the comment and the marker: the marker is what routes a phrase-free reply on an issue, and the
1529
- // comment is what tells a pull request's author to reply, so no answer they invite can meet the old key. Every
1530
- // park takes this path, a resumed run that parks again included. Guarded although the seam never throws: a
1531
- // fault here must not cost the person the comment that says how to continue.
1532
- if (parked) await Promise.resolve().then(releaseParkedWindow).catch(() => {});
1533
- if (parked) await comment(job, parkedComment(job.target));
1534
- // The forge marker the receiver routes a phrase-free reply on: set on a park, cleared when a run on a key that
1535
- // could park ends without parking. Never throws, and decides nothing: the sidecar promotion just wrote does.
1536
- if (parked) await parkedLabel(job, true);
2736
+ const approvalParked = exitReason === "awaiting-approval" && Array.isArray(job.gates) && job.gates.includes(approvalMetadata?.action);
2737
+ const parked = (exitReason === "awaiting-input" || approvalParked) && job.awaitInput === true;
2738
+ const promoted = prepared.session ? promoteSession(prepared.session, { piVersion, context, parked, ...(sourceRunId ? { parkedSource: { runId: sourceRunId } } : {}) }) : null;
2739
+ if (parked && !approvalParked && promoted?.promoted === true && deps.questionCaptureEnabled?.() === true && deps.questionGroupStore && sourceRunId) {
2740
+ try {
2741
+ const context = typeof prepared.questionRedact === "function"
2742
+ ? (deps.readQuestionContext ?? readQuestionContext)({ path: join(prepared.session.hostDir, "current.jsonl"), redact: prepared.questionRedact })
2743
+ : { status: "source-unavailable" };
2744
+ const captured = await deps.questionGroupStore.capture({ runId: sourceRunId, scopeId: touchScopeOf(job, hostName), project: pickupProject ?? null, repo: job.repo ?? null, at: deps.touchNow?.() ?? Number(now), text: context.text ?? null, sourceStatus: context.status === "ready" ? null : context.status, contextTruncated: context.truncated === true, redact: (text) => text, ...(Number.isSafeInteger(job.trigger?.matched?.index) ? { triggerIndex: job.trigger.matched.index } : {}) });
2745
+ if (!captured?.ok || captured.status !== "unassigned") log("question_capture_partial", { status: captured?.status ?? "source-unavailable" });
2746
+ } catch { log("question_source_unavailable", {}); }
2747
+ }
2748
+
2749
+ if (deps.approvalRequest) {
2750
+ const ended = await deps.approvalRuntime.finish({ requestId: deps.approvalRequest.requestId, jobId: deps.approvalJobId });
2751
+ if (!ended.ok) return { outcome: "policy", reason: "approval-finish-unavailable", exitCode: code, turns, tokens, budgetReserved: true, session: mergeSession(prepared, session, promoted) };
2752
+ }
2753
+ let approvalFacts = null;
2754
+ if (approvalParked) {
2755
+ // A failed publication must not re-run a paid container. Keep the saved transcript and name the repair.
2756
+ const fingerprint = promoted?.promoted === true && deps.approvalSessionCheck?.fingerprint ? deps.approvalSessionCheck.fingerprint({ sessionKey: prepared.session.key, ...approvalMetadata }) : null;
2757
+ const approvalSession = fingerprint?.ok ? { ...prepared.session, digest: fingerprint.digest } : prepared.session;
2758
+ const published = promoted?.promoted === true && isParked(job) ? await deps.approvalRuntime.afterPark({ job: { ...job, id: deps.queueJobId ?? sourceRunId }, session: approvalSession, metadata: approvalMetadata, rawJob: deps.rawApprovalJob ?? job }) : { refused: "approval-session-not-saved" };
2759
+ if (!published.ok) {
2760
+ // Issue #761: `{ newer: true }`, the live triggers file was written by a newer pi-dispatch, so the declaration
2761
+ // this park would be armed under cannot be judged here. NOT deferred, unlike the free gates: the container has
2762
+ // run and been paid for, and a deferral would run it again. No request is published, so nothing can decide or
2763
+ // resume this park on this build; recorded as the declaration that could not be read, exactly as an
2764
+ // unreadable file is, never as a store fault, and said in its own sentence, which names the upgrade.
2765
+ if (published.newer === true) log("approval_park_newer_release", {});
2766
+ await comment(job, published.newer === true ? APPROVAL_PARK_NEWER_COMMENT : APPROVAL_COMMENTS["approval-state-unavailable"]);
2767
+ return { outcome: "policy", reason: published.newer === true ? "approval-policy-unavailable" : (published.refused ?? "approval-state-unavailable"), exitCode: code, turns, tokens, budgetReserved: true, session: mergeSession(prepared, session, promoted) };
2768
+ }
2769
+ approvalFacts = published.audit;
2770
+ }
2771
+ // THE PARK SEQUENCE (issue #638 reordered it): promote, marker ON, release the window, take the held replies,
2772
+ // then the comment. Every step closes a gap the one before it leaves open:
2773
+ // - the marker AFTER the promotion (a reply it routes must find the `parked` mark and the transcript) and
2774
+ // BEFORE the release: a phrase-free reply in between routes as a parked reply, meets the window this run
2775
+ // still holds, and is HELD (the keeper still registers the run), so no reply slips past both;
2776
+ // - the release before the take, so the taken reply is enqueued WITH the window (held-settle.mjs) and becomes
2777
+ // the one job every later answer is coalesced against, never a second resumed run beside it;
2778
+ // - the comment LAST, because it invites an answer: with a reply taken it says the run continues
2779
+ // (HELD_TAKEN_COMMENT) instead of asking for one.
2780
+ // The marker stays on when a reply was taken; the resumed run clears it when it ends without parking, as any
2781
+ // run on the key does. Every park takes this path, a resumed run that parks again included. Each seam never
2782
+ // throws and is guarded anyway: a fault must not cost the person the comment that says how to continue. The
2783
+ // marker: set on a park, cleared when a run on a key that could park ends without parking; it decides nothing,
2784
+ // the sidecar the promotion just wrote does.
2785
+ if (parked) await parkedLabel(job, true, approvalParked);
1537
2786
  else if (job.awaitInput === true || job.parkedReply === true) await parkedLabel(job, false);
2787
+ if (parked) await Promise.resolve().then(releaseParkedWindow).catch(() => {});
2788
+ const taken = parked ? await Promise.resolve().then(takeHeldReplies).catch(() => null) : null;
2789
+ if (parked && !approvalParked && promoted?.promoted === true && deps.answerDraftRuntime && sourceRunId) {
2790
+ await deps.answerDraftRuntime.afterPark({ job, prepared, runId: sourceRunId, project: pickupProject, startedAt: now.getTime(), taken: Boolean(taken?.taken || Number(taken?.resumed) > 0) }).catch(() => log("answer_draft_enqueue_unavailable", {}));
2791
+ }
2792
+ // `taken` "other": a reply that got in between the release and the take already is the resumed job, so the
2793
+ // comment says the run continues there too, never "Reply here" (review round 2).
2794
+ if (parked) await comment(job, approvalParked ? APPROVAL_COMMENTS["awaiting-approval"] : taken?.taken === "here" || taken?.taken === "other" || Number(taken?.resumed) > 0 ? HELD_TAKEN_COMMENT : parkedComment(job.target));
1538
2795
  return {
1539
2796
  outcome: "completed",
1540
- ...(parked ? { reason: "awaiting-input" } : {}),
2797
+ ...(parked ? { reason: approvalParked ? "awaiting-approval" : "awaiting-input" } : {}),
2798
+ ...(approvalFacts ? { approval: approvalFacts } : {}),
1541
2799
  exitCode: code,
1542
2800
  turns,
1543
2801
  tokens,
@@ -1556,6 +2814,7 @@ export async function runJob(job, deps) {
1556
2814
  // record's `plan` is null otherwise, and every other result is unchanged.
1557
2815
  ...(plan ? { plan } : {}),
1558
2816
  ...(resources ? { resources } : {}),
2817
+ ...(progress ? { progress } : {}),
1559
2818
  };
1560
2819
  }
1561
2820
  case EXIT_POLICY: {
@@ -1582,7 +2841,7 @@ export async function runJob(job, deps) {
1582
2841
  // `why`. parseExitWhy keeps only a member of the closed COST_CAP_WHYS off a `cost-cap` line that said code 2,
1583
2842
  // and it rides only beside a `cost-cap` reason, so a forged line can at worst name the wrong rule of three.
1584
2843
  const why = reason === "cost-cap" && COST_CAP_WHYS.includes(exitWhy) ? { why: exitWhy } : {};
1585
- return { outcome: "policy", reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...why, ...(resources ? { resources } : {}) };
2844
+ return { outcome: "policy", reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...why, ...(resources ? { resources } : {}), ...(progress ? { progress } : {}) };
1586
2845
  }
1587
2846
  case EXIT_INFRA:
1588
2847
  // NO comment on any infra throw, here or in the catch: an InfraRetry may be retried and
@@ -1590,7 +2849,7 @@ export async function runJob(job, deps) {
1590
2849
  // the whole infra class lives at the terminal seam -- start.mjs's failed listener, guarded on
1591
2850
  // BullMQ's own finishedOn -- which also catches the stall-kill and wait-gate paths this
1592
2851
  // function never sees (issue #288).
1593
- throw new InfraRetry(`infra failure, container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources });
2852
+ return await recoverOrThrow(new InfraRetry(`infra failure, container exit ${code}`, { reason: CONTAINER_INFRA_EXIT, exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources }));
1594
2853
  default:
1595
2854
  // THE RUNTIME NEVER HANDED CONTROL TO THE RUNNER, in whatever integers this venue spells that
1596
2855
  // (issue #227). For docker it is 125 (`docker run` itself failed), 126 (the entrypoint exists
@@ -1608,7 +2867,7 @@ export async function runJob(job, deps) {
1608
2867
  // No `dollars` here: the hold is still standing, and the catch refunds it whole with the job-count slots.
1609
2868
  throw new InfraRetry(`the runtime could not start the container, exit ${code}`, { reason: "container-never-started", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), resources });
1610
2869
  }
1611
- throw new InfraRetry(`unknown container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources });
2870
+ return await recoverOrThrow(new InfraRetry(`unknown container exit ${code}`, { reason: CONTAINER_EXIT_UNKNOWN, exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources }));
1612
2871
  }
1613
2872
  } catch (e) {
1614
2873
  // A CONFIG-tagged throw is a determinate policy refusal wearing an exception, and issue #310 is the
@@ -1642,14 +2901,26 @@ export async function runJob(job, deps) {
1642
2901
  const hold = dollarHold;
1643
2902
  dollarHold = null;
1644
2903
  if ((e?.piDispatchConfig === true && !containerRan) || isNeverStartedRetry(e)) {
1645
- const { applied, of } = await releaseDollars(redis, hold, { log });
2904
+ // Issue #645: a RECORDED hold is given back only by the call that moves it `held` -> `released` in the run's
2905
+ // ledger; any other answer means someone else finished it or the ledger cannot say, and giving it back again
2906
+ // could refund money twice, so it stays at the floor. An unrecorded hold (its record failed) is given back
2907
+ // directly, as every hold was before: no one else can see it.
2908
+ const releasable = !holdRecorded || (await runBudgetLedger.release(runId, durable.seq)) === "moved";
2909
+ if (!releasable) log("run_budget_release_skipped", { seq: durable.seq });
2910
+ const { applied, of } = releasable ? await releaseDollars(redis, hold, { log }) : { applied: 0, of: 1 };
1646
2911
  const heldModel = (hold.ledgers ?? []).some((l) => modelPrefixesOf(modelDollars).has(l.keyPrefix));
1647
- dollars = dollarsRecord({ reservedMicros: hold.amountMicros, settledMicros: applied === of ? 0 : hold.amountMicros, basis: applied === of ? "refunded" : "floor", modelBasis: heldModel ? (applied === of ? "refunded" : "floor") : null });
2912
+ // Issue #761: along the whole hold, which the release walked: a key it gave back holds 0 of this job's money, so
2913
+ // a release that stopped part-way (the record then states the whole reservation) leaves those windows out.
2914
+ const reservedIn = deploymentWindowsOf(hold, { applied, target: 0, recorded: applied === of ? 0 : hold.amountMicros, valkey: valkeyTag });
2915
+ dollars = dollarsRecord({ reservedMicros: hold.amountMicros, settledMicros: applied === of ? 0 : hold.amountMicros, basis: applied === of ? "refunded" : "floor", modelBasis: heldModel ? (applied === of ? "refunded" : "floor") : null, reservedIn });
1648
2916
  } else {
1649
2917
  log("dollar_hold_unsettled", { reservedMicros: hold.amountMicros, error: e instanceof InfraRetry ? "infra-retry" : (e?.name ?? "error") });
1650
2918
  const heldModel = (hold.ledgers ?? []).some((l) => modelPrefixesOf(modelDollars).has(l.keyPrefix));
1651
- dollars = dollarsRecord({ reservedMicros: hold.amountMicros, settledMicros: hold.amountMicros, basis: "floor", modelBasis: heldModel ? "floor" : null });
2919
+ // Issue #761: nothing moved a key, so each holds the whole reservation, which is what this record states.
2920
+ dollars = dollarsRecord({ reservedMicros: hold.amountMicros, settledMicros: hold.amountMicros, basis: "floor", modelBasis: heldModel ? "floor" : null, reservedIn: deploymentWindowsOf(hold, { applied: 0, target: hold.amountMicros, recorded: hold.amountMicros, valkey: valkeyTag }) });
1652
2921
  }
2922
+ // A failed give-back becomes a settled floor, not a running job's reservation.
2923
+ publishDollarHold(null);
1653
2924
  }
1654
2925
  // The record reads `dollars` off the error on every throw path (buildRecord), so a retried or failed attempt says
1655
2926
  // what its windows were charged.
@@ -1660,6 +2931,14 @@ export async function runJob(job, deps) {
1660
2931
  // a frozen error keeps its own fields; the log lines above are the record of the hold
1661
2932
  }
1662
2933
  }
2934
+ // Issue #644: the same for the run's progress, so a retried or failed attempt's record says where it got to.
2935
+ if (progress !== null && e !== null && typeof e === "object" && e.progress === undefined) {
2936
+ try {
2937
+ e.progress = progress;
2938
+ } catch {
2939
+ // a frozen error keeps its own fields; progress is display only
2940
+ }
2941
+ }
1663
2942
 
1664
2943
  if (e?.piDispatchConfig === true && !containerRan) {
1665
2944
  // GUARDED, and the `held` list is the record of what actually happened. `releaseLedgers` is a loop of
@@ -1682,7 +2961,7 @@ export async function runJob(job, deps) {
1682
2961
  // classification that has ALREADY released the budget, and the job would escape as whatever the
1683
2962
  // comment threw, with the ledger refunded and the record saying something else entirely. The
1684
2963
  // shipped adapter never throws; this makes that a property of the arm rather than of the wiring.
1685
- await comment(job, "Refused: this deployment is misconfigured, so the job could not be started. Ask the operator to run `pi-dispatch doctor`. Not run.").catch(() => {});
2964
+ await comment(job, REFUSAL_COMMENTS["config-refused"]).catch(() => {});
1686
2965
  log("refused_config", { kind: job.kind ?? null, refunded });
1687
2966
  // budgetReserved reflects the LEDGER: whether the GLOBAL slot is still held after the refund (`globalHeld`).
1688
2967
  return { outcome: "policy", reason: "config-refused", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: globalHeld(), ...(dollars ? { dollars } : {}) }; // return => not retried
@@ -1693,10 +2972,13 @@ export async function runJob(job, deps) {
1693
2972
  // here (exit-1 infra, unknown exit) means the container ran and legitimately spent its slot,
1694
2973
  // so `reason` gates the release to the never-started case only. It releases only what `held` still
1695
2974
  // lists and run once per invocation; a BullMQ retry reserves afresh, so this cannot double-release.
1696
- if (isNeverStartedRetry(e)) {
2975
+ if (isNeverStartedRetry(e) && !slotOwnedByRun) {
1697
2976
  // All-or-none (issues #242 and #499 part B): a never-started container follows EVERY job-count reserve, so
1698
2977
  // every held ledger refunds together, last first, through the one helper -- and a scoped or project refusal
1699
2978
  // returned above with the ledgers before it already given back.
2979
+ // NOT for an armed run's slot (issue #645, `slotOwnedByRun`): the run's ledger says the slot is held, so the
2980
+ // retry carries it instead of reserving; refunding it here would let that retry run on a slot nothing counts.
2981
+ // The cost: a run whose every attempt never started keeps one slot counted, the safe side.
1700
2982
  await refundLedgers("container-never-started");
1701
2983
  }
1702
2984
  // After the refund, so it says what the ledger holds: false when never-started gave the global slot back, true
@@ -1704,18 +2986,13 @@ export async function runJob(job, deps) {
1704
2986
  if (e instanceof InfraRetry) e.budgetReserved = globalHeld();
1705
2987
  throw e;
1706
2988
  } finally {
2989
+ if (prepared && job.watch) await collectWatch({ job, prepared, completed: watchCompleted, policyRefused: watchPolicyRefused, usage: watchUsage }).catch(() => {});
2990
+ if (prepared && job.questionGrouping) await deps.questionGroupRuntime?.collect({ job, prepared, completed: watchCompleted }).catch(() => {});
2991
+ if (prepared && job.answerDraft) await deps.answerDraftRuntime?.collect({ job, prepared, completed: watchCompleted, result: answerUsage }).catch(() => {});
1707
2992
  if (prepared) await cleanup(prepared).catch(() => {});
1708
2993
  }
1709
2994
  }
1710
2995
 
1711
- /**
1712
- * The reason a job retried for the podman venue's rootless network keeper carries (issue #458, PR #463 round 2): a
1713
- * fixed token, as every run-record reason is, and the key the terminal comment is chosen by.
1714
- */
1715
- export const NETNS_KEEPER_NOT_HOLDING = "netns-keeper-not-holding";
1716
- /** Issue #476: a job held on a young keeper that kept restarting, or stayed young past the hold's bound. */
1717
- export const NETNS_KEEPER_CRASH_LOOP = "netns-keeper-crash-loop";
1718
-
1719
2996
  /** Thrown for the retryable (infra) class only. The BullMQ processor lets this propagate to retry. */
1720
2997
  /**
1721
2998
  * The one `session` object the run record carries, from the host's intent and the container's report
@@ -1736,6 +3013,13 @@ export const NETNS_KEEPER_CRASH_LOOP = "netns-keeper-crash-loop";
1736
3013
  function mergeSession(prepared, fromRunner, promoted = null) {
1737
3014
  const host = prepared?.session;
1738
3015
  if (!host && !fromRunner) return null;
3016
+ // Issue #643: a job that is not a resume job now runs on a live transcript file (`prepared.live`), which the runner
3017
+ // opens empty and reports as `absent`. That is a fact about the live file, not about resuming, and this object
3018
+ // describes resuming: such a job's record says `disabled`, exactly what the runner said for it before the live file
3019
+ // existed, so its record is byte-identical to one written before the feature. Narrow on purpose, the live file and
3020
+ // the runner's token exactly `absent`: any other report is carried as it always was, and resume jobs (`host`
3021
+ // present) never reach this line.
3022
+ if (!host && prepared?.live && fromRunner?.resumed === false && fromRunner.reason === "absent") return { resumed: false, reason: "disabled", bytes: null };
1739
3023
  // A HOST GATE THAT REFUSED OUTRANKS THE RUNNER'S `absent`, and without this rule it never reached a
1740
3024
  // record at all. A refused read stages a 0-byte file rather than nothing (session-store.mjs, where the
1741
3025
  // reasoning is pi's EEXIST race), the container is handed that file either way, and pi opens it and
@@ -1777,6 +3061,25 @@ const SECRET_FAILURES = {
1777
3061
  nul: "printed a value containing a NUL byte, which cannot survive the container's argv",
1778
3062
  };
1779
3063
 
3064
+ /**
3065
+ * The bundle resolver's failures (issue #704, secret-bundles.mjs), our words, the `SECRET_FAILURES` rule: an unknown
3066
+ * code degrades to the caller's generic sentence. None names a variable or a value: this text is posted publicly.
3067
+ */
3068
+ const BUNDLE_FAILURES = {
3069
+ exit: "refused the bundle's reference",
3070
+ empty: "printed no variable",
3071
+ overflow: "printed more than the bundle size cap allows",
3072
+ nul: "printed output containing a NUL byte",
3073
+ malformed: "printed output that is not in the format its profile declares",
3074
+ quoted: "printed a quoted dotenv value, and the dotenv grammar takes no quotes (declare the profile as json instead)",
3075
+ "not-string": "printed a JSON value that is not a string",
3076
+ "bad-name": "printed a name that is not an environment variable name",
3077
+ duplicate: "printed one variable twice",
3078
+ "empty-value": "printed a variable with an empty value",
3079
+ "value-too-large": "printed a value larger than one secret may be",
3080
+ "too-many": "printed more variables than one bundle may carry",
3081
+ };
3082
+
1780
3083
  /**
1781
3084
  * Is this throw the "the runner never ran" retry (issue #227), whichever path raised it: a never-started exit
1782
3085
  * (`isNeverStartedExit`), a spawn fault, or a pre-start gate? The catch's refunds (the job-count slots and, issue
@@ -1787,10 +3090,13 @@ function isNeverStartedRetry(e) {
1787
3090
  }
1788
3091
 
1789
3092
  export class InfraRetry extends Error {
1790
- constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved, dollars, resources } = {}) {
3093
+ constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved, dollars, resources, loaded } = {}) {
1791
3094
  super(message, cause ? { cause } : undefined);
1792
3095
  this.name = "InfraRetry";
1793
3096
  this.piDispatchRetry = true;
3097
+ // The message default is never recorded or passed on as text: the record's door (`recordReasonOf`) records a
3098
+ // reason that is no token as `worker-error`, and the failure hook flattens one to `infra`. A throw that knows its
3099
+ // cause passes a token.
1794
3100
  this.reason = reason ?? message;
1795
3101
  this.exitCode = exitCode ?? null;
1796
3102
  this.turns = turns ?? null;
@@ -1811,6 +3117,9 @@ export class InfraRetry extends Error {
1811
3117
  this.dollars = dollars ?? null;
1812
3118
  // Issue #596: what the container used, off its exit line, or null; set on a throw after a container ran.
1813
3119
  this.resources = resources ?? null;
3120
+ // Issue #800: what the run loaded, for a throw that REPLACES another (`runJob` sets it on the first), or that comes out
3121
+ // of the run's wrapper before the processor noted it. Absent otherwise, so an error that holds none says so.
3122
+ if (loaded !== undefined && loaded !== null) this.loaded = loaded;
1814
3123
  }
1815
3124
  }
1816
3125
 
@@ -1841,4 +3150,72 @@ export class NetnsKeeperYoungHold extends InfraRetry {
1841
3150
  }
1842
3151
  }
1843
3152
 
3153
+ /**
3154
+ * A recovery job held while the run it continues still has a container running on this host (issue #645). An
3155
+ * `InfraRetry` with `container-never-started`, so any path that does not know it still retries and refunds nothing it did
3156
+ * not take; `makeProcessor` knows it by `recoveryWaitMs`, and moves the job to the delayed set for that long without
3157
+ * spending an attempt (a recovery job has one), the move the two holds above make.
3158
+ */
3159
+ /**
3160
+ * A job whose project's secret bundle names a profile this host does not serve, on a multi-host worker's SHARED queue
3161
+ * (issue #704 review). An `InfraRetry`, so any path that does not know it still retries; `makeProcessor` knows it by
3162
+ * `bundleProfileHold`, and moves the job to the delayed set for the never-fits gate's recheck without spending an
3163
+ * attempt, so a host that declares the profile can take it.
3164
+ */
3165
+ export class SecretBundleProfileHold extends InfraRetry {
3166
+ constructor(message, options) {
3167
+ super(message, options);
3168
+ this.name = "SecretBundleProfileHold";
3169
+ this.bundleProfileHold = true;
3170
+ }
3171
+ }
3172
+
3173
+ /**
3174
+ * The HTTP status of a failed pull-request head read, or null: octokit's `status`, else the `returned <code>` the
3175
+ * fetch-based forge hosts put in their InfraRetry message (status only, never the body). Read only to tell a
3176
+ * determinate 401/403/404 from a fault worth retrying.
3177
+ */
3178
+ function headReadStatus(error) {
3179
+ if (Number.isInteger(error?.status)) return error.status;
3180
+ const m = /\breturned (\d{3})\b/.exec(String(error?.message ?? ""));
3181
+ return m ? Number(m[1]) : null;
3182
+ }
3183
+
3184
+ /**
3185
+ * Whether a failed head read is a RATE LIMIT (issue #704 review round 2): octokit's error carries the response headers,
3186
+ * and GitHub's primary limit answers 403 with `x-ratelimit-remaining: 0`, its secondary limit 403 or 429 with a
3187
+ * `retry-after`. A header is read by name only, never echoed. The fetch-based hosts surface no headers, and their
3188
+ * 429 is already outside the determinate set.
3189
+ */
3190
+ function headReadRateLimited(error) {
3191
+ const headers = error?.response?.headers;
3192
+ if (headers === null || typeof headers !== "object") return false;
3193
+ const get = (name) => (typeof headers.get === "function" ? headers.get(name) : headers[name]);
3194
+ const remaining = get("x-ratelimit-remaining");
3195
+ return String(remaining ?? "") === "0" || (get("retry-after") !== undefined && get("retry-after") !== null);
3196
+ }
3197
+
3198
+ /**
3199
+ * A job this worker must not run because something it reads was written by a newer pi-dispatch: the triggers file a
3200
+ * free gate reads live, or a checkpoint (`DES-STORE-VERSION-WHEN-NEEDED`). The processor's catch DEFERS it
3201
+ * (`newer-release`) rather than count an attempt, so the job waits for a worker that can read it, spending nothing.
3202
+ * `what` names the store, for the log line only.
3203
+ */
3204
+ export class NewerReleaseHold extends InfraRetry {
3205
+ constructor(what) {
3206
+ super(`${what} was written by a newer pi-dispatch`, { budgetReserved: false });
3207
+ this.name = "NewerReleaseHold";
3208
+ this.newerRelease = true;
3209
+ this.what = what;
3210
+ }
3211
+ }
3212
+
3213
+ export class RecoveryWaitHold extends InfraRetry {
3214
+ constructor(message, { waitMs = RECOVERY_WAIT_MS, ...options } = {}) {
3215
+ super(message, options);
3216
+ this.name = "RecoveryWaitHold";
3217
+ this.recoveryWaitMs = waitMs;
3218
+ }
3219
+ }
3220
+
1844
3221
  export { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY };