@edgehero/pi-dispatch 4.3.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/.env.example +94 -4
  2. package/README.md +20 -7
  3. package/deploy/console.service +52 -0
  4. package/deploy/docker-compose.yml +3 -1
  5. package/deploy/pi-dispatch-valkey.container +4 -2
  6. package/deploy/sandbox-gateway.service +23 -0
  7. package/deploy/worker-env-wrapper.cmd +6 -0
  8. package/deploy/worker-env-wrapper.sh +10 -1
  9. package/package.json +117 -19
  10. package/src/agent-snapshot.mjs +49 -0
  11. package/src/answerer-runtime.mjs +194 -0
  12. package/src/answerer.mjs +178 -0
  13. package/src/approval-policy.mjs +76 -0
  14. package/src/approval-runtime.mjs +188 -0
  15. package/src/approval-session.mjs +137 -0
  16. package/src/approval-state.mjs +170 -0
  17. package/src/auth-cli.mjs +86 -0
  18. package/src/auth-connections.mjs +145 -0
  19. package/src/auth-policy.mjs +32 -0
  20. package/src/auth-store.mjs +97 -0
  21. package/src/azure-host.mjs +66 -4
  22. package/src/azure-prompt.mjs +5 -4
  23. package/src/backend-conformance.mjs +34 -2
  24. package/src/backend-local.mjs +103 -34
  25. package/src/backend-openshell.mjs +1540 -0
  26. package/src/backend-podman.mjs +63 -2
  27. package/src/backend-registry.mjs +61 -4
  28. package/src/backends.mjs +395 -6
  29. package/src/bullmq-internals.mjs +208 -0
  30. package/src/canary-cli.mjs +28 -0
  31. package/src/canary-lock.mjs +41 -0
  32. package/src/canary-runtime.mjs +237 -0
  33. package/src/canary.mjs +53 -0
  34. package/src/cancel-cli.mjs +47 -34
  35. package/src/cancel-state.mjs +213 -15
  36. package/src/capabilities.mjs +36 -1
  37. package/src/capacity-cli.mjs +28 -8
  38. package/src/capacity-records.mjs +75 -16
  39. package/src/capacity.mjs +65 -0
  40. package/src/chat-cli.mjs +288 -0
  41. package/src/chat-fork.mjs +547 -0
  42. package/src/chat-mode.mjs +156 -0
  43. package/src/chat-view.mjs +720 -0
  44. package/src/checkpoint-paths.mjs +113 -0
  45. package/src/cli.mjs +128 -41
  46. package/src/comment-tables.mjs +322 -0
  47. package/src/community-auth-adapters.mjs +87 -0
  48. package/src/community-auth-helper.mjs +76 -0
  49. package/src/community-auth-metadata.mjs +6 -0
  50. package/src/community-auth-output-pins.mjs +38 -0
  51. package/src/community-auth-patches.mjs +168 -0
  52. package/src/community-auth-pins.mjs +38 -0
  53. package/src/config.mjs +199 -30
  54. package/src/connection.mjs +83 -7
  55. package/src/console-answer.mjs +113 -0
  56. package/src/console-audit.mjs +604 -0
  57. package/src/console-bootstrap.mjs +159 -0
  58. package/src/console-client.mjs +848 -0
  59. package/src/console-probe.mjs +106 -0
  60. package/src/console-tokens.mjs +459 -0
  61. package/src/container-names.mjs +51 -0
  62. package/src/container-spec.mjs +20 -1
  63. package/src/cpu-reserve.mjs +4 -1
  64. package/src/defer-reasons.mjs +200 -0
  65. package/src/defer-write.mjs +208 -0
  66. package/src/deployment-venue.mjs +23 -6
  67. package/src/docker-run.mjs +11 -0
  68. package/src/doctor-bundles.mjs +130 -0
  69. package/src/doctor-model-proposals.mjs +109 -0
  70. package/src/doctor-openshell.mjs +238 -0
  71. package/src/doctor-results.mjs +602 -0
  72. package/src/doctor-spawner.mjs +215 -0
  73. package/src/doctor.mjs +1663 -198
  74. package/src/dollar-budget.mjs +212 -3
  75. package/src/endpoint-status.mjs +100 -0
  76. package/src/env-allowlist.mjs +103 -9
  77. package/src/env-file.mjs +6 -4
  78. package/src/file-expect.mjs +47 -0
  79. package/src/forgejo-auth.mjs +4 -0
  80. package/src/forgejo-host.mjs +127 -31
  81. package/src/forgejo-prompt.mjs +5 -4
  82. package/src/forges.mjs +73 -15
  83. package/src/fork-job.mjs +58 -0
  84. package/src/fork-policy.mjs +23 -0
  85. package/src/fork-store.mjs +338 -0
  86. package/src/get-token.mjs +1 -1
  87. package/src/git-dirty.mjs +119 -14
  88. package/src/git-hardening.mjs +85 -0
  89. package/src/github-host.mjs +31 -6
  90. package/src/github-prompt.mjs +7 -16
  91. package/src/gitlab-host.mjs +42 -3
  92. package/src/gitlab-prompt.mjs +5 -4
  93. package/src/held-document.mjs +23 -0
  94. package/src/held-replies.mjs +325 -0
  95. package/src/held-settle.mjs +266 -0
  96. package/src/hook-runs.mjs +188 -0
  97. package/src/host-budget.mjs +8 -5
  98. package/src/host-clone.mjs +72 -0
  99. package/src/host-loads.mjs +557 -0
  100. package/src/host-registry.mjs +66 -6
  101. package/src/host-telemetry.mjs +92 -0
  102. package/src/http-body.mjs +78 -0
  103. package/src/image-preflight.mjs +156 -13
  104. package/src/import-pi.mjs +60 -172
  105. package/src/inbox-relay.mjs +482 -0
  106. package/src/inbox-settle.mjs +196 -0
  107. package/src/index.mjs +740 -129
  108. package/src/init.mjs +35 -5
  109. package/src/job-data-version.mjs +75 -0
  110. package/src/job-id.mjs +30 -15
  111. package/src/job-layout.mjs +22 -0
  112. package/src/job-size.mjs +16 -0
  113. package/src/live-jobs.mjs +14 -6
  114. package/src/live-probes.mjs +6 -2
  115. package/src/live-transcript.mjs +872 -0
  116. package/src/loaded-record.mjs +609 -0
  117. package/src/mcp-config.mjs +1 -1
  118. package/src/mcp-preflight.mjs +36 -5
  119. package/src/model-attempts.mjs +84 -0
  120. package/src/model-catalog.mjs +83 -0
  121. package/src/model-cooldowns.mjs +22 -0
  122. package/src/model-failover-protocol.mjs +11 -0
  123. package/src/model-failover.mjs +134 -0
  124. package/src/model-observation.mjs +50 -0
  125. package/src/model-proposal-protocol.mjs +21 -0
  126. package/src/model-proposals.mjs +75 -0
  127. package/src/model-ref.mjs +67 -10
  128. package/src/model-roles-file.mjs +24 -0
  129. package/src/model-roles.mjs +73 -0
  130. package/src/model-status.mjs +34 -0
  131. package/src/models-json.mjs +4 -1
  132. package/src/money.mjs +1 -1
  133. package/src/oauth-openai.mjs +121 -0
  134. package/src/older-state.mjs +249 -0
  135. package/src/on-failure.mjs +17 -13
  136. package/src/open-browser.mjs +31 -2
  137. package/src/opener-cli.mjs +135 -0
  138. package/src/opener-protocol.mjs +27 -0
  139. package/src/opener-terminal.mjs +128 -0
  140. package/src/openshell-cli.mjs +155 -0
  141. package/src/openshell-policy.mjs +163 -0
  142. package/src/outbox.mjs +14 -0
  143. package/src/output-cap.mjs +1 -0
  144. package/src/package-canary-record.mjs +26 -0
  145. package/src/package-canary.mjs +135 -0
  146. package/src/package-names.mjs +9 -0
  147. package/src/package-stage.mjs +418 -0
  148. package/src/packages.mjs +24 -16
  149. package/src/parked.mjs +2 -1
  150. package/src/pause-windows.mjs +136 -17
  151. package/src/podman-stack.mjs +40 -14
  152. package/src/portfolio-snapshot.mjs +26 -7
  153. package/src/prepare-github.mjs +41 -77
  154. package/src/prepare-local.mjs +32 -2
  155. package/src/prepare.mjs +45 -11
  156. package/src/private-store-boundary.mjs +70 -0
  157. package/src/processor.mjs +1554 -167
  158. package/src/projects.mjs +23 -0
  159. package/src/prompt-fence.mjs +18 -0
  160. package/src/provider-key.mjs +7 -0
  161. package/src/provider-steering.mjs +1 -0
  162. package/src/question-context.mjs +35 -0
  163. package/src/question-group-runtime.mjs +91 -0
  164. package/src/question-groups.mjs +171 -0
  165. package/src/queue.mjs +296 -55
  166. package/src/reason-catalog.mjs +448 -0
  167. package/src/receiver-status.mjs +230 -0
  168. package/src/record-reason.mjs +37 -0
  169. package/src/recovery-queue.mjs +566 -0
  170. package/src/redact-secrets.mjs +219 -0
  171. package/src/reserve-estimate.mjs +101 -0
  172. package/src/reserved-env.mjs +36 -1
  173. package/src/retention.mjs +67 -0
  174. package/src/run-budget-ledger.mjs +283 -0
  175. package/src/run-checkpoint.mjs +627 -0
  176. package/src/run-container.mjs +749 -33
  177. package/src/run-history.mjs +791 -45
  178. package/src/run-mirror.mjs +75 -7
  179. package/src/run-progress.mjs +751 -0
  180. package/src/runtime-observations.mjs +3 -1
  181. package/src/runtime-settings.mjs +272 -37
  182. package/src/sandbox-cli.mjs +78 -2
  183. package/src/sandbox-deployment.mjs +51 -0
  184. package/src/sandbox-gateway-protocol.mjs +378 -0
  185. package/src/sandbox-hosts.mjs +100 -0
  186. package/src/sandbox-inventory.mjs +83 -0
  187. package/src/sandbox-policy.mjs +225 -0
  188. package/src/sandbox-prepare-requests.mjs +145 -0
  189. package/src/sandbox-prepare.mjs +285 -0
  190. package/src/sandbox-refs.mjs +70 -0
  191. package/src/sandbox-session-monitor.mjs +4 -0
  192. package/src/sandbox-session-watch.mjs +124 -0
  193. package/src/sandbox-sessions.mjs +161 -0
  194. package/src/sandbox-store.mjs +63 -6
  195. package/src/sandbox-workspace-guard.mjs +39 -0
  196. package/src/sandbox.mjs +27 -200
  197. package/src/scheduler-stall-guard.mjs +14 -8
  198. package/src/schedules.mjs +19 -6
  199. package/src/secret-bundles.mjs +245 -0
  200. package/src/secret-profiles.mjs +103 -3
  201. package/src/secrets.mjs +19 -9
  202. package/src/service-env.mjs +38 -5
  203. package/src/service.mjs +638 -106
  204. package/src/session-key.mjs +60 -0
  205. package/src/session-store.mjs +168 -10
  206. package/src/setup-contract.mjs +40 -0
  207. package/src/setup-egress.mjs +113 -0
  208. package/src/setup-host-entry.mjs +33 -0
  209. package/src/setup-host.mjs +242 -0
  210. package/src/setup-lock.mjs +41 -0
  211. package/src/size-records.mjs +15 -3
  212. package/src/start.mjs +1353 -135
  213. package/src/status-comment.mjs +415 -0
  214. package/src/stderr-transcript.mjs +318 -0
  215. package/src/steer-ack.mjs +122 -0
  216. package/src/steer-cli.mjs +213 -0
  217. package/src/steer-framing.mjs +185 -0
  218. package/src/steer-inbox.mjs +693 -0
  219. package/src/store-version.mjs +139 -0
  220. package/src/subscription-config.mjs +56 -0
  221. package/src/subscription-limits.mjs +3 -0
  222. package/src/subscription-quota.mjs +259 -0
  223. package/src/subscription-runtime.mjs +123 -0
  224. package/src/subscription-status.mjs +42 -0
  225. package/src/subscription-wait.mjs +11 -0
  226. package/src/subscriptions.mjs +7 -1
  227. package/src/thinking-level.mjs +19 -0
  228. package/src/touch-outbox.mjs +61 -0
  229. package/src/touch-state.mjs +133 -0
  230. package/src/triggers-file.mjs +299 -39
  231. package/src/triggers.mjs +782 -40
  232. package/src/trust-paths.mjs +171 -0
  233. package/src/up.mjs +398 -51
  234. package/src/updates-cli.mjs +61 -0
  235. package/src/updates-look.mjs +127 -0
  236. package/src/updates-reconcile.mjs +182 -0
  237. package/src/updates-subscription-adapters.mjs +8 -0
  238. package/src/updates.mjs +252 -0
  239. package/src/valkey-auth.mjs +32 -9
  240. package/src/valkey-endpoint.mjs +14 -4
  241. package/src/wait-for.mjs +1 -1
  242. package/src/wait-state.mjs +62 -3
  243. package/src/watch-board.mjs +38 -0
  244. package/src/watch-closer.mjs +1 -1
  245. package/src/watch-contract.mjs +102 -0
  246. package/src/watch-decision.mjs +78 -0
  247. package/src/watch-drainer.mjs +14 -0
  248. package/src/watch-issue-adapter.mjs +44 -0
  249. package/src/watch-issue.mjs +111 -0
  250. package/src/watch-record.mjs +10 -0
  251. package/src/watch-runtime.mjs +167 -0
  252. package/src/watch-store.mjs +183 -0
  253. package/src/watch-trace.mjs +34 -0
  254. package/src/workspace-checkpoint.mjs +1750 -0
  255. package/src/zero-touch-fixes.mjs +99 -0
  256. package/src/zero-touch.mjs +225 -0
  257. package/src/repeat-slot.mjs +0 -13
package/src/start.mjs CHANGED
@@ -1,3 +1,32 @@
1
+ import { subscriptionRoutes } from "./subscription-runtime.mjs";
2
+ import { configuredSubscriptions } from "./subscription-config.mjs";
3
+ import { publishSubscriptionStatus } from "./subscription-status.mjs";
4
+ import { readBusy as canaryBusy } from "./canary-lock.mjs";
5
+ import { makeReadModelRoles } from "./model-roles-file.mjs";
6
+ import { makeWatchStore } from "./watch-store.mjs";
7
+ import { makeTouchStore } from "./touch-state.mjs";
8
+ import { makeTouchOutbox } from "./touch-outbox.mjs";
9
+ import { makeQuestionGroupStore } from "./question-groups.mjs";
10
+ import { makeQuestionGroupRuntime } from "./question-group-runtime.mjs";
11
+ import { makeAnswerDraftStore } from "./answerer.mjs";
12
+ import { makeAnswerDraftRuntime, readAnswerPending } from "./answerer-runtime.mjs";
13
+ import { forgeApiRoot } from "./forges.mjs";
14
+ import { enqueueAnswerDraftJob } from "./queue.mjs";
15
+ import { makeWatchRuntime } from "./watch-runtime.mjs";
16
+ import { makeWatchDrainer } from "./watch-drainer.mjs";
17
+ import { makeWatchIssuePublisher } from "./watch-issue.mjs";
18
+ import { makeWatchIssueAdapter } from "./watch-issue-adapter.mjs";
19
+ import { checkFailoverCandidate, failoverEgressRequired } from "./model-failover.mjs";
20
+ import { parseTriggers, parseTriggersDocument, TRIGGERS_FILE_VERSION } from "./triggers.mjs";
21
+ import { watchDurationMs } from "./watch-contract.mjs";
22
+ import { makeApprovalSessionCheck } from "./approval-session.mjs";
23
+ import { APPROVAL_FINISHED_TTL_MS, makeApprovalStore } from "./approval-state.mjs";
24
+ import { makeApprovalRuntime, approvalExpiryJobId } from "./approval-runtime.mjs";
25
+ import { makeCheckApprovalPolicy } from "./approval-policy.mjs";
26
+ import { APPROVAL_LABEL } from "./parked.mjs";
27
+ import { createForkStore } from "./fork-store.mjs";
28
+ import { makeCheckForkPolicy } from "./fork-policy.mjs";
29
+ import { thinkingLevelAllowed } from "./thinking-level.mjs";
1
30
  import { readdirSync, readFileSync, statSync, watch } from "node:fs";
2
31
  import { lookup as dnsLookup } from "node:dns/promises";
3
32
  import { homedir, networkInterfaces, release as osRelease, userInfo } from "node:os";
@@ -8,7 +37,9 @@ import { valkeyClientContext } from "./valkey-endpoint.mjs";
8
37
  import { authRefusalFor, makeRedisClient, onValkeyError, parseConnection, valkeyAuthState, valkeyPasswordFor } from "./connection.mjs";
9
38
  import { reconcileGated, reloadSchedules } from "./cron.mjs";
10
39
  import { makeGitHubAuth } from "./get-token.mjs";
11
- import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING } from "./processor.mjs";
40
+ import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, checkDraftAnswerSource } from "./processor.mjs";
41
+ import { consoleDraftOf } from "./console-answer.mjs";
42
+ import { FAILED_COMMENT, FAILED_COMMENT_BY_REASON, RECOVERY_HOST_LOST_COMMENT } from "./comment-tables.mjs";
12
43
  import { transientError } from "./transient.mjs";
13
44
  import { makeGitHubHost } from "./github-host.mjs";
14
45
  import { githubFailureFields } from "./octokit-log.mjs";
@@ -24,8 +55,10 @@ import { MAX_SLOTS, loadModelEndpoints, modelEndpointsPath, readOverlayModels }
24
55
  import { builtinModel, checkModelsKnown } from "./model-catalog.mjs";
25
56
  import { capabilityTokens, serializeCaps } from "./capabilities.mjs";
26
57
  import { cronFingerprint } from "./fingerprint.mjs";
27
- import { makeHostRegistry } from "./host-registry.mjs";
58
+ import { makeHostRegistry, readLiveHosts } from "./host-registry.mjs";
59
+ import { endpointStatusFields, ENDPOINT_STATUS_VERSION } from "./endpoint-status.mjs";
28
60
  import { liveJobsFields, liveJobsOf } from "./live-jobs.mjs";
61
+ import { makeSandboxTelemetry, parseBudgetFrom, parseCpuReserve } from "./host-telemetry.mjs";
29
62
  import { budgetField, readUserServiceLimits } from "./host-budget.mjs";
30
63
  import { makeCpuReserve, reservePlan } from "./cpu-reserve.mjs";
31
64
  import { makeImagePreflight } from "./image-preflight.mjs";
@@ -35,44 +68,73 @@ import { unenforcedSizeFlags } from "./job-size.mjs";
35
68
  import { makeCollectChain } from "./outbox.mjs";
36
69
  import { makeCollectPlan } from "./outbox-plan.mjs";
37
70
  import { makePortfolioSnapshot } from "./portfolio-snapshot.mjs";
38
- import { containerPackagePaths, readStageManifest } from "./packages.mjs";
71
+ import { containerPackagePaths, PACKAGES_SUBDIR, readStageManifest, STAGE_MANIFEST } from "./packages.mjs";
39
72
  import { makeCleanup, makeForgePreparers, makePrepareWorkspace } from "./prepare.mjs";
40
- import { listRunningSandboxes, makeSandboxNetworkSweeper, makeSandboxRuntimeWatch } from "./sandbox.mjs";
73
+ import { listRunningSandboxes, makeSandboxNetworkSweeper, makeSandboxRuntimeWatch, sandboxSyncRefusal } from "./sandbox.mjs";
74
+ import { createSandboxPreparer } from "./sandbox-prepare.mjs";
75
+ import { checkSandboxWorkspace } from "./sandbox-workspace-guard.mjs";
76
+ import { startSandboxPrepareWorker } from "./sandbox-prepare-requests.mjs";
77
+ import { makeSandboxInventory } from "./sandbox-inventory.mjs";
41
78
  import { makeRetentionSweep } from "./retention-sweep.mjs";
42
- import { makeSandboxReaper } from "./sandbox-store.mjs";
79
+ import { FORGE_JOB_RETENTION_S } from "./retention.mjs";
80
+ import { makeDoctorSpawner } from "./doctor-spawner.mjs";
81
+ import { hostHasPending, publishCanaryFacts, UPDATES_LOOK_EVERY_HOURS } from "./updates.mjs";
82
+ import { makePackageCanaryRuntime } from "./package-canary.mjs";
83
+
84
+ /** How often a host asks whether a person wants something of it, and how long applying it may take: several npm installs on a slow link. */
85
+ const UPDATES_RECONCILE_EVERY_MS = 2 * 60 * 1000;
86
+ const UPDATES_RECONCILE_TIMEOUT_MS = 15 * 60 * 1000;
87
+ import { fileURLToPath } from "node:url";
88
+ import { listSandboxes, sandboxDeadline, makeSandboxReaper } from "./sandbox-store.mjs";
43
89
  import { makeSessionStore } from "./session-store.mjs";
44
90
  import { scrubCredentials } from "./redact.mjs";
45
- import { makeCheckOnceSpent, makeCheckPortfolioFlag, makeCheckWaitSkew, makeDisarmOnce } from "./triggers-file.mjs";
91
+ import { makeCheckOnceSpent, makeCheckPortfolioFlag, makeCheckWaitSkew, makeDisarmOnce, makeFindSecretBundle } from "./triggers-file.mjs";
92
+ import { makeBundleResolver } from "./secret-bundles.mjs";
46
93
  import { WATCH_DEBOUNCE_MS, changedWhileArming, makeWatchCloser, readBeforeArming } from "./watch-closer.mjs";
47
- import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
48
- import { checkProjectRows, danglingProjectRows, dollarRowsWithoutCap, loadScopedLimits, SCOPED_LIMITS_VERSION, scopeClaimRows } from "./scoped-limits.mjs";
94
+ import { loadPauseWindows, PAUSE_WINDOWS_VERSION, pauseUntilMs } from "./pause-windows.mjs";
95
+ import { checkProjectRows, danglingProjectRows, dollarRowsWithoutCap, makeInFlight, loadScopedLimits, SCOPED_LIMITS_VERSION, scopeClaimRows } from "./scoped-limits.mjs";
49
96
  import { escapeControls, loadProjects, projectOf, projectsFingerprint } from "./projects.mjs";
50
97
  import { envelopeDigest, envelopeInsideJobPaths, loadEnvelopeChecked } from "./envelope.mjs";
51
98
  import { NO_ENVELOPE_FINGERPRINT, makeAllocationAudit, makeAllocationLogReaper, makeAllocationState } from "./allocation.mjs";
99
+ import { makeConsoleAuditReaper } from "./console-audit.mjs";
52
100
  import { optionalUsdMicros } from "./money.mjs";
53
101
  import { makeOnFailure } from "./on-failure.mjs";
102
+ import { makeHookRunsPublisher } from "./hook-runs.mjs";
54
103
  import { makeWaitChecker } from "./wait-check.mjs";
55
104
  import { makeWaitState } from "./wait-state.mjs";
56
- import { hostQueueName, makeQueue } from "./queue.mjs";
57
- import { endpointShown, execDockerBounded, makeContainerGone, makeDockerEndpointResolver, makeJobContainerLister, makeLocalBackend, makeReaper, makeStopContainer, quotedShown } from "./backend-local.mjs";
105
+ import { hostQueueName, makeQueue, enqueueWatchJob, discoverHostQueuesReporting, QUEUE } from "./queue.mjs";
106
+ import { endpointShown, execDockerBounded, makeDockerEndpointResolver, makeJobContainerLister, makeLocalBackend, makeReaper, makeStopContainer, quotedShown } from "./backend-local.mjs";
58
107
  import { NETNS_KEEPER_MIN_AGE_MS, NETNS_KEEPER_YOUNG_MARGIN_MS, runtimeFromFacts } from "./netns-keeper.mjs";
59
108
  import { makeBackendRegistry, reapAll, resolveBackendName } from "./backend-registry.mjs";
60
- import { DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_SERVICE_LOCAL, backendFor, isPerMachineHost, observationRefusalIsTransient, observationRefusals, unobservedFloor } from "./backends.mjs";
109
+ import { DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, OPENSHELL_BACKEND, OPENSHELL_BIND_BACKEND, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_SERVICE_LOCAL, backendFor, hostBackendsField, isPerMachineHost, observationRefusalIsTransient, observationRefusals, unobservedFloor } from "./backends.mjs";
110
+ import { makeOpenshellBackend, makeOpenshellReaper, openshellBootVersions } from "./backend-openshell.mjs";
61
111
  import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_INFO_TIMEOUT_MS, cachedPodmanInfo, decidePodmanJobUser, makePodmanBackend, makePodmanInfoReader, makePodmanReaper, observePodman, podmanConfRefusal, unavailableFor, podmanConfWidening, podmanJobUserRefusal, resolvePodmanImageUser } from "./backend-podman.mjs";
62
- import { PODMAN_RESTART_HOLD_EXPIRED, makePodmanServiceReader, onceFs, makeRootfulMemory, observeHost, observeRootfulConf, readRootfulService, rootfulConfRefusal, rootfulConfRetries, rootfulUnreadList, runtimeObservationKey } from "./runtime-observations.mjs";
112
+ import { makePodmanServiceReader, onceFs, makeRootfulMemory, observeHost, observeRootfulConf, readRootfulService, rootfulConfRefusal, rootfulConfRetries, rootfulUnreadList, runtimeObservationKey } from "./runtime-observations.mjs";
63
113
 
64
114
  import { makeRunContainer } from "./run-container.mjs";
65
115
  import { piProviders, providerKeyCandidates, resolveProviderCredential } from "./env-allowlist.mjs";
66
116
  import { parseAllowlist } from "./egress-allowlist.mjs";
67
- import { MCP_TRANSIENT_READ_CODES, jobProviders, makeMcpCheck, providerKeyNames } from "./mcp-preflight.mjs";
68
- import { RESERVED_ENV_NAMES } from "./triggers.mjs";
69
- import { makeSecretsResolver } from "./secrets.mjs";
70
- import { buildRecord, EXIT_OOM_KILLED, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, newerRecord, RUNNER_POLICY_REASONS, sanitizeJobId, UNREADABLE_RECORD } from "./run-history.mjs";
117
+ import { makeMcpCheck, reservedJobNames } from "./mcp-preflight.mjs";
118
+ import { makeSecretsResolver, realExecutable } from "./secrets.mjs";
119
+ import { servableSecretProfiles } from "./secret-profiles.mjs";
120
+ import { buildRecord, EXIT_OOM_KILLED, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, newerRecord, recordOfThisJob, recordReadable, RUNNER_POLICY_REASONS, sanitizeJobId, UNREADABLE_RECORD } from "./run-history.mjs";
71
121
  import { makeRunMirror, readMirroredRecord } from "./run-mirror.mjs";
72
- import { readOverlay, resolveSettings } from "./runtime-settings.mjs";
122
+ import { makeLiveTranscriptOpener } from "./live-transcript.mjs";
123
+ import { makeInboxRelay } from "./inbox-relay.mjs";
124
+ import { makeLateForge, settleInbox } from "./inbox-settle.mjs";
125
+ import { makeHeldSettle } from "./held-settle.mjs";
126
+ import { readProgressKey } from "./run-progress.mjs";
127
+ import { makeStatusCommenter, makeStatusCommentRegistry, renderStatusText, STATUS_OUTCOMES, STATUS_RETRYING } from "./status-comment.mjs";
128
+ import { readOverlay, resolveSettings, SETTINGS_VERSION } from "./runtime-settings.mjs";
129
+ import { JOB_DATA_VERSION } from "./job-data-version.mjs";
73
130
  import { usdFingerprint } from "./dollar-fingerprint.mjs";
74
131
  import { authoredCron, envelopeJobPaths, loadSchedules, servedSchedules } from "./schedules.mjs";
75
132
  import { makeStallGuard } from "./scheduler-stall-guard.mjs";
133
+ import { makeWorkspaceCheckpointer } from "./workspace-checkpoint.mjs";
134
+ import { makeRunBudgetLedger } from "./run-budget-ledger.mjs";
135
+ import { normalizeHost } from "./run-checkpoint.mjs";
136
+ import { isRecoveryArmed, makeHostLostSweep, makeRecoveryEnqueuer, makeSalvager, pointedJobDirs, runIdOf, sweepOrphanJobDirs } from "./recovery-queue.mjs";
137
+ import { finishModelAttempt, modelAttemptIdOf } from "./model-attempts.mjs";
76
138
 
77
139
 
78
140
  /** How long boot will wait for `docker image inspect` before shipping without a digest. */
@@ -142,6 +204,18 @@ const WORKER_VERSION = (() => {
142
204
  // the same handle); re-exported here so every existing importer keeps its address.
143
205
  export { makeWatchCloser } from "./watch-closer.mjs";
144
206
 
207
+ /** The bound on `recordRun`'s final status edit for an attempt that registered no commenter, as the commenter's own. */
208
+ const STATUS_UNREGISTERED_EDIT_MS = 4_000;
209
+
210
+ /**
211
+ * The approval expiry job's options (issue #649): its id is the request's own (`approvalExpiryJobId`), and it is a
212
+ * forge-kind job built from the request's `resumeJob`, so it is kept as long as a forge job (`FORGE_JOB_RETENTION_S`,
213
+ * in SECONDS: BullMQ's `removeOn*.age`), the window in which its id still answers a second enqueue as the same job.
214
+ */
215
+ export function approvalExpiryJobOpts(jobId) {
216
+ return { jobId, attempts: 2, backoff: { type: "exponential", delay: 5000 }, removeOnComplete: { age: FORGE_JOB_RETENTION_S }, removeOnFail: { age: FORGE_JOB_RETENTION_S } };
217
+ }
218
+
145
219
  /**
146
220
  * Race a read against a fuse, and CLEAN THE FUSE UP whichever side wins (issue #300). The fuse is
147
221
  * unref'd, deliberately: it exists so a wedged docker daemon cannot hold boot, and it must never itself
@@ -174,14 +248,175 @@ export async function settleWithin(promise, ms, fallback) {
174
248
  }
175
249
  }
176
250
 
251
+ /**
252
+ * How long a queue's close waits for each of its two steps before it stops waiting (issue #813): first for the
253
+ * connection's own start, then for the close itself, after which the queue is disconnected. Five seconds, the bound the
254
+ * backend registry refusal already gave the same handles, which reads it from here. Against a live server the start
255
+ * settles in milliseconds (it is the INFO below) and a queue's close in about as long, so neither bound is reached by a
256
+ * queue's close that would have ended. A Worker's close also waits for the job it is running, so the one Worker closed
257
+ * this way gets a longer close step (`SANDBOX_CONTROL_CLOSE_BOUND_MS`).
258
+ */
259
+ export const QUEUE_CLOSE_BOUND_MS = 5_000;
260
+
261
+ /**
262
+ * The sandbox control Worker's bound for its CLOSE step (issue #813); its start keeps `QUEUE_CLOSE_BOUND_MS`. A
263
+ * Worker's close waits for the job it is running, and nothing aborts that job first: the control Worker is not among
264
+ * the workers `createWorker`'s stop cancels, and a human's preparation (a clone, a checkout) is not a paid run to cut
265
+ * short. Under the queues' five seconds a preparation still running was cut off with Valkey up (measured: an 8 s job
266
+ * left `active` at 5001 ms, where the unbounded close waited 8006 ms and the job completed). Twenty seconds, so the two
267
+ * waits together (25 s) stay inside the 30 s that systemd (`TimeoutStopSec`) and launchd (`ExitTimeOut`) give a stop
268
+ * (`service.mjs`). nssm waits 15 s after its Ctrl-C (`AppStopMethodConsole 15000`), so on Windows a preparation past
269
+ * about 15 s is still ended by nssm's later stop methods, as it was under the unbounded wait. One that outlasts this
270
+ * bound is disconnected like a queue, its job left `active` for BullMQ's stalled check.
271
+ */
272
+ export const SANDBOX_CONTROL_CLOSE_BOUND_MS = 20_000;
273
+
274
+ const TIMED_OUT = Symbol("timed out");
275
+
276
+ /**
277
+ * `promise`'s own outcome, `{ value }` or `{ error }`, or `TIMED_OUT` once `ms` has passed; never rejects. The timer is
278
+ * REF'D, unlike `settleWithin`'s fuse, and cleared whichever side settles: a queue is closed where nothing else may hold
279
+ * the event loop (a refused boot draining to its exit code, a test's teardown), and an unref'd timer awaited there never
280
+ * fires, so node would abandon the await with the queue still open. Cleared, it holds the loop no longer than the wait.
281
+ */
282
+ async function outcomeWithin(promise, ms) {
283
+ let timer = null;
284
+ try {
285
+ return await Promise.race([
286
+ Promise.resolve(promise).then(
287
+ (value) => ({ value }),
288
+ (error) => ({ error }),
289
+ ),
290
+ new Promise((resolve) => {
291
+ timer = setTimeout(() => resolve(TIMED_OUT), ms);
292
+ }),
293
+ ]);
294
+ } finally {
295
+ clearTimeout(timer);
296
+ }
297
+ }
298
+
299
+ /**
300
+ * The close `closeQueueSettled` and `settleCloseOf` share. It waits for the queue's start (`waitUntilReady`), at most
301
+ * `readyMs`, its failure ignored; then runs `close`, at most `closeMs`. A close that settled in time answers with its
302
+ * own value or rejection. One that did not is left pending and the queue is disconnected, as the backend registry
303
+ * refusal does: a queue whose connection came up closes by sending QUIT and awaiting the reply, which a server that
304
+ * stopped answering never sends. The disconnect is not awaited (BullMQ's awaits the start, which may never come), and
305
+ * its rejection is observed so it cannot surface as an unhandled one.
306
+ *
307
+ * After the close, whatever its outcome, each of the queue's connections gets a no-op `error` listener. BullMQ's
308
+ * `RedisConnection` constructor re-emits its start's rejection as an `error` on itself (`initializing.catch`), and its
309
+ * close strips every listener from it first (`removeAllListeners`). When the close lands during a connect attempt
310
+ * after a failed one, on a connection that never became ready, the start then rejects with the failed attempt's error
311
+ * and the emit finds no listener: an unhandled rejection (measured: now and then against a refused port, and every
312
+ * close after about 10 s against a host that drops packets). Attached before the close, the listener would be stripped
313
+ * with the rest. Each in a try: the connections are BullMQ's own objects, and nothing here may replace the close's
314
+ * answer.
315
+ */
316
+ async function closeWithin(queue, close, { readyMs, closeMs }) {
317
+ await outcomeWithin(Promise.resolve().then(() => queue.waitUntilReady?.()), readyMs);
318
+ const closed = await outcomeWithin(Promise.resolve().then(close), closeMs);
319
+ for (const connection of [queue.connection, queue.blockingConnection]) {
320
+ try {
321
+ connection?.on?.("error", () => {});
322
+ } catch {
323
+ // a connection that takes no listener has none to re-emit to
324
+ }
325
+ }
326
+ if (closed !== TIMED_OUT) {
327
+ if ("error" in closed) throw closed.error;
328
+ return closed.value;
329
+ }
330
+ try {
331
+ void Promise.resolve(queue.disconnect?.()).catch(() => {});
332
+ } catch {
333
+ // a queue with nothing left to drop
334
+ }
335
+ return undefined;
336
+ }
337
+
338
+ /**
339
+ * Closes a BullMQ Queue only after its connection's own start has settled, and never throws. BullMQ's
340
+ * `RedisConnection.init` sends an INFO (the server's version and type) that nothing in BullMQ awaits; a queue closed
341
+ * while that INFO is still pending on a fail-fast client (`enableOfflineQueue: false`) rejects it as an UNHANDLED
342
+ * rejection, inside BullMQ ("Stream isn't writeable and enableOfflineQueue options is false", measured in CI on the
343
+ * boot that refuses right after making the reconcile queue, issue #645's PR). Waiting for `waitUntilReady` (its failure
344
+ * swallowed here) lets that INFO land or fail inside the init's own handling first.
345
+ *
346
+ * BOUNDED, by `QUEUE_CLOSE_BOUND_MS` (issue #813). A long-lived queue's start never ends when nothing answers
347
+ * VALKEY_URL: BullMQ's `waitUntilReady` resolves on `ready` and rejects only on `end`, and the client retries forever,
348
+ * so the unbounded wait held a stopping worker until its supervisor killed it, and the release workflow's suite for six
349
+ * hours. A fail-fast client's start does end, but not always inside the bound: against a host that drops packets the
350
+ * reconcile queue's client gives up after about 6.4 s (three 2 s connect timeouts and two 200 ms retry delays,
351
+ * `parseConnection`). Once the bound has passed the queue is closed anyway: BullMQ's close of a connection still
352
+ * starting disconnects it and swallows the start's own rejection, and the start's re-emitted `error` meets the
353
+ * listener `closeWithin` leaves behind. It is the hazard `receiver/src/start.mjs` avoids by releasing the raw client
354
+ * instead. Exported for the test, with the bound as an option for the same reason.
355
+ */
356
+ export async function closeQueueSettled(queue, { boundMs = QUEUE_CLOSE_BOUND_MS } = {}) {
357
+ if (!queue) return;
358
+ try {
359
+ await closeWithin(queue, () => queue.close?.(), { readyMs: boundMs, closeMs: boundMs });
360
+ } catch {}
361
+ }
362
+
363
+ /**
364
+ * Routes a queue's own `close` through `closeQueueSettled`'s bounded wait, in place, so every closer that holds the
365
+ * queue (the closer list, the worker's shutdown) gets the settled close without a wrapper object. Unlike
366
+ * `closeQueueSettled` it keeps the close's own answer, a rejection included, when the close settles in time. Idempotent,
367
+ * and a queue with no `close` is left as it is. `closeBoundMs` bounds the close step alone, `boundMs` by default: a
368
+ * Worker whose close waits for a job it must let finish passes a longer one (`SANDBOX_CONTROL_CLOSE_BOUND_MS`).
369
+ */
370
+ export function settleCloseOf(queue, { boundMs = QUEUE_CLOSE_BOUND_MS, closeBoundMs = boundMs } = {}) {
371
+ if (!queue || typeof queue.close !== "function" || queue.close.settled === true) return queue;
372
+ const close = queue.close.bind(queue);
373
+ const settled = (...args) => closeWithin(queue, () => close(...args), { readyMs: boundMs, closeMs: closeBoundMs });
374
+ settled.settled = true;
375
+ queue.close = settled;
376
+ return queue;
377
+ }
378
+
379
+ /**
380
+ * The triggers file's live reload (issue #761), EXPORTED apart from its watcher like `reloadPauseWindows`. The cron
381
+ * reload is `reloadSchedules`'s, unchanged: a file that does not load keeps the running schedules
382
+ * (`schedules_reload_invalid`). One state rides beside it, `state.newer`: SET when the loader refused the file as one
383
+ * a newer pi-dispatch wrote (its configError's `.newer`), logged `triggers_reload_newer`, and CLEARED the next time the
384
+ * file loads. An edit that breaks the file leaves it as it was: that says nothing about what the newer file held. The
385
+ * processor defers every job while it is set (`triggersNewer`), since the last good copy every reader would fall back
386
+ * to is no longer what the operator wrote. `reload` and `load` are injectable for tests; the rest is
387
+ * `reloadSchedules`' own options.
388
+ */
389
+ export async function reloadTriggersFile(config, queue, state, { reload = reloadSchedules, load = loadSchedules, ...options } = {}) {
390
+ let loaded = false;
391
+ let thrown = null;
392
+ const loadFn = (cfg, opts) => {
393
+ try {
394
+ const schedules = load(cfg, opts);
395
+ loaded = true;
396
+ return schedules;
397
+ } catch (error) {
398
+ thrown = error;
399
+ throw error;
400
+ }
401
+ };
402
+ const result = await reload(config, queue, { ...options, loadFn });
403
+ if (loaded) {
404
+ state.newer = false;
405
+ } else if (thrown?.newer === true) {
406
+ state.newer = true;
407
+ options.log?.("triggers_reload_newer", { reason: thrown?.message ?? String(thrown) });
408
+ }
409
+ return result;
410
+ }
411
+
177
412
  /**
178
413
  * Watch the DIRECTORY holding the triggers file (robust to the admin's atomic tmp+rename, which swaps the
179
414
  * inode a file-watch would lose), debounce, and re-reconcile the cron schedulers on change via
180
- * `reloadSchedules`. Best-effort: a platform without `fs.watch` logs and the worker keeps its boot-time
181
- * schedulers. The FSWatcher is unref'd (the debounce it arms is NOT), and the returned closer is what
182
- * `startWorker` registers so the watch dies with the worker that armed it (issue #295).
415
+ * `reloadTriggersFile` (which keeps `state.newer`, issue #761). Best-effort: a platform without `fs.watch` logs and
416
+ * the worker keeps its boot-time schedulers. The FSWatcher is unref'd (the debounce it arms is NOT), and the returned
417
+ * closer is what `startWorker` registers so the watch dies with the worker that armed it (issue #295).
183
418
  */
184
- function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot, afterReload = () => {}) {
419
+ function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot, afterReload = () => {}, state = { newer: false }, watchFn = watch) {
185
420
  const path = config.triggersFile;
186
421
  const dir = dirname(path) || ".";
187
422
  const file = basename(path);
@@ -193,13 +428,13 @@ function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot,
193
428
  // (issue #386). A baseline taken here would measure the arming instead.
194
429
  readBeforeArming(handles, readFile, atBoot);
195
430
  try {
196
- handles.watcher = watch(dir, (_event, changed) => {
431
+ handles.watcher = watchFn(dir, (_event, changed) => {
197
432
  if (handles.closed) return; // see makeWatchCloser: by construction, not by a delivery rule
198
433
  if (changed && changed !== file) return; // only our file (a null name -> reload to be safe)
199
434
  clearTimeout(handles.timer);
200
435
  // Issue #504 part B: a cron folder or a skills dir an edit adds is a new job path, so the envelope's place is
201
436
  // judged again after every triggers reload.
202
- handles.timer = setTimeout(() => void reloadSchedules(config, queue, { log: closer.reloadLog, ref, registry, tz, fleet }).then(() => afterReload(closer.reloadLog)), WATCH_DEBOUNCE_MS);
437
+ handles.timer = setTimeout(() => void reloadTriggersFile(config, queue, state, { log: closer.reloadLog, ref, registry, tz, fleet }).then(() => afterReload(closer.reloadLog)), WATCH_DEBOUNCE_MS);
203
438
  });
204
439
  handles.watcher.unref?.();
205
440
  log("triggers_watching", { path });
@@ -211,35 +446,54 @@ function watchTriggersFile(config, queue, log, ref, registry, tz, fleet, atBoot,
211
446
  // changed.
212
447
  if (changedWhileArming(handles, readFile)) {
213
448
  log("triggers_reread_after_arming", { path });
214
- void reloadSchedules(config, queue, { log: closer.reloadLog, ref, registry, tz, fleet }).then(() => afterReload(closer.reloadLog));
449
+ void reloadTriggersFile(config, queue, state, { log: closer.reloadLog, ref, registry, tz, fleet }).then(() => afterReload(closer.reloadLog));
215
450
  }
216
451
  return closer;
217
452
  }
218
453
 
454
+ /**
455
+ * The pause-windows reload, EXPORTED apart from its watcher so its two states are testable without fs.watch (the
456
+ * scoped-limits reload's precedent). A good file replaces `ref.current` and clears `ref.newer`. A bad edit keeps the
457
+ * last-good windows in effect (OQ-008 live-edit safety), logged `pause_windows_reload_invalid`: the pause gate never
458
+ * loses its config to a typo.
459
+ *
460
+ * A file written by a newer pi-dispatch (issue #761, the loader's configError with `.newer`) keeps the last-good
461
+ * windows too, and ALSO sets `ref.newer`, logged `pause_windows_reload_newer`: it is no typo, and those windows are not
462
+ * what the operator wrote any more, since the newer file may pause a job they would run. The processor defers every job
463
+ * (`newer-release`) while it is set, and only the next good reload clears it: an edit that then breaks the file says
464
+ * nothing about what the newer one paused.
465
+ */
466
+ export function reloadPauseWindows(config, ref, log, { load = loadPauseWindows } = {}) {
467
+ try {
468
+ ref.current = load(config);
469
+ ref.newer = false;
470
+ log("pause_windows_reloaded", { count: ref.current.length });
471
+ } catch (err) {
472
+ if (err?.newer === true) {
473
+ ref.newer = true;
474
+ log("pause_windows_reload_newer", { reason: err?.message });
475
+ } else {
476
+ log("pause_windows_reload_invalid", { reason: err?.message });
477
+ }
478
+ }
479
+ }
480
+
219
481
  /**
220
482
  * Watch the DIRECTORY holding the pause-windows file (same atomic-rename robustness as the triggers watch)
221
- * and hot-swap the in-memory windows in `ref.current` on change. A bad edit keeps the last-good windows in
222
- * effect (OQ-008 live-edit safety) — the pause gate never loses its config to a typo. Best-effort; the
483
+ * and hot-swap the in-memory windows in `ref.current` on change (`reloadPauseWindows`). Best-effort; the
223
484
  * FSWatcher is unref'd and the returned closer stops the watch with the worker (issue #295).
224
485
  */
225
- function watchPauseWindowsFile(config, ref, log, atBoot) {
486
+ function watchPauseWindowsFile(config, ref, log, atBoot, watchFn = watch) {
226
487
  const path = config.pauseWindowsFile;
227
488
  const dir = dirname(path) || ".";
228
489
  const file = basename(path);
229
490
  const handles = { watcher: null, timer: null, closed: false };
230
491
  const closer = makeWatchCloser(handles, log);
231
- const reload = () => {
232
- try {
233
- ref.current = loadPauseWindows(config);
234
- closer.reloadLog("pause_windows_reloaded", { count: ref.current.length });
235
- } catch (err) {
236
- closer.reloadLog("pause_windows_reload_invalid", { reason: err?.message });
237
- }
238
- };
492
+ const reload = () => reloadPauseWindows(config, ref, closer.reloadLog);
239
493
  const readFile = () => readFileSync(path, "utf8");
240
494
  readBeforeArming(handles, readFile, atBoot); // the BOOT LOAD's own bytes; see watch-closer (issue #386)
241
495
  try {
242
- handles.watcher = watch(dir, (_event, changed) => {
496
+ handles.watcher = watchFn(dir, (_event, changed) => {
243
497
  if (handles.closed) return; // see makeWatchCloser: by construction, not by a delivery rule
244
498
  if (changed && changed !== file) return;
245
499
  clearTimeout(handles.timer);
@@ -563,6 +817,36 @@ function watchModelEndpointsFile(config, ref, log, atBoot) {
563
817
  * per job (mintToken throws configError) rather than blocking startup. Collaborators are injectable
564
818
  * (defaulting to the real ones) so the wiring is testable offline with no Redis and no forge.
565
819
  */
820
+ /**
821
+ * Whether a status comment for `jobId` could contain `phrase` (issue #644). The receiver starts a job on a comment that
822
+ * merely CONTAINS the phrase (receiver filter.mjs `body.includes`), so a bot comment matching its own deployment's phrase
823
+ * (a phrase like "pi-dispatch" or "Turn") would be read as a new request on every edit. The text is fixed words, the job
824
+ * id and integers, so this renders it for every word form it can take (singular and plural counts, each failure part,
825
+ * each phase form, running, every outcome and the retry) and compares with every digit run collapsed to one `0` on both sides: the
826
+ * integers are the only part that varies, and a phrase holding a number matches any number there. Conservative on
827
+ * purpose: a false yes costs one comment, a false no a comment loop.
828
+ */
829
+ export function statusTextMayMatch(phrase, jobId, render = renderStatusText) {
830
+ if (typeof phrase !== "string" || phrase === "") return false;
831
+ const norm = (t) => String(t).replace(/[0-9]+/g, "0");
832
+ const needle = norm(phrase);
833
+ for (const c of [1, 2]) {
834
+ for (const failed of [0, c]) {
835
+ for (const interrupted of [0, c]) {
836
+ for (const phase of [null, { index: c - 1, done: null, total: null }, { index: c - 1, done: c, total: null }, { index: c - 1, done: c, total: c }]) {
837
+ for (const phases of [[], Array.from({ length: c + 1 }, () => "")]) {
838
+ for (const outcome of [undefined, ...STATUS_OUTCOMES]) {
839
+ const snapshot = { turn: c, steps: c, failed, interrupted, phase, phases };
840
+ if (norm(render({ jobId, snapshot, outcome })).includes(needle)) return true;
841
+ }
842
+ }
843
+ }
844
+ }
845
+ }
846
+ }
847
+ return false;
848
+ }
849
+
566
850
  export async function startWorker(
567
851
  env = process.env,
568
852
  {
@@ -582,6 +866,7 @@ export async function startWorker(
582
866
  makeAuth = makeGitHubAuth,
583
867
  makeHost = makeGitHubHost,
584
868
  createWorkerFn = createWorker,
869
+ makeRuntimeQueue: makeRuntimeQueueFn = makeQueue,
585
870
  makeCpuReserve: makeCpuReserveFn = makeCpuReserve,
586
871
  makeReaper: makeReaperFn = makeReaper,
587
872
  makeBackendRegistry: makeBackendRegistryFn = makeBackendRegistry,
@@ -592,16 +877,40 @@ export async function startWorker(
592
877
  makeLogSink: makeLogSinkFn = makeLogSink,
593
878
  makeRecordWriter: makeRecordWriterFn = makeRecordWriter,
594
879
  makeRunMirror: makeRunMirrorFn = makeRunMirror,
880
+ // Issue #761: the portfolio snapshot's factory, a seam so a wiring test sees the run index cap it is handed.
881
+ makePortfolioSnapshot: makePortfolioSnapshotFn = makePortfolioSnapshot,
595
882
  // The fleet copy's by-id read, seamed so a wiring test can answer it without a live mirror.
596
883
  readMirroredRecord: readMirroredRecordFn = readMirroredRecord,
597
884
  makeLogReaper: makeLogReaperFn = makeLogReaper,
598
885
  makeSandboxReaper: makeSandboxReaperFn = makeSandboxReaper,
886
+ createSandboxPreparer: createSandboxPreparerFn = createSandboxPreparer,
887
+ startSandboxPrepareWorker: startSandboxPrepareWorkerFn = startSandboxPrepareWorker,
888
+ makeSandboxInventory: makeSandboxInventoryFn = makeSandboxInventory,
889
+ listSandboxes: listSandboxesFn = listSandboxes,
599
890
  makeSandboxNetworkSweeper: makeSandboxNetworkSweeperFn = makeSandboxNetworkSweeper,
600
891
  // Issue #429: the one call that asks a runtime which sandboxes are open, seamed so a wiring test can drive the real
601
892
  // sandbox reaper against a real retention root without spawning docker or podman.
602
893
  listRunningSandboxes: listRunningSandboxesFn = listRunningSandboxes,
603
894
  makeRetentionSweep: makeRetentionSweepFn = makeRetentionSweep,
895
+ makeTouchStore: makeTouchStoreFn = makeTouchStore,
896
+ makeTouchOutbox: makeTouchOutboxFn = makeTouchOutbox,
897
+ makeQuestionGroupStore: makeQuestionGroupStoreFn = makeQuestionGroupStore,
898
+ makeQuestionGroupRuntime: makeQuestionGroupRuntimeFn = makeQuestionGroupRuntime,
899
+ // Issue #712: the worker's own doctor runs, a seam so a wiring test sees the interval and never spawns a doctor.
900
+ makeDoctorSpawner: makeDoctorSpawnerFn = makeDoctorSpawner,
901
+ makeUpdatesRunner: makeUpdatesRunnerFn = makeDoctorSpawner,
604
902
  makeRunContainer: makeRunContainerFn = makeRunContainer,
903
+ // Issue #643: the preparer factory, a seam so a test can see what the wiring hands it (`stageLive`).
904
+ makePrepareWorkspace: makePrepareWorkspaceFn = makePrepareWorkspace,
905
+ // Issue #644: the opener and the status commenter factories, seams so a wiring test sees the switches and the
906
+ // forge calls the wiring hands them.
907
+ makeLiveTranscriptOpener: makeLiveTranscriptOpenerFn = makeLiveTranscriptOpener,
908
+ // Issue #646: the steer inbox relay's factory, a seam so a wiring test sees the client and host it is bound to.
909
+ makeInboxRelay: makeInboxRelayFn = makeInboxRelay,
910
+ makeStatusCommenter: makeStatusCommenterFn = makeStatusCommenter,
911
+ // Issue #644: the progress key's bounded read, a seam so a wiring test can hand `recordRun` an earlier attempt's
912
+ // comment id without a Valkey.
913
+ readProgressKey: readProgressKeyFn = readProgressKey,
605
914
  makeSecretsResolver: makeSecretsResolverFn = makeSecretsResolver,
606
915
  makeImagePreflight: makeImagePreflightFn = makeImagePreflight,
607
916
  makeScopeClaimSweeper: makeScopeClaimSweeperFn = makeScopeClaimSweeper,
@@ -636,6 +945,11 @@ export async function startWorker(
636
945
  readPodmanInfo: readPodmanInfoFn = makePodmanInfoReader(),
637
946
  makePodmanReaper: makePodmanReaperFn = makePodmanReaper,
638
947
  makePodmanBackend: makePodmanBackendFn = makePodmanBackend,
948
+ // Issue #684: the openshell venue's three, podman's pattern: the boot's version read, its reaper and its bundle,
949
+ // each a seam because the real ones spawn `openshell`, and only while the venue is blessed.
950
+ readOpenshellVersions: readOpenshellVersionsFn = openshellBootVersions,
951
+ makeOpenshellReaper: makeOpenshellReaperFn = makeOpenshellReaper,
952
+ makeOpenshellBackend: makeOpenshellBackendFn = makeOpenshellBackend,
639
953
  makeGitLabAuth: makeGitLabAuthFn = makeGitLabAuth,
640
954
  makeGitLabHost: makeGitLabHostFn = makeGitLabHost,
641
955
  makeForgejoAuth: makeForgejoAuthFn = makeForgejoAuth,
@@ -663,6 +977,9 @@ export async function startWorker(
663
977
  // literal address every Valkey client of this worker then connects to. A seam: the real one reads /proc, probes
664
978
  // this host's addresses and resolves the name, none of which belongs in a wiring test.
665
979
  judgeValkey = defaultJudgeValkey,
980
+ // `fs.watch` for the triggers and pause-windows watchers, so a test can deliver a change itself: macOS can lose a
981
+ // real event outright (the rebuilt FSEvents stream, watch-closer.mjs), which is not what those tests are about.
982
+ watchFs = watch,
666
983
  // The scoped-limits watcher, injectable so a test can see what it is armed with (PR #549's review: the cap
667
984
  // thunk the reload warning reads). Production passes nothing and gets the real watcher.
668
985
  watchScopedLimits: watchScopedLimitsFn = watchScopedLimitsFile,
@@ -670,6 +987,21 @@ export async function startWorker(
670
987
  watchProjects: watchProjectsFn = watchProjectsFile,
671
988
  // The envelope watcher (issue #504 part B), injectable for the same reason.
672
989
  watchEnvelope: watchEnvelopeFn = watchEnvelopeFile,
990
+ // Issue #645 (durable runs): the checkpoint collaborators, seams so a wiring test sees what the boot hands each and
991
+ // what the processor and the opener are given. `loadRunCheckpoint` is the module holding the run checkpointer and
992
+ // the pointer reads (run-checkpoint.mjs), imported at boot rather than statically, so a test can hand in a double.
993
+ makeWorkspaceCheckpointer: makeWorkspaceCheckpointerFn = makeWorkspaceCheckpointer,
994
+ loadRunCheckpoint = () => import("./run-checkpoint.mjs"),
995
+ makeRunBudgetLedger: makeRunBudgetLedgerFn = makeRunBudgetLedger,
996
+ makeRecoveryEnqueuer: makeRecoveryEnqueuerFn = makeRecoveryEnqueuer,
997
+ // The host-lost sweep's factory: a wiring test reaches its `onLost` (the comment, the page, the record) without a
998
+ // live queue, since the harness's clients are closed by the time a test could drive a sweep.
999
+ makeHostLostSweep: makeHostLostSweepFn = makeHostLostSweep,
1000
+ // The boot sweep of orphaned job directories, a seam so a wiring test drives it against a directory of its own.
1001
+ sweepOrphanJobDirs: sweepOrphanJobDirsFn = sweepOrphanJobDirs,
1002
+ // Issue #812: the expiry a finished job's model attempt key is given, a seam so a wiring test sees which id the
1003
+ // listeners below hand it, since the harness's client is closed by the time a test drives one.
1004
+ finishModelAttempt: finishModelAttemptFn = finishModelAttempt,
673
1005
  } = {},
674
1006
  ) {
675
1007
  const config = loadConfigFn(env);
@@ -684,6 +1016,12 @@ export async function startWorker(
684
1016
  // read, its reaper and its bundle exist only while it is blessed, so a deployment that never chose it spawns no
685
1017
  // `podman` and pays nothing for it being in the table.
686
1018
  const podmanBlessed = config.backends.includes(PODMAN_BACKEND);
1019
+ // Issue #684: and for the openshell venue, blessable since this issue: its CLI is spawned only while it is blessed.
1020
+ // Issue #686 part 3: either of its two modes, which share the gateway, the CLI, the version read and the boot reaper;
1021
+ // each mode's own bundle is built only while that mode is blessed.
1022
+ const openshellCopyBlessed = config.backends.includes(OPENSHELL_BACKEND);
1023
+ const openshellBindBlessed = config.backends.includes(OPENSHELL_BIND_BACKEND);
1024
+ const openshellBlessed = openshellCopyBlessed || openshellBindBlessed;
687
1025
  // Issue #354: an ABSENT `observationPreflight` admits every job, which is the right answer only for a venue whose
688
1026
  // words hold without observing anything. A venue whose table entry names an `observedBy` and carries no preflight
689
1027
  // would have those words hold with nothing looking, and a floor naming one would pass on capability alone: the
@@ -719,7 +1057,10 @@ export async function startWorker(
719
1057
  // worker judges it itself, before it contacts Valkey at all, with the same function (`resolveWorkerValkey`), and
720
1058
  // every client below connects to the judged literal address, never to a name resolved again later. A refusal is a
721
1059
  // configError (exit 2, not restarted into the same answer); nothing answering is a plain error (exit 1, restarted).
722
- const valkey = await judgeValkey({ url: config.valkeyUrl, venues: { localUsed: localBlessed, podmanUsed: podmanBlessed }, env });
1060
+ // `openshellUsed` (issue #707, review round 2 of #715): an openshell venue on Podman is the queue's engine where no
1061
+ // `local` or `podman` venue is (`openshellSetupOf`), and the judgement refuses root's listener there as `up` does. Left
1062
+ // out, as it was, the rule saw no openshell venue and the boot took a root-held Valkey `up` had refused.
1063
+ const valkey = await judgeValkey({ url: config.valkeyUrl, venues: { localUsed: localBlessed, podmanUsed: podmanBlessed, openshellUsed: openshellBlessed }, env });
723
1064
  for (const note of valkey.notes ?? []) log("valkey_note", { note });
724
1065
  if (valkey.pinned) log("valkey_pinned", { address: valkey.pinned.address, port: valkey.pinned.port, heldBy: valkey.pinned.heldBy });
725
1066
  // Every client below connects through connection.mjs' JudgedConnector, which judges the (pinned) address again on
@@ -751,11 +1092,16 @@ export async function startWorker(
751
1092
  },
752
1093
  });
753
1094
  const schedules = { current: loadSchedules(config, { fleet: config.workerNameDeclared, ...recording("triggers", config.triggersFile) }) };
1095
+ // Issue #761: whether the triggers file on disk was written by a newer pi-dispatch, kept by the triggers watcher's
1096
+ // reload (`reloadTriggersFile`). At boot such a file is the loader's configError, so it starts false.
1097
+ const triggersState = { newer: false };
754
1098
 
755
1099
  // REQ-SCOPED-PAUSE-WINDOWS: load + validate the pause-windows file with the operator present and before any
756
1100
  // Valkey contact, so a malformed file refuses startup (configError) rather than silently disabling scoped
757
1101
  // pauses. Held in a mutable ref so the live-reload watcher can hot-swap it. [] means no scoped pauses.
758
- const pauseWindows = { current: loadPauseWindows(config, recording("pauseWindows", config.pauseWindowsFile)) };
1102
+ // `newer` (issue #761) is set by a live reload that met a file written by a newer pi-dispatch, which keeps these
1103
+ // windows (`reloadPauseWindows`); at boot such a file is the loader's configError, so it starts false.
1104
+ const pauseWindows = { current: loadPauseWindows(config, recording("pauseWindows", config.pauseWindowsFile)), newer: false };
759
1105
 
760
1106
  // Issue #242: same posture for the scoped-limits file -- fail-loud with the operator present, mutable
761
1107
  // ref for the live-reload watcher, [] when unset (the folder mutex is code and needs no file).
@@ -889,6 +1235,18 @@ export async function startWorker(
889
1235
 
890
1236
  const bootRefusal = jobUserBootRefusal(bootDecision, config.defaultBackend);
891
1237
  if (bootRefusal) throw configError(bootRefusal);
1238
+ // Issue #684: the openshell venue's releases, read once at boot (the half #683 left to the blessing). A CLI or a gateway
1239
+ // at another release than the one this venue was measured on refuses the boot (exit 2); a CLI that does not answer is
1240
+ // restarted into (exit 1); a gateway that does not answer refuses nothing, since every job's preflight reads it again.
1241
+ // Issue #712: whether the gateway is on this host, as the boot read it, for the registry row's `backends` word.
1242
+ // `unknown` while not read (a gateway down at boot): a word published per beat must not spawn the CLI per beat.
1243
+ let openshellGatewayWord = "unknown";
1244
+ if (openshellBlessed) {
1245
+ const versions = await readOpenshellVersionsFn({ gateway: config.openshell.gateway });
1246
+ if (versions?.refusal) throw versions.transient ? new Error(versions.refusal) : configError(versions.refusal);
1247
+ if (versions?.gatewayDown) log("openshell_gateway_down_at_boot", { gateway: config.openshell.gateway });
1248
+ if (typeof versions?.gatewayLocal === "boolean") openshellGatewayWord = versions.gatewayLocal ? "local" : "remote";
1249
+ }
892
1250
  // Issue #448: where a local job runs on rootful Podman's Docker API service on this host, the containers.conf that
893
1251
  // service reads (the system chain, root's own, the unit's CONTAINERS_CONF) and whether the running service started
894
1252
  // before it last changed. A VENUE refusal, as the podman venue's #428 one is and for its reason: with egress off no
@@ -1176,12 +1534,18 @@ export async function startWorker(
1176
1534
  // boot reaper for a venue it does not hold.
1177
1535
  // Issue #354: podman's the same way and for the same reason, `makeReaper` with `bin: "podman"`, so the sweep lists
1178
1536
  // the store this account's jobs actually ran in. Docker's listing says nothing about it, and the reverse.
1537
+ // Issue #684: the openshell venue's, by its owner label: crashed jobs' sandboxes, then their providers. Issue #686 part
1538
+ // 3: ONE reaper for both modes, registered under each blessed name and run once: the owner label is shared, a crash's
1539
+ // leftover of either mode is an orphan alike, and at boot no job of this process is live in either.
1540
+ const openshellReap = openshellBlessed ? makeOpenshellReaperFn({ gateway: config.openshell.gateway, log }) : null;
1179
1541
  backendReaps = {
1180
1542
  ...(localBlessed ? { [DEFAULT_BACKEND]: makeReaperFn({ log }) } : {}),
1181
1543
  ...(podmanBlessed ? { [PODMAN_BACKEND]: makePodmanReaperFn({ log }) } : {}),
1544
+ ...(openshellCopyBlessed ? { [OPENSHELL_BACKEND]: openshellReap } : {}),
1545
+ ...(openshellBindBlessed ? { [OPENSHELL_BIND_BACKEND]: openshellReap } : {}),
1182
1546
  ...Object.fromEntries(extraBackends.map((b) => [b?.name, b?.reap])),
1183
1547
  };
1184
- const swept = (await reapAll(Object.values(backendReaps), { log }))?.reaped === true;
1548
+ const swept = (await reapAll([...new Set(Object.values(backendReaps))], { log }))?.reaped === true;
1185
1549
  // PROVEN FOR THE WHOLE HOST, or not at all (issue #354). `reapAll` is conservative over the reapers it is
1186
1550
  // handed, and two gaps sit outside that. A BLESSED venue with no reaper here is refused by the registry, but
1187
1551
  // only further down, after the scope sweep has already acted on this answer. And a host without `local` can
@@ -1216,6 +1580,13 @@ export async function startWorker(
1216
1580
  // reaper above never reaches (it reaps the top-level `.log` and `.json` only). Never throws, so no double wrap.
1217
1581
  const reapAllocationLogs = makeAllocationLogReaper({ logsDir: config.logsDir, retentionDays: config.logRetentionDays, log });
1218
1582
  reapAllocationLogs();
1583
+ // Issue #640: the operator console's audit files (`console-audit/YYYY-MM.jsonl`). The worker runs it, not the console,
1584
+ // because the worker always runs and the console may never; it never reaps the newest file (the chain's head). Never
1585
+ // throws. Issue #761: on its OWN retention (`PI_CONSOLE_AUDIT_RETENTION_DAYS`, which inherits the run history's when
1586
+ // unset), so shortening the run history for privacy does not shorten the security trail. The periodic sweep below
1587
+ // re-runs this closure, so boot and every tick read the one value.
1588
+ const reapConsoleAudit = makeConsoleAuditReaper({ logsDir: config.logsDir, retentionDays: config.consoleAuditRetentionDays, log });
1589
+ reapConsoleAudit();
1219
1590
 
1220
1591
  // REQ-RESURRECTABLE-SANDBOX: sweep retained per-job directories past their window, so what `cleanup`
1221
1592
  // kept for re-opening stays bounded. Third in the row and deliberately its own sweep -- a different
@@ -1264,9 +1635,17 @@ export async function startWorker(
1264
1635
  // One raw Redis client, shared by the budget (via the worker) and the scheduler stall guard, so it is
1265
1636
  // hoisted out of the createWorkerFn arg object.
1266
1637
  const redis = makeRedisClient(valkey.url, { servername: valkey.servername, context: valkeyContext });
1638
+ const subscriptions = configuredSubscriptions({ config, redis, now: () => Number(now()) });
1267
1639
  // Its errors as one message-only line (PR #475's review, round 2), like every Queue and Worker's: without a listener
1268
1640
  // ioredis printed a stack per reconnect attempt.
1269
1641
  onValkeyError(redis, "shared client");
1642
+ // Accounting is repaired independently of work: publication failure must never replay a human action.
1643
+ const touchStore = makeTouchStoreFn({ redis, now, retentionDays: config.logRetentionDays });
1644
+ const touchOutbox = makeTouchOutboxFn({ dir: join(config.logsDir, "touch-outbox"), store: touchStore, log });
1645
+ const touchInitialized = await touchStore.initialize();
1646
+ if (!touchInitialized.ok) log("touch_tracking_unavailable", { reason: touchInitialized.error });
1647
+ await touchOutbox.repair();
1648
+ const questionGroupStore = makeQuestionGroupStoreFn({ redis, now, retentionDays: config.logRetentionDays });
1270
1649
 
1271
1650
  // Issue #504 part B: the applied split lives in Valkey (`alloc:plan`), shared by every host. Only with an envelope.
1272
1651
  // The boot reconcile seeds the neutral split when there is none, expires a plan past its life and re-bases on an
@@ -1311,7 +1690,9 @@ export async function startWorker(
1311
1690
  // collector enqueues chained children onto it -- the same pi-jobs queue, so one handle serves both.
1312
1691
  // Non-failFast: a long-lived handle rides out a Valkey blip. Registered as an extraCloser so shutdown
1313
1692
  // drains it after the worker.
1314
- const runtimeQueue = makeQueue(valkeyConn());
1693
+ // Install the settled close before ANY later boot refusal can reach these handles. Waiting until
1694
+ // the worker handoff left the backend registry's refusal racing BullMQ's initial INFO (#664).
1695
+ const runtimeQueue = settleCloseOf(makeRuntimeQueueFn(valkeyConn()));
1315
1696
 
1316
1697
  // THE HOST QUEUE (issue #57): work only this machine can do, because the folder lives here.
1317
1698
  //
@@ -1327,7 +1708,80 @@ export async function startWorker(
1327
1708
  // live triggers-file edit lands on the same queue the boot reconcile used; otherwise the shared
1328
1709
  // runtime queue, exactly as before. Registered as an extraCloser only when it is a NEW handle --
1329
1710
  // closing `runtimeQueue` twice would be closing another owner's connection.
1330
- const cronQueue = hostQueue ? makeQueue(valkeyConn(), { name: hostQueue }) : runtimeQueue;
1711
+ const cronQueue = hostQueue ? settleCloseOf(makeRuntimeQueueFn(valkeyConn(), { name: hostQueue })) : runtimeQueue;
1712
+
1713
+ // Issue #645 (durable runs): the checkpoint collaborators, built once here and handed to the processor and the live
1714
+ // transcript opener. Only with a store (`config.checkpointsDir`, PI_CHECKPOINTS_DIR else <PI_SESSIONS_DIR>/.checkpoints):
1715
+ // without one an armed job is refused by the processor before it spends (`recover-store-unset`), so nothing below may
1716
+ // pretend to checkpoint into nowhere. The run checkpointer's module is loaded unconditionally, because its pointer read
1717
+ // is what the STALL listener needs on every host of a fleet, store or not: the host that notices a stall is any host.
1718
+ let runCheckpointModule = null;
1719
+ try {
1720
+ runCheckpointModule = await loadRunCheckpoint();
1721
+ } catch (err) {
1722
+ // Fail loudly: every armed job on this host then falls back to today's path, and the operator is told why at boot.
1723
+ log("run_checkpoint_unavailable", { reason: scrubCredentials(err?.message) });
1724
+ }
1725
+ const durableHost = config.workerName;
1726
+ // `snapshotTimeoutMs` 20 s, not the module's 60 s default: a LIVE checkpoint snapshots a PAUSED container, so this bound
1727
+ // is how long a paid job can sit frozen (its provider stream stalled, its tool timers running on the clock). A tree too
1728
+ // big to walk in 20 s is skipped (`timeout`) and the run goes on, with its exit checkpoint as the fallback; 60 s of
1729
+ // freeze per checkpoint would be long enough for a provider to drop the stream the agent is waiting on.
1730
+ const forkStore = config.sessionsDir ? createForkStore({ dir: join(config.sessionsDir, ".fork-sources"), now, hostName: durableHost, hostQueue }) : null;
1731
+ const workspaceCheckpointer = config.checkpointsDir ? makeWorkspaceCheckpointerFn({ storeDir: config.checkpointsDir, now, log, hostName: durableHost, limits: { snapshotTimeoutMs: 20_000 } }) : null;
1732
+ const runCheckpointer =
1733
+ workspaceCheckpointer && typeof runCheckpointModule?.makeRunCheckpointer === "function"
1734
+ ? runCheckpointModule.makeRunCheckpointer({ forkStore, checkpointer: workspaceCheckpointer, redis, hostName: durableHost, hostQueue, now, log })
1735
+ : null;
1736
+ // The per-run dollar ledger (run-budget-ledger.mjs): on every host, since a recovery settles the crashed attempt's hold
1737
+ // wherever it runs, and an unarmed job never calls it.
1738
+ const runBudgetLedger = makeRunBudgetLedgerFn({ redis, now, log });
1739
+ const readCheckpointPointer = async ({ runId }) => {
1740
+ if (typeof runCheckpointModule?.readCheckpointPointer !== "function") return null;
1741
+ try {
1742
+ return await runCheckpointModule.readCheckpointPointer({ redis, runId, timeoutMs: 2_000 });
1743
+ } catch (err) {
1744
+ log("checkpoint_pointer_read_failed", { code: typeof err?.code === "string" ? err.code : "error" });
1745
+ return null;
1746
+ }
1747
+ };
1748
+ // The queues a recovery can be routed to: the shared one, this host's own, or another host's (the crashed host's,
1749
+ // when THIS host noticed the stall). Another host's handle is opened on first use and joins the closers then; the
1750
+ // array is read at shutdown, not at handoff, which is what makes a late push sound (see `extraClosers` below).
1751
+ const otherHostQueues = new Map();
1752
+ const recoveryQueues = {
1753
+ shared: () => runtimeQueue,
1754
+ host: (name) => {
1755
+ if (name === hostQueue) return cronQueue;
1756
+ let q = otherHostQueues.get(name);
1757
+ if (!q) {
1758
+ q = makeQueue(valkeyConn(), { name });
1759
+ otherHostQueues.set(name, q);
1760
+ extraClosers.push({ close: () => closeQueueSettled(q) });
1761
+ }
1762
+ return q;
1763
+ },
1764
+ };
1765
+ const enqueueRecovery = makeRecoveryEnqueuerFn({ redis, queues: recoveryQueues, hostQueueName, now, log });
1766
+ // Issue #686 part 3, review round 1 (#714): on the openshell-bind venue a lost attempt's job dir is a sandbox's bind,
1767
+ // and that sandbox may still be writing it, so the salvage first has the venue end it and confirm it gone. Resolved at
1768
+ // call time: the bundle is built further down, and a salvage runs only once jobs are consumed.
1769
+ const salvageCheckpoint = makeSalvager({ jobsDir: config.jobsDir, hostName: durableHost, workspaceCheckpointer, runCheckpointer, log, settleJobDir: (name) => (openshellBindBackend ? openshellBindBackend.settleJobDir(name) : true) });
1770
+ // The boot sweep of orphaned `jobs/job-*` directories: a crashed worker's processor never cleaned up after its jobs.
1771
+ // Here, before `createWorker` starts consuming, so no job of THIS process owns any of them; the age bound (the job
1772
+ // timeout plus the abort grace's margin) covers another process on this host. A directory a checkpoint pointer of this
1773
+ // host still names is kept for its recovery's salvage, and when those pointers cannot be read the sweep is skipped
1774
+ // rather than run blind.
1775
+ // The pointer read is BOUNDED: a Valkey that does not answer must not hold the boot (every command of this client waits
1776
+ // for a connection rather than failing), and an unanswered read is the same skip as a failed one.
1777
+ try {
1778
+ const keep = workspaceCheckpointer ? await settleWithin(pointedJobDirs({ redis, hostName: durableHost }), 5_000, null) : new Set();
1779
+ if (keep === null) throw new Error("the checkpoint pointers were not read within 5 s");
1780
+ await sweepOrphanJobDirsFn({ jobsDir: config.jobsDir, olderThanMs: JOB_TIMEOUT_MS + 5 * 60 * 1000, now, keep, log });
1781
+ } catch (err) {
1782
+ log("job_dir_sweep_skipped", { reason: scrubCredentials(err?.message) });
1783
+ }
1784
+ log("durable_runs", { checkpointsDir: config.checkpointsDir ?? null, checkpointer: Boolean(runCheckpointer), pointerReads: typeof runCheckpointModule?.readCheckpointPointer === "function" });
1331
1785
 
1332
1786
  // REQ-LOCAL-JOB-VISIBILITY durable run history, all host-side. The raw `.log` sink is gated on
1333
1787
  // captureJobLogs (raw container output is user-authored data, opt-in per no-pii-in-logs); the id-only
@@ -1371,8 +1825,138 @@ export async function startWorker(
1371
1825
  // a worker name: an unnamed one is a single host, its own files ARE the whole history, and a mirror
1372
1826
  // would be bytes nothing reads. That is also what keeps a single-host deployment byte-identical, since
1373
1827
  // no job then issues a single extra Valkey command.
1374
- const runMirror = config.workerNameDeclared ? makeRunMirrorFn({ redis, retentionDays: config.logRetentionDays, log }) : null;
1375
- const recordRun = ({ job, result, error, startedAt, endedAt, project, size = null, capacity = null, earlier = null }) => {
1828
+ // Issue #761: trimmed to this host's `PI_RUN_MIRROR_MAX_RUNS`; every writer trims the shared index to its own cap, so a
1829
+ // fleet's lowest wins, and each trim raises the horizon a reader sees the cut by.
1830
+ const runMirror = config.workerNameDeclared ? makeRunMirrorFn({ redis, retentionDays: config.logRetentionDays, indexMax: config.runMirrorMaxRuns, log }) : null;
1831
+ // Issue #644: the forge status comment's two calls, the `comment` adapter's shape below (forgeFor, the auth re-resolve,
1832
+ // a token minted per call) and its rule: they NEVER throw, since the comment decides nothing. `statusPost` answers
1833
+ // `{ id }` or null; `statusEdit` answers `{ ok: true }`, `{ stop }` for a 403, 404 or 429 (the comment is gone, the
1834
+ // token may not edit it, or the forge is rate limiting: editing on would only repeat the refusal) and `{ ok: false }`
1835
+ // for any other failure, which the commenter tries again on its next edit. Logs carry the job id and the status.
1836
+ const draftReplyMayPost = async job => {
1837
+ if (!consoleDraftOf(job?.console)) return true;
1838
+ try { return await checkDraftAnswerSource(job, { answerDraftStore, answerDraftRuntime }) === null; }
1839
+ catch { return false; }
1840
+ };
1841
+ const statusAuth = async (job) => {
1842
+ const forge = forgeFor(job);
1843
+ if (!forge?.host) return null;
1844
+ let auth = forge.auth ?? null;
1845
+ if (!auth) {
1846
+ try {
1847
+ auth = await ensureAuth(job.kind);
1848
+ } catch {
1849
+ auth = null;
1850
+ }
1851
+ }
1852
+ return auth ? { forge, auth } : null;
1853
+ };
1854
+ const statusPost = async (job, text) => {
1855
+ if (job?.answerDraft || job?.purpose === "answer-draft") return null;
1856
+ try {
1857
+ if (!await draftReplyMayPost(job)) return null;
1858
+ const ready = await statusAuth(job);
1859
+ if (!ready || typeof ready.forge.host.postStatusComment !== "function") return null;
1860
+ const token = await ready.auth.mintToken(job);
1861
+ const posted = await ready.forge.host.postStatusComment(job, job.target, text, token);
1862
+ return typeof posted?.id === "string" && posted.id !== "" ? { id: posted.id } : null;
1863
+ } catch (err) {
1864
+ log("status_comment_failed", { jobId: job?.id, op: "post", status: Number.isSafeInteger(err?.status) ? err.status : null });
1865
+ return null;
1866
+ }
1867
+ };
1868
+ const statusEdit = async (job, id, text) => {
1869
+ if (job?.answerDraft || job?.purpose === "answer-draft") return { stop: "answer-draft" };
1870
+ try {
1871
+ if (!await draftReplyMayPost(job)) return { stop: "answer-source-changed" };
1872
+ const ready = await statusAuth(job);
1873
+ if (!ready) return { ok: false }; // auth unreachable just now: the next edit asks again
1874
+ if (typeof ready.forge.host.editStatusComment !== "function") return { stop: "no-edit" };
1875
+ const token = await ready.auth.mintToken(job);
1876
+ await ready.forge.host.editStatusComment(job, job.target, id, text, token);
1877
+ return { ok: true };
1878
+ } catch (err) {
1879
+ if (err?.status === 403 || err?.status === 404 || err?.status === 429) return { stop: `http-${err.status}` };
1880
+ log("status_comment_failed", { jobId: job?.id, op: "edit", status: Number.isSafeInteger(err?.status) ? err.status : null });
1881
+ return { ok: false };
1882
+ }
1883
+ };
1884
+ // The commenter for one job, asked by the opener only when its trigger set `run.statusComment` and progress is on;
1885
+ // null for a job with no forge to post to (a local or cron job without a target) or a host that cannot edit, so such
1886
+ // a job runs with no comment rather than with a comment it could never update.
1887
+ // The commenters of the running jobs, by job id: the sink hands each its final snapshot when the container ends, and
1888
+ // `recordRun` below makes the final edit with the record's outcome (status-comment.mjs says why not the exit's).
1889
+ const statusComments = makeStatusCommentRegistry();
1890
+ const makeProgressCommenter = ({ job, jobId, commentId, onId, finalTimeoutMs }) => {
1891
+ const host = forgeFor(job)?.host;
1892
+ if (!host || typeof host.editStatusComment !== "function" || typeof host.postStatusComment !== "function" || !job?.target) {
1893
+ log("status_comment_skipped", { jobId: job?.id ?? null, reason: "no-forge-target" });
1894
+ return null;
1895
+ }
1896
+ // The comment must never trigger a job of its own: skipped, and said once for this job, when its text could hold
1897
+ // the phrase the job was started with. A job started otherwise carries no phrase (`matched.phrase` is the comment
1898
+ // route's), so a deployment whose comment trigger has a colliding phrase is caught only on its comment jobs.
1899
+ const phrase = job?.trigger?.matched?.phrase;
1900
+ if (statusTextMayMatch(phrase, jobId)) {
1901
+ log("status_comment_skipped", { jobId: job?.id ?? null, reason: "phrase-collision" });
1902
+ return null;
1903
+ }
1904
+ const commenter = makeStatusCommenterFn({ jobId, post: (text) => statusPost(job, text), edit: (id, text) => statusEdit(job, id, text), commentId, onId, log, ...(finalTimeoutMs ? { finalTimeoutMs } : {}) });
1905
+ return commenter ? statusComments.register(jobId, commenter) : null;
1906
+ };
1907
+ // Issue #643 (DES-LIVE-TRANSCRIPT-STREAM): every running job's transcript, redacted, to its own capped Valkey stream,
1908
+ // on every deployment unless PI_LIVE_TRANSCRIPTS=0. Not gated on a worker name, unlike the mirror above: the panel on
1909
+ // a single host reads the same stream. Handed to every venue's runContainer, which publish only beside a staged file,
1910
+ // or, on openshell (issue #686), from the exec's stderr records.
1911
+ //
1912
+ // Issue #644 (DES-RUN-PROGRESS): the same opener reads each job's progress off the same file unless PI_RUN_PROGRESS=0,
1913
+ // so it is built when either switch is on; with transcripts off it tails the file and writes no stream.
1914
+ //
1915
+ // Issue #645: the run checkpointer is the file's third reader (an armed job's quiet points schedule its checkpoints), so
1916
+ // with a checkpointer the opener is built even with both switches off; it then tails the file for the checkpoint alone
1917
+ // (no stream, no progress key). `checkpoint` is called per attempt with the open() arguments the attempt supplies.
1918
+ //
1919
+ // Issue #646: a steer-armed job's delivery acks are read off the same file (steer-ack.mjs, the fourth reader), so with a
1920
+ // Valkey client the opener is built even with every other reader off (`steer: true`); a job that is not armed hands no
1921
+ // ack consumer and is tailed exactly as before (or not at all).
1922
+ const steerWired = Boolean(redis);
1923
+ const openLiveTranscript =
1924
+ config.liveTranscripts || config.runProgress || runCheckpointer || steerWired
1925
+ ? makeLiveTranscriptOpenerFn({ redis, log, transcripts: config.liveTranscripts, progress: config.runProgress, makeCommenter: makeProgressCommenter, ...(runCheckpointer ? { checkpoint: (args) => runCheckpointer.open(args) } : {}), ...(steerWired ? { steer: true } : {}) })
1926
+ : null;
1927
+ // Issue #646 (DES-STEER-VIA-READONLY-INBOX-MOUNT): the inbox relay's opener, handed to every venue's runContainer, which
1928
+ // opens one per steer-ARMED attempt (on openshell, issue #686, one writing frames on the exec's stdin). Bound here to the worker's client and to the host name the run checkpointer
1929
+ // writes (`durableHost`), the one this host is known by in the fleet: the open key names it.
1930
+ const openInboxRelay = steerWired ? (args) => makeInboxRelayFn({ redis, host: durableHost, now, log, ...args,
1931
+ onDelivered: async ({ runId, id, at }) => {
1932
+ const intent = await touchOutbox.begin({ runId, cause: "steer", at, nonce: id });
1933
+ const result = await touchOutbox.applied(intent);
1934
+ if (!result.ok) log("touch_record_pending", { reason: result.error });
1935
+ },
1936
+ }) : null;
1937
+ // Issue #644: the final edit for an attempt that registered no commenter, so nothing else would edit the comment again.
1938
+ // That is an attempt refused before its container (a cost cap, a policy gate) after an earlier attempt posted the
1939
+ // comment and left it saying "attempt ended, retrying": with no edit here it said so for good. The comment's id is
1940
+ // the one the earlier attempt's final write put in the progress key; an attempt 1 refused before its container
1941
+ // finds none (nothing was posted) and edits nothing. One edit, from the record's outcome and the key's last snapshot
1942
+ // (the earlier attempt's counters, the last the run reached), bounded and never throwing; skipped, as the commenter
1943
+ // is, when the text could hold the job's own phrase.
1944
+ const settleUnregisteredStatus = async (job, outcome) => {
1945
+ try {
1946
+ const data = job?.data ?? {};
1947
+ // Issue #645: the progress key and the comment are the RUN's, so a recovery (`.rN`) finds the comment its run posted.
1948
+ const runId = runIdOf(job);
1949
+ if (statusTextMayMatch(data.trigger?.matched?.phrase, runId)) return;
1950
+ const key = await readProgressKeyFn({ redis, jobId: runId });
1951
+ if (!key?.commentId) return;
1952
+ const text = renderStatusText({ jobId: runId, snapshot: key.snapshot, outcome });
1953
+ const done = await settleWithin(statusEdit(data, key.commentId, text), STATUS_UNREGISTERED_EDIT_MS, null);
1954
+ if (done === null) log("status_comment_final_timeout", { jobId: job.id });
1955
+ } catch {
1956
+ // The comment decides nothing: a failure here costs a stale comment, never the record path.
1957
+ }
1958
+ };
1959
+ const recordRun = ({ job, result, error, startedAt, endedAt, project, size = null, capacity = null, earlier = null, maxTurns = null }) => {
1376
1960
  // The project (issue #499) was resolved at the pickup gate and rides here as `project` (an id or null), so a live
1377
1961
  // edit of projects.json mid-run cannot make the record disagree with what the job was counted against. A record
1378
1962
  // path that ends BEFORE the pickup gate (the wait gate's refusals) passes none, and resolves from the live ref
@@ -1382,8 +1966,24 @@ export async function startWorker(
1382
1966
  // four `recordRun` call sites byte-unchanged and `buildRecord` a pure function of its arguments.
1383
1967
  // The default venue rides the same way and for the same reason (#277): it is the value the registry
1384
1968
  // below is built with, so the record resolves a job's venue exactly as dispatch does.
1385
- const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId, size, capacity, earlier });
1969
+ const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId, size, capacity, earlier, maxTurns, touchTracking: true });
1386
1970
  writeRecord(record);
1971
+ // Issue #644: the status comment's last edit, from the record's classification. An infrastructure failure the queue
1972
+ // will retry says the attempt ended and is being retried, not "finished": BullMQ retries an error that is not
1973
+ // unrecoverable while attempts are left, and this record is written while the attempt still counts as running
1974
+ // (`attemptsMade` is the attempts finished before it). Never throws, never awaited; a job with no commenter is a
1975
+ // no-op.
1976
+ try {
1977
+ const attempts = Number.isSafeInteger(job?.opts?.attempts) ? job.opts.attempts : 1;
1978
+ // Issue #645: an attempt whose run continues as a recovery (`recovery-enqueued`) is retrying too, not failed: the
1979
+ // recovery picks the same comment up (it is the run's) and ends it with its own outcome.
1980
+ const retrying = (!result && error instanceof InfraRetry && record.attempt < attempts) || record.reason === "recovery-enqueued";
1981
+ const outcome = retrying ? STATUS_RETRYING : record.outcome;
1982
+ // The commenters are registered by the RUN id (the opener is handed it), so a `.rN` record settles its run's.
1983
+ const runId = runIdOf(job);
1984
+ if (statusComments.has(runId)) void statusComments.settle(runId, outcome);
1985
+ else if (job?.data?.statusComment === true) void settleUnregisteredStatus(job, outcome);
1986
+ } catch {}
1387
1987
  // STRICTLY AFTER the file, and deliberately not awaited. After, because a crash between the two must
1388
1988
  // leave a record with no fleet row rather than a fleet row with no record -- the mirror is a VIEW,
1389
1989
  // and a view that can outlive its source is a second source of truth. Not awaited, because this is
@@ -1408,17 +2008,39 @@ export async function startWorker(
1408
2008
  readRecord,
1409
2009
  readMirrored: runMirror ? (jobId) => readMirroredRecordFn(redis, sanitizeJobId(jobId)) : null,
1410
2010
  });
2011
+ // Issue #645: a run's record by job id, this host's file then the fleet's copy where a mirror is armed, for the
2012
+ // processor's recovery gate (a recovery whose base run already settled is superseded, not run). A recovery may land on
2013
+ // a host that never ran its base attempt, so a local-only read would miss exactly the fleet case. Never rejects;
2014
+ // unreadable or unreachable is null.
2015
+ // `keep`, when given, is the rule the caller's record must pass (the processor's: this run's settled record, issue
2016
+ // #811), applied to EACH side: this host's file answers only when it passes, and the fleet's copy is asked otherwise.
2017
+ // Without it a stale record an older job left in this host's file under a reused id would be the answer, and the
2018
+ // settled record in the mirror would never be looked at. Without `keep`, any record this host holds answers, as before.
2019
+ const readRunRecord = async (jobId, { keep = null } = {}) => {
2020
+ const kept = (record) => (record && record !== UNREADABLE_RECORD && (typeof keep !== "function" || keep(record)) ? record : null);
2021
+ const local = kept(readRecord(jobId));
2022
+ if (local) return local;
2023
+ if (!runMirror) return null;
2024
+ const mirrored = await readMirroredRecordFn(redis, sanitizeJobId(jobId)).catch(() => null);
2025
+ return kept(mirrored);
2026
+ };
1411
2027
  // The record a retry (or a pickup after a stall) is about to replace (issue #599), read BOTH ways: this host's file and
1412
2028
  // the fleet's copy where a mirror is armed. Both, because each host keeps its own file: a job that ran on A, then B,
1413
2029
  // then A again finds A's stale first attempt locally while B's second is in the mirror. `newerRecord` keeps the higher
1414
- // attempt (a tie: the later end). Its slot interval rides the new record as `earlier`. Never rejects: unreadable or
1415
- // unreachable is no record from that side.
1416
- const previousRecord = async (jobId) => {
2030
+ // attempt (a tie: the later end). Its slot interval rides the new record as `earlier`. The failed listener's lost-lock
2031
+ // check reads a job's own failed record through it too, for the same reason (issue #761). Never rejects: unreadable
2032
+ // or unreachable is no record from that side.
2033
+ // `keep`, when given, is the rule a caller's record must pass (`recordOfThisJob` for one attempt, pure and never
2034
+ // throwing), applied to EACH side before the higher attempt is chosen. Applied after, it would judge only the side
2035
+ // chosen: ids are reused, so a stale record an older job left at a higher attempt would win on attempt and then fail
2036
+ // the rule, and this attempt's own record on the other side would never be looked at.
2037
+ const previousRecord = async (jobId, { keep = null } = {}) => {
2038
+ const kept = (record) => (record === null || typeof keep !== "function" || keep(record) ? record : null);
1417
2039
  const read = readRecord(jobId);
1418
- const local = read === UNREADABLE_RECORD ? null : read;
2040
+ const local = kept(read === UNREADABLE_RECORD ? null : read);
1419
2041
  if (!runMirror) return local;
1420
2042
  const mirrored = await readMirroredRecordFn(redis, sanitizeJobId(jobId)).catch(() => null);
1421
- return newerRecord(local, mirrored === UNREADABLE_RECORD ? null : mirrored);
2043
+ return newerRecord(local, kept(mirrored === UNREADABLE_RECORD ? null : mirrored));
1422
2044
  };
1423
2045
 
1424
2046
  // INT-CONFIG-OVERLAY-CONTRACT: the worker reads the runtime-settings overlay at EACH job start, so this
@@ -1432,6 +2054,12 @@ export async function startWorker(
1432
2054
  // (issue #501), so an overlay window with an env per-job cap is valid. `secretProfiles` rides alongside the
1433
2055
  // effective keys; that function says why.
1434
2056
  const res = resolveSettings(config, readOverlay(settingsFile, { log }));
2057
+ // Issue #761: a file written by a newer pi-dispatch keeps `newer: true`, so the processor DEFERS the job
2058
+ // (`newer-release`) instead of refusing it as settings-overlay-invalid. Its own line: it is no broken file.
2059
+ if (res.invalid && res.newer === true) {
2060
+ log("settings_overlay_newer", { reason: res.invalid, settingsFile });
2061
+ return { invalid: res.invalid, newer: true };
2062
+ }
1435
2063
  if (res.invalid) {
1436
2064
  log("settings_overlay_invalid", { reason: res.invalid, settingsFile });
1437
2065
  return { invalid: res.invalid };
@@ -1460,6 +2088,52 @@ export async function startWorker(
1460
2088
  // (`OQ-009`), so the working tree it needs is the one this machine just used: routing it anywhere
1461
2089
  // else would enqueue a job only this host can run onto a queue every host drains.
1462
2090
  const collectChain = makeCollectChain({ queue: cronQueue, config, log });
2091
+ const watchStore = makeWatchStore(redis, { now });
2092
+ const watchIssueAdapter = makeWatchIssueAdapter({ getContext: async target => {
2093
+ const auth = forges[target.kind]?.auth ?? await ensureAuth(target.kind);
2094
+ if (!auth) throw configError("watch: configure the target forge credentials");
2095
+ const token = await auth.mintToken({ kind: target.kind, repo: target.repo, permissions: { issues: "write", metadata: "read" } });
2096
+ return { apiUrl: target.kind === "github" ? "https://api.github.com" : config[target.kind]?.apiUrl, token };
2097
+ } });
2098
+ const watchPublisher = makeWatchIssuePublisher({ redis, adapter: watchIssueAdapter, now,
2099
+ canClose: async mapping => {
2100
+ const doc = parseTriggersDocument(readFileSync(onceTriggersFile, "utf8"), onceTriggersFile);
2101
+ const rule = doc.triggers.find(t => t.on.type === "watch" && t.on.id === mapping.ruleId && t.on.source === mapping.source);
2102
+ const target = rule?.run.issue;
2103
+ return Boolean(doc.watchSources?.[mapping.source] && target?.closeAfter && target.kind === mapping.target.kind && target.repo === mapping.target.repo && watchDurationMs(target.closeAfter) === mapping.closeAfterMs);
2104
+ },
2105
+ reportClose: async (outcome, mapping) => { if (mapping?.ruleId) await watchStore.recordIssueOutcome(mapping.ruleId, outcome); },
2106
+ });
2107
+ const watchRuntime = makeWatchRuntime({ store: watchStore, triggersPath: onceTriggersFile, now, log, getSettings, pauseUntil: (job, at) => pauseUntilMs(pauseWindows.current, job, at), enqueue: (fields, opts) => enqueueWatchJob(cronQueue, fields, { now, ...opts }), publishIssue: watchPublisher });
2108
+ const questionGroupRuntime = makeQuestionGroupRuntimeFn({ store: questionGroupStore, triggersPath: onceTriggersFile, getSettings,
2109
+ resolveClassifier: ({ provider, id }) => {
2110
+ const models = readOverlayModels(config.globalPiDir);
2111
+ const entry = models?.providers?.[provider];
2112
+ const custom = entry?.models?.find((model) => model.id === id);
2113
+ if (custom) return custom;
2114
+ const base = builtinModel(provider, id);
2115
+ const override = entry?.modelOverrides?.[id];
2116
+ return base ? { ...base, ...override, cost: { ...base.cost, ...override?.cost } } : null;
2117
+ },
2118
+ });
2119
+ const answerDraftStore = makeAnswerDraftStore({ redis, now, retentionDays: config.logRetentionDays });
2120
+ const answerDraftRuntime = makeAnswerDraftRuntime({ store: answerDraftStore, triggersPath: onceTriggersFile, now, log,
2121
+ enqueue: fields => enqueueAnswerDraftJob(runtimeQueue, fields),
2122
+ readParkedInfo: sessionStore.readParkedInfo,
2123
+ readApproval: source => approvalStore.find(source),
2124
+ readPendingAnswer: async source => {
2125
+ const found = await discoverHostQueuesReporting(redis);
2126
+ if (found.incomplete || found.names.length > 64) return { unknown: true };
2127
+ const queues = [runtimeQueue, ...found.names.map(name => recoveryQueues.host(name))];
2128
+ return readAnswerPending({ queues, source, runId: source.runId });
2129
+ },
2130
+ // The API root is the forge table's (`forgeApiRoot`), the one the console binds a completed draft to. Raw, as
2131
+ // configured: the runner fetches under it and checks Forgejo's issue_url against it.
2132
+ identity: async job => {
2133
+ const auth = forgeFor(job)?.auth ?? await ensureAuth(job.kind);
2134
+ return auth ? { selfId: job.kind === "azure" ? auth.selfId?.id : auth.selfId, apiUrl: forgeApiRoot(job.kind, config) } : null;
2135
+ },
2136
+ });
1463
2137
 
1464
2138
  // Issue #505. One live-file check, shared by the pickup gate, the snapshot and the plan collector, so the three ask one
1465
2139
  // file by one rule: the same file and rule as the one-shot checks below (`onceTriggersFile`). `pi-dispatch run
@@ -1471,7 +2145,8 @@ export async function startWorker(
1471
2145
  const collectPlan = makeCollectPlan({ allocation: allocationState, governing: governingNow, projects: () => projects.current, checkPortfolioFlag, log });
1472
2146
  // The snapshot a confirmed portfolio job reads as /job/portfolio.json, built at prepare. The run counts are complete
1473
2147
  // only with the run mirror, which a declared worker name arms (the `runMirror` rule below).
1474
- const portfolioSnapshot = makePortfolioSnapshot({ checkPortfolioFlag, governing: governingNow, projects: () => projects.current, limits: () => scopedLimits.current, allocation: allocationState, redis, logsDir: config.logsDir, mirror: config.workerNameDeclared === true, log });
2148
+ // `indexMax` (issue #761): the snapshot reads the whole index, which this host's own trim keeps at or under its cap.
2149
+ const portfolioSnapshot = makePortfolioSnapshotFn({ checkPortfolioFlag, governing: governingNow, projects: () => projects.current, limits: () => scopedLimits.current, allocation: allocationState, redis, logsDir: config.logsDir, mirror: config.workerNameDeclared === true, indexMax: config.runMirrorMaxRuns, log });
1475
2150
 
1476
2151
  // REQ-GLOBAL-PI-OVERLAY staged packages: read the operator's stage manifest at EACH job start, like
1477
2152
  // getSettings above and the pause-window ref below.
@@ -1499,6 +2174,21 @@ export async function startWorker(
1499
2174
  if (lastPackageKey !== null && key !== lastPackageKey) log("packages_stage_changed", { count: key === "" ? 0 : key.split(":").length });
1500
2175
  lastPackageKey = key;
1501
2176
  };
2177
+ // Issue #800: the name and version each path of the last good read stood for, so the rows a job's record names are the
2178
+ // read that chose its paths. Beside last-known-good, and replaced with it.
2179
+ let stagedRowByPath = new Map();
2180
+ // Issue #800: whether the set this returned says what is staged here, which the record needs before it calls a name absent. `readStageManifest` answers null
2181
+ // for a manifest that is not there and for one that does not read, and only the first is a stage with nothing in it: with no last-good set the second returns
2182
+ // [] too, and a package's real name would be recorded as one this host does not hold.
2183
+ let stageRead = true;
2184
+ const manifestIsAbsent = () => {
2185
+ try {
2186
+ statSync(join(config.globalPiDir, PACKAGES_SUBDIR, STAGE_MANIFEST));
2187
+ return false;
2188
+ } catch (error) {
2189
+ return error?.code === "ENOENT";
2190
+ }
2191
+ };
1502
2192
  const getPackagePaths = () => {
1503
2193
  if (!config.globalPiDir) return [];
1504
2194
  const staged = readStageManifest({ globalPiDir: config.globalPiDir });
@@ -1508,13 +2198,18 @@ export async function startWorker(
1508
2198
  return lastGoodPackagePaths;
1509
2199
  }
1510
2200
  notePackageKey("");
2201
+ stageRead = manifestIsAbsent();
1511
2202
  return [];
1512
2203
  }
2204
+ stageRead = true;
1513
2205
  const paths = containerPackagePaths(staged);
1514
2206
  notePackageKey(paths.join(":"));
1515
2207
  lastGoodPackagePaths = paths;
2208
+ stagedRowByPath = new Map(staged.packages.map((p) => [containerPackagePaths({ packages: [p] })[0], { name: p.name, version: p.version }]));
1516
2209
  return paths;
1517
2210
  };
2211
+ getPackagePaths.rowOf = (path) => stagedRowByPath.get(path) ?? null;
2212
+ getPackagePaths.stageRead = () => stageRead;
1518
2213
  // One read at boot, for the same log line the boot read always emitted, and to seed last-known-good.
1519
2214
  if (config.globalPiDir && !readStageManifest({ globalPiDir: config.globalPiDir })) log("packages_manifest_absent", { overlay: config.globalPiDir });
1520
2215
  getPackagePaths();
@@ -1547,6 +2242,8 @@ export async function startWorker(
1547
2242
  egress: config.egress,
1548
2243
  egressProxy: config.egressProxy,
1549
2244
  openJobLog,
2245
+ ...(openLiveTranscript ? { openLiveTranscript } : {}),
2246
+ ...(openInboxRelay ? { openInboxRelay } : {}),
1550
2247
  globalPiDir: config.globalPiDir,
1551
2248
  allowGlobalExtensions: config.allowGlobalExtensions,
1552
2249
  packagePaths: getPackagePaths,
@@ -1555,6 +2252,8 @@ export async function startWorker(
1555
2252
  forgeHosts: { gitlab: config.gitlab?.apiUrl ?? null, forgejo: config.forgejo?.apiUrl ?? null, azure: config.azure?.orgUrl ?? null },
1556
2253
  backendFloor: config.backendFloor,
1557
2254
  reap: backendReaps[PODMAN_BACKEND],
2255
+ // Issue #681: the budget's leftover listing, through the same seam local's takes.
2256
+ listJobContainers: listJobContainersFn("podman"),
1558
2257
  readInfo: podmanInfo,
1559
2258
  platform: jobUserIdentity.platform,
1560
2259
  euid: jobUserIdentity.euid,
@@ -1593,8 +2292,52 @@ export async function startWorker(
1593
2292
  // Its own sentence (PR #463 round 3): the job-shaped one says "this job is retried", and at boot there is no job.
1594
2293
  if (typeof bootKeeper?.keeper === "string") log("netns_keeper_not_holding_at_boot", { reason: bootKeeper.keeperAtBoot ?? bootKeeper.keeper });
1595
2294
  }
1596
- // Every bundle this boot built beside local's, in registration order: the podman one first, then the injected ones.
1597
- const builtBackends = [...(podmanBackend ? [podmanBackend] : []), ...extraBackends];
2295
+ // Issue #684: the openshell bundle, while blessed. Its policy reads the same two sources the squid venues' proxy reads
2296
+ // and the MCP check reads (this folder's egress-allowlist.conf, a missing one admitting nothing, and the pickup's
2297
+ // endpoint snapshot, or the boot-loaded one where a job carries none); its reaper is the boot sweep's own.
2298
+ // ONE reader of this folder's egress-allowlist.conf for the MCP check and the openshell policy, so the two cannot
2299
+ // disagree about what the proxy admits. No allowlist here: every host outside a declared endpoint is unreachable,
2300
+ // which it is. Any other failure is the caller's to classify (a transient errno is retried).
2301
+ const readEgressAllowlist = () => {
2302
+ try {
2303
+ return parseAllowlist(readFileSync(join(process.cwd(), "egress-allowlist.conf"), "utf8"));
2304
+ } catch (error) {
2305
+ if (error?.code === "ENOENT") return [];
2306
+ throw error;
2307
+ }
2308
+ };
2309
+ // Issue #686 part 3: one option set, so the two modes cannot drift apart in anything but their mode (and their key in
2310
+ // the boot sweep's map, which holds one reaper for both).
2311
+ const readOpenshellEndpoints = () => modelEndpoints.current ?? [];
2312
+ const openshellOptions = (mode) => ({
2313
+ mode,
2314
+ image: config.jobImage,
2315
+ // Review round 1 (#714): the bind probe's empty source directory is made under the jobs dir.
2316
+ jobsDir: config.jobsDir,
2317
+ engine: config.openshell.engine,
2318
+ gateway: config.openshell.gateway,
2319
+ readAllowlist: readEgressAllowlist,
2320
+ readEndpoints: readOpenshellEndpoints,
2321
+ hostEnv: env,
2322
+ openJobLog,
2323
+ // Issue #686: the same opener the binding venues get, fed from the exec's stderr records, and the same inbox relay
2324
+ // opener, whose relay writes a steered job's messages as frames on the exec's stdin there (part 2).
2325
+ ...(openLiveTranscript ? { openLiveTranscript } : {}),
2326
+ ...(openInboxRelay ? { openInboxRelay } : {}),
2327
+ globalPiDir: config.globalPiDir,
2328
+ allowGlobalExtensions: config.allowGlobalExtensions,
2329
+ packagePaths: getPackagePaths,
2330
+ forwardEnv: config.forwardEnv,
2331
+ authFromPi: config.authFromPi,
2332
+ forgeHosts: { gitlab: config.gitlab?.apiUrl ?? null, forgejo: config.forgejo?.apiUrl ?? null, azure: config.azure?.orgUrl ?? null },
2333
+ reap: backendReaps[mode === "bind" ? OPENSHELL_BIND_BACKEND : OPENSHELL_BACKEND],
2334
+ log,
2335
+ });
2336
+ const openshellBackend = openshellCopyBlessed ? makeOpenshellBackendFn(openshellOptions("copy")) : null;
2337
+ const openshellBindBackend = openshellBindBlessed ? makeOpenshellBackendFn(openshellOptions("bind")) : null;
2338
+ // Every bundle this boot built beside local's, in registration order: podman, openshell, openshell-bind, then the
2339
+ // injected ones.
2340
+ const builtBackends = [...(podmanBackend ? [podmanBackend] : []), ...(openshellBackend ? [openshellBackend] : []), ...(openshellBindBackend ? [openshellBindBackend] : []), ...extraBackends];
1598
2341
  // BOUNDED, because `.catch()` cannot rescue a promise that never settles: `runDocker` resolves only on
1599
2342
  // the child's `close` or `error` and has no timeout of its own, so a wedged daemon would hang boot
1600
2343
  // here. This read is a nicety -- a digest for the boot line and the registry -- and a nicety may
@@ -1629,7 +2372,10 @@ export async function startWorker(
1629
2372
  }
1630
2373
  // Resolved once: `Intl` is not free, and this value cannot change without a restart.
1631
2374
  const hostTz = Intl.DateTimeFormat().resolvedOptions().timeZone ?? "";
2375
+ const endpointSlots = makeInFlight();
2376
+ const endpointStatusNow = () => endpointStatusFields(modelEndpoints.current, endpointSlots, config.workerNameDeclared);
1632
2377
  const registry = makeHostRegistryFn({ redis, name: config.workerName, log });
2378
+ const retainedNow = makeSandboxTelemetry({ list: () => listSandboxesFn({ sandboxDir: config.sandboxDir, throwOnReadFailure: true }), deadline: sandboxDeadline, retentionHours: config.sandboxRetentionHours, canOpen: (manifest) => !sandboxSyncRefusal({ jobId: manifest.jobId, manifest, blessed: config.backends }), now, log });
1633
2379
  // Issue #596, phase 2: one integer of the host budget's snapshot for the beat, or "" before the worker exists.
1634
2380
  const snapshotField = (key) => {
1635
2381
  const snap = worker?.hostBudget?.snapshot?.();
@@ -1645,7 +2391,7 @@ export async function startWorker(
1645
2391
  return [];
1646
2392
  }
1647
2393
  };
1648
- return liveJobsFields(liveJobsOf({ running: read(() => worker?.runningJobs?.list?.()), budgetEntries: read(() => worker?.hostBudget?.entries?.()) }));
2394
+ return liveJobsFields(liveJobsOf({ defaultVenue: config.defaultBackend, running: read(() => worker?.runningJobs?.list?.()), budgetEntries: read(() => worker?.hostBudget?.entries?.()) }));
1649
2395
  };
1650
2396
  // NOT awaited, and that is load-bearing rather than an optimisation. `makeRedisClient` sets
1651
2397
  // `maxRetriesPerRequest: null` -- required for BullMQ's blocking connections -- which means a command
@@ -1671,7 +2417,20 @@ export async function startWorker(
1671
2417
  //
1672
2418
  // Recomputed per beat rather than frozen at boot, for the reason the digest is: a host that gains a
1673
2419
  // profile on restart must start attracting that work within one beat, and one that loses it must stop.
1674
- caps: () => serializeCaps(capabilityTokens(config)),
2420
+ //
2421
+ // Issue #704: the secret half is the profiles this host would RESOLVE, not only the env-declared ones. The
2422
+ // overlay is read here each beat (without a log, `fpUsd`'s reason), so a profile declared in the panel is
2423
+ // advertised within one beat of the write when its resolver sits inside PI_SECRET_RESOLVER_ROOTS, and
2424
+ // never when it does not. A name declared in both sources makes this host refuse every secrets job, so it
2425
+ // then advertises no secret profile at all. An overlay the job path refuses advertises the env profiles,
2426
+ // the row it always published; every job on such a host refuses as settings-overlay-invalid anyway.
2427
+ caps: () => {
2428
+ // `resolveSettings`, the job path's own rule (getSettings): an overlay that refuses every job as
2429
+ // settings-overlay-invalid, by its own validation or by the merged dollar rule, contributes no profile.
2430
+ const settings = resolveSettings(config, readOverlay(settingsFile));
2431
+ const served = servableSecretProfiles({ envProfiles: config.secretProfiles, overlayProfiles: settings.invalid ? {} : (settings.secretProfiles ?? {}), roots: config.secretResolverRoots, realExecutablePath: realExecutable });
2432
+ return serializeCaps(capabilityTokens({ secretProfiles: served.ambiguous ? {} : served.profiles, waitProfiles: config.waitProfiles }));
2433
+ },
1675
2434
  // The host's IANA zone, because a cron PATTERN carries none: `triggers.json` has no `tz` field and
1676
2435
  // BullMQ hands the pattern to cron-parser with no zone, so it resolves in each worker's LOCAL time.
1677
2436
  // On one host that is exactly what an operator means; on two in different zones the same pattern is
@@ -1703,9 +2462,21 @@ export async function startWorker(
1703
2462
  // Issue #596: the highest scoped-limits version this build reads, so doctor can name a worker that would keep its
1704
2463
  // last good file (so no size and no later edit to the file applies on it) once the file is version 3. An integer.
1705
2464
  limitsVersion: SCOPED_LIMITS_VERSION,
2465
+ // Issue #761 (DES-STORE-VERSION-WHEN-NEEDED): the highest version this build reads of the settings overlay, the pause
2466
+ // windows file, the triggers file and a queued job's data, integers, on `limitsVersion`'s model: only doctor reads
2467
+ // them, to name a peer that would defer (or, from before these fields, misread) what this host's files now need.
2468
+ settingsVersion: SETTINGS_VERSION,
2469
+ pauseWindowsVersion: PAUSE_WINDOWS_VERSION,
2470
+ triggersVersion: TRIGGERS_FILE_VERSION,
2471
+ jobDataVersion: JOB_DATA_VERSION,
1706
2472
  // Issue #596, phase 2: this host's budget and what its jobs hold against it, integers (MiB and hundredths of a CPU),
1707
2473
  // `off` for a budget switched off and "" while unknown, so doctor and a forge job's never-fits check can read which
1708
2474
  // host a size fits on. Thunks over the worker's one budget, so every beat says what is held now.
2475
+ sandboxes: () => retainedNow().sandboxes,
2476
+ sandboxesMore: () => retainedNow().sandboxesMore,
2477
+ sandboxRetentionHours: () => retainedNow().sandboxRetentionHours,
2478
+ budgetFrom: () => { const value = parseBudgetFrom(worker?.hostBudget?.detail?.()); return value ? JSON.stringify(value) : ""; },
2479
+ cpuReserve: () => JSON.stringify(parseCpuReserve(cpuReserve.states?.()) ?? []),
1709
2480
  budgetMemMiB: () => budgetField(worker?.hostBudget?.current().memMiB ?? null),
1710
2481
  budgetCpuCenti: () => budgetField(worker?.hostBudget?.current().cpuCenti ?? null),
1711
2482
  usedMemMiB: () => snapshotField("usedMemMiB"),
@@ -1721,6 +2492,10 @@ export async function startWorker(
1721
2492
  waiters: () => snapshotField("waiters"),
1722
2493
  jobs: () => (worker ? liveJobsNow().jobs : ""),
1723
2494
  jobsMore: () => (worker ? liveJobsNow().jobsMore : ""),
2495
+ endpointsVersion: ENDPOINT_STATUS_VERSION,
2496
+ endpointsStartedAt: endpointStatusNow().endpointsStartedAt,
2497
+ endpoints: () => endpointStatusNow().endpoints,
2498
+ endpointsMore: () => endpointStatusNow().endpointsMore,
1724
2499
  budgetHolds: () => snapshotField("holds"),
1725
2500
  budgetOrphans: () => snapshotField("orphans"),
1726
2501
  // whether the boot listing of the job containers left from before this worker started has been read
@@ -1731,6 +2506,10 @@ export async function startWorker(
1731
2506
  if (typeof snap?.seeded !== "boolean") return "";
1732
2507
  return snap.seeded ? "listed" : `unlisted:${(snap.unseeded ?? []).join(",")}`;
1733
2508
  },
2509
+ // Issue #712: the venues this host blesses (the first is its default), its floor and its OpenShell engine and
2510
+ // gateway word, for the console's and the panel's Venues view (`hostBackendsField`: names and fixed words only).
2511
+ // A value, not a thunk: every part is boot configuration or the boot's gateway read, none changes without a restart.
2512
+ backends: hostBackendsField({ backends: config.backends, backendFloor: config.backendFloor, openshellEngine: config.openshell?.engine ?? null, gatewayWord: openshellGatewayWord }),
1734
2513
  });
1735
2514
 
1736
2515
 
@@ -1864,6 +2643,8 @@ export async function startWorker(
1864
2643
  // job containers, and an unproven answer must never free a slot.
1865
2644
  stopContainer: makeStopContainer(),
1866
2645
  reap: backendReaps[DEFAULT_BACKEND],
2646
+ // Issue #681: the host budget's leftover listing rides on the bundle, so the boot seed asks each venue its own.
2647
+ listJobContainers: listJobContainersFn("docker"),
1867
2648
  // One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
1868
2649
  // image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
1869
2650
  // `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.
@@ -1884,6 +2665,8 @@ export async function startWorker(
1884
2665
  egress: config.egress, // REQ-EGRESS-ALLOWLIST: the per-job network and the proxy variables
1885
2666
  egressProxy: config.egressProxy,
1886
2667
  openJobLog,
2668
+ openLiveTranscript, // issues #643 and #644: null when PI_LIVE_TRANSCRIPTS=0 and PI_RUN_PROGRESS=0 (and no Valkey client, #646)
2669
+ openInboxRelay, // issue #646: the steer inbox relay, null without a Valkey client
1887
2670
  globalPiDir: config.globalPiDir, // REQ-GLOBAL-PI-OVERLAY: :ro overlay mount when configured
1888
2671
  allowGlobalExtensions: config.allowGlobalExtensions,
1889
2672
  // REQ-GLOBAL-PI-OVERLAY: staged package paths; every job receives them unless its trigger set
@@ -1941,11 +2724,14 @@ export async function startWorker(
1941
2724
  const opened = [registry, runtimeQueue, ...(cronQueue !== runtimeQueue ? [cronQueue] : [])];
1942
2725
  // Bounded, then forced: a queue whose connection came up closes by sending QUIT and awaiting the reply, and against a
1943
2726
  // server that stopped answering that wait never ends (measured through a stalling proxy), so the refusal itself would
1944
- // never travel. Five seconds covers the host registry's own bounded close; whatever is still open is disconnected.
1945
- await settleWithin(Promise.allSettled(opened.map((handle) => Promise.resolve().then(() => handle?.close?.()))), 5_000);
2727
+ // never travel. `QUEUE_CLOSE_BOUND_MS` (five seconds) covers the host registry's own bounded close, and is the bound
2728
+ // each queue's own close waits for each of its steps; whatever is still open is disconnected.
2729
+ await settleWithin(Promise.allSettled(opened.map((handle) => Promise.resolve().then(() => handle?.close?.()))), QUEUE_CLOSE_BOUND_MS);
1946
2730
  for (const handle of opened) {
1947
2731
  try {
1948
- handle?.disconnect?.();
2732
+ // BullMQ's disconnect awaits initialization and can reject asynchronously when that start failed.
2733
+ // Observe that cleanup rejection without extending the bound or replacing the boot's refusal.
2734
+ void Promise.resolve(handle?.disconnect?.()).catch(() => {});
1949
2735
  } catch {
1950
2736
  // a handle with nothing left to drop
1951
2737
  }
@@ -1965,10 +2751,23 @@ export async function startWorker(
1965
2751
  // a spread, a freeze, a snapshot inside `createWorker` -- un-registers the watchers in SILENCE and puts
1966
2752
  // issue #295 back. Append only: two tests pin `[0]` as the runtime queue and `[1]` as the registry.
1967
2753
  const extraClosers = [runtimeQueue, registry, ...(cronQueue === runtimeQueue ? [] : [cronQueue])];
2754
+ if (subscriptions) {
2755
+ const publish = () => publishSubscriptionStatus(redis, config.workerName, subscriptions.runtime).catch(() => log("subscription_status_unavailable", {}));
2756
+ await publish();
2757
+ const tick = setInterval(publish, 15_000);
2758
+ tick.unref?.();
2759
+ extraClosers.push({ close: () => clearInterval(tick) });
2760
+ }
2761
+ // A first source can arrive through live configuration after boot. Recovery and quiet-close
2762
+ // draining must already exist then, even when the initial source registry was empty.
2763
+ extraClosers.push(makeWatchDrainer({ runtime: watchRuntime, log }));
2764
+ extraClosers.push(makeWatchDrainer({ runtime: answerDraftRuntime, log: () => log("answer_handoff_unavailable", {}) }));
1968
2765
 
1969
2766
  // HOISTED out of the deps literal (issue #288): the terminal failed listener below needs the same
1970
2767
  // adapter the processor gets, and two bodies would drift exactly where drift costs a public comment.
1971
2768
  const comment = async (job, text) => {
2769
+ if (job?.answerDraft || job?.purpose === "answer-draft") return;
2770
+ if (!await draftReplyMayPost(job)) return;
1972
2771
  // Best-effort: the processor awaits comment() inside its try, so a rejection here would
1973
2772
  // corrupt the job outcome and could drive a wrong retry / second PR (CONST-RETRY-INFRA-ONLY).
1974
2773
  // This adapter NEVER throws.
@@ -2002,11 +2801,27 @@ export async function startWorker(
2002
2801
  log("comment", { jobId: job?.id, text });
2003
2802
  };
2004
2803
 
2804
+ // Issue #646 (DES-STEER-LATE-MESSAGE-RESUME-OR-REPLY): what a late forge message becomes at a steer-armed run's final
2805
+ // outcome, for all three settle sites (the processor, the host-lost sweep, the failed listener): the run's semantic
2806
+ // window released, then a resumed job on the queue that ran the run or one fixed reply through `comment` above. The
2807
+ // recovery queues are the same handles a #645 recovery is routed through, so a host queue is reached by its name.
2808
+ const lateForge = makeLateForge({ redis, queues: recoveryQueues, comment, log, now, canPublish: draftReplyMayPost });
2809
+ // The host-lost sweep's run ended on a host that is gone, so its resumed job goes to the shared queue, never to that
2810
+ // host's queue, where nothing would ever take it.
2811
+ const lateForgeShared = (entries, context) => lateForge(entries, { ...context, queueName: QUEUE });
2812
+ // Issue #638 (DES-AWAIT-INPUT-ENDS-THE-JOB): what an awaitInput run's held replies become. `take` runs inside the
2813
+ // processor's park sequence and starts the first recent one as the resumed job, with the window; the settle at the
2814
+ // same three final-outcome sites is the backstop that drops them for any run that did not park (held-settle.mjs).
2815
+ // The host-lost sweep's run can never have parked, so there it only ever drops.
2816
+ const heldSettle = makeHeldSettle({ redis, queues: recoveryQueues, comment, log, now });
2817
+
2005
2818
  // Issue #625: the forge's parked marker, set on a park and cleared when it no longer applies. comment()'s shape and its
2006
2819
  // rule: it NEVER throws, because the marker decides nothing (the session store's sidecar does) and a forge that refuses
2007
2820
  // the label must not flip an outcome. A local job has no forge and no marker. Issues only: a pull request's reply is
2008
2821
  // routed on the phrase (parked.mjs).
2009
- const parkedLabel = async (job, on) => {
2822
+ const parkedLabel = async (job, on, approval = false) => {
2823
+ if (job?.answerDraft || job?.purpose === "answer-draft") return;
2824
+ if (!await draftReplyMayPost(job)) return;
2010
2825
  const forge = forgeFor(job);
2011
2826
  if (!forge?.host?.setParkedLabel || job?.target?.type !== "issue") return;
2012
2827
  try {
@@ -2014,11 +2829,58 @@ export async function startWorker(
2014
2829
  if (!auth) return;
2015
2830
  const token = await auth.mintToken(job);
2016
2831
  await forge.host.setParkedLabel(job, job.target, on, token);
2832
+ if (approval || !on) await forge.host.setParkedLabel(job, job.target, on && approval, token, APPROVAL_LABEL);
2017
2833
  } catch (err) {
2018
2834
  log("parked_label_failed", { jobId: job?.id, on, reason: err?.message });
2019
2835
  }
2020
2836
  };
2021
2837
 
2838
+ const approvalSessionCheck = makeApprovalSessionCheck({ sessionsDir: config.sessionsDir, maxBytes: config.sessionMaxBytes, ttlDays: config.sessionsTtlDays, maxAgeDays: config.sessionMaxAgeDays, maxResumeChain: config.sessionMaxResumeChain, maxContextPct: config.sessionMaxContextPct, now });
2839
+ const approvalStore = makeApprovalStore({ redis, now });
2840
+ const approvalRuntime = makeApprovalRuntime({
2841
+ store: approvalStore,
2842
+ touchStore,
2843
+ touchOutbox,
2844
+ checkPolicy: makeCheckApprovalPolicy({ triggersPath: onceTriggersFile, parseTriggers }),
2845
+ now,
2846
+ host: durableHost,
2847
+ hostQueue,
2848
+ requireSessionDigest: true,
2849
+ enqueueExpiry: async (request) => {
2850
+ const queue = hostQueue ? cronQueue : runtimeQueue;
2851
+ const jobId = approvalExpiryJobId(request.requestId);
2852
+ await queue.add(request.kind, {
2853
+ ...request.resumeJob,
2854
+ approvalRequestId: request.requestId,
2855
+ trigger: { ...request.resumeJob.trigger, deliveryId: `expiry-${request.requestId}`, comment: { body: "/decline" } },
2856
+ }, approvalExpiryJobOpts(jobId));
2857
+ return { ok: true };
2858
+ },
2859
+ comment: async (request, text) => {
2860
+ // One expiry comment per request: the mark lives as long as the request's finished state (MILLISECONDS, PX).
2861
+ if (await redis.set(`run:approval:expiry-comment:${request.requestId}`, "1", "PX", APPROVAL_FINISHED_TTL_MS, "NX") === "OK") await comment(request.resumeJob, text);
2862
+ },
2863
+ });
2864
+
2865
+ let approvalSweep = null;
2866
+ const startApprovalSweep = () => {
2867
+ if (approvalSweep) return;
2868
+ approvalSweep = makeRetentionSweepFn({
2869
+ reapers: [{ name: "approval", reap: async () => {
2870
+ const result = await approvalRuntime.sweep();
2871
+ if (!result.ok) log("approval_expiry_unavailable", { failures: result.failures?.length ?? 1 });
2872
+ } }], intervalMs: 60_000, log,
2873
+ });
2874
+ approvalSweep.start();
2875
+ extraClosers.push(approvalSweep);
2876
+ };
2877
+ const publishApproval = approvalRuntime.afterPark;
2878
+ approvalRuntime.afterPark = async (options) => {
2879
+ const result = await publishApproval(options);
2880
+ if (result.ok) startApprovalSweep();
2881
+ return result;
2882
+ };
2883
+
2022
2884
  // The operator's failure hook (issue #288, INT-ON-FAILURE-HOOK-CONTRACT). Constructed ONLY when the
2023
2885
  // knob is set: an unset PI_ON_FAILURE builds nothing, spawns nothing, and logs nothing -- the
2024
2886
  // byte-identical guarantee. Fired from the two terminal listeners below, never from the processor:
@@ -2027,8 +2889,10 @@ export async function startWorker(
2027
2889
  // `hostEnv: env`, not the default process.env -- the #309 rule every spawner in this file follows
2028
2890
  // (runContainer, resolveSecrets): a subprocess runs with THIS worker's env, identical on the real
2029
2891
  // path and divergent only under an injected one, which is exactly where the difference would hide.
2892
+ const hookRuns = config.onFailure ? makeHookRunsPublisher({ redis, name: config.workerName, now, log }) : null;
2893
+ if (hookRuns) extraClosers.push(hookRuns);
2030
2894
  const onFailure = config.onFailure
2031
- ? makeOnFailure({ command: config.onFailure, timeoutMs: config.onFailureTimeoutMs, host: config.workerName ?? "", hostEnv: env, log })
2895
+ ? makeOnFailure({ command: config.onFailure, timeoutMs: config.onFailureTimeoutMs, host: config.workerName ?? "", hostEnv: env, log, publish: hookRuns.publish })
2032
2896
  : null;
2033
2897
  // Which POLICY reasons page the operator. Paid terminals only: worker-abort, runner-policy and every
2034
2898
  // named runner reason cost a container and ended wrong. The runner's named reasons are SPREAD from
@@ -2043,24 +2907,45 @@ export async function startWorker(
2043
2907
  // One predicate for the completed listener and the lost-lock path below, so a record replays exactly the page its
2044
2908
  // result would have sent.
2045
2909
  const pagesAsPolicy = (result) => Boolean(onFailure) && result?.outcome === "policy" && HOOK_POLICY_REASONS.has(result.reason) && result.budgetReserved !== false;
2046
- // The infra-terminal sentence (issue #288). FIXED, never err.message: the message classes that reach
2047
- // a failedReason carry host paths and library words (the #310 record), and for a local job this text
2048
- // lands verbatim in the service log through the adapter's stdout fallthrough. The worker log already
2049
- // holds the truncated reason on the job_failed line beside it.
2050
- const FAILED_COMMENT = "Failed: an error stopped this job and it will not be retried further. Ask the operator to check the worker log.";
2051
- // Issue #458 (PR #463 round 2): a job that never started because the podman venue's rootless network keeper did not
2052
- // hold says so, since the generic line sends an operator to a log that only says the job failed. Fixed text keyed by
2053
- // the fixed reason token, never the error's own words: a forge comment carries nothing a host read produced.
2054
- const FAILED_COMMENT_BY_REASON = Object.freeze({
2055
- // Neutral on the cause (PR #463 round 3): a keeper that is not running and a proxy that must restart after it both land here.
2056
- [NETNS_KEEPER_NOT_HOLDING]: "Failed before it started: this worker's rootless Podman egress proxy did not pass its pre-start check (its rootless network keeper, pi-dispatch-netns-keeper), so the job was retried and never run. Nothing was spent. Ask the operator to run `pi-dispatch doctor` on the worker for the exact fix.",
2057
- // Issue #476: a job held on a young keeper that kept restarting while it waited.
2058
- [NETNS_KEEPER_CRASH_LOOP]: "Failed before it started: this worker's rootless network keeper (pi-dispatch-netns-keeper), which its Podman egress proxy needs, kept restarting while the job waited for it, so the job was retried and never run. Nothing was spent. Ask the operator to run `pi-dispatch doctor` on the worker and read the keeper's log (`journalctl --user -u pi-dispatch-netns-keeper.service`).",
2059
- // Issue #448 (gate round 2 of PR #473): the hold's own ending, so the comment names the cause and the fix.
2060
- [PODMAN_RESTART_HOLD_EXPIRED]: "Failed before it started: rootful Podman's service on this worker kept running with a containers.conf older than its files (or a file's change time stayed ahead of the host's clock) for an hour, so the job was held and never run. Nothing was spent. Ask the operator to restart it while no local job runs (`sudo systemctl restart podman.service`), then run the job again.",
2910
+ // Issue #645: the one RETURNED failure that pages. A recovery whose container ran, ended infra and could not queue
2911
+ // another recovery returns `failed`/`recovery-failed` (the processor posts its own sentence), so it reaches the
2912
+ // completed listener and never the failed one: without this it would be the only paid terminal that pages nobody.
2913
+ const pagesAsReturnedFailure = (result) => Boolean(onFailure) && result?.outcome === "failed" && result.reason === "recovery-failed";
2914
+ // The infra-terminal sentences (FAILED_COMMENT, FAILED_COMMENT_BY_REASON, issue #288) and the host-lost one
2915
+ // (RECOVERY_HOST_LOST_COMMENT, issue #645) are module constants in comment-tables.mjs since issue #710, with why each is fixed.
2916
+ // Fleet-wide, on every host: whichever host's sweep claims a due recovery first tells its run. The record says it too,
2917
+ // so the panel's history shows the run ended rather than a recovery that vanished.
2918
+ const sweepHostLost = makeHostLostSweepFn({
2919
+ redis,
2920
+ queues: recoveryQueues,
2921
+ now,
2922
+ log,
2923
+ // The registry's live rows (host-registry.mjs heartbeats, HOST_TTL_MS): a recovery whose host is still beating is
2924
+ // delayed (a pause window, a full host), not lost. Names in the run checkpointer's spelling, as the pointer has them;
2925
+ // null when the registry could not be read, which extends rather than fails.
2926
+ liveHosts: async () => {
2927
+ const read = await readLiveHosts(redis, { now, prune: false });
2928
+ if (read?.unreachable || !Array.isArray(read?.hosts)) return null;
2929
+ return new Set(read.hosts.map((h) => normalizeHost(h.name)));
2930
+ },
2931
+ onLost: async (job) => {
2932
+ const at = new Date(now()).toISOString();
2933
+ // Issue #646: the run's final outcome, so its inbox is closed and counted here too (index.mjs's rule for every
2934
+ // record it writes); a steer job's waiting messages are then `late`, never left `queued` until the key expires.
2935
+ const lostError = { reason: "recovery-host-lost" };
2936
+ if (job?.data?.steer === true) {
2937
+ const steered = await settleInbox({ redis, job, error: lostError, log, onLateForge: lateForgeShared });
2938
+ if (steered) lostError.steered = steered;
2939
+ }
2940
+ if (job?.data?.awaitInput === true) await heldSettle({ job, error: lostError, queueName: QUEUE });
2941
+ recordRun({ job: { id: job.id, name: job.name, data: job.data, opts: job.opts, attemptsMade: 0, timestamp: job.timestamp }, error: lostError, startedAt: null, endedAt: at });
2942
+ await comment({ ...job.data, id: job.id }, RECOVERY_HOST_LOST_COMMENT);
2943
+ onFailure?.({ jobId: job.id, outcome: "failed", reason: "recovery-host-lost" });
2944
+ },
2061
2945
  });
2062
2946
 
2063
2947
  const worker = createWorkerFn({
2948
+ endpointSlots,
2064
2949
  connection: valkeyConn(),
2065
2950
  // Issue #596, phase 2 (DES-HOST-BUDGET): the host budget's inputs. `createWorker` builds the ONE budget from them and
2066
2951
  // shares it between both queues' processors. The settings and the default size were refused at boot if bad.
@@ -2069,7 +2954,10 @@ export async function startWorker(
2069
2954
  jobDefault: { memMiB: config.jobSize.memMiB, cpuCenti: config.jobSize.cpuCenti },
2070
2955
  readFacts: readHostFacts,
2071
2956
  onRefresh: syncCpuReserve,
2072
- containerGone: containerGoneFn ?? makeContainerGone({ binOf: (venue) => (resolveBackendName(venue ?? {}, config.defaultBackend) === PODMAN_BACKEND ? "podman" : "docker") }),
2957
+ // Issue #681: asked of the venue that holds the container, through the registry. This used to pick the CLI from the
2958
+ // venue's name and send every venue but podman to docker, so a third venue's orphan was either freed while it still
2959
+ // ran (docker lists no such name) or never freed (no docker on the host).
2960
+ containerGone: containerGoneFn ?? backends.containerGone,
2073
2961
  // AFTER the boot reaper (above), every blessed venue's remaining job containers, seeded into the ledger
2074
2962
  // as orphans from their size labels. PER VENUE (gate round 2 of phase 2): a venue that cannot be listed
2075
2963
  // stops only its own jobs until a tick reads it, because a container nobody counted is an overcommit, while the
@@ -2078,18 +2966,26 @@ export async function startWorker(
2078
2966
  // cause that refuses a boot (the venue's boot-refusing set: every job on it is refused before a container, so this
2079
2967
  // worker starts none there). A cause decided from one unreadable answer is re-decided per job and can clear, so
2080
2968
  // it leaves the venue unread.
2969
+ // Issue #681: every BLESSED registered venue, each listed by its own bundle; it used to be local and podman by name,
2970
+ // so a third venue was never seeded and its leftover containers were never counted. Only local and podman have a
2971
+ // boot job-user decision; another venue's holds-none rule is ENOENT alone.
2081
2972
  survivors: Object.fromEntries(
2082
- [...(localBlessed ? [[DEFAULT_BACKEND, "docker", bootDecision, BOOT_REFUSING_JOB_USER_CAUSES]] : []), ...(podmanBlessed ? [[PODMAN_BACKEND, "podman", bootPodmanDecision, PODMAN_BOOT_REFUSING_CAUSES]] : [])].map(([backend, bin, decision, stable]) => [
2083
- backend,
2084
- async () => {
2085
- try {
2086
- return (await listJobContainersFn(bin)()).map((c) => ({ ...c, venue: { backend } }));
2087
- } catch (error) {
2088
- if (error?.code === "ENOENT" || (decision?.mode === "unmappable" && stable.has(decision.cause))) return [];
2089
- throw error;
2090
- }
2091
- },
2092
- ]),
2973
+ Object.entries(backends.listers())
2974
+ .filter(([backend]) => config.backends.includes(backend))
2975
+ .map(([backend, list]) => {
2976
+ const [decision, stable] = backend === DEFAULT_BACKEND ? [bootDecision, BOOT_REFUSING_JOB_USER_CAUSES] : backend === PODMAN_BACKEND ? [bootPodmanDecision, PODMAN_BOOT_REFUSING_CAUSES] : [null, new Set()];
2977
+ return [
2978
+ backend,
2979
+ async () => {
2980
+ try {
2981
+ return (await list()).map((c) => ({ ...c, venue: { backend } }));
2982
+ } catch (error) {
2983
+ if (error?.code === "ENOENT" || (decision?.mode === "unmappable" && stable.has(decision.cause))) return [];
2984
+ throw error;
2985
+ }
2986
+ },
2987
+ ];
2988
+ }),
2093
2989
  ),
2094
2990
  defaultVenue: config.defaultBackend,
2095
2991
  now,
@@ -2110,6 +3006,7 @@ export async function startWorker(
2110
3006
  name: config.workerName,
2111
3007
  concurrency: bootConcurrency,
2112
3008
  getSettings,
3009
+ readModelRoles: makeReadModelRoles({ triggersPath: onceTriggersFile, log }),
2113
3010
  redis,
2114
3011
  recordRun,
2115
3012
  settledRecord,
@@ -2118,6 +3015,12 @@ export async function startWorker(
2118
3015
  // REQ-SCOPED-PAUSE-WINDOWS: the processor defers a job whose folder/repo is inside an active window.
2119
3016
  // Reads the live-reloaded ref, so an operator edit takes effect on the next job without a restart.
2120
3017
  pauseUntil: (job, now) => pauseUntilMs(pauseWindows.current, job, now),
3018
+ // Issue #761: while the file on disk was written by a newer pi-dispatch, `pauseUntil` above reads the last good
3019
+ // windows, which may not pause what the file pauses, so every job waits for this host's upgrade instead.
3020
+ pauseWindowsNewer: () => pauseWindows.newer === true,
3021
+ // Issue #761: while the triggers file on disk was written by a newer pi-dispatch, every reader of it holds a last
3022
+ // good copy or refuses, so every job waits for this host's upgrade instead (the processor's first gates).
3023
+ triggersNewer: () => triggersState.newer === true,
2121
3024
  // Issue #242: the scoped-limits snapshot the pickup gate and the scoped budget read, once per
2122
3025
  // pickup, from the live-reloaded ref -- same next-job grain as pauseUntil above.
2123
3026
  scopedLimits: () => scopedLimits.current,
@@ -2174,6 +3077,26 @@ export async function startWorker(
2174
3077
  deps: {
2175
3078
  collectChain,
2176
3079
  collectPlan,
3080
+ // Issue #646: the late-message branches at the processor's settle (index.mjs `settleSteer`).
3081
+ onLateForge: lateForge,
3082
+ watchRuntime,
3083
+ questionGroupRuntime,
3084
+ questionGroupStore,
3085
+ questionCaptureEnabled: () => {
3086
+ try { return parseTriggersDocument(readFileSync(onceTriggersFile, "utf8"), onceTriggersFile).zeroTouch?.capture === true; }
3087
+ catch { log("question_capture_unavailable", { reason: "configuration-unreadable" }); return false; }
3088
+ },
3089
+ touchStore,
3090
+ touchOutbox,
3091
+ touchNow: now,
3092
+ collectWatch: watchRuntime.collect,
3093
+ // Issue #638: held replies at the processor's settle (index.mjs `settleSteer` calls it first). Wired, it also arms
3094
+ // the keeper that makes an awaitInput run reachable while it runs.
3095
+ settleHeld: heldSettle,
3096
+ // The take inside the park sequence (processor.mjs), which starts the held answer before the parked comment.
3097
+ takeHeld: heldSettle.take,
3098
+ answerDraftRuntime,
3099
+ answerDraftStore,
2177
3100
  // The one-shot pre-spend check (issue #231): reads the same file the disarm writes, refuses
2178
3101
  // only on a FOREIGN positive mark (index.mjs binds the real queue jobId so a retry of the
2179
3102
  // spending delivery is excused). In the compose topology this check is the once-enforcement
@@ -2191,24 +3114,53 @@ export async function startWorker(
2191
3114
  // A PROBE: whatever it resolves is dropped on the floor. The credential itself is read where it always
2192
3115
  // was, inside buildContainerEnv, so no live key is ever in scope in the processor.
2193
3116
  // Issue #503: `modelEndpoints` is the pickup's snapshot, the same one runContainer hands buildContainerEnv.
3117
+ subscriptionRuntime: subscriptions?.runtime ?? null,
3118
+ subscriptionAuthRequired: async (job) => {
3119
+ if (!subscriptions) return false;
3120
+ for (const ref of subscriptionRoutes(job)) if ((await subscriptions.auth.check(ref.provider)).managed) return true;
3121
+ return false;
3122
+ },
2194
3123
  checkProviderCredential: (job, { modelEndpoints = null } = {}) => {
2195
- try {
2196
- resolveProviderCredential({ provider: job.provider, hostEnv: env, authFromPi: config.authFromPi, forwardEnv: config.forwardEnv, modelEndpoints });
2197
- return { ok: true };
2198
- } catch (error) {
2199
- // Only OUR determinate refusal. Anything else (a bug here, an fs fault the module does not model)
2200
- // must not become a policy refusal on the operator's issue: it rethrows into runJob's catch, which
2201
- // classifies it the way it always did.
2202
- // Issue #503: a transient overlay read at this pickup is no verdict; the processor retries it as infra.
2203
- if (error?.piDispatchTransient === true) return { ok: false, unavailable: error.code ?? "unreadable" };
2204
- if (error?.piDispatchConfig !== true) throw error;
2205
- return { ok: false, message: error.message };
2206
- }
3124
+ const legacy = () => {
3125
+ try {
3126
+ resolveProviderCredential({ provider: job.provider, hostEnv: env, authFromPi: config.authFromPi, forwardEnv: job.answerDraft || job.watch || job.questionGrouping ? [] : config.forwardEnv, modelEndpoints });
3127
+ return { ok: true };
3128
+ } catch (error) {
3129
+ // Only OUR determinate refusal. Anything else (a bug here, an fs fault the module does not model)
3130
+ // must not become a policy refusal on the operator's issue: it rethrows into runJob's catch, which
3131
+ // classifies it the way it always did.
3132
+ // Issue #503: a transient overlay read at this pickup is no verdict; the processor retries it as infra.
3133
+ if (error?.piDispatchTransient === true) return { ok: false, unavailable: error.code ?? "unreadable" };
3134
+ if (error?.piDispatchConfig !== true) throw error;
3135
+ return { ok: false, message: error.message };
3136
+ }
3137
+ };
3138
+ return subscriptions ? subscriptions.auth.check(job.provider).then((checked) => checked.managed ? checked : legacy()) : legacy();
2207
3139
  },
2208
3140
  // Issue #502. The model-exists gate: pi's builtin catalog, then the overlay models.json, read at most once per
2209
3141
  // job and only when a model is not builtin, so the operator's edits apply without a restart. The SAME file
2210
3142
  // the endpoint gate reads, through the same reader, so absent, unreadable and unparseable mean one thing.
2211
3143
  checkModelsKnown: (refs) => checkModelsKnown(refs, { readOverlay: () => readOverlayModels(config.globalPiDir) }),
3144
+ checkFailoverCandidate: (job, { models, endpoints }) => {
3145
+ const egress = failoverEgressRequired(job, { egress: config.egress, defaultBackend: config.backends[0] });
3146
+ return checkFailoverCandidate(job, {
3147
+ models, endpoints, egress, allowlist: egress ? readEgressAllowlist() : [],
3148
+ credential: async (candidate, { modelEndpoints }) => {
3149
+ if (subscriptions) {
3150
+ const checked = await subscriptions.auth.check(candidate.provider);
3151
+ if (checked.managed) return checked;
3152
+ }
3153
+ try {
3154
+ resolveProviderCredential({ provider: candidate.provider, hostEnv: env, authFromPi: config.authFromPi, forwardEnv: config.forwardEnv, modelEndpoints });
3155
+ return { ok: true };
3156
+ } catch (error) {
3157
+ if (error?.piDispatchTransient) return { unavailable: true };
3158
+ if (error?.piDispatchConfig) return { ok: false };
3159
+ throw error;
3160
+ }
3161
+ },
3162
+ });
3163
+ },
2212
3164
  // Issue #503 part 7: the builtin catalog's model object, for the zero-rated check that lets a job on local
2213
3165
  // zero-rated models reserve nothing in the dollar windows (processor.mjs, `zeroRatedVerdict`).
2214
3166
  builtinModel,
@@ -2242,9 +3194,9 @@ export async function startWorker(
2242
3194
  const held = await runtimeQueue.getJob(id);
2243
3195
  if (!held) return false;
2244
3196
  // EXISTENCE IS NOT LIVENESS, and the difference decides whether a target stays deafened:
2245
- // `removeOnComplete`/`removeOnFail` keep a finished job's hash for 31 days, so a holder that
2246
- // can never wake again would answer "still waiting" for a month. Only a state it can still be
2247
- // picked up from counts.
3197
+ // `removeOnComplete`/`removeOnFail` keep a finished job's hash for `FORGE_JOB_RETENTION_S`, so a
3198
+ // holder that can never wake again would answer "still waiting" for a month. Only a state it can
3199
+ // still be picked up from counts.
2248
3200
  const state = await held.getState();
2249
3201
  return state === "delayed" || state === "waiting" || state === "active" || state === "prioritized" || state === "waiting-children";
2250
3202
  },
@@ -2255,10 +3207,17 @@ export async function startWorker(
2255
3207
  // `local`'s are the closures built beside its bundle below, unchanged.
2256
3208
  observationPreflight: backends.observationPreflight,
2257
3209
  jobUserPreflight: backends.jobUserPreflight,
3210
+ // Issue #704 part 3: the venue's judgement of a project secret bundle, dispatched like the two above.
3211
+ bundleGate: backends.bundleGate,
2258
3212
  // Completed-only, so a policy or infra exit leaves the canonical transcript byte-identical and a
2259
3213
  // retry starts from what the first attempt did (CONST-RETRY-INFRA-ONLY).
2260
3214
  promoteSession: sessionStore.promoteSession,
3215
+ // Issue #645: a recovery stages its checkpoint's transcript as the job's session (`reason: "recovered"`).
3216
+ ...(typeof sessionStore.stageRecovered === "function" ? { stageRecovered: sessionStore.stageRecovered } : {}),
2261
3217
  isParked: sessionStore.isParked,
3218
+ readParkedInfo: sessionStore.readParkedInfo,
3219
+ approvalRuntime,
3220
+ approvalSessionCheck,
2262
3221
  parkedLabel,
2263
3222
  // The same value the store above was built from, passed explicitly so the processor's fail-closed
2264
3223
  // `run.resume` gate answers from THIS config rather than from its own env default. Identical on the
@@ -2278,6 +3237,30 @@ export async function startWorker(
2278
3237
  // injected env they were not, which is the divergence this file already calls out by name for
2279
3238
  // sessionsDir. One deployment value, one place, same rule as the profiles below it.
2280
3239
  resolveSecrets: makeSecretsResolverFn({ envProfiles: config.secretProfiles, roots: config.secretResolverRoots, timeoutMs: config.secretResolveTimeoutMs, forwardEnv: config.forwardEnv, hostEnv: env, log }),
3240
+ // Issue #704, project secret bundles. The binding is read from the SAME triggers file the one-shot, skew and
3241
+ // portfolio checks read (`onceTriggersFile`), by the worker, never carried by the receiver, and answered from
3242
+ // the last good read while the file is broken (triggers-file.mjs says why). The resolver is the run.secrets
3243
+ // seam's sibling, with the same roots, timeout, forwarded names and host env.
3244
+ // `bundlesDeclared`: whether this host declares any bundle profile, env or panel. On such a host a MISSING
3245
+ // triggers file is a failure (last good read, else retry), never "no bundles" (triggers-file.mjs says why).
3246
+ canaryBusy: () => canaryBusy(redis),
3247
+ findSecretBundle: makeFindSecretBundle({
3248
+ triggersPath: onceTriggersFile,
3249
+ log,
3250
+ bundlesDeclared: () => {
3251
+ if (Object.keys(config.secretBundleProfiles ?? {}).length > 0) return true;
3252
+ const settings = getSettings();
3253
+ return !settings.invalid && Object.keys(settings.secretBundles ?? {}).length > 0;
3254
+ },
3255
+ }),
3256
+ secretBundles: makeBundleResolver({ envBundles: config.secretBundleProfiles ?? {}, roots: config.secretResolverRoots, timeoutMs: config.secretResolveTimeoutMs, forwardEnv: config.forwardEnv, hostEnv: env, log }),
3257
+ // The fork check's seam: the job's own forge host, the one prepare hands run.resume's fork gate. A forge with
3258
+ // no host (none configured for that kind) answers nothing, which the processor reads as a fork: no bundle.
3259
+ resolvePullRequestHead: async (job, token) => {
3260
+ const host = forgeFor(job)?.host;
3261
+ if (typeof host?.resolvePullRequestHead !== "function") return null;
3262
+ return await host.resolvePullRequestHead(job, token);
3263
+ },
2281
3264
  // REQ-MCP-OPERATOR-SERVERS: the overlay's mcp.json read once per job that names a server, never at boot (an
2282
3265
  // operator who re-runs import-pi must not have to restart the worker). The allowlist is the one the proxy
2283
3266
  // mounts, this folder's egress-allowlist.conf on both venues (up.mjs, podman-stack.mjs), and the endpoints
@@ -2289,29 +3272,9 @@ export async function startWorker(
2289
3272
  egress: config.egress,
2290
3273
  // DES-FORGE-MCP: the GitLab a gitlab job's forge server calls, the same URL its clone and its glab use.
2291
3274
  gitlabUrl: config.gitlab?.apiUrl ?? null,
2292
- // The credentials the job is set up to spend with (providerKeyNames). A read of the overlay models.json that
2293
- // may pass on a retry is thrown to the check, which retries the job; any other failure adds no overlay
2294
- // names, and the model gates, which ran first, refuse every job such a file breaks.
2295
- reserved: (job) => {
2296
- let overlayModels = null;
2297
- try {
2298
- overlayModels = readOverlayModels(config.globalPiDir);
2299
- } catch (error) {
2300
- if (MCP_TRANSIENT_READ_CODES.has(error?.code)) throw error;
2301
- }
2302
- const providers = jobProviders(job, { defaultProvider: config.provider, allowedModels: config.allowedModels, everyProvider: piProviders() });
2303
- return new Set([...RESERVED_ENV_NAMES, ...providerKeyNames({ providers, candidatesOf: providerKeyCandidates, overlayModels })]);
2304
- },
2305
- allowlist: () => {
2306
- try {
2307
- return parseAllowlist(readFileSync(join(process.cwd(), "egress-allowlist.conf"), "utf8"));
2308
- } catch (error) {
2309
- // No allowlist here: every server outside a declared endpoint is unreachable, which it is. Any other
2310
- // failure is the check's to classify (a transient errno is retried).
2311
- if (error?.code === "ENOENT") return [];
2312
- throw error;
2313
- }
2314
- },
3275
+ // The credentials the job is set up to spend with (`reservedJobNames`, which the console's preview asks too).
3276
+ reserved: (job) => reservedJobNames(job, { readModels: () => readOverlayModels(config.globalPiDir), defaultProvider: config.provider, allowedModels: config.allowedModels, everyProvider: piProviders(), candidatesOf: providerKeyCandidates }),
3277
+ allowlist: readEgressAllowlist,
2315
3278
  endpoints: () => modelEndpoints.current,
2316
3279
  }),
2317
3280
  // #227. What PI_BACKENDS blessed, so a trigger naming an unblessed venue refuses pre-spend. The
@@ -2329,8 +3292,14 @@ export async function startWorker(
2329
3292
  // Off the REGISTRY's surface, like every other per-job backend fact. Rebuilding it at the call site
2330
3293
  // is how a fact ends up dispatched on one path and hardcoded on another.
2331
3294
  neverStartedExits: backends.neverStartedExits,
2332
- prepareWorkspace: makePrepareWorkspace({
3295
+ checkSandboxWorkspace: (job) => checkSandboxWorkspace(job, { sandboxDir: config.sandboxDir }),
3296
+ packageCanary: makePackageCanaryRuntime({ globalPiDir: config.globalPiDir, env, host: config.workerName, log, publishFacts: (facts) => publishCanaryFacts(redis, facts) }),
3297
+ prepareWorkspace: makePrepareWorkspaceFn({
2333
3298
  jobsDir: config.jobsDir,
3299
+ // Issue #643: with PI_LIVE_TRANSCRIPTS=0 no live file is staged either, so a job without a resume transcript
3300
+ // runs exactly as it did before the feature (its session in memory, no /session mount). Since issue #644 the
3301
+ // progress reader needs the same file, so it is left out only when both switches are off.
3302
+ ...(config.liveTranscripts || config.runProgress ? {} : { stageLive: null }),
2334
3303
  // Issue #464: the boot's own check, asked again before every job against the same env.
2335
3304
  ensureDir: (dir) => ensureJobsDirFn(dir, { env }),
2336
3305
  forgeFor,
@@ -2341,7 +3310,7 @@ export async function startWorker(
2341
3310
  defaultBackend: config.defaultBackend,
2342
3311
  // Issue #504 part B: a local job's folder is resolved at prepare and mounted resolved; one named inside a run
2343
3312
  // root or a cron folder must still resolve inside it, and none may be the envelope's folder or above it.
2344
- localPlacement: { jobPaths: envelopeJobPathsNow, envelopeFile: config.envelopeFile ?? null },
3313
+ localPlacement: { jobPaths: envelopeJobPathsNow, envelopeFile: config.envelopeFile ?? null, sandboxDir: config.sandboxDir, authStoreDir: subscriptions?.storeDir ?? null },
2345
3314
  // Issue #505: /job/portfolio.json for a confirmed portfolio job.
2346
3315
  portfolioSnapshot,
2347
3316
  preparers: makeForgePreparers({ gitlabApiUrl: config.gitlab?.apiUrl ?? null, forgejoApiUrl: config.forgejo?.apiUrl ?? null, azureOrgUrl: config.azure?.orgUrl ?? null }),
@@ -2395,6 +3364,21 @@ export async function startWorker(
2395
3364
  if (!host) throw configError(`no forge host is configured for job kind ${JSON.stringify(job?.kind)}`);
2396
3365
  return await host.isDefaultBranchProtected(job, token);
2397
3366
  },
3367
+ // Issue #645 (durable runs), built above. Null where this host has no store (`recover-store-unset` refuses an
3368
+ // armed job before it spends); the ledger, the pointer read and the enqueue exist on every host.
3369
+ workspaceCheckpointer,
3370
+ forkStore,
3371
+ checkForkPolicy: makeCheckForkPolicy({ triggersPath: onceTriggersFile }),
3372
+ checkThinking: (job) => thinkingLevelAllowed(job, { models: readOverlayModels(config.globalPiDir), builtinModel }),
3373
+ runCheckpointer,
3374
+ runBudgetLedger,
3375
+ readCheckpointPointer,
3376
+ enqueueRecovery,
3377
+ // A recovery landing on the host that crashed takes one more checkpoint from the crashed attempt's job directory
3378
+ // before it restores (recovery-queue.mjs `makeSalvager`): `salvageCheckpoint({ runId, pointer, attempt })`.
3379
+ salvageCheckpoint,
3380
+ readRecord: readRunRecord,
3381
+ hostName: durableHost,
2398
3382
  },
2399
3383
  });
2400
3384
 
@@ -2406,6 +3390,15 @@ export async function startWorker(
2406
3390
  // the live Worker held the loop forever. The catch below covers the REGION, not a list of calls,
2407
3391
  // so a step added tomorrow is covered the day it is added.
2408
3392
  try {
3393
+ // Human preparation uses its own bounded control worker, outside the paid processor and its slot ledgers.
3394
+ const prepareSandbox = createSandboxPreparerFn({ config, env, forgeFor, readTriggers: () => readFileSync(onceTriggersFile, "utf8"), projects: () => projects.current, limits: () => scopedLimits.current, now });
3395
+ const sandboxControl = startSandboxPrepareWorkerFn({ connection: valkeyConn(), host: config.workerName, config, prepare: prepareSandbox, now, log });
3396
+ // Its close waits for the preparation it is running, which nothing aborts first, so that step gets the longer bound.
3397
+ settleCloseOf(sandboxControl, { closeBoundMs: SANDBOX_CONTROL_CLOSE_BOUND_MS });
3398
+ extraClosers.push(sandboxControl);
3399
+ const sandboxInventory = makeSandboxInventoryFn({ redis, host: config.workerName, read: retainedNow, settings: { image: config.jobImage, defaultBackend: config.defaultBackend, backends: config.backends, egress: config.egress, egressProxy: config.egressProxy, sandboxRetentionHours: config.sandboxRetentionHours, sandboxPinDays: config.sandboxPinDays, jobMemory: env.PI_JOB_MEMORY, jobCpus: env.PI_JOB_CPUS }, now, log });
3400
+ sandboxInventory.start();
3401
+ extraClosers.push(sandboxInventory);
2409
3402
 
2410
3403
  // REQ-LOCAL-JOB-VISIBILITY: exactly one terminal line per job, carrying the job id and outcome,
2411
3404
  // where the operator is already looking. This is the local counterpart of the GitHub issue
@@ -2419,7 +3412,19 @@ export async function startWorker(
2419
3412
  // BOTH workers, or a cron job on the host queue produces no `job_completed` line at all -- and
2420
3413
  // REQ-LOCAL-JOB-VISIBILITY's whole point is that a missing line is what tells a human a run did nothing.
2421
3414
  const allWorkers = [worker, ...(worker.hostWorker ? [worker.hostWorker] : [])];
3415
+ // Issue #812: a finished job's model attempt key gets its expiry (model-attempts.mjs), on the queue's own word that
3416
+ // the job finished: this event, and `failed` with `finishedOn` below. Fire and forget, inside the existing listener
3417
+ // bodies: an expiry that fails leaves the key as it was, without one, and the fixed line says so. The id is the
3418
+ // processor's (index.mjs `modelAttemptId`); a job with no queue name to read names no key, so nothing is sent.
3419
+ const finishAttempt = (w, job) => {
3420
+ const queueName = job?.queueName ?? w.name;
3421
+ if (typeof queueName !== "string" || queueName === "" || job?.id === undefined || job?.id === null) return;
3422
+ void Promise.resolve()
3423
+ .then(() => finishModelAttemptFn(redis, modelAttemptIdOf(queueName, job.id)))
3424
+ .catch(() => log("model_attempt_finish_failed", { jobId: String(job.id) }));
3425
+ };
2422
3426
  for (const w of allWorkers) w.on("completed", (job, result) => {
3427
+ finishAttempt(w, job);
2423
3428
  log("job_completed", { jobId: job?.id, outcome: result?.outcome, ...(result?.reason ? { reason: result.reason } : {}) });
2424
3429
  // The hook's POLICY half (issue #288): worker-abort, runner-policy and provider-auth-refused RETURN, so they land here
2425
3430
  // and never in the failed listener -- a failed-only mount would miss exactly the paid terminals
@@ -2431,11 +3436,26 @@ export async function startWorker(
2431
3436
  if (pagesAsPolicy(result)) {
2432
3437
  onFailure({ jobId: job?.id, outcome: "policy", reason: result.reason });
2433
3438
  }
3439
+ if (pagesAsReturnedFailure(result)) onFailure({ jobId: job?.id, outcome: "failed", reason: result.reason });
2434
3440
  });
2435
3441
  // The failed listener's body, for a failure the queue decided. Split out only so the lost-lock check below can
2436
3442
  // run it after an await; a failure of any other reason runs it synchronously, exactly as before.
2437
- const onFailed = (job, err) => {
2438
- log("job_failed", { jobId: job?.id, attempt: job?.attemptsMade, reason: String(err?.message ?? err).slice(0, 120) });
3443
+ //
3444
+ // `settledElsewhere` (issue #761) is a terminal failure whose attempt another host already ended and recorded,
3445
+ // in a record this build can read only by its core (the lost-lock check below says which). It runs the terminal
3446
+ // duties below, so nothing the job held is left open: the processor never closes the approval claim of a failed
3447
+ // attempt, so without this one its target would refuse new requests `target-pending` for good. Two things
3448
+ // differ: the terminal line is `job_lost_lock_after_completion` with the outcome alone, and nothing is posted.
3449
+ // The page is the same as any terminal failure's, and a stall's error carries no reason, so the hook's argv reads
3450
+ // `infra`. The run's inbox and held replies close as the run's end says (below). What the host that ran the
3451
+ // attempt may already have done is safe to do again: a second close of an inbox or of held replies acts on
3452
+ // nothing, and the draft's refusal writes the same state.
3453
+ const approvalFinalizers = new Set();
3454
+ let approvalFinalizerCloser = false;
3455
+ const onFailed = (job, err, { settledElsewhere = false } = {}) => {
3456
+ // Exactly one terminal line either way (REQ-LOCAL-JOB-VISIBILITY).
3457
+ if (settledElsewhere) log("job_lost_lock_after_completion", { jobId: job?.id, outcome: "failed" });
3458
+ else log("job_failed", { jobId: job?.id, attempt: job?.attemptsMade, reason: String(err?.message ?? err).slice(0, 120) });
2439
3459
  // The one reason whose sentence is the fix itself (issue #458): logged WHOLE, beside the cut line above.
2440
3460
  if (err?.reason === NETNS_KEEPER_NOT_HOLDING || err?.reason === NETNS_KEEPER_CRASH_LOOP) log("job_failed_netns_keeper", { jobId: job?.id, attempt: job?.attemptsMade, reason: String(err?.message ?? "") });
2441
3461
  // The TERMINAL failed attempt only (issue #288): `finishedOn` is set by BullMQ's own move on the
@@ -2446,7 +3466,37 @@ export async function startWorker(
2446
3466
  // never sees: the stall-kill (maxStalledCount 0 fails a crashed worker's job at next pickup)
2447
3467
  // and the wait-gate rethrow that escapes above the processor's catch.
2448
3468
  if (job?.finishedOn) {
2449
- void comment({ ...job.data, id: job.id }, (typeof err?.reason === "string" && Object.hasOwn(FAILED_COMMENT_BY_REASON, err.reason) ? FAILED_COMMENT_BY_REASON[err.reason] : null) ?? FAILED_COMMENT);
3469
+ if (job.data?.answerDraft) void answerDraftRuntime.refused(job.data, { reason: "failed" }).catch(() => log("answer_terminal_unavailable", {}));
3470
+ // An exhausted retry has no future container to consume its claim. Close only this queue job's
3471
+ // authority, never another job's, and let a new authorized run request a fresh approval.
3472
+ if (job.data?.approvalRequestId) {
3473
+ const finishing = settleWithin(Promise.resolve().then(() => approvalRuntime.finish({ requestId: job.data.approvalRequestId, jobId: String(job.id) })), 2500, { error: "approval-finish-timeout" })
3474
+ .then((result) => { if (!result?.ok) log("approval_terminal_finish_failed", { jobId: String(job.id), reason: "approval-finish-unavailable" }); })
3475
+ .catch(() => log("approval_terminal_finish_failed", { jobId: String(job.id), reason: "approval-finish-unavailable" }))
3476
+ .finally(() => approvalFinalizers.delete(finishing));
3477
+ approvalFinalizers.add(finishing);
3478
+ if (!approvalFinalizerCloser) {
3479
+ approvalFinalizerCloser = true;
3480
+ extraClosers.push({ close: () => Promise.allSettled([...approvalFinalizers]) });
3481
+ }
3482
+ }
3483
+ // Issue #646, review finding F2: a TERMINAL failure the processor never saw (a stall-kill of a crashed worker's
3484
+ // job with no recovery enqueued, a rethrow above the processor's catch) is the run's final outcome too, so its
3485
+ // inbox is closed here and what waits becomes `late` instead of sitting `queued` until the key expires. A
3486
+ // terminal failure the processor did see was settled before its record: this second close returns the same entries
3487
+ // with `first: false` (steer-inbox.mjs closeInbox), so settleInbox answers nothing again (review round 1, F2).
3488
+ // Both closes are the RUN's, keyed by its run id, which a #645 recovery keeps. A run settled elsewhere may have
3489
+ // been handed to a recovery there (a `recovery-enqueued` this build cannot read), and that recovery must find
3490
+ // both open (inbox-settle.mjs `isFinalOutcome`): closing them would answer its waiting messages as late, on the
3491
+ // forge, about a run that goes on. So they close then only for a job no recovery can continue, whose run ended
3492
+ // with it.
3493
+ if (!settledElsewhere || !isRecoveryArmed(job.data)) {
3494
+ if (job.data?.steer === true) void settleInbox({ redis, job, error: { reason: typeof err?.reason === "string" ? err.reason : "infra" }, log, onLateForge: lateForge });
3495
+ // Issue #638: the same for an awaitInput run's held replies; a failure never parked, so they are dropped, and
3496
+ // a second close (the processor settled first) acts on nothing.
3497
+ if (job.data?.awaitInput === true) void heldSettle({ job, error: { reason: typeof err?.reason === "string" ? err.reason : "infra" } });
3498
+ }
3499
+ if (!settledElsewhere) void comment({ ...job.data, id: job.id }, (typeof err?.reason === "string" && Object.hasOwn(FAILED_COMMENT_BY_REASON, err.reason) ? FAILED_COMMENT_BY_REASON[err.reason] : null) ?? FAILED_COMMENT);
2450
3500
  onFailure?.({ jobId: job?.id, outcome: "failed", reason: typeof err?.reason === "string" ? err.reason : "infra" });
2451
3501
  }
2452
3502
  };
@@ -2459,7 +3509,59 @@ export async function startWorker(
2459
3509
  // the completed listener would have sent for that record (a paid policy stop) is sent here instead, because
2460
3510
  // that listener never saw the first finish. No record, an earlier attempt's, a `failed` one, or a lookup fault
2461
3511
  // keeps the failure path below: the comment and the page.
3512
+ // Issue #645 (durable runs): a terminal stall of an ARMED job whose run has a checkpoint pointer is not the end of
3513
+ // the run. The worker that ran it died (that is what a stall is), so the processor never got to take its infra-exit
3514
+ // path; this listener takes it instead: the recovery is enqueued (routed to the host holding the checkpoint), the
3515
+ // crashed attempt's record is written (`failed`, `recovery-enqueued`: nothing else would write it, its processor is
3516
+ // gone), and the failure comment and on_failure are SUPPRESSED, because the run is continuing. Anything short of an
3517
+ // enqueued recovery (no pointer, the cap, an enqueue that failed) answers false, and the caller runs today's path:
3518
+ // the comment and the page, loudly, never a run that silently stops.
3519
+ const recoverStall = async (job) => {
3520
+ if (!isRecoveryArmed(job?.data)) return false;
3521
+ const runId = runIdOf(job);
3522
+ const pointer = await readCheckpointPointer({ runId });
3523
+ if (!pointer) {
3524
+ log("job_recovery_skipped", { jobId: job.id, reason: "no-checkpoint" });
3525
+ return false;
3526
+ }
3527
+ // Issue #662: the crashed attempt's start and how far it got, off the run's progress key, read BEFORE the enqueue,
3528
+ // because the recovery's first progress write replaces that key with its own empty snapshot. NOT `job.processedOn`:
3529
+ // BullMQ's stall check does not fail a stalled job outright, it moves it back to wait with the deferred failure
3530
+ // marked (`moveStalledJobsToWait` sets `defa`), and the next pickup's `prepareJobForProcessing` stamps a fresh
3531
+ // `processedOn` before it fails the job, so `processedOn` here is the re-pick time (a three-minute attempt read
3532
+ // as 23 ms long in the #645 lab). The key's `startedAt` is the attempt's own, written by its writer, and it is
3533
+ // taken only when three checks pass: the key has not ended (a crash never writes the final snapshot), its start is
3534
+ // not after now, and its start is no earlier than this job's enqueue time (`job.timestamp`). The last one ties the
3535
+ // key to THIS job: the key is the RUN's, so a recovery that died before its own first write (or ran on a host with
3536
+ // PI_RUN_PROGRESS=0) would otherwise find the base attempt's unended key and record a start from before the
3537
+ // recovery existed, with the base's counts. A retry of the same job id cannot be told apart by this (its
3538
+ // timestamp is the job's first enqueue); its writer's first write replaces the key, so only a retry that died
3539
+ // before writing would show the earlier attempt's start. Anything else is null, an honest unknown. Bounded and
3540
+ // never throwing (`readProgressKey`); the key is display only, so it decides nothing here either.
3541
+ const at0 = now();
3542
+ const key = await Promise.resolve().then(() => readProgressKeyFn({ redis, jobId: runId })).catch(() => null);
3543
+ const snap = key?.snapshot;
3544
+ const own = snap !== null && typeof snap === "object" && snap.ended === false && Number.isSafeInteger(snap.startedAt) && snap.startedAt > 0 && snap.startedAt <= at0 && Number.isSafeInteger(job.timestamp) && snap.startedAt >= job.timestamp;
3545
+ const crashedProgress = own ? { turns: snap.turn, steps: snap.steps, failed: snap.failed, interrupted: snap.interrupted, replayed: snap.replayed } : null;
3546
+ const r = await enqueueRecovery({ job, cause: "stall", pointer });
3547
+ if (!r?.enqueued) return false;
3548
+ // `attemptsMade` already counts this attempt here (moveToFailed added one), and `buildRecord` adds one for a record
3549
+ // written while processing, so the record is handed the count from before the move: it then names THIS attempt.
3550
+ const at = now();
3551
+ recordRun({
3552
+ job: { id: job.id, name: job.name, data: job.data, opts: job.opts, attemptsMade: Math.max(0, (Number.isSafeInteger(job.attemptsMade) ? job.attemptsMade : 1) - 1), timestamp: job.timestamp },
3553
+ // The snapshot's integer counts, rebuilt by `buildRecord` (`recordedProgress`): a bad value is dropped, never copied.
3554
+ result: { outcome: "failed", reason: "recovery-enqueued", ...(crashedProgress !== null ? { progress: crashedProgress } : {}) },
3555
+ startedAt: own ? new Date(snap.startedAt).toISOString() : null,
3556
+ endedAt: new Date(at).toISOString(),
3557
+ });
3558
+ log("job_failed", { jobId: job.id, attempt: job.attemptsMade, reason: "recovery-enqueued" });
3559
+ return true;
3560
+ };
2462
3561
  for (const w of allWorkers) w.on("failed", (job, err) => {
3562
+ // `finishedOn` is the queue's word that no retry follows (`onFailed` reads it the same way), so the expiry is set
3563
+ // here, before any path below returns: the terminal failure, the stall and the lost-lock checks alike.
3564
+ if (job?.finishedOn) finishAttempt(w, job);
2463
3565
  if (!(job?.finishedOn && err?.message === STALLED_FAILED_REASON)) return onFailed(job, err);
2464
3566
  void (async () => {
2465
3567
  // `attemptsMade` is read AFTER BullMQ's moveToFailed added one, so it equals the attempt number the
@@ -2468,9 +3570,54 @@ export async function startWorker(
2468
3570
  // can see why the record did not suppress it (a clock skew past the tolerance reads `older-than-job`).
2469
3571
  const onReject = (reason, source) => log("job_lost_lock_record_rejected", { jobId: job.id, reason, source });
2470
3572
  const record = await settledRecord(job.id, { attempt: job.attemptsMade, since: job.timestamp, onReject });
2471
- if (!record) return onFailed(job, err);
2472
- log("job_lost_lock_after_completion", { jobId: job.id, outcome: record.outcome, ...(record.reason ? { reason: record.reason } : {}) });
2473
- if (pagesAsPolicy(record)) onFailure({ jobId: job.id, outcome: "policy", reason: record.reason });
3573
+ if (!record) {
3574
+ // Issue #645: a RETURNED failure of this attempt that then lost its lock. `settledRecord` refuses every
3575
+ // `failed` record (a failure is the failure path's to report), but two returned failures already said
3576
+ // everything: `recovery-failed` posted its own sentence (so only its page is owed, once), and
3577
+ // `recovery-enqueued` handed the run to a recovery (nothing is owed). Running the failure path for either
3578
+ // would post a second failure comment, the second one false for a run that continues.
3579
+ // Read BOTH ways (`previousRecord`), never this host's file alone: on a fleet this host may hold an earlier
3580
+ // attempt's file while the host that ran THIS attempt wrote its record to the mirror. And each side held to
3581
+ // the rule every lookup of a job's own record shares (`recordOfThisJob`: the id, this attempt, and not older
3582
+ // than the job) BEFORE the higher attempt is kept, because ids are reused: a failed record an older job left
3583
+ // under the same id must not stand in for this attempt's and take away its recovery or its comment, and,
3584
+ // at a higher attempt, must not hide this attempt's own record on the other side either.
3585
+ const keep = (record) => recordOfThisJob(record, { jobId: job.id, attempt: job.attemptsMade, since: job.timestamp });
3586
+ const own = await previousRecord(job.id, { keep }).catch(() => null);
3587
+ const ownFailure = own !== null && own.outcome === "failed";
3588
+ // Issue #761: this attempt's `failed` record, written by a newer pi-dispatch. Its id, attempt, times and
3589
+ // outcome are stable core fields, so it says that the attempt ended on that host and what became of it was
3590
+ // decided there. Its `reason` is not core, so this build cannot tell `recovery-enqueued` (the run continues,
3591
+ // nothing is owed) from `recovery-failed` (its sentence posted, its page owed) from an ordinary failure (the
3592
+ // comment and the page owed). The failure path below fits only the last, and breaks the first:
3593
+ // `recoverStall` would ask for the newer host's recovery again and write this build's record over the newer
3594
+ // one, and the comment would say "will not be retried further" of a run that continues. So the terminal
3595
+ // failure path runs without its comment (`settledElsewhere`): every duty that closes what the job held, and
3596
+ // one page, which is owed or harmless in all three cases, where a wrong comment is wrong on the forge, in
3597
+ // front of everyone. An ordinary failure's comment is what this costs, and only while the fleet runs two
3598
+ // releases.
3599
+ if (ownFailure && !recordReadable(own)) return onFailed(job, err, { settledElsewhere: true });
3600
+ if (ownFailure && (own.reason === "recovery-failed" || own.reason === "recovery-enqueued")) {
3601
+ log("job_lost_lock_after_completion", { jobId: job.id, outcome: own.outcome, reason: own.reason });
3602
+ if (pagesAsReturnedFailure(own)) onFailure({ jobId: job.id, outcome: "failed", reason: own.reason });
3603
+ return;
3604
+ }
3605
+ // No settled record: the attempt died running. A run that can continue from its checkpoint does; any
3606
+ // fault on that path falls through to the failure comment and the page.
3607
+ let recovered = false;
3608
+ try {
3609
+ recovered = await recoverStall(job);
3610
+ } catch (e) {
3611
+ log("job_recovery_enqueue_failed", { jobId: job.id, code: typeof e?.code === "string" ? e.code : "error" });
3612
+ }
3613
+ if (recovered) return;
3614
+ return onFailed(job, err);
3615
+ }
3616
+ // Issue #761: a record a newer pi-dispatch wrote is read for its stable core only; its reason may mean
3617
+ // something else there, so it is neither logged nor paged on.
3618
+ const readable = recordReadable(record);
3619
+ log("job_lost_lock_after_completion", { jobId: job.id, outcome: record.outcome, ...(readable && record.reason ? { reason: record.reason } : {}) });
3620
+ if (readable && pagesAsPolicy(record)) onFailure({ jobId: job.id, outcome: "policy", reason: record.reason });
2474
3621
  })().catch(() => {}); // `settledRecord` never rejects; this only keeps a throwing log line from going unhandled
2475
3622
  });
2476
3623
 
@@ -2511,7 +3658,7 @@ export async function startWorker(
2511
3658
  const r = await reconcileGated(rq, served, { registry, log, tz: hostTz, authored: authoredCron(config) });
2512
3659
  log("schedules_installed", { installed: r.installed, removed: r.removed, ...(unserved.length > 0 && { unserved: unserved.length }) });
2513
3660
  } finally {
2514
- await rq.close().catch(() => {});
3661
+ await closeQueueSettled(rq);
2515
3662
  }
2516
3663
  } else {
2517
3664
  log("schedules_installed", { installed: 0, removed: 0, ...(unserved.length > 0 && { unserved: unserved.length }) });
@@ -2522,13 +3669,13 @@ export async function startWorker(
2522
3669
  // Only when a triggers file is configured; best-effort; a bad edit keeps the running schedulers. The
2523
3670
  // closer each of the three returns joins `extraClosers`, so the watch stops with the worker (issue #295).
2524
3671
  if (config.triggersFile) {
2525
- extraClosers.push(watchTriggersFile(config, cronQueue, log, schedules, registry, hostTz, config.workerNameDeclared, atBoot.triggers, checkEnvelopePlace));
3672
+ extraClosers.push(watchTriggersFile(config, cronQueue, log, schedules, registry, hostTz, config.workerNameDeclared, atBoot.triggers, checkEnvelopePlace, triggersState, watchFs));
2526
3673
  }
2527
3674
 
2528
3675
  // REQ-SCOPED-PAUSE-WINDOWS live edit: watch the pause-windows file and hot-swap the in-memory windows, so
2529
3676
  // an operator's add/delete of a pause window takes effect without a worker restart. A bad edit is kept out.
2530
3677
  if (config.pauseWindowsFile) {
2531
- extraClosers.push(watchPauseWindowsFile(config, pauseWindows, log, atBoot.pauseWindows));
3678
+ extraClosers.push(watchPauseWindowsFile(config, pauseWindows, log, atBoot.pauseWindows, watchFs));
2532
3679
  }
2533
3680
 
2534
3681
  // Issue #499 part B: the two files a project row joins reload as a pair; the deployment cap rides along so a limits
@@ -2556,6 +3703,15 @@ export async function startWorker(
2556
3703
  extraClosers.push(watchModelEndpointsFile(config, modelEndpoints, log, atBoot.modelEndpoints));
2557
3704
  }
2558
3705
 
3706
+ // Arm expiry only for declarations this host serves, including after a live edit first publishes a request.
3707
+ let declaresApprovals = false;
3708
+ try { declaresApprovals = parseTriggers(readFileSync(onceTriggersFile, "utf8"), onceTriggersFile).some((entry) => entry.run?.gates); } catch {}
3709
+ const retainedApprovals = await settleWithin(approvalStore.hasPending(durableHost), 2500, { error: "store-unavailable" });
3710
+ if (!retainedApprovals.ok) log("approval_expiry_scan_unavailable", { reason: "store-unavailable" });
3711
+ // A revoked declaration still has unanswered requests to retire. On an unreadable index, arm the retrying
3712
+ // sweep and name the uncertainty; do not silently abandon a durable deadline.
3713
+ if (declaresApprovals || retainedApprovals.pending || !retainedApprovals.ok) startApprovalSweep();
3714
+
2559
3715
  // issue #292 / OQ-007: re-run the three retention sweeps on a timer, because the supported deployment
2560
3716
  // is a service that restarts only on failure, so the healthy worker was the one that never re-swept.
2561
3717
  // Armed HERE, at the end of boot beside the watches: all three closures exist, boot's own sweeps have
@@ -2567,14 +3723,38 @@ export async function startWorker(
2567
3723
  // reaper. It is registered in `extraClosers` for issue #295's finding that UNREF'D IS NOT CLEANED UP --
2568
3724
  // this handle holds an `rmSync`, and a tick landing mid-drain could delete a retained workspace behind a
2569
3725
  // worker that already reported a clean shutdown.
3726
+ // Issue #645: the two durable-run sweeps, run once here at boot and then on every tick. The host-lost sweep on every
3727
+ // host (a fleet's recoveries are judged by whichever host gets there first); the checkpoint store's retention only
3728
+ // where there is a store. A fault is a log line, never a refused boot: both are backstops over a TTL.
3729
+ // The store sweep is given NO `active` list, deliberately: it removes a run's store only when its newest manifest is
3730
+ // older than the store TTL (7 days), and a run running here has written to its store within the job timeout (30
3731
+ // minutes: primeBase at its start, a checkpoint at most a minute apart), so no running run can be judged stale and an
3732
+ // active list could change no verdict. A run waiting for its recovery is bounded the same way: the host-lost sweep
3733
+ // gives up on it at RECOVERY_MAX_WAIT_MS, which IS that TTL. (The processor's running list holds job ids, not run
3734
+ // ids, so passing it would also name a `.rN` job's store wrongly.)
3735
+ const reapForkSources = forkStore ? () => forkStore.sweep() : null;
3736
+ const reapCheckpoints = workspaceCheckpointer ? async () => void (await workspaceCheckpointer.sweep()) : null;
3737
+ const reapHostLost = async () => void (await sweepHostLost());
3738
+ // NOT awaited: the host-lost sweep's first command waits for Valkey however long it is away, and the boot must not.
3739
+ // Each settles into its own log line; neither can reject unhandled.
3740
+ for (const [name, reap] of [["recovery_host_lost", reapHostLost], ...(reapForkSources ? [["fork_source", reapForkSources]] : []), ...(reapCheckpoints ? [["checkpoint", reapCheckpoints]] : [])]) {
3741
+ void Promise.resolve()
3742
+ .then(reap)
3743
+ .catch((err) => log(`${name}_reaper_skipped`, { reason: scrubCredentials(err?.message) }));
3744
+ }
2570
3745
  if (config.sweepIntervalHours > 0) {
2571
3746
  const sweep = makeRetentionSweepFn({
2572
3747
  reapers: [
2573
3748
  { name: "log", reap: reapLogs },
2574
3749
  // Issue #504 part B: where boot runs it, right after the run history it sits beside.
2575
3750
  { name: "allocation_log", reap: reapAllocationLogs },
3751
+ { name: "console_audit", reap: reapConsoleAudit },
3752
+ { name: "touch_outbox", reap: () => touchOutbox.repair() },
2576
3753
  { name: "sandbox", reap: reapSandboxes },
2577
3754
  { name: "session", reap: () => sessionStore.reapSessions() },
3755
+ ...(reapForkSources ? [{ name: "fork_source", reap: reapForkSources }] : []),
3756
+ { name: "recovery_host_lost", reap: reapHostLost },
3757
+ ...(reapCheckpoints ? [{ name: "checkpoint", reap: reapCheckpoints }] : []),
2578
3758
  ],
2579
3759
  intervalMs: config.sweepIntervalHours * 3600000,
2580
3760
  log,
@@ -2583,6 +3763,37 @@ export async function startWorker(
2583
3763
  extraClosers.push(sweep);
2584
3764
  }
2585
3765
 
3766
+ // Issue #712 (`DES-DOCTOR-RESULTS-PUBLISHED`): `pi-dispatch doctor --publish` as a child, shortly after boot and every
3767
+ // PI_DOCTOR_INTERVAL_HOURS, so every host publishes its checks whether or not anyone runs doctor there. After the
3768
+ // registry has started (above), so the first run's fleet lines see this host's own row. At 0 nothing is constructed,
3769
+ // the sweep's posture: no timer, no closer, no child is reachable.
3770
+ if (config.doctorIntervalHours > 0) {
3771
+ const doctorRuns = makeDoctorSpawnerFn({
3772
+ cliPath: fileURLToPath(new URL("./cli.mjs", import.meta.url)),
3773
+ intervalMs: config.doctorIntervalHours * 3600000,
3774
+ log,
3775
+ });
3776
+ doctorRuns.start();
3777
+ extraClosers.push(doctorRuns);
3778
+ }
3779
+
3780
+ // Issue #796 (`DES-PACKAGE-UPDATES-PREPARED-NOT-APPLIED`): the look, off until PI_UPDATES_LOOK=1, and each host applying what a person asked for. Children for
3781
+ // doctor's reason: neither belongs on the loop that renews paid job locks. The reconcile starts nothing until a person asked for something, and reaches a
3782
+ // registry only then, which is why it needs no switch of its own: until then it is one free Valkey read every two minutes.
3783
+ const cliPath = fileURLToPath(new URL("./cli.mjs", import.meta.url));
3784
+ if (config.updatesLook) {
3785
+ const look = makeUpdatesRunnerFn({ cliPath, intervalMs: UPDATES_LOOK_EVERY_HOURS * 3600000, log, args: ["updates", "look"], events: { overlapped: "updates_look_overlapped", spawnFailed: "updates_look_failed", finished: "updates_look_finished" } });
3786
+ look.start();
3787
+ extraClosers.push(look);
3788
+ }
3789
+ const reconcile = makeUpdatesRunnerFn({
3790
+ cliPath, intervalMs: UPDATES_RECONCILE_EVERY_MS, bootDelayMs: UPDATES_RECONCILE_EVERY_MS, timeoutMs: UPDATES_RECONCILE_TIMEOUT_MS, log, args: ["updates", "reconcile"],
3791
+ events: { overlapped: "updates_reconcile_overlapped", spawnFailed: "updates_reconcile_failed", finished: "updates_reconcile_finished" },
3792
+ shouldRun: () => hostHasPending(redis, config.workerName),
3793
+ });
3794
+ reconcile.start();
3795
+ extraClosers.push(reconcile);
3796
+
2586
3797
  log("worker_started", {
2587
3798
  queue: "pi-jobs",
2588
3799
  // Issue #278: the service's OWN answer to which daemon its jobs go to. `doctor` reads its caller's
@@ -2606,6 +3817,7 @@ export async function startWorker(
2606
3817
  imageDigest: bootImage.imageDigest ?? null, // two hosts on two builds of one tag used to emit byte-identical boot lines
2607
3818
  concurrency: bootConcurrency, // the slot count the Worker is actually constructed with (overlay may raise/lower it)
2608
3819
  sweepIntervalHours: config.sweepIntervalHours, // 0 = boot-only sweeps, this version's pre-#292 behaviour
3820
+ doctorIntervalHours: config.doctorIntervalHours, // issue #712: 0 = the worker runs no doctor of its own
2609
3821
  dailyCap: config.dailyCap,
2610
3822
  weeklyCap: config.weeklyCap, // null when the weekly window is disabled
2611
3823
  monthlyCap: config.monthlyCap, // null when the monthly window is disabled
@@ -2626,7 +3838,13 @@ export async function startWorker(
2626
3838
  settingsFile: config.settingsFile,
2627
3839
  captureJobLogs: config.captureJobLogs,
2628
3840
  logRetentionDays: config.logRetentionDays,
3841
+ consoleAuditRetentionDays: config.consoleAuditRetentionDays, // issue #761: the console audit trail's own; 0 = kept forever
3842
+ runMirrorMaxRuns: config.runMirrorMaxRuns, // issue #761: the shared run index's cap this host trims to
2629
3843
  sandboxRetentionHours: config.sandboxRetentionHours, // 0 = retention off; a run's directory is deleted as before
3844
+ // Issue #645 (durable runs): where checkpoints go (null = none: an armed job is refused before it spends) and
3845
+ // whether this host takes them (the store AND the run checkpointer); doctor reads the same directory.
3846
+ checkpointsDir: config.checkpointsDir ?? null,
3847
+ durableRuns: Boolean(runCheckpointer),
2630
3848
  });
2631
3849
  return worker;
2632
3850
  } catch (err) {