@edgehero/pi-dispatch 4.3.1 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (260) hide show
  1. package/.env.example +84 -3
  2. package/README.md +21 -7
  3. package/deploy/console.service +52 -0
  4. package/deploy/docker-compose.yml +3 -1
  5. package/deploy/pi-dispatch-valkey.container +4 -2
  6. package/deploy/sandbox-gateway.service +23 -0
  7. package/deploy/worker-env-wrapper.cmd +6 -0
  8. package/deploy/worker-env-wrapper.sh +10 -1
  9. package/package.json +118 -19
  10. package/src/agent-snapshot.mjs +49 -0
  11. package/src/answerer-runtime.mjs +194 -0
  12. package/src/answerer.mjs +178 -0
  13. package/src/approval-policy.mjs +76 -0
  14. package/src/approval-runtime.mjs +188 -0
  15. package/src/approval-session.mjs +137 -0
  16. package/src/approval-state.mjs +170 -0
  17. package/src/auth-cli.mjs +86 -0
  18. package/src/auth-connections.mjs +145 -0
  19. package/src/auth-policy.mjs +32 -0
  20. package/src/auth-store.mjs +97 -0
  21. package/src/azure-host.mjs +66 -4
  22. package/src/azure-prompt.mjs +5 -4
  23. package/src/backend-conformance.mjs +34 -2
  24. package/src/backend-local.mjs +103 -34
  25. package/src/backend-openshell.mjs +1540 -0
  26. package/src/backend-podman.mjs +63 -2
  27. package/src/backend-registry.mjs +61 -4
  28. package/src/backends.mjs +395 -6
  29. package/src/bullmq-internals.mjs +208 -0
  30. package/src/canary-cli.mjs +28 -0
  31. package/src/canary-lock.mjs +41 -0
  32. package/src/canary-runtime.mjs +237 -0
  33. package/src/canary.mjs +53 -0
  34. package/src/cancel-cli.mjs +47 -34
  35. package/src/cancel-state.mjs +213 -15
  36. package/src/capabilities.mjs +36 -1
  37. package/src/capacity-cli.mjs +28 -8
  38. package/src/capacity-records.mjs +75 -16
  39. package/src/capacity.mjs +65 -0
  40. package/src/chat-cli.mjs +288 -0
  41. package/src/chat-fork.mjs +547 -0
  42. package/src/chat-mode.mjs +156 -0
  43. package/src/chat-view.mjs +720 -0
  44. package/src/checkpoint-paths.mjs +113 -0
  45. package/src/cli.mjs +136 -41
  46. package/src/comment-tables.mjs +322 -0
  47. package/src/community-auth-adapters.mjs +87 -0
  48. package/src/community-auth-helper.mjs +76 -0
  49. package/src/community-auth-metadata.mjs +6 -0
  50. package/src/community-auth-output-pins.mjs +38 -0
  51. package/src/community-auth-patches.mjs +168 -0
  52. package/src/community-auth-pins.mjs +38 -0
  53. package/src/config.mjs +190 -29
  54. package/src/connection.mjs +83 -7
  55. package/src/console-answer.mjs +113 -0
  56. package/src/console-audit.mjs +604 -0
  57. package/src/console-bootstrap.mjs +325 -0
  58. package/src/console-client.mjs +848 -0
  59. package/src/console-devices.mjs +372 -0
  60. package/src/console-install.mjs +239 -0
  61. package/src/console-open.mjs +99 -0
  62. package/src/console-probe.mjs +106 -0
  63. package/src/console-tokens.mjs +459 -0
  64. package/src/container-names.mjs +51 -0
  65. package/src/container-spec.mjs +20 -1
  66. package/src/cpu-reserve.mjs +4 -1
  67. package/src/defer-reasons.mjs +200 -0
  68. package/src/defer-write.mjs +208 -0
  69. package/src/deployment-venue.mjs +23 -6
  70. package/src/docker-run.mjs +11 -0
  71. package/src/doctor-bundles.mjs +130 -0
  72. package/src/doctor-model-proposals.mjs +109 -0
  73. package/src/doctor-openshell.mjs +238 -0
  74. package/src/doctor-results.mjs +602 -0
  75. package/src/doctor-spawner.mjs +215 -0
  76. package/src/doctor.mjs +1657 -196
  77. package/src/dollar-budget.mjs +212 -3
  78. package/src/endpoint-status.mjs +100 -0
  79. package/src/env-allowlist.mjs +96 -6
  80. package/src/env-file.mjs +6 -4
  81. package/src/envelope.mjs +5 -3
  82. package/src/file-expect.mjs +47 -0
  83. package/src/forgejo-host.mjs +42 -5
  84. package/src/forgejo-prompt.mjs +5 -4
  85. package/src/forges.mjs +33 -3
  86. package/src/fork-job.mjs +58 -0
  87. package/src/fork-policy.mjs +23 -0
  88. package/src/fork-store.mjs +338 -0
  89. package/src/get-token.mjs +1 -1
  90. package/src/git-dirty.mjs +119 -14
  91. package/src/git-hardening.mjs +85 -0
  92. package/src/github-host.mjs +31 -6
  93. package/src/github-prompt.mjs +7 -16
  94. package/src/gitlab-host.mjs +42 -3
  95. package/src/gitlab-prompt.mjs +5 -4
  96. package/src/held-document.mjs +23 -0
  97. package/src/held-replies.mjs +325 -0
  98. package/src/held-settle.mjs +266 -0
  99. package/src/hook-runs.mjs +188 -0
  100. package/src/host-budget.mjs +8 -5
  101. package/src/host-clone.mjs +72 -0
  102. package/src/host-loads.mjs +557 -0
  103. package/src/host-registry.mjs +66 -6
  104. package/src/host-telemetry.mjs +92 -0
  105. package/src/http-body.mjs +78 -0
  106. package/src/image-preflight.mjs +156 -13
  107. package/src/import-pi.mjs +60 -172
  108. package/src/inbox-relay.mjs +482 -0
  109. package/src/inbox-settle.mjs +196 -0
  110. package/src/index.mjs +734 -129
  111. package/src/init.mjs +35 -5
  112. package/src/job-data-version.mjs +75 -0
  113. package/src/job-id.mjs +30 -15
  114. package/src/job-layout.mjs +22 -0
  115. package/src/job-size.mjs +16 -0
  116. package/src/live-jobs.mjs +14 -6
  117. package/src/live-probes.mjs +6 -2
  118. package/src/live-transcript.mjs +872 -0
  119. package/src/loaded-record.mjs +609 -0
  120. package/src/mcp-config.mjs +4 -2
  121. package/src/mcp-preflight.mjs +36 -5
  122. package/src/model-attempts.mjs +84 -0
  123. package/src/model-catalog.mjs +83 -0
  124. package/src/model-cooldowns.mjs +22 -0
  125. package/src/model-failover-protocol.mjs +11 -0
  126. package/src/model-failover.mjs +134 -0
  127. package/src/model-observation.mjs +50 -0
  128. package/src/model-proposal-protocol.mjs +21 -0
  129. package/src/model-proposals.mjs +75 -0
  130. package/src/model-ref.mjs +67 -10
  131. package/src/model-roles-file.mjs +24 -0
  132. package/src/model-roles.mjs +73 -0
  133. package/src/model-status.mjs +34 -0
  134. package/src/models-json.mjs +12 -2
  135. package/src/money.mjs +1 -1
  136. package/src/oauth-openai.mjs +121 -0
  137. package/src/older-state.mjs +249 -0
  138. package/src/on-failure.mjs +17 -13
  139. package/src/open-browser.mjs +31 -2
  140. package/src/opener-cli.mjs +135 -0
  141. package/src/opener-protocol.mjs +27 -0
  142. package/src/opener-terminal.mjs +128 -0
  143. package/src/openshell-cli.mjs +155 -0
  144. package/src/openshell-policy.mjs +163 -0
  145. package/src/outbox.mjs +14 -0
  146. package/src/output-cap.mjs +1 -0
  147. package/src/package-canary-record.mjs +26 -0
  148. package/src/package-canary.mjs +135 -0
  149. package/src/package-names.mjs +9 -0
  150. package/src/package-stage.mjs +418 -0
  151. package/src/packages.mjs +24 -16
  152. package/src/parked.mjs +2 -1
  153. package/src/pause-windows.mjs +136 -17
  154. package/src/podman-stack.mjs +40 -14
  155. package/src/portfolio-snapshot.mjs +26 -7
  156. package/src/prepare-github.mjs +41 -77
  157. package/src/prepare-local.mjs +32 -2
  158. package/src/prepare.mjs +45 -11
  159. package/src/private-store-boundary.mjs +70 -0
  160. package/src/processor.mjs +1550 -173
  161. package/src/projects.mjs +23 -0
  162. package/src/prompt-fence.mjs +18 -0
  163. package/src/provider-key.mjs +7 -0
  164. package/src/provider-steering.mjs +1 -0
  165. package/src/question-context.mjs +35 -0
  166. package/src/question-group-runtime.mjs +91 -0
  167. package/src/question-groups.mjs +171 -0
  168. package/src/queue.mjs +264 -55
  169. package/src/reason-catalog.mjs +448 -0
  170. package/src/receiver-status.mjs +230 -0
  171. package/src/record-reason.mjs +37 -0
  172. package/src/recovery-queue.mjs +566 -0
  173. package/src/redact-secrets.mjs +219 -0
  174. package/src/reserve-estimate.mjs +101 -0
  175. package/src/reserved-env.mjs +36 -1
  176. package/src/retention.mjs +67 -0
  177. package/src/run-budget-ledger.mjs +283 -0
  178. package/src/run-checkpoint.mjs +627 -0
  179. package/src/run-container.mjs +749 -33
  180. package/src/run-history.mjs +791 -45
  181. package/src/run-mirror.mjs +75 -7
  182. package/src/run-progress.mjs +751 -0
  183. package/src/runtime-observations.mjs +3 -1
  184. package/src/runtime-settings.mjs +272 -37
  185. package/src/sandbox-cli.mjs +78 -2
  186. package/src/sandbox-deployment.mjs +51 -0
  187. package/src/sandbox-gateway-protocol.mjs +378 -0
  188. package/src/sandbox-hosts.mjs +100 -0
  189. package/src/sandbox-inventory.mjs +83 -0
  190. package/src/sandbox-policy.mjs +225 -0
  191. package/src/sandbox-prepare-requests.mjs +145 -0
  192. package/src/sandbox-prepare.mjs +285 -0
  193. package/src/sandbox-refs.mjs +70 -0
  194. package/src/sandbox-session-monitor.mjs +4 -0
  195. package/src/sandbox-session-watch.mjs +124 -0
  196. package/src/sandbox-sessions.mjs +161 -0
  197. package/src/sandbox-store.mjs +63 -6
  198. package/src/sandbox-workspace-guard.mjs +39 -0
  199. package/src/sandbox.mjs +27 -200
  200. package/src/scheduler-stall-guard.mjs +14 -8
  201. package/src/schedules.mjs +19 -6
  202. package/src/secret-bundles.mjs +245 -0
  203. package/src/secret-profiles.mjs +103 -3
  204. package/src/secrets.mjs +19 -9
  205. package/src/service-env.mjs +38 -5
  206. package/src/service.mjs +638 -106
  207. package/src/session-key.mjs +60 -0
  208. package/src/session-store.mjs +168 -10
  209. package/src/setup-contract.mjs +40 -0
  210. package/src/setup-egress.mjs +113 -0
  211. package/src/setup-host-entry.mjs +36 -0
  212. package/src/setup-host.mjs +242 -0
  213. package/src/setup-lock.mjs +41 -0
  214. package/src/size-records.mjs +15 -3
  215. package/src/start.mjs +1353 -135
  216. package/src/status-comment.mjs +415 -0
  217. package/src/stderr-transcript.mjs +318 -0
  218. package/src/steer-ack.mjs +122 -0
  219. package/src/steer-cli.mjs +213 -0
  220. package/src/steer-framing.mjs +185 -0
  221. package/src/steer-inbox.mjs +693 -0
  222. package/src/store-version.mjs +139 -0
  223. package/src/subscription-config.mjs +56 -0
  224. package/src/subscription-limits.mjs +3 -0
  225. package/src/subscription-quota.mjs +259 -0
  226. package/src/subscription-runtime.mjs +123 -0
  227. package/src/subscription-status.mjs +42 -0
  228. package/src/subscription-wait.mjs +11 -0
  229. package/src/subscriptions.mjs +7 -1
  230. package/src/thinking-level.mjs +19 -0
  231. package/src/touch-outbox.mjs +61 -0
  232. package/src/touch-state.mjs +133 -0
  233. package/src/triggers-file.mjs +299 -39
  234. package/src/triggers.mjs +780 -38
  235. package/src/trust-paths.mjs +171 -0
  236. package/src/up.mjs +402 -53
  237. package/src/updates-cli.mjs +61 -0
  238. package/src/updates-look.mjs +127 -0
  239. package/src/updates-reconcile.mjs +182 -0
  240. package/src/updates-subscription-adapters.mjs +8 -0
  241. package/src/updates.mjs +252 -0
  242. package/src/valkey-auth.mjs +32 -9
  243. package/src/valkey-endpoint.mjs +14 -4
  244. package/src/wait-for.mjs +1 -1
  245. package/src/wait-state.mjs +62 -3
  246. package/src/watch-board.mjs +38 -0
  247. package/src/watch-closer.mjs +1 -1
  248. package/src/watch-contract.mjs +102 -0
  249. package/src/watch-decision.mjs +78 -0
  250. package/src/watch-drainer.mjs +14 -0
  251. package/src/watch-issue-adapter.mjs +44 -0
  252. package/src/watch-issue.mjs +111 -0
  253. package/src/watch-record.mjs +10 -0
  254. package/src/watch-runtime.mjs +167 -0
  255. package/src/watch-store.mjs +183 -0
  256. package/src/watch-trace.mjs +34 -0
  257. package/src/workspace-checkpoint.mjs +1750 -0
  258. package/src/zero-touch-fixes.mjs +99 -0
  259. package/src/zero-touch.mjs +225 -0
  260. package/src/repeat-slot.mjs +0 -13
package/src/init.mjs CHANGED
@@ -10,7 +10,8 @@
10
10
  import { existsSync, copyFileSync, lstatSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
11
11
  import { fileURLToPath } from "node:url";
12
12
  import { join } from "node:path";
13
- import { parseBackendList, venuesOf } from "./backends.mjs";
13
+ import { openshellSetupOf, parseBackendList, venuesOf } from "./backends.mjs";
14
+ import { OPENSHELL_BREW_INSTALL, OPENSHELL_GATEWAY_UNIT, OPENSHELL_NO_WINDOWS, OPENSHELL_RELEASE_URL, OPENSHELL_TESTED_VERSION } from "./openshell-cli.mjs";
14
15
  import { deploymentVenueEnv } from "./deployment-venue.mjs";
15
16
  import { setEnvKeyIfEmpty } from "./env-file.mjs";
16
17
  import { PACKAGED_EGRESS_PROXY_CONF } from "./egress-conf-copy.mjs";
@@ -167,7 +168,7 @@ export function runInit(cwd = process.cwd(), deps = {}) {
167
168
  const code = results.some(([verb]) => verb === "refused") ? 1 : 0;
168
169
  if (deps.steps === false) return code;
169
170
  if (deps.venues) {
170
- out(nextSteps(deps.venues, { platform }));
171
+ out(nextSteps(deps.venues, { platform, env: deps.venueEnv ?? env }));
171
172
  return code;
172
173
  }
173
174
  // Decided exactly as `up` decides it (issue #453, gate round 1): a next-steps ladder for a venue `up` would refuse to
@@ -184,7 +185,7 @@ export function runInit(cwd = process.cwd(), deps = {}) {
184
185
  out(`\nNext: which venue this deployment runs is unknown, so no steps are shown: ${err.message}. Fix PI_BACKENDS, then run \`pi-dispatch init\` again for them (it keeps every file above).\n`);
185
186
  return code;
186
187
  }
187
- out(nextSteps(venuesOf(venue.env), { platform }));
188
+ out(nextSteps(venuesOf(venue.env), { platform, env: venue.env }));
188
189
  return code;
189
190
  }
190
191
 
@@ -250,8 +251,14 @@ function lstatOrNull(fs, path) {
250
251
  * the job image into this account's own store, the venue key and provider key in `.env` (what `up` and `service
251
252
  * install` read), the stack as Quadlet units, doctor, and the worker as a user service.
252
253
  */
253
- export function nextSteps(venues = { localUsed: true, podmanUsed: false }, { platform = "linux" } = {}) {
254
- if (venues.podmanUsed && !venues.localUsed) return platform === "linux" ? PODMAN_NEXT_STEPS : PODMAN_OFF_LINUX;
254
+ export function nextSteps(venues = { localUsed: true, podmanUsed: false }, { platform = "linux", env = {} } = {}) {
255
+ // Issue #707: the openshell venues' ladder, by the rule `up` and `service install` follow (`openshellSetupOf`). Alone,
256
+ // a ladder of their own; beside `local` or `podman`, that venue's ladder and one line saying what `up` adds for them.
257
+ const openshell = openshellSetupOf(venues, env);
258
+ if (openshell.used && platform === "win32") return `\nNext: ${OPENSHELL_NO_WINDOWS}.\n`;
259
+ if (openshell.used && !venues.localUsed && !venues.podmanUsed) return openshellNextSteps(openshell, platform);
260
+ const also = openshell.used ? `\nThe openshell venue too: install OpenShell ${OPENSHELL_TESTED_VERSION} (${OPENSHELL_RELEASE_URL}); \`pi-dispatch up\` then checks its gateway,\noffers to start and register it, and puts the job image into ${openshell.engine ?? "its engine"}, the gateway's engine.\n` : "";
261
+ if (venues.podmanUsed && !venues.localUsed) return platform === "linux" ? `${PODMAN_NEXT_STEPS}${also}` : PODMAN_OFF_LINUX;
255
262
  return `
256
263
  Next:
257
264
  1. docker pull ghcr.io/edgehero/pi-job:latest && docker tag ghcr.io/edgehero/pi-job:latest pi-job:latest
@@ -263,6 +270,29 @@ Next:
263
270
 
264
271
  Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch
265
272
  (or let the panel do all of the above: /dispatch setup walks these steps with a consent per action)
273
+ ${also}`;
274
+ }
275
+
276
+ /**
277
+ * The openshell venues alone (issue #707): OpenShell is installed by the operator (`up` never installs it), then `up`
278
+ * checks its gateway and offers to start and register it, puts the job image into the gateway's engine and runs Valkey
279
+ * there, and `service install` runs the worker as a unit that pulls the gateway's in (a LaunchAgent on macOS, beside the
280
+ * release formula's Homebrew service).
281
+ */
282
+ function openshellNextSteps(openshell, platform) {
283
+ const pad = (cmd) => cmd.padEnd(52);
284
+ // The release formula on macOS (review round 1 of #715): homebrew-core's `openshell` defines no Homebrew service.
285
+ const install = platform === "darwin" ? `\`${OPENSHELL_BREW_INSTALL}\`, the release formula, whose service runs the gateway` : OPENSHELL_RELEASE_URL;
286
+ const service = platform === "darwin" ? "# the worker as a LaunchAgent; `brew services start openshell` keeps the gateway up" : `# the worker as a user service that pulls in ${OPENSHELL_GATEWAY_UNIT}`;
287
+ return `
288
+ Next (the openshell venue on ${openshell.engine ?? "its engine"}). First install OpenShell ${OPENSHELL_TESTED_VERSION}, the release the venue was measured
289
+ on (${install}); \`pi-dispatch up\` checks it and installs nothing:
290
+ 1. ${pad("edit .env")}# PI_BACKENDS=openshell (PI_OPENSHELL_ENGINE=podman for Podman), and ANTHROPIC_API_KEY (or your provider's key)
291
+ 2. ${pad("pi-dispatch up")}# the gateway (offers to start and register it), the job image into ${openshell.engine ?? "its engine"}, Valkey there
292
+ 3. ${pad("pi-dispatch doctor")}# verify the gateway, its engine, Landlock, the image and the key
293
+ 4. ${pad("pi-dispatch service install")}${service}
294
+
295
+ Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch
266
296
  `;
267
297
  }
268
298
 
@@ -0,0 +1,75 @@
1
+ /**
2
+ * The version of a queued job's data (issue #761, `DES-STORE-VERSION-WHEN-NEEDED`). A LEAF apart from the shared rule,
3
+ * so the receiver can stamp the jobs it stores for later (`fallbackJobOf`) with the same rule the worker's builders use.
4
+ *
5
+ * Jobs outlive an upgrade: a forge job is kept `FORGE_JOB_RETENTION_DAYS` (retention.mjs), a recovery may wait up to 7
6
+ * days, held and parked runs wait as long as their conditions do. A field a newer release adds to job data is DROPPED
7
+ * by an older worker's builders and IGNORED by its processor, which for a field that narrows a job (a model list, a
8
+ * cost cap) is a wider job, run and paid for. `trigger-skew` catches that only for the fields it lists, and only for
9
+ * forge jobs. So a field whose absence an older worker would misread declares the version that introduced it here, the
10
+ * builders stamp `dataVersion` with the lowest version the data needs, and a worker that meets a version above its own
11
+ * DEFERS the job (`newer-release`) rather than run it: the job waits, with its reason shown, for a worker that can read
12
+ * it, and nothing is spent or lost.
13
+ *
14
+ * Unversioned on purpose: answer-draft, watch and question-grouping jobs carry only references their runtime resolves at
15
+ * pickup, and sandbox prepare requests live 30 minutes on their own host's queue.
16
+ */
17
+ import { neededVersion, valueNeededVersion } from "./store-version.mjs";
18
+
19
+ /** The highest job data version this build reads. */
20
+ export const JOB_DATA_VERSION = 3;
21
+
22
+ /**
23
+ * Each job data field that needs a version above 1, with the version that introduced it.
24
+ * `packageCanary` (issue #796): an older worker would ignore it and run the trigger's ordinary job in its place, a paid run that tried nothing and reported on nothing.
25
+ * `excludeExtensions` and `excludeExtensionTools` (issue #800): an older worker would ignore either and run the job with what its trigger takes away, on a clean record.
26
+ * At 3 and not 2, because a version says what its reader knows and a worker that reads 2 knows `packageCanary` and not these: stamped 2, such a worker
27
+ * reads the job as its own, runs it with both fields ignored, and rebuilds a stored message without them. A new field takes a version no running worker reads.
28
+ */
29
+ export const JOB_DATA_FIELD_VERSIONS = Object.freeze({ packageCanary: 2, excludeExtensions: 3, excludeExtensionTools: 3 });
30
+
31
+ /**
32
+ * Each EXISTING top-level field whose range a later release widens, as `field -> (value) => version` (the shared rule,
33
+ * `valueNeededVersion`): a new condition kind in `waitFor`, say, or a new `kind` (a forge this build has no credentials
34
+ * for, refused `config-refused` at the mint). An older worker meeting the wider value would refuse the job (a comment, a
35
+ * record, a disarmed one-shot) where a stamp makes it defer. Empty until one does.
36
+ */
37
+ export const JOB_DATA_VALUE_VERSIONS = Object.freeze({});
38
+
39
+ /** The field that carries the version. */
40
+ export const JOB_DATA_VERSION_FIELD = "dataVersion";
41
+
42
+ /** The lowest version `data` needs, from its top-level fields and their values: 1 while nothing in it needs more. */
43
+ export function jobDataVersionFor(data, { table = JOB_DATA_FIELD_VERSIONS, valueTable = JOB_DATA_VALUE_VERSIONS } = {}) {
44
+ if (data === null || typeof data !== "object") return 1;
45
+ const { [JOB_DATA_VERSION_FIELD]: _version, ...fields } = data;
46
+ return Math.max(neededVersion(Object.keys(fields), table), valueNeededVersion(fields, valueTable));
47
+ }
48
+
49
+ /**
50
+ * Whether `data` carries a `dataVersion` this build cannot read, as `{ version }`, else null: a version newer than
51
+ * `supported` (a newer worker's job: `{ version: N }`), or a value that is not a whole number of 1 or more (`null`,
52
+ * `"1"`, `1.5`, `0`: `{ version: null }`), which no release writes.
53
+ * A missing `dataVersion` is version 1. The caller defers such a job, malformed included: the hold spends nothing and
54
+ * an operator can cancel it, while running it would guess at data nothing here wrote. It never reads the rest first.
55
+ */
56
+ export function jobDataNewer(data, { supported = JOB_DATA_VERSION } = {}) {
57
+ if (data === null || typeof data !== "object" || !Object.hasOwn(data, JOB_DATA_VERSION_FIELD)) return null;
58
+ const version = data[JOB_DATA_VERSION_FIELD];
59
+ if (Number.isSafeInteger(version) && version >= 1 && version <= supported) return null;
60
+ return { version: Number.isSafeInteger(version) && version >= 1 ? version : null };
61
+ }
62
+
63
+ /**
64
+ * `data` with `dataVersion` set to the version it needs: the higher of the version it arrived with (a job a newer
65
+ * release stored, re-enqueued here) and the one its fields and values need, and no field at all while that is 1. Throws
66
+ * a RangeError when the data arrived with a version this build cannot read (`jobDataNewer`: newer, or malformed), so a
67
+ * builder never re-enqueues a newer job under a lower version and erases the evidence its fields were dropped.
68
+ */
69
+ export function withJobDataVersion(data, { table = JOB_DATA_FIELD_VERSIONS, valueTable = JOB_DATA_VALUE_VERSIONS, supported = JOB_DATA_VERSION } = {}) {
70
+ if (jobDataNewer(data, { supported })) throw new RangeError(`job data carries a ${JOB_DATA_VERSION_FIELD} this build cannot read (newer than ${supported}, or not a whole number of 1 or more), so it cannot rebuild it`);
71
+ const declared = data?.[JOB_DATA_VERSION_FIELD];
72
+ const needed = Math.max(Number.isSafeInteger(declared) ? declared : 1, jobDataVersionFor(data, { table, valueTable }));
73
+ const { [JOB_DATA_VERSION_FIELD]: _drop, ...rest } = data ?? {};
74
+ return needed > 1 ? { ...rest, [JOB_DATA_VERSION_FIELD]: needed } : rest;
75
+ }
package/src/job-id.mjs CHANGED
@@ -1,4 +1,5 @@
1
1
  import { createHash } from "node:crypto";
2
+ import { MANUAL_JOB_PREFIX, customJobIdAllowed } from "./bullmq-internals.mjs";
2
3
  import { configError } from "./config.mjs";
3
4
  import { forgeSpec } from "./forges.mjs";
4
5
 
@@ -12,11 +13,13 @@ import { forgeSpec } from "./forges.mjs";
12
13
  * Kept free of any bullmq import so the dedup logic is testable everywhere, not only where the
13
14
  * queue's dependencies are installed.
14
15
  */
15
- export function localJobId({ folder, flow, task, minute, provider, model, models }) {
16
+ export function localJobId({ folder, flow, task, minute, provider, model, models, fallbackModels }) {
16
17
  // The model fields join the key (issue #544): two runs that differ only in the model are two runs, and with the
17
18
  // key blind to them the second was dropped as a duplicate. Appended only when one is set, so a job that names no
18
19
  // model keeps the id it always had. `models` is JSON so a list cannot collide with a differently split one.
19
- const modelFields = provider === undefined && model === undefined && models === undefined ? [] : [provider ?? "", model ?? "", models === undefined ? "" : JSON.stringify(models)];
20
+ const modelFields = provider === undefined && model === undefined && models === undefined && fallbackModels === undefined ? [] : [provider ?? "", model ?? "", models === undefined ? "" : JSON.stringify(models)];
21
+ // The ordered fallback policy changes which model may run, so it is part of local identity too.
22
+ if (fallbackModels !== undefined) modelFields.push(JSON.stringify(fallbackModels));
20
23
  // NUL-delimited so {folder:'a',task:'bc'} and {folder:'ab',task:'c'} cannot collide.
21
24
  const digest = createHash("sha256").update([folder, flow ?? "", task ?? "", minute, ...modelFields].join("\0")).digest("hex");
22
25
  return `local-${digest.slice(0, 16)}`;
@@ -29,15 +32,27 @@ export function localJobId({ folder, flow, task, minute, provider, model, models
29
32
  * must not pay twice). A later minute is a new id, so a deliberate re-run is never blocked.
30
33
  *
31
34
  * The shape is BullMQ's, not a choice: a custom id that holds a `:` is refused unless it splits into exactly three
32
- * parts (bullmq `Job.addJob`, "Custom Id cannot contain :"), which is how its own `repeat:<id>:<millis>` passes. A cron
33
- * id can hold no `:` (the triggers loader refuses one), so this always splits into three. An ISO time would add two
34
- * more. The `manual:` prefix keeps it apart from a scheduled run's `repeat:` id, which is what tells the event writer
35
- * (`localEventContext`) there is no scheduled instant to report.
35
+ * parts (`customJobIdAllowed`, BullMQ's own rule), which is how its own `repeat:<id>:<millis>` passes. A cron id can
36
+ * hold no `:` (the triggers loader refuses one), so this always splits into three; one that does is refused here, by
37
+ * that rule, before the queue throws it. An ISO time would add two more. The `manual:` prefix keeps it apart from a
38
+ * scheduled run's `repeat:` id, which is what tells the event writer (`localEventContext`) there is no scheduled
39
+ * instant to report.
36
40
  */
37
41
  export function manualTriggerJobId({ triggerId, now }) {
38
- if (typeof triggerId !== "string" || triggerId === "" || triggerId.includes(":")) throw new TypeError("a cron trigger id is a non-empty string with no ':'");
39
- const millis = Math.floor(now.getTime() / 60_000) * 60_000;
40
- return `manual:${triggerId}:${millis}`;
42
+ const refused = () => new TypeError("a cron trigger id is a non-empty string with no ':'");
43
+ if (typeof triggerId !== "string" || triggerId === "") throw refused();
44
+ const id = `${MANUAL_JOB_PREFIX}:${triggerId}:${Math.floor(now.getTime() / 60_000) * 60_000}`;
45
+ if (!customJobIdAllowed(id)) throw refused();
46
+ return id;
47
+ }
48
+
49
+ /**
50
+ * A canary's jobId (issue #796): the same shape, so every reader that attributes a run to its trigger still does, one millisecond past the minute, so a hand fire in the same minute
51
+ * cannot swallow it, nor it the fire. They are different jobs running different code.
52
+ */
53
+ export function manualCanaryJobId({ triggerId, now }) {
54
+ const base = manualTriggerJobId({ triggerId, now });
55
+ return base.replace(/\d+$/, (minute) => String(Number(minute) + 1));
41
56
  }
42
57
 
43
58
  /**
@@ -62,9 +77,9 @@ export function chainedJobId({ parentJobId, flow, task }) {
62
77
 
63
78
  /**
64
79
  * The deterministic jobId for a GitHub-triggered job: the `X-GitHub-Delivery` GUID, prefixed.
65
- * BullMQ's dedup is `EXISTS jobId`, so a redelivered webhook (GitHub retries on timeout) carries
66
- * the same GUID, resolves to the same id, and the duplicate paid run is rejected --
67
- * REQ-DEDUP-BY-DELIVERY-GUID.
80
+ * BullMQ's dedup is `EXISTS jobId`, so a redelivered webhook (GitHub never resends one on its own; a
81
+ * person redelivers it from the hook's Recent Deliveries or the REST API) carries the same GUID, resolves
82
+ * to the same id, and the duplicate paid run is rejected -- REQ-DEDUP-BY-DELIVERY-GUID.
68
83
  *
69
84
  * Kept free of any bullmq import (mirrors localJobId) so the dedup key is derivable everywhere, not
70
85
  * only where the queue's dependencies are installed. A missing GUID is a caller bug -- the receiver
@@ -77,9 +92,9 @@ export function deliveryJobId(guid) {
77
92
 
78
93
  /**
79
94
  * The same thing for a GitLab-triggered job: the `webhook-id` (or its older name `Idempotency-Key`),
80
- * prefixed `gl-`. GitLab keeps that value CONSTANT across its own retries, which is the exact property
81
- * REQ-DEDUP-BY-DELIVERY-GUID needs, so the guarantee is the same one -- and retention-bounded in the same
82
- * way.
95
+ * prefixed `gl-`. GitLab keeps that value CONSTANT when a delivery is resent ("Resend Request" or the API;
96
+ * GitLab never resends on its own), which is the exact property REQ-DEDUP-BY-DELIVERY-GUID needs, so the
97
+ * guarantee is the same one -- and retention-bounded in the same way.
83
98
  */
84
99
  export function gitlabDeliveryJobId(id) {
85
100
  return forgeDeliveryJobId("gitlab", id);
@@ -0,0 +1,22 @@
1
+ /**
2
+ * The worker's copy of the container's file layouts (issue #683, INT-CONTAINER-JOB-INPUTS): `mounts` for a venue that
3
+ * binds the job's directories (`local`, `podman`), `sandbox` for one that copies them under the image's WORKDIR
4
+ * (`openshell`). The runner reads the same table (image/runner/src/layout.mjs) by the name the worker writes into
5
+ * PI_JOB_LAYOUT; the two files live on different hosts at run time, so the copy is forced, and
6
+ * `worker/test/job-layout.test.mjs` holds them identical. CHANGE BOTH, IN THE SAME COMMIT.
7
+ *
8
+ * The `mounts` row is exactly container-spec.mjs's mount targets, which predate this file; that test pins them too, so
9
+ * a binding venue's argv is byte-identical to one built before the table existed.
10
+ */
11
+ export const LAYOUTS = Object.freeze({
12
+ mounts: Object.freeze({ job: "/job", workspace: "/workspace", session: "/session", outbox: "/outbox", inbox: "/inbox", global: "/opt/pi-global" }),
13
+ sandbox: Object.freeze({ job: "/sandbox/job", workspace: "/sandbox/workspace", session: "/sandbox/session", outbox: "/sandbox/outbox", inbox: null, global: "/sandbox/pi-global" }),
14
+ });
15
+
16
+ export const DEFAULT_LAYOUT = "mounts";
17
+
18
+ /** The layout `name` names, refusing one the table does not hold: a spec built on a guessed tree reads nothing. */
19
+ export function layoutFor(name = DEFAULT_LAYOUT) {
20
+ if (typeof name !== "string" || !Object.hasOwn(LAYOUTS, name)) throw new Error(`unknown job layout ${JSON.stringify(name)} (known: ${Object.keys(LAYOUTS).join(", ")})`);
21
+ return LAYOUTS[name];
22
+ }
package/src/job-size.mjs CHANGED
@@ -90,6 +90,22 @@ export const DEFAULT_JOB_CPUS = "2";
90
90
  /** Where a resolved size came from: the job's project row, the deployment's `PI_JOB_*` settings, or the built-in default. */
91
91
  export const JOB_SIZE_SOURCES = Object.freeze(["project", "env", "default"]);
92
92
 
93
+ /** pi 1.4.0's QuickJS heap limit per codemode worker, held against its artifact by codemode-workers.pinned.test.mjs. */
94
+ export const CODEMODE_WORKER_HEAP_MIB = 256;
95
+
96
+ /**
97
+ * Issue #693: reserve half the job's memory for pi, tools and worker overhead; divide the rest by pi's QuickJS
98
+ * heap limit. This is a concurrency sizing heuristic, NOT an RSS guarantee: the VM heap excludes worker overhead,
99
+ * and extensions may create sessions in other processes. The container's memory bound remains the hard limit.
100
+ * At the 512 MiB floor one worker fits; sizes between whole slots round down rather than borrowing the reserve.
101
+ */
102
+ export function codemodeWorkersOf(memMiB) {
103
+ if (!Number.isSafeInteger(memMiB) || memMiB < JOB_MEMORY_FLOOR_MIB || memMiB > JOB_MEMORY_CEILING_MIB) {
104
+ throw new Error("codemode worker sizing requires a valid job memory in MiB");
105
+ }
106
+ return Math.max(1, Math.floor(memMiB / (2 * CODEMODE_WORKER_HEAP_MIB)));
107
+ }
108
+
93
109
  /** The scope prefix of a project row, as `scoped-limits.mjs` spells it (`PROJECT_SCOPE_PREFIX`; a test holds them equal). */
94
110
  const PROJECT_ROW_PREFIX = "project:";
95
111
 
package/src/live-jobs.mjs CHANGED
@@ -21,7 +21,10 @@
21
21
  * carried and cannot satisfy it. No quote, backslash, slash or control character can reach the JSON, so the
22
22
  * writer's path-shape check never refuses the field.
23
23
  * - `p` is a project id (`isProjectId`) or null; `runs:rec:*` already carries project ids.
24
- * - `m`, `c` and `at` are integers (MiB, hundredths of a CPU, epoch millis), `o` is 1.
24
+ * - `m`, `c`, `at` and orphan stop `s` are integers (MiB, hundredths of a CPU, epoch millis), `o` is 1;
25
+ * `v` is the orphan runtime from this build's closed venue table.
26
+ * - `h` and `ha` are the actual dollar reservation (micro-dollars) and its epoch instant. A zero is known zero;
27
+ * absent fields mean an older host or an orphan whose reservation this process cannot know, never zero.
25
28
  *
26
29
  * ORDER AND SIZE. Oldest first (by `at`, then id), at most `LIVE_JOBS_MAX`: the jobs that have held a slot longest
27
30
  * carry the most busy time and are the ones an operator looks for. The rest are counted in `jobsMore`, never dropped
@@ -29,6 +32,7 @@
29
32
  */
30
33
 
31
34
  import { createHash } from "node:crypto";
35
+ import { BACKEND_NAMES } from "./backends.mjs";
32
36
  import { isProjectId } from "./project-id.mjs";
33
37
 
34
38
  /** The most jobs one row lists; the rest are counted (`jobsMore`). */
@@ -38,6 +42,7 @@ export const LIVE_JOB_ID_RE = /^[A-Za-z0-9._:-]{1,128}$/;
38
42
  /** The most a `jobs` value may be before a reader declines to parse it: 32 entries at their longest fit well inside. */
39
43
  export const LIVE_JOBS_MAX_BYTES = 16 * 1024;
40
44
 
45
+ const nonnegativeInt = (v) => Number.isSafeInteger(v) && v >= 0;
41
46
  const positiveInt = (v) => Number.isSafeInteger(v) && v > 0;
42
47
 
43
48
  /** A job id as it may be published: itself when inside `LIVE_JOB_ID_RE`, else its digest; null for no id. */
@@ -48,11 +53,11 @@ export function publishedJobId(id) {
48
53
  }
49
54
 
50
55
  /**
51
- * The list a row publishes, before the cap: `[{ id, p, m, c, at, o? }]`, oldest first. `running` is the in-flight
56
+ * The list a row publishes, before the cap: `[{ id, p, m, c, at, o?, v?, s? }]`, oldest first. `running` is the in-flight
52
57
  * map's entries (`{ id, project, memMiB, cpuCenti, at }`), `budgetEntries` the host budget's ledger (`entries()`), of
53
58
  * which only orphans are taken. An entry with no id or no instant is left out and counted in `skipped`.
54
59
  */
55
- export function liveJobsOf({ running = [], budgetEntries = [] } = {}) {
60
+ export function liveJobsOf({ running = [], budgetEntries = [], defaultVenue = null } = {}) {
56
61
  const out = [];
57
62
  const seen = new Set();
58
63
  let skipped = 0;
@@ -64,7 +69,8 @@ export function liveJobsOf({ running = [], budgetEntries = [] } = {}) {
64
69
  }
65
70
  if (seen.has(id)) return; // one job, two pickups (or running and orphaned): listed once
66
71
  seen.add(id);
67
- out.push({ id, p: isProjectId(e.project) ? e.project : null, ...(positiveInt(e.memMiB) ? { m: e.memMiB } : {}), ...(positiveInt(e.cpuCenti) ? { c: e.cpuCenti } : {}), at: e.at, ...(orphan ? { o: 1 } : {}) });
72
+ const venue = typeof e.orphan?.venue === "string" ? e.orphan.venue : (e.orphan?.venue && typeof e.orphan.venue === "object" ? (e.orphan.venue.backend ?? defaultVenue) : null);
73
+ out.push({ id, p: isProjectId(e.project) ? e.project : null, ...(positiveInt(e.memMiB) ? { m: e.memMiB } : {}), ...(positiveInt(e.cpuCenti) ? { c: e.cpuCenti } : {}), at: e.at, ...(nonnegativeInt(e.holdMicros) && positiveInt(e.holdAt) ? { h: e.holdMicros, ha: e.holdAt } : {}), ...(orphan ? { o: 1, ...(BACKEND_NAMES.includes(venue) ? { v: venue } : {}), ...(positiveInt(e.orphan?.since) ? { s: e.orphan.since } : {}) } : {}) });
68
74
  };
69
75
  // Running first, so a job both running and in the ledger as an orphan (it cannot be: the gate defers it) reads as running.
70
76
  for (const e of Array.isArray(running) ? running : []) push(e, false);
@@ -116,11 +122,13 @@ function parseList(list) {
116
122
  const ok = e !== null && typeof e === "object" && !Array.isArray(e);
117
123
  const m = ok ? optionalSize(e.m) : undefined;
118
124
  const c = ok ? optionalSize(e.c) : undefined;
119
- if (!ok || typeof e.id !== "string" || !LIVE_JOB_ID_RE.test(e.id) || !(e.p === undefined || e.p === null || isProjectId(e.p)) || m === undefined || c === undefined || !positiveInt(e.at) || !(e.o === undefined || e.o === 1 || typeof e.o === "boolean")) {
125
+ const venue = ok && (e.v === undefined || e.v === null || BACKEND_NAMES.includes(e.v));
126
+ const since = ok && (e.s === undefined || e.s === null || positiveInt(e.s));
127
+ if (!ok || !venue || !since || typeof e.id !== "string" || !LIVE_JOB_ID_RE.test(e.id) || !(e.p === undefined || e.p === null || isProjectId(e.p)) || m === undefined || c === undefined || !positiveInt(e.at) || !(e.o === undefined || e.o === 1 || typeof e.o === "boolean")) {
120
128
  dropped++;
121
129
  continue;
122
130
  }
123
- jobs.push({ id: e.id, p: e.p ?? null, m, c, at: e.at, o: e.o === 1 || e.o === true });
131
+ jobs.push({ id: e.id, p: e.p ?? null, m, c, at: e.at, o: e.o === 1 || e.o === true, ...(nonnegativeInt(e.h) && positiveInt(e.ha) ? { h: e.h, ha: e.ha } : {}), ...(e.v != null ? { v: e.v } : {}), ...(e.s != null ? { s: e.s } : {}) });
124
132
  }
125
133
  return { jobs, dropped };
126
134
  }
@@ -767,8 +767,11 @@ export async function runLiveProbes({
767
767
  // caller's and already knows its binary. `isLocal` is the gate on both endpoint reads: docker's is the observed
768
768
  // endpoint, a podman venue's is `serviceIsRemote === false`, which has no `.local` to read. The gate still decides
769
769
  // on the value it is handed and must answer exactly `true`; anything else runs nothing.
770
- buildArgs = buildDockerRunArgs,
771
- bin = "docker",
770
+ // Issue #681: both REQUIRED here, never a docker default. A venue's doctor that forgot to pass them would read back
771
+ // docker's containers and print the verdict as its own. The four builders keep docker's builder as their default,
772
+ // since only this entry point reaches them with a venue.
773
+ buildArgs,
774
+ bin,
772
775
  isLocal = (observed) => observed?.local === true,
773
776
  // Issue #452, gate round 3: the runtime as this pass already read it (`{ podman, rootless, version } | null`), for the
774
777
  // detach gate, so doctor asks the daemon once per run. Absent, the gate reads it itself.
@@ -782,6 +785,7 @@ export async function runLiveProbes({
782
785
  cgroupParent = CGROUP_PARENT,
783
786
  cpuBudgetCenti = null,
784
787
  }) {
788
+ if (typeof buildArgs !== "function" || typeof bin !== "string" || bin === "") throw new TypeError("runLiveProbes: the venue's buildArgs and bin are required (issue #681)");
785
789
  const notRun = (reason) => ({ ran: false, reason, verdicts: [], notes: [], swept: [] });
786
790
  const notLocal = notRun(`this shell's ${bin} CLI is not observed to point at this host, so bind paths and .Mounts would describe another machine; nothing was run`);
787
791
  if (isLocal(endpoint) !== true) return notLocal;