@edgehero/pi-dispatch 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +94 -4
- package/README.md +20 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +117 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +128 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +199 -30
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +159 -0
- package/src/console-client.mjs +848 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1663 -198
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +103 -9
- package/src/env-file.mjs +6 -4
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-auth.mjs +4 -0
- package/src/forgejo-host.mjs +127 -31
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +73 -15
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +740 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +1 -1
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +4 -1
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1554 -167
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +296 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +33 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +782 -40
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +398 -51
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
package/src/init.mjs
CHANGED
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
import { existsSync, copyFileSync, lstatSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
11
11
|
import { fileURLToPath } from "node:url";
|
|
12
12
|
import { join } from "node:path";
|
|
13
|
-
import { parseBackendList, venuesOf } from "./backends.mjs";
|
|
13
|
+
import { openshellSetupOf, parseBackendList, venuesOf } from "./backends.mjs";
|
|
14
|
+
import { OPENSHELL_BREW_INSTALL, OPENSHELL_GATEWAY_UNIT, OPENSHELL_NO_WINDOWS, OPENSHELL_RELEASE_URL, OPENSHELL_TESTED_VERSION } from "./openshell-cli.mjs";
|
|
14
15
|
import { deploymentVenueEnv } from "./deployment-venue.mjs";
|
|
15
16
|
import { setEnvKeyIfEmpty } from "./env-file.mjs";
|
|
16
17
|
import { PACKAGED_EGRESS_PROXY_CONF } from "./egress-conf-copy.mjs";
|
|
@@ -167,7 +168,7 @@ export function runInit(cwd = process.cwd(), deps = {}) {
|
|
|
167
168
|
const code = results.some(([verb]) => verb === "refused") ? 1 : 0;
|
|
168
169
|
if (deps.steps === false) return code;
|
|
169
170
|
if (deps.venues) {
|
|
170
|
-
out(nextSteps(deps.venues, { platform }));
|
|
171
|
+
out(nextSteps(deps.venues, { platform, env: deps.venueEnv ?? env }));
|
|
171
172
|
return code;
|
|
172
173
|
}
|
|
173
174
|
// Decided exactly as `up` decides it (issue #453, gate round 1): a next-steps ladder for a venue `up` would refuse to
|
|
@@ -184,7 +185,7 @@ export function runInit(cwd = process.cwd(), deps = {}) {
|
|
|
184
185
|
out(`\nNext: which venue this deployment runs is unknown, so no steps are shown: ${err.message}. Fix PI_BACKENDS, then run \`pi-dispatch init\` again for them (it keeps every file above).\n`);
|
|
185
186
|
return code;
|
|
186
187
|
}
|
|
187
|
-
out(nextSteps(venuesOf(venue.env), { platform }));
|
|
188
|
+
out(nextSteps(venuesOf(venue.env), { platform, env: venue.env }));
|
|
188
189
|
return code;
|
|
189
190
|
}
|
|
190
191
|
|
|
@@ -250,8 +251,14 @@ function lstatOrNull(fs, path) {
|
|
|
250
251
|
* the job image into this account's own store, the venue key and provider key in `.env` (what `up` and `service
|
|
251
252
|
* install` read), the stack as Quadlet units, doctor, and the worker as a user service.
|
|
252
253
|
*/
|
|
253
|
-
export function nextSteps(venues = { localUsed: true, podmanUsed: false }, { platform = "linux" } = {}) {
|
|
254
|
-
|
|
254
|
+
export function nextSteps(venues = { localUsed: true, podmanUsed: false }, { platform = "linux", env = {} } = {}) {
|
|
255
|
+
// Issue #707: the openshell venues' ladder, by the rule `up` and `service install` follow (`openshellSetupOf`). Alone,
|
|
256
|
+
// a ladder of their own; beside `local` or `podman`, that venue's ladder and one line saying what `up` adds for them.
|
|
257
|
+
const openshell = openshellSetupOf(venues, env);
|
|
258
|
+
if (openshell.used && platform === "win32") return `\nNext: ${OPENSHELL_NO_WINDOWS}.\n`;
|
|
259
|
+
if (openshell.used && !venues.localUsed && !venues.podmanUsed) return openshellNextSteps(openshell, platform);
|
|
260
|
+
const also = openshell.used ? `\nThe openshell venue too: install OpenShell ${OPENSHELL_TESTED_VERSION} (${OPENSHELL_RELEASE_URL}); \`pi-dispatch up\` then checks its gateway,\noffers to start and register it, and puts the job image into ${openshell.engine ?? "its engine"}, the gateway's engine.\n` : "";
|
|
261
|
+
if (venues.podmanUsed && !venues.localUsed) return platform === "linux" ? `${PODMAN_NEXT_STEPS}${also}` : PODMAN_OFF_LINUX;
|
|
255
262
|
return `
|
|
256
263
|
Next:
|
|
257
264
|
1. docker pull ghcr.io/edgehero/pi-job:latest && docker tag ghcr.io/edgehero/pi-job:latest pi-job:latest
|
|
@@ -263,6 +270,29 @@ Next:
|
|
|
263
270
|
|
|
264
271
|
Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch
|
|
265
272
|
(or let the panel do all of the above: /dispatch setup walks these steps with a consent per action)
|
|
273
|
+
${also}`;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* The openshell venues alone (issue #707): OpenShell is installed by the operator (`up` never installs it), then `up`
|
|
278
|
+
* checks its gateway and offers to start and register it, puts the job image into the gateway's engine and runs Valkey
|
|
279
|
+
* there, and `service install` runs the worker as a unit that pulls the gateway's in (a LaunchAgent on macOS, beside the
|
|
280
|
+
* release formula's Homebrew service).
|
|
281
|
+
*/
|
|
282
|
+
function openshellNextSteps(openshell, platform) {
|
|
283
|
+
const pad = (cmd) => cmd.padEnd(52);
|
|
284
|
+
// The release formula on macOS (review round 1 of #715): homebrew-core's `openshell` defines no Homebrew service.
|
|
285
|
+
const install = platform === "darwin" ? `\`${OPENSHELL_BREW_INSTALL}\`, the release formula, whose service runs the gateway` : OPENSHELL_RELEASE_URL;
|
|
286
|
+
const service = platform === "darwin" ? "# the worker as a LaunchAgent; `brew services start openshell` keeps the gateway up" : `# the worker as a user service that pulls in ${OPENSHELL_GATEWAY_UNIT}`;
|
|
287
|
+
return `
|
|
288
|
+
Next (the openshell venue on ${openshell.engine ?? "its engine"}). First install OpenShell ${OPENSHELL_TESTED_VERSION}, the release the venue was measured
|
|
289
|
+
on (${install}); \`pi-dispatch up\` checks it and installs nothing:
|
|
290
|
+
1. ${pad("edit .env")}# PI_BACKENDS=openshell (PI_OPENSHELL_ENGINE=podman for Podman), and ANTHROPIC_API_KEY (or your provider's key)
|
|
291
|
+
2. ${pad("pi-dispatch up")}# the gateway (offers to start and register it), the job image into ${openshell.engine ?? "its engine"}, Valkey there
|
|
292
|
+
3. ${pad("pi-dispatch doctor")}# verify the gateway, its engine, Landlock, the image and the key
|
|
293
|
+
4. ${pad("pi-dispatch service install")}${service}
|
|
294
|
+
|
|
295
|
+
Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch
|
|
266
296
|
`;
|
|
267
297
|
}
|
|
268
298
|
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The version of a queued job's data (issue #761, `DES-STORE-VERSION-WHEN-NEEDED`). A LEAF apart from the shared rule,
|
|
3
|
+
* so the receiver can stamp the jobs it stores for later (`fallbackJobOf`) with the same rule the worker's builders use.
|
|
4
|
+
*
|
|
5
|
+
* Jobs outlive an upgrade: a forge job is kept `FORGE_JOB_RETENTION_DAYS` (retention.mjs), a recovery may wait up to 7
|
|
6
|
+
* days, held and parked runs wait as long as their conditions do. A field a newer release adds to job data is DROPPED
|
|
7
|
+
* by an older worker's builders and IGNORED by its processor, which for a field that narrows a job (a model list, a
|
|
8
|
+
* cost cap) is a wider job, run and paid for. `trigger-skew` catches that only for the fields it lists, and only for
|
|
9
|
+
* forge jobs. So a field whose absence an older worker would misread declares the version that introduced it here, the
|
|
10
|
+
* builders stamp `dataVersion` with the lowest version the data needs, and a worker that meets a version above its own
|
|
11
|
+
* DEFERS the job (`newer-release`) rather than run it: the job waits, with its reason shown, for a worker that can read
|
|
12
|
+
* it, and nothing is spent or lost.
|
|
13
|
+
*
|
|
14
|
+
* Unversioned on purpose: answer-draft, watch and question-grouping jobs carry only references their runtime resolves at
|
|
15
|
+
* pickup, and sandbox prepare requests live 30 minutes on their own host's queue.
|
|
16
|
+
*/
|
|
17
|
+
import { neededVersion, valueNeededVersion } from "./store-version.mjs";
|
|
18
|
+
|
|
19
|
+
/** The highest job data version this build reads. */
|
|
20
|
+
export const JOB_DATA_VERSION = 3;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Each job data field that needs a version above 1, with the version that introduced it.
|
|
24
|
+
* `packageCanary` (issue #796): an older worker would ignore it and run the trigger's ordinary job in its place, a paid run that tried nothing and reported on nothing.
|
|
25
|
+
* `excludeExtensions` and `excludeExtensionTools` (issue #800): an older worker would ignore either and run the job with what its trigger takes away, on a clean record.
|
|
26
|
+
* At 3 and not 2, because a version says what its reader knows and a worker that reads 2 knows `packageCanary` and not these: stamped 2, such a worker
|
|
27
|
+
* reads the job as its own, runs it with both fields ignored, and rebuilds a stored message without them. A new field takes a version no running worker reads.
|
|
28
|
+
*/
|
|
29
|
+
export const JOB_DATA_FIELD_VERSIONS = Object.freeze({ packageCanary: 2, excludeExtensions: 3, excludeExtensionTools: 3 });
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Each EXISTING top-level field whose range a later release widens, as `field -> (value) => version` (the shared rule,
|
|
33
|
+
* `valueNeededVersion`): a new condition kind in `waitFor`, say, or a new `kind` (a forge this build has no credentials
|
|
34
|
+
* for, refused `config-refused` at the mint). An older worker meeting the wider value would refuse the job (a comment, a
|
|
35
|
+
* record, a disarmed one-shot) where a stamp makes it defer. Empty until one does.
|
|
36
|
+
*/
|
|
37
|
+
export const JOB_DATA_VALUE_VERSIONS = Object.freeze({});
|
|
38
|
+
|
|
39
|
+
/** The field that carries the version. */
|
|
40
|
+
export const JOB_DATA_VERSION_FIELD = "dataVersion";
|
|
41
|
+
|
|
42
|
+
/** The lowest version `data` needs, from its top-level fields and their values: 1 while nothing in it needs more. */
|
|
43
|
+
export function jobDataVersionFor(data, { table = JOB_DATA_FIELD_VERSIONS, valueTable = JOB_DATA_VALUE_VERSIONS } = {}) {
|
|
44
|
+
if (data === null || typeof data !== "object") return 1;
|
|
45
|
+
const { [JOB_DATA_VERSION_FIELD]: _version, ...fields } = data;
|
|
46
|
+
return Math.max(neededVersion(Object.keys(fields), table), valueNeededVersion(fields, valueTable));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Whether `data` carries a `dataVersion` this build cannot read, as `{ version }`, else null: a version newer than
|
|
51
|
+
* `supported` (a newer worker's job: `{ version: N }`), or a value that is not a whole number of 1 or more (`null`,
|
|
52
|
+
* `"1"`, `1.5`, `0`: `{ version: null }`), which no release writes.
|
|
53
|
+
* A missing `dataVersion` is version 1. The caller defers such a job, malformed included: the hold spends nothing and
|
|
54
|
+
* an operator can cancel it, while running it would guess at data nothing here wrote. It never reads the rest first.
|
|
55
|
+
*/
|
|
56
|
+
export function jobDataNewer(data, { supported = JOB_DATA_VERSION } = {}) {
|
|
57
|
+
if (data === null || typeof data !== "object" || !Object.hasOwn(data, JOB_DATA_VERSION_FIELD)) return null;
|
|
58
|
+
const version = data[JOB_DATA_VERSION_FIELD];
|
|
59
|
+
if (Number.isSafeInteger(version) && version >= 1 && version <= supported) return null;
|
|
60
|
+
return { version: Number.isSafeInteger(version) && version >= 1 ? version : null };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* `data` with `dataVersion` set to the version it needs: the higher of the version it arrived with (a job a newer
|
|
65
|
+
* release stored, re-enqueued here) and the one its fields and values need, and no field at all while that is 1. Throws
|
|
66
|
+
* a RangeError when the data arrived with a version this build cannot read (`jobDataNewer`: newer, or malformed), so a
|
|
67
|
+
* builder never re-enqueues a newer job under a lower version and erases the evidence its fields were dropped.
|
|
68
|
+
*/
|
|
69
|
+
export function withJobDataVersion(data, { table = JOB_DATA_FIELD_VERSIONS, valueTable = JOB_DATA_VALUE_VERSIONS, supported = JOB_DATA_VERSION } = {}) {
|
|
70
|
+
if (jobDataNewer(data, { supported })) throw new RangeError(`job data carries a ${JOB_DATA_VERSION_FIELD} this build cannot read (newer than ${supported}, or not a whole number of 1 or more), so it cannot rebuild it`);
|
|
71
|
+
const declared = data?.[JOB_DATA_VERSION_FIELD];
|
|
72
|
+
const needed = Math.max(Number.isSafeInteger(declared) ? declared : 1, jobDataVersionFor(data, { table, valueTable }));
|
|
73
|
+
const { [JOB_DATA_VERSION_FIELD]: _drop, ...rest } = data ?? {};
|
|
74
|
+
return needed > 1 ? { ...rest, [JOB_DATA_VERSION_FIELD]: needed } : rest;
|
|
75
|
+
}
|
package/src/job-id.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { MANUAL_JOB_PREFIX, customJobIdAllowed } from "./bullmq-internals.mjs";
|
|
2
3
|
import { configError } from "./config.mjs";
|
|
3
4
|
import { forgeSpec } from "./forges.mjs";
|
|
4
5
|
|
|
@@ -12,11 +13,13 @@ import { forgeSpec } from "./forges.mjs";
|
|
|
12
13
|
* Kept free of any bullmq import so the dedup logic is testable everywhere, not only where the
|
|
13
14
|
* queue's dependencies are installed.
|
|
14
15
|
*/
|
|
15
|
-
export function localJobId({ folder, flow, task, minute, provider, model, models }) {
|
|
16
|
+
export function localJobId({ folder, flow, task, minute, provider, model, models, fallbackModels }) {
|
|
16
17
|
// The model fields join the key (issue #544): two runs that differ only in the model are two runs, and with the
|
|
17
18
|
// key blind to them the second was dropped as a duplicate. Appended only when one is set, so a job that names no
|
|
18
19
|
// model keeps the id it always had. `models` is JSON so a list cannot collide with a differently split one.
|
|
19
|
-
const modelFields = provider === undefined && model === undefined && models === undefined ? [] : [provider ?? "", model ?? "", models === undefined ? "" : JSON.stringify(models)];
|
|
20
|
+
const modelFields = provider === undefined && model === undefined && models === undefined && fallbackModels === undefined ? [] : [provider ?? "", model ?? "", models === undefined ? "" : JSON.stringify(models)];
|
|
21
|
+
// The ordered fallback policy changes which model may run, so it is part of local identity too.
|
|
22
|
+
if (fallbackModels !== undefined) modelFields.push(JSON.stringify(fallbackModels));
|
|
20
23
|
// NUL-delimited so {folder:'a',task:'bc'} and {folder:'ab',task:'c'} cannot collide.
|
|
21
24
|
const digest = createHash("sha256").update([folder, flow ?? "", task ?? "", minute, ...modelFields].join("\0")).digest("hex");
|
|
22
25
|
return `local-${digest.slice(0, 16)}`;
|
|
@@ -29,15 +32,27 @@ export function localJobId({ folder, flow, task, minute, provider, model, models
|
|
|
29
32
|
* must not pay twice). A later minute is a new id, so a deliberate re-run is never blocked.
|
|
30
33
|
*
|
|
31
34
|
* The shape is BullMQ's, not a choice: a custom id that holds a `:` is refused unless it splits into exactly three
|
|
32
|
-
* parts (
|
|
33
|
-
*
|
|
34
|
-
* more. The `manual:` prefix keeps it apart from a
|
|
35
|
-
* (`localEventContext`) there is no scheduled
|
|
35
|
+
* parts (`customJobIdAllowed`, BullMQ's own rule), which is how its own `repeat:<id>:<millis>` passes. A cron id can
|
|
36
|
+
* hold no `:` (the triggers loader refuses one), so this always splits into three; one that does is refused here, by
|
|
37
|
+
* that rule, before the queue throws it. An ISO time would add two more. The `manual:` prefix keeps it apart from a
|
|
38
|
+
* scheduled run's `repeat:` id, which is what tells the event writer (`localEventContext`) there is no scheduled
|
|
39
|
+
* instant to report.
|
|
36
40
|
*/
|
|
37
41
|
export function manualTriggerJobId({ triggerId, now }) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
42
|
+
const refused = () => new TypeError("a cron trigger id is a non-empty string with no ':'");
|
|
43
|
+
if (typeof triggerId !== "string" || triggerId === "") throw refused();
|
|
44
|
+
const id = `${MANUAL_JOB_PREFIX}:${triggerId}:${Math.floor(now.getTime() / 60_000) * 60_000}`;
|
|
45
|
+
if (!customJobIdAllowed(id)) throw refused();
|
|
46
|
+
return id;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* A canary's jobId (issue #796): the same shape, so every reader that attributes a run to its trigger still does, one millisecond past the minute, so a hand fire in the same minute
|
|
51
|
+
* cannot swallow it, nor it the fire. They are different jobs running different code.
|
|
52
|
+
*/
|
|
53
|
+
export function manualCanaryJobId({ triggerId, now }) {
|
|
54
|
+
const base = manualTriggerJobId({ triggerId, now });
|
|
55
|
+
return base.replace(/\d+$/, (minute) => String(Number(minute) + 1));
|
|
41
56
|
}
|
|
42
57
|
|
|
43
58
|
/**
|
|
@@ -62,9 +77,9 @@ export function chainedJobId({ parentJobId, flow, task }) {
|
|
|
62
77
|
|
|
63
78
|
/**
|
|
64
79
|
* The deterministic jobId for a GitHub-triggered job: the `X-GitHub-Delivery` GUID, prefixed.
|
|
65
|
-
* BullMQ's dedup is `EXISTS jobId`, so a redelivered webhook (GitHub
|
|
66
|
-
*
|
|
67
|
-
* REQ-DEDUP-BY-DELIVERY-GUID.
|
|
80
|
+
* BullMQ's dedup is `EXISTS jobId`, so a redelivered webhook (GitHub never resends one on its own; a
|
|
81
|
+
* person redelivers it from the hook's Recent Deliveries or the REST API) carries the same GUID, resolves
|
|
82
|
+
* to the same id, and the duplicate paid run is rejected -- REQ-DEDUP-BY-DELIVERY-GUID.
|
|
68
83
|
*
|
|
69
84
|
* Kept free of any bullmq import (mirrors localJobId) so the dedup key is derivable everywhere, not
|
|
70
85
|
* only where the queue's dependencies are installed. A missing GUID is a caller bug -- the receiver
|
|
@@ -77,9 +92,9 @@ export function deliveryJobId(guid) {
|
|
|
77
92
|
|
|
78
93
|
/**
|
|
79
94
|
* The same thing for a GitLab-triggered job: the `webhook-id` (or its older name `Idempotency-Key`),
|
|
80
|
-
* prefixed `gl-`. GitLab keeps that value CONSTANT
|
|
81
|
-
* REQ-DEDUP-BY-DELIVERY-GUID needs, so the
|
|
82
|
-
* way.
|
|
95
|
+
* prefixed `gl-`. GitLab keeps that value CONSTANT when a delivery is resent ("Resend Request" or the API;
|
|
96
|
+
* GitLab never resends on its own), which is the exact property REQ-DEDUP-BY-DELIVERY-GUID needs, so the
|
|
97
|
+
* guarantee is the same one -- and retention-bounded in the same way.
|
|
83
98
|
*/
|
|
84
99
|
export function gitlabDeliveryJobId(id) {
|
|
85
100
|
return forgeDeliveryJobId("gitlab", id);
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The worker's copy of the container's file layouts (issue #683, INT-CONTAINER-JOB-INPUTS): `mounts` for a venue that
|
|
3
|
+
* binds the job's directories (`local`, `podman`), `sandbox` for one that copies them under the image's WORKDIR
|
|
4
|
+
* (`openshell`). The runner reads the same table (image/runner/src/layout.mjs) by the name the worker writes into
|
|
5
|
+
* PI_JOB_LAYOUT; the two files live on different hosts at run time, so the copy is forced, and
|
|
6
|
+
* `worker/test/job-layout.test.mjs` holds them identical. CHANGE BOTH, IN THE SAME COMMIT.
|
|
7
|
+
*
|
|
8
|
+
* The `mounts` row is exactly container-spec.mjs's mount targets, which predate this file; that test pins them too, so
|
|
9
|
+
* a binding venue's argv is byte-identical to one built before the table existed.
|
|
10
|
+
*/
|
|
11
|
+
export const LAYOUTS = Object.freeze({
|
|
12
|
+
mounts: Object.freeze({ job: "/job", workspace: "/workspace", session: "/session", outbox: "/outbox", inbox: "/inbox", global: "/opt/pi-global" }),
|
|
13
|
+
sandbox: Object.freeze({ job: "/sandbox/job", workspace: "/sandbox/workspace", session: "/sandbox/session", outbox: "/sandbox/outbox", inbox: null, global: "/sandbox/pi-global" }),
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
export const DEFAULT_LAYOUT = "mounts";
|
|
17
|
+
|
|
18
|
+
/** The layout `name` names, refusing one the table does not hold: a spec built on a guessed tree reads nothing. */
|
|
19
|
+
export function layoutFor(name = DEFAULT_LAYOUT) {
|
|
20
|
+
if (typeof name !== "string" || !Object.hasOwn(LAYOUTS, name)) throw new Error(`unknown job layout ${JSON.stringify(name)} (known: ${Object.keys(LAYOUTS).join(", ")})`);
|
|
21
|
+
return LAYOUTS[name];
|
|
22
|
+
}
|
package/src/job-size.mjs
CHANGED
|
@@ -90,6 +90,22 @@ export const DEFAULT_JOB_CPUS = "2";
|
|
|
90
90
|
/** Where a resolved size came from: the job's project row, the deployment's `PI_JOB_*` settings, or the built-in default. */
|
|
91
91
|
export const JOB_SIZE_SOURCES = Object.freeze(["project", "env", "default"]);
|
|
92
92
|
|
|
93
|
+
/** pi 1.4.0's QuickJS heap limit per codemode worker, held against its artifact by codemode-workers.pinned.test.mjs. */
|
|
94
|
+
export const CODEMODE_WORKER_HEAP_MIB = 256;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Issue #693: reserve half the job's memory for pi, tools and worker overhead; divide the rest by pi's QuickJS
|
|
98
|
+
* heap limit. This is a concurrency sizing heuristic, NOT an RSS guarantee: the VM heap excludes worker overhead,
|
|
99
|
+
* and extensions may create sessions in other processes. The container's memory bound remains the hard limit.
|
|
100
|
+
* At the 512 MiB floor one worker fits; sizes between whole slots round down rather than borrowing the reserve.
|
|
101
|
+
*/
|
|
102
|
+
export function codemodeWorkersOf(memMiB) {
|
|
103
|
+
if (!Number.isSafeInteger(memMiB) || memMiB < JOB_MEMORY_FLOOR_MIB || memMiB > JOB_MEMORY_CEILING_MIB) {
|
|
104
|
+
throw new Error("codemode worker sizing requires a valid job memory in MiB");
|
|
105
|
+
}
|
|
106
|
+
return Math.max(1, Math.floor(memMiB / (2 * CODEMODE_WORKER_HEAP_MIB)));
|
|
107
|
+
}
|
|
108
|
+
|
|
93
109
|
/** The scope prefix of a project row, as `scoped-limits.mjs` spells it (`PROJECT_SCOPE_PREFIX`; a test holds them equal). */
|
|
94
110
|
const PROJECT_ROW_PREFIX = "project:";
|
|
95
111
|
|
package/src/live-jobs.mjs
CHANGED
|
@@ -21,7 +21,10 @@
|
|
|
21
21
|
* carried and cannot satisfy it. No quote, backslash, slash or control character can reach the JSON, so the
|
|
22
22
|
* writer's path-shape check never refuses the field.
|
|
23
23
|
* - `p` is a project id (`isProjectId`) or null; `runs:rec:*` already carries project ids.
|
|
24
|
-
* - `m`, `c` and `
|
|
24
|
+
* - `m`, `c`, `at` and orphan stop `s` are integers (MiB, hundredths of a CPU, epoch millis), `o` is 1;
|
|
25
|
+
* `v` is the orphan runtime from this build's closed venue table.
|
|
26
|
+
* - `h` and `ha` are the actual dollar reservation (micro-dollars) and its epoch instant. A zero is known zero;
|
|
27
|
+
* absent fields mean an older host or an orphan whose reservation this process cannot know, never zero.
|
|
25
28
|
*
|
|
26
29
|
* ORDER AND SIZE. Oldest first (by `at`, then id), at most `LIVE_JOBS_MAX`: the jobs that have held a slot longest
|
|
27
30
|
* carry the most busy time and are the ones an operator looks for. The rest are counted in `jobsMore`, never dropped
|
|
@@ -29,6 +32,7 @@
|
|
|
29
32
|
*/
|
|
30
33
|
|
|
31
34
|
import { createHash } from "node:crypto";
|
|
35
|
+
import { BACKEND_NAMES } from "./backends.mjs";
|
|
32
36
|
import { isProjectId } from "./project-id.mjs";
|
|
33
37
|
|
|
34
38
|
/** The most jobs one row lists; the rest are counted (`jobsMore`). */
|
|
@@ -38,6 +42,7 @@ export const LIVE_JOB_ID_RE = /^[A-Za-z0-9._:-]{1,128}$/;
|
|
|
38
42
|
/** The most a `jobs` value may be before a reader declines to parse it: 32 entries at their longest fit well inside. */
|
|
39
43
|
export const LIVE_JOBS_MAX_BYTES = 16 * 1024;
|
|
40
44
|
|
|
45
|
+
const nonnegativeInt = (v) => Number.isSafeInteger(v) && v >= 0;
|
|
41
46
|
const positiveInt = (v) => Number.isSafeInteger(v) && v > 0;
|
|
42
47
|
|
|
43
48
|
/** A job id as it may be published: itself when inside `LIVE_JOB_ID_RE`, else its digest; null for no id. */
|
|
@@ -48,11 +53,11 @@ export function publishedJobId(id) {
|
|
|
48
53
|
}
|
|
49
54
|
|
|
50
55
|
/**
|
|
51
|
-
* The list a row publishes, before the cap: `[{ id, p, m, c, at, o? }]`, oldest first. `running` is the in-flight
|
|
56
|
+
* The list a row publishes, before the cap: `[{ id, p, m, c, at, o?, v?, s? }]`, oldest first. `running` is the in-flight
|
|
52
57
|
* map's entries (`{ id, project, memMiB, cpuCenti, at }`), `budgetEntries` the host budget's ledger (`entries()`), of
|
|
53
58
|
* which only orphans are taken. An entry with no id or no instant is left out and counted in `skipped`.
|
|
54
59
|
*/
|
|
55
|
-
export function liveJobsOf({ running = [], budgetEntries = [] } = {}) {
|
|
60
|
+
export function liveJobsOf({ running = [], budgetEntries = [], defaultVenue = null } = {}) {
|
|
56
61
|
const out = [];
|
|
57
62
|
const seen = new Set();
|
|
58
63
|
let skipped = 0;
|
|
@@ -64,7 +69,8 @@ export function liveJobsOf({ running = [], budgetEntries = [] } = {}) {
|
|
|
64
69
|
}
|
|
65
70
|
if (seen.has(id)) return; // one job, two pickups (or running and orphaned): listed once
|
|
66
71
|
seen.add(id);
|
|
67
|
-
|
|
72
|
+
const venue = typeof e.orphan?.venue === "string" ? e.orphan.venue : (e.orphan?.venue && typeof e.orphan.venue === "object" ? (e.orphan.venue.backend ?? defaultVenue) : null);
|
|
73
|
+
out.push({ id, p: isProjectId(e.project) ? e.project : null, ...(positiveInt(e.memMiB) ? { m: e.memMiB } : {}), ...(positiveInt(e.cpuCenti) ? { c: e.cpuCenti } : {}), at: e.at, ...(nonnegativeInt(e.holdMicros) && positiveInt(e.holdAt) ? { h: e.holdMicros, ha: e.holdAt } : {}), ...(orphan ? { o: 1, ...(BACKEND_NAMES.includes(venue) ? { v: venue } : {}), ...(positiveInt(e.orphan?.since) ? { s: e.orphan.since } : {}) } : {}) });
|
|
68
74
|
};
|
|
69
75
|
// Running first, so a job both running and in the ledger as an orphan (it cannot be: the gate defers it) reads as running.
|
|
70
76
|
for (const e of Array.isArray(running) ? running : []) push(e, false);
|
|
@@ -116,11 +122,13 @@ function parseList(list) {
|
|
|
116
122
|
const ok = e !== null && typeof e === "object" && !Array.isArray(e);
|
|
117
123
|
const m = ok ? optionalSize(e.m) : undefined;
|
|
118
124
|
const c = ok ? optionalSize(e.c) : undefined;
|
|
119
|
-
|
|
125
|
+
const venue = ok && (e.v === undefined || e.v === null || BACKEND_NAMES.includes(e.v));
|
|
126
|
+
const since = ok && (e.s === undefined || e.s === null || positiveInt(e.s));
|
|
127
|
+
if (!ok || !venue || !since || typeof e.id !== "string" || !LIVE_JOB_ID_RE.test(e.id) || !(e.p === undefined || e.p === null || isProjectId(e.p)) || m === undefined || c === undefined || !positiveInt(e.at) || !(e.o === undefined || e.o === 1 || typeof e.o === "boolean")) {
|
|
120
128
|
dropped++;
|
|
121
129
|
continue;
|
|
122
130
|
}
|
|
123
|
-
jobs.push({ id: e.id, p: e.p ?? null, m, c, at: e.at, o: e.o === 1 || e.o === true });
|
|
131
|
+
jobs.push({ id: e.id, p: e.p ?? null, m, c, at: e.at, o: e.o === 1 || e.o === true, ...(nonnegativeInt(e.h) && positiveInt(e.ha) ? { h: e.h, ha: e.ha } : {}), ...(e.v != null ? { v: e.v } : {}), ...(e.s != null ? { s: e.s } : {}) });
|
|
124
132
|
}
|
|
125
133
|
return { jobs, dropped };
|
|
126
134
|
}
|
package/src/live-probes.mjs
CHANGED
|
@@ -767,8 +767,11 @@ export async function runLiveProbes({
|
|
|
767
767
|
// caller's and already knows its binary. `isLocal` is the gate on both endpoint reads: docker's is the observed
|
|
768
768
|
// endpoint, a podman venue's is `serviceIsRemote === false`, which has no `.local` to read. The gate still decides
|
|
769
769
|
// on the value it is handed and must answer exactly `true`; anything else runs nothing.
|
|
770
|
-
|
|
771
|
-
|
|
770
|
+
// Issue #681: both REQUIRED here, never a docker default. A venue's doctor that forgot to pass them would read back
|
|
771
|
+
// docker's containers and print the verdict as its own. The four builders keep docker's builder as their default,
|
|
772
|
+
// since only this entry point reaches them with a venue.
|
|
773
|
+
buildArgs,
|
|
774
|
+
bin,
|
|
772
775
|
isLocal = (observed) => observed?.local === true,
|
|
773
776
|
// Issue #452, gate round 3: the runtime as this pass already read it (`{ podman, rootless, version } | null`), for the
|
|
774
777
|
// detach gate, so doctor asks the daemon once per run. Absent, the gate reads it itself.
|
|
@@ -782,6 +785,7 @@ export async function runLiveProbes({
|
|
|
782
785
|
cgroupParent = CGROUP_PARENT,
|
|
783
786
|
cpuBudgetCenti = null,
|
|
784
787
|
}) {
|
|
788
|
+
if (typeof buildArgs !== "function" || typeof bin !== "string" || bin === "") throw new TypeError("runLiveProbes: the venue's buildArgs and bin are required (issue #681)");
|
|
785
789
|
const notRun = (reason) => ({ ran: false, reason, verdicts: [], notes: [], swept: [] });
|
|
786
790
|
const notLocal = notRun(`this shell's ${bin} CLI is not observed to point at this host, so bind paths and .Mounts would describe another machine; nothing was run`);
|
|
787
791
|
if (isLocal(endpoint) !== true) return notLocal;
|