@edgehero/pi-dispatch 2.1.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/.env.example +41 -5
  2. package/README.md +11 -5
  3. package/deploy/docker-compose.yml +12 -0
  4. package/deploy/egress-proxy.conf +28 -3
  5. package/deploy/pi-dispatch-egress-proxy.container +8 -2
  6. package/package.json +9 -2
  7. package/src/allocation.mjs +731 -0
  8. package/src/backends.mjs +243 -0
  9. package/src/budget.mjs +40 -4
  10. package/src/cli.mjs +222 -11
  11. package/src/config.mjs +126 -5
  12. package/src/daemon-facts.mjs +3 -0
  13. package/src/deployment-venue.mjs +1 -0
  14. package/src/doctor.mjs +2280 -195
  15. package/src/dollar-budget.mjs +373 -0
  16. package/src/dollar-fingerprint.mjs +83 -0
  17. package/src/egress-cli.mjs +316 -0
  18. package/src/egress-proxy-state.mjs +35 -5
  19. package/src/egress.mjs +12 -0
  20. package/src/env-allowlist.mjs +125 -6
  21. package/src/env-file.mjs +194 -25
  22. package/src/envelope.mjs +413 -0
  23. package/src/exit-code.mjs +22 -0
  24. package/src/fleet-lease.mjs +85 -25
  25. package/src/get-token.mjs +16 -5
  26. package/src/git-dirty.mjs +67 -0
  27. package/src/github-app-setup.mjs +6 -3
  28. package/src/github-host.mjs +5 -3
  29. package/src/host-pi.mjs +1 -1
  30. package/src/identity.mjs +2 -1
  31. package/src/image-preflight.mjs +98 -24
  32. package/src/image-ref.mjs +37 -0
  33. package/src/import-pi.mjs +4 -2
  34. package/src/index.mjs +407 -62
  35. package/src/init.mjs +18 -0
  36. package/src/job-id.mjs +26 -3
  37. package/src/live-probes.mjs +24 -9
  38. package/src/model-catalog.mjs +297 -0
  39. package/src/model-endpoints.mjs +671 -0
  40. package/src/model-ref.mjs +151 -0
  41. package/src/models-json.mjs +268 -0
  42. package/src/money.mjs +144 -0
  43. package/src/octokit-log.mjs +65 -0
  44. package/src/outbox-plan.mjs +218 -0
  45. package/src/outbox.mjs +29 -9
  46. package/src/output-cap.mjs +157 -0
  47. package/src/pause-windows.mjs +81 -2
  48. package/src/pi-model-loader.mjs +77 -0
  49. package/src/podman-stack.mjs +16 -3
  50. package/src/portfolio-snapshot.mjs +304 -0
  51. package/src/prepare-local.mjs +247 -12
  52. package/src/prepare.mjs +35 -3
  53. package/src/priorities.mjs +569 -0
  54. package/src/processor.mjs +599 -170
  55. package/src/project-id.mjs +17 -0
  56. package/src/projects.mjs +238 -0
  57. package/src/provider-steering.mjs +179 -65
  58. package/src/queue.mjs +111 -6
  59. package/src/reserved-env.mjs +31 -0
  60. package/src/run-container.mjs +59 -5
  61. package/src/run-history.mjs +379 -24
  62. package/src/run-mirror.mjs +30 -0
  63. package/src/runtime-settings.mjs +104 -9
  64. package/src/schedules.mjs +33 -1
  65. package/src/scoped-limits.mjs +447 -27
  66. package/src/service.mjs +15 -4
  67. package/src/session-store.mjs +131 -6
  68. package/src/start.mjs +528 -40
  69. package/src/triggers-file.mjs +65 -4
  70. package/src/triggers.mjs +141 -11
  71. package/src/up.mjs +308 -34
  72. package/src/valkey-endpoint.mjs +3 -2
package/src/doctor.mjs CHANGED
@@ -11,7 +11,7 @@
11
11
  * operator's FULL-scope gh login, which then reaches every token-carrying job container — the opposite of
12
12
  * the App path's per-repo short-lived tokens (CONST-TOKEN-SCOPED-PER-JOB) — so doctor names the scopes it
13
13
  * carries; and gh is preflighted inside the job image, since a token that works host-side but not
14
- * in-container fails jobs mid-run, not at submit. Token values travel via the spawn env only, never argv.
14
+ * in-container fails jobs mid-run, not at submit. Token values travel via the spawn env or stdin, never argv.
15
15
  *
16
16
  * Issue #80 adds the RECEIVER's half of the preflight. Doctor runs on the worker host, but the triggers
17
17
  * file names forges whose deliveries only ever arrive if the receiver can boot -- and the receiver is
@@ -57,10 +57,18 @@ import { basename, dirname, isAbsolute, join, delimiter, posix, resolve, win32 }
57
57
  import { fileURLToPath } from "node:url";
58
58
  import { spawn as nodeSpawn } from "node:child_process";
59
59
  import { randomBytes } from "node:crypto";
60
- import { accountTempRoot, defaultLogsDir, defaultSandboxDir, defaultSettingsFile, defaultWorkerName, globalExtensionsEnabled, jobsDirOwnerFix, jobsDirPath, sandboxDirOwnerFix, legacyTempStateDir, logsDirPath, pauseWindowsFilePath, safeHomeDir, scopedLimitsFilePath, settingsFilePath, underOsTempDir } from "./config.mjs";
60
+ import { DEFAULT_MODEL, DEFAULT_PROVIDER, DEFAULT_VALKEY_URL, accountTempRoot, allowedModelsFrom, defaultLogsDir, defaultSandboxDir, defaultSettingsFile, defaultWorkerName, globalExtensionsEnabled, jobsDirOwnerFix, jobsDirPath, sandboxDirOwnerFix, legacyTempStateDir, logsDirPath, modelEndpointsFilePath, delimitedList, envelopeFilePath, pauseWindowsFilePath, projectsFilePath, safeHomeDir, scopedLimitsFilePath, settingsFilePath, underOsTempDir } from "./config.mjs";
61
61
  import { SYSTEMD_HAZARD_SHAPES, decodeEnvFile, envFileHazard, envValueShown, quotedRegions, readEnvAssignments, renderEnvValue, envFileWrapperInternal, wrapperInternalSentence } from "./env-file.mjs";
62
- import { canonicalScope, loadScopedLimits, parseScopedLimits } from "./scoped-limits.mjs";
63
- import { loadPauseWindows } from "./pause-windows.mjs";
62
+ import { canonicalScope, danglingProjectRows, dollarRowsBelowJobCap, dollarRowsWithoutCap, isModelScope, isProjectScope, loadScopedLimits, parseScopedLimits } from "./scoped-limits.mjs";
63
+ import { EMPTY_PROJECTS_FINGERPRINT, loadProjects, projectsFingerprint } from "./projects.mjs";
64
+ import { parseModelsJson, stripBom, stripJsonComments } from "./models-json.mjs";
65
+ import { isTransientOverlayRead, overlayProviderProblem } from "./model-catalog.mjs";
66
+ import { EMPTY_USD_FINGERPRINT, usdFingerprint } from "./dollar-fingerprint.mjs";
67
+ import { splitModelEntry } from "./model-ref.mjs";
68
+ import { ignoredOutputCapModels, outputCapView, outputUnboundable } from "./output-cap.mjs";
69
+ import { KEYLESS_API_KEY, KEYLESS_HOW, MODEL_ENDPOINTS_FILE_NAME, MODEL_ENDPOINTS_INCLUDE_NAME, MODEL_ENDPOINT_ID_RE, OVERLAY_LINK_FIX, OVERLAY_NOT_A_FILE_FIX, RENAMED_PROVIDERS, baseUrlTarget, keylessVerdict, loadModelEndpoints, providerRenameHint, readOverlayModels, renderEndpointsInclude, unreportedUsageModels } from "./model-endpoints.mjs";
70
+ import { declaredEndpointsIn, endpointsDeclaredIn, reloadCommand, rulesFileIncludes, rulesPredateEndpointsLine } from "./egress-cli.mjs";
71
+ import { loadPauseWindows, parseScopeString } from "./pause-windows.mjs";
64
72
  import { WAIT_AFTER_MAX_DEFAULT_MS, afterInstantMs, parseWaitProfiles } from "./wait-for.mjs";
65
73
  import { isForgeKind } from "./forges.mjs";
66
74
  import { findLiteralSecret, ADMIN_RE } from "./import-pi.mjs";
@@ -72,15 +80,15 @@ import { GIT_READ_FLAGS } from "./git-hardening.mjs";
72
80
  import { resolveBackendName } from "./backend-registry.mjs";
73
81
  import { deploymentVenueEnv, sharedShellIgnored } from "./deployment-venue.mjs";
74
82
  import { readDeploymentEnv, resolveServiceEnv, serviceEnvFileOf, serviceEnvLoader } from "./service-env.mjs";
75
- import { imageRefProblem } from "./image-ref.mjs";
76
- import { PROXY_STATE_FORMAT, parseProxyState, shippedProxyDrift } from "./egress-proxy-state.mjs";
83
+ import { imageRefProblem, jobImageFix, pullOffered } from "./image-ref.mjs";
84
+ import { PROXY_STATE_FORMAT, parseProxyState, rulesIncludeEndpoints, shippedProxyDrift } from "./egress-proxy-state.mjs";
77
85
  import { PACKAGED_EGRESS_PROXY_CONF, judgeProxyConfCopy, packageCopyName, readPackagedProxyConf } from "./egress-conf-copy.mjs";
78
- import { ABSENT, ASSERTED, DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, OBSERVATION_FIX, OBSERVATIONS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_ROOTFUL_WIDENING_KEYS, PODMAN_SERVICE_LOCAL, PROPERTY_NAMES, RUNTIME_ADDS_NO_MOUNTS, declarationOf, floorShortfall, parseBackendFloor, parseBackendList, unarmedFloor, unobservedFloor, venuesOf } from "./backends.mjs";
79
- import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_FIRST_START_TIMEOUT_MS, PODMAN_INFO_TIMEOUT_MS, PODMAN_JOB_USER_FIX, decidePodmanJobUser, makePodmanInfoReader, observePodman, podmanConfFix, podmanConfWidening, resolvePodmanImageUser } from "./backend-podman.mjs";
86
+ import { ABSENT, ASSERTED, DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, OBSERVATION_FIX, OBSERVATIONS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_ROOTFUL_WIDENING_KEYS, PODMAN_SERVICE_LOCAL, PROPERTY_NAMES, RUNTIME_ADDS_NO_MOUNTS, HOST_ROUTE_LAN, HOST_ROUTE_REFUTED, HOST_ROUTE_WORKS, declarationOf, floorShortfall, hostRouteFor, isProxyLocalHost, parseBackendFloor, parseBackendList, unarmedFloor, unobservedFloor, venuesOf } from "./backends.mjs";
87
+ import { PODMAN_BOOT_REFUSING_CAUSES, PODMAN_FIRST_START_TIMEOUT_MS, PODMAN_INFO_TIMEOUT_MS, PODMAN_JOB_USER_FIX, decidePodmanJobUser, makePodmanInfoReader, observePodman, observeRootlessNetns, podmanConfFix, podmanConfWidening, resolvePodmanImageUser } from "./backend-podman.mjs";
80
88
  import { PODMAN_PINNED_FLAGS, buildPodmanRunArgs, containerSpec, podmanArgsFromSpec } from "./docker-run.mjs";
81
89
  import { PODMAN_SERVICE_TIMEOUT_MS, PODMAN_SERVICE_UNIT, makePodmanServiceReader, observeHost, observeRootfulConf, readRootfulService, rootfulConfFix, rootfulConfRetries, rootfulConfResidual, rootfulUnreadList } from "./runtime-observations.mjs";
82
90
  import { endpointShown, makeDockerEndpointResolver, quotedShown } from "./backend-local.mjs";
83
- import { DEFAULT_EGRESS_PROXY, STOPPED_PROXY_STATES, EGRESS_CANARY_NET_PREFIX, EGRESS_CANARY_PROBE_PREFIX, egressArmed, egressCanaryNetwork, egressCanaryProbe, egressEnv, egressProxyName, networkEndpoints, removeNetworkOrSay } from "./egress.mjs";
91
+ import { DEFAULT_EGRESS_PROXY, STOPPED_PROXY_STATES, EGRESS_CANARY_NET_PREFIX, EGRESS_CANARY_PROBE_PREFIX, EGRESS_ENDPOINT_PROBE_PREFIX, egressArmed, egressCanaryNetwork, egressCanaryProbe, egressEndpointProbe, egressEnv, egressProxyName, egressProxyUrl, networkEndpoints, removeNetworkOrSay } from "./egress.mjs";
84
92
  import { detachBlockedSentence, makeDetachGate, runtimeFromFacts } from "./netns-keeper.mjs";
85
93
  import { runLiveProbes } from "./live-probes.mjs";
86
94
  import { VALKEY_PASSWORD_KEY, VALKEY_PASSWORD_HOWTO, VALKEY_PORT_KEY, isLoopbackHost, valkeyPasswordProblem, valkeyPortConflict } from "./valkey-auth.mjs";
@@ -95,7 +103,14 @@ import { parseSecretProfiles } from "./secret-profiles.mjs";
95
103
  // can share them: doctor NAMES a variable and env-allowlist WRITES one, and they must never differ.
96
104
  import { apiKeyVariable, nonApiKeyKind } from "./provider-key.mjs";
97
105
  import { parseTriggers } from "./triggers.mjs";
98
- import { cronPlacement } from "./schedules.mjs";
106
+ import { readOverlay, resolveSettings } from "./runtime-settings.mjs";
107
+ import { DOLLAR_KEY_PREFIX } from "./dollar-budget.mjs";
108
+ import { dayKey, monthKey, weekKey } from "./budget.mjs";
109
+ import { DOLLAR_ENV_NAMES, DOLLAR_SETTING_KEYS, checkDollarInvariant, effectiveCostCapMicros, formatMicros, optionalUsdMicros, parseUsdMicros } from "./money.mjs";
110
+ import { cronPlacement, envelopeJobPaths } from "./schedules.mjs";
111
+ import { envelopeDigest, loadEnvelopeChecked } from "./envelope.mjs";
112
+ import { OTHER } from "./priorities.mjs";
113
+ import { ALLOC_PLAN_KEY, NO_ENVELOPE_FINGERPRINT } from "./allocation.mjs";
99
114
 
100
115
  const NODE_FLOOR = [22, 19]; // pi's engine floor (22.19.0)
101
116
 
@@ -117,15 +132,28 @@ const PODMAN_BOUNDS_FIX = Object.freeze({
117
132
 
118
133
  // The podman venue's pull of the deployment's default job image into this account's store (issue #433): the fix line's
119
134
  // words and `--fix`'s prompt, one string so the two cannot name different commands.
120
- const PODMAN_JOB_IMAGE_PULL = "podman pull ghcr.io/edgehero/pi-job:latest && podman tag ghcr.io/edgehero/pi-job:latest pi-job:latest";
135
+ const PODMAN_JOB_IMAGE_PULL = jobImageFix("podman", "pi-job:latest");
121
136
 
122
137
  // The fix for a trigger-named image without the runner entrypoint, one sentence for either runtime (issue #433).
123
138
  const TRIGGER_IMAGE_ENTRYPOINT_FIX = "build your job image FROM this repo's image/Dockerfile so it keeps /entrypoint.sh -- an image without the runner can exit 0 without ever starting the agent, and the queue records that as success (docs/job-image.md)";
124
139
 
125
140
  // The in-image gh probe's argv after the runtime's name, shared by docker and podman (issue #433) so the two cannot
126
- // drift: the token rides value-less `-e` flags, which the CLI fills from the spawn env, so it never enters argv
127
- // (visible in `ps`) and never reaches doctor's output. Not the job builder's argv: the probe asks whether the image's gh
128
- // can reach the forge with this token, which is a question about the image and the network, not about a job's bounds.
141
+ // drift. Not the job builder's argv: the probe asks whether the image's gh can reach the forge with this token, which is
142
+ // a question about the image and the network, not about a job's bounds.
143
+ //
144
+ // The token rides STDIN (issue #521), never the container's environment. It used to ride value-less `-e` flags, which
145
+ // kept it out of argv and out of doctor's output but put it in the container create request, and Docker Desktop's
146
+ // backend log (`~/Library/Containers/com.docker.docker/Data/log/host/com.docker.backend.log`) writes that request,
147
+ // environment included, to disk in plain text. Measured on 2026-10-02 (Docker Desktop 4.37.2, engine 27.4.0): one
148
+ // doctor run left a dummy token in that log twice (GH_TOKEN and GITHUB_TOKEN) with `-e`, and zero times with stdin.
149
+ // This token is the operator's own, so unlike a job's minted one it does not expire within the hour. `-i` attaches
150
+ // stdin, the entrypoint becomes `sh`, and the script reads one line and exports it to the `gh auth status` it execs,
151
+ // so the value lives only in that process's memory. Rejected: `--env-file` (the CLI expands it into the same create
152
+ // request) and a mounted file (a host temp file that must be written, protected and removed, and on rootless Podman
153
+ // made readable across a uid map, to say what one pipe says). An image without `sh` (or `gh`) fails the probe with
154
+ // exit 126 or 127, and the line says gh could not be run inside the job image; an image built FROM image/Dockerfile
155
+ // has both. `gh auth status` stays separate argv words after the script's `$0`, so the probe still reads as what it
156
+ // runs.
129
157
  //
130
158
  // EXCEPT the venue's pins, on podman (review round 1): the account's containers.conf may default what the argv does not
131
159
  // name, and two of those defaults hand the probe far more than the token. `env_host = true` (which the podman venue
@@ -133,8 +161,9 @@ const TRIGGER_IMAGE_ENTRYPOINT_FIX = "build your job image FROM this repo's imag
133
161
  // provider keys, GITHUB_PAT and WEBHOOK_SECRET with it, and `http_proxy` copies the proxy variables. So a podman probe
134
162
  // carries `PODMAN_PINNED_FLAGS` whole, the same array every podman job carries: `--env-host=false` and
135
163
  // `--http-proxy=false` for that, and the private namespaces so the probe is no less contained than a job. docker has no
136
- // such defaults to pin (its CLI forwards only what `-e` names), so its argv is what it always was.
137
- const ghProbeArgs = (image, bin) => ["run", "--rm", "--pull=never", ...(bin === "podman" ? PODMAN_PINNED_FLAGS : []), "-e", "GH_TOKEN", "-e", "GITHUB_TOKEN", "--entrypoint", "gh", image, "auth", "status"];
164
+ // such defaults to pin (its CLI forwards only what `-e` names, and the probe names nothing), so its argv carries no pins.
165
+ const GH_PROBE_SCRIPT = 'read -r t; export GH_TOKEN="$t" GITHUB_TOKEN="$t"; exec "$@"';
166
+ const ghProbeArgs = (image, bin) => ["run", "--rm", "-i", "--pull=never", ...(bin === "podman" ? PODMAN_PINNED_FLAGS : []), "--entrypoint", "sh", image, "-c", GH_PROBE_SCRIPT, "sh", "gh", "auth", "status"];
138
167
 
139
168
 
140
169
  // Issue #471: `shellVars` is THIS shell's environment and is read in exactly the places a test pins (the venue and
@@ -179,6 +208,13 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
179
208
  // without uninstalling a dependency. Threaded like every other seam: a seam collectChecks honours
180
209
  // and runDoctor silently drops is a seam that cannot pin an EXIT CODE, only a check object.
181
210
  providerOracle = defaultProviderOracle,
211
+ // Issues #501 and #502: the worker's model catalog and pi's own model loader, both imported lazily. Seams, so a test
212
+ // can drive the not-installed arm and a disagreement without editing pi. Undefined means the defaults.
213
+ modelCatalog,
214
+ piModelLoader,
215
+ dollarKeysExist,
216
+ // Issue #504 part B: the applied split's envelope digest (`alloc:plan`), read once. Undefined means the default.
217
+ readAppliedSplit,
182
218
  // --live (issue #278, INT-LIVE-PROBE-CONTRACT): read the backend declarations back off short-lived real containers.
183
219
  // STRICTLY `=== true`, so only the CLI's own flag arms it: a truthy string from a caller that forwarded an
184
220
  // option bag runs nothing. The fs, PID-liveness and nonce are seams so the sequence is driven without Docker.
@@ -214,6 +250,17 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
214
250
  proxyFilesExist,
215
251
  // PR #488's review: whether a path of the proxy's two files is a DIRECTORY, which docker would mount as the file.
216
252
  proxyFileIsDirectory,
253
+ // Issue #503: whether the shipped proxy's third mount is needed (`proxyIncludeNeeds`), for tests.
254
+ includeNeeds,
255
+ // Issue #503 (part 6): the declared model endpoints as the service reads them, and this host's own LAN IPv4
256
+ // addresses for the measured route table, both for tests. The real reads are the defaults where they are used.
257
+ declaredEndpoints,
258
+ hostAddresses,
259
+ // Issue #552: the overlay models.json's read for the credential-free line, for tests that need an errno a real
260
+ // file cannot give (EIO, EMFILE). The real read is the default where it is used.
261
+ readOverlayFile,
262
+ // Issue #556: the overlay models.json's lstat, for tests that need a socket or a device a test cannot make.
263
+ lstatOverlayFile,
217
264
  // Issue #484: how a copy of the proxy's rules is read, and the installed package's copy it is compared with.
218
265
  readProxyConf,
219
266
  readPackagedProxyConf: readPackagedConf,
@@ -328,7 +375,7 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
328
375
  return { ...(await valkeyAuthState(url, { context, withoutPassword })), passwordSet: Boolean(sent.password), from: sent.from };
329
376
  }
330
377
  : null;
331
- const seams = { cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
378
+ const seams = { cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, ...(modelCatalog ? { modelCatalog } : {}), ...(piModelLoader ? { piModelLoader } : {}), ...(dollarKeysExist ? { dollarKeysExist } : {}), ...(readAppliedSplit ? { readAppliedSplit } : {}), fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(includeNeeds ? { includeNeeds } : {}), ...(declaredEndpoints ? { declaredEndpoints } : {}), ...(readOverlayFile ? { readOverlayFile } : {}), ...(lstatOverlayFile ? { lstatOverlayFile } : {}), ...(hostAddresses ? { hostAddresses } : {}), ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
332
379
  // Issue #471: every other service key, resolved ONCE for the whole run (the fix pass's re-collect and `--live` judge the
333
380
  // same resolution). THE RULE (PR #474's round cap, after three rounds of trust patches): no program doctor starts is
334
381
  // handed anything from `.env`. Every child gets this shell's own environment, the one it had before #471; a `.env`
@@ -518,7 +565,7 @@ function asText(content, enc) {
518
565
  * set is a key the other reads back the same way -- and it is asked for THIS PLATFORM's loader, because
519
566
  * the three loaders of this file disagree and a blended reading is wrong for every deployment at once.
520
567
  */
521
- export const ENV_FILE_READABLE_KEYS = Object.freeze(["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE"]);
568
+ export const ENV_FILE_READABLE_KEYS = Object.freeze(["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE", "PI_PROJECTS_FILE", "PI_MODEL_ENDPOINTS_FILE", "PI_ENVELOPE_FILE"]);
522
569
 
523
570
  /**
524
571
  * The keys doctor takes from the deployment's `.env` as the SERVICE's values (issue #453, and since issue #471 every key
@@ -540,7 +587,7 @@ export const ENV_FILE_READABLE_KEYS = Object.freeze(["PI_PAUSE_WINDOWS_FILE", "P
540
587
  export const GITHUB_SERVICE_KEYS = Object.freeze(["GITHUB_AUTH_SOURCE", "GITHUB_APP_ID", "GITHUB_APP_INSTALLATION_ID", "GITHUB_APP_PRIVATE_KEY_PATH", "GITHUB_APP_PRIVATE_KEY"]);
541
588
  /** Issue #471: the worker's settings doctor judges, which it read from this shell alone while the service read them from
542
589
  * `.env`. TEMP is TMPDIR's twin in the worker's temp root; PI_CODING_AGENT_DIR is where the worker reads auth.json. */
543
- export const WORKER_SERVICE_KEYS = Object.freeze(["PI_JOB_IMAGE", "PI_TRIGGERS_FILE", "PI_LOGS_DIR", "PI_SETTINGS_FILE", "PI_SESSIONS_DIR", "PI_SESSIONS_TTL_DAYS", "PI_SESSION_MAX_AGE_DAYS", "PI_SESSION_MAX_CONTEXT_PCT", "PI_SESSION_MAX_RESUME_CHAIN", "PI_GLOBAL_PI_DIR", "PI_GLOBAL_ALLOW_EXTENSIONS", "PI_FORWARD_ENV", "PI_AUTH_FROM_PI", "PI_CODING_AGENT_DIR", "PI_BACKEND_FLOOR", "PI_SECRET_PROFILES", "PI_SECRET_RESOLVER_ROOTS", "PI_WAIT_PROFILES", "PI_WAIT_AFTER_MAX_MS", "PI_SANDBOX_RETENTION_HOURS", "GITHUB_PAT_VAR", "TEMP"]);
590
+ export const WORKER_SERVICE_KEYS = Object.freeze(["PI_JOB_IMAGE", "PI_TRIGGERS_FILE", "PI_LOGS_DIR", "PI_SETTINGS_FILE", "PI_SESSIONS_DIR", "PI_SESSIONS_TTL_DAYS", "PI_SESSION_MAX_AGE_DAYS", "PI_SESSION_MAX_CONTEXT_PCT", "PI_SESSION_MAX_RESUME_CHAIN", "PI_GLOBAL_PI_DIR", "PI_GLOBAL_ALLOW_EXTENSIONS", "PI_FORWARD_ENV", "PI_AUTH_FROM_PI", "PI_CODING_AGENT_DIR", "PI_BACKEND_FLOOR", "PI_SECRET_PROFILES", "PI_SECRET_RESOLVER_ROOTS", "PI_WAIT_PROFILES", "PI_WAIT_AFTER_MAX_MS", "PI_SANDBOX_RETENTION_HOURS", "PI_ALLOWED_MODELS", "PI_DISPATCH_RUN_ROOTS", "GITHUB_PAT_VAR", "TEMP", ...Object.values(DOLLAR_ENV_NAMES)]);
544
591
  /** Issue #471: the receiver's keys doctor judges its boot by (the receiver's unit reads the same `.env`). */
545
592
  export const RECEIVER_SERVICE_KEYS = Object.freeze(["WEBHOOK_SECRET", "RECEIVER_PORT", "GITLAB_TOKEN", "GITLAB_URL", "GITLAB_WEBHOOK_MODE", "GITLAB_WEBHOOK_SECRET", "FORGEJO_URL", "FORGEJO_TOKEN", "FORGEJO_WEBHOOK_SECRET", "AZURE_ORG_URL", "AZURE_TOKEN", "AZURE_WEBHOOK_MODE", "AZURE_WEBHOOK_SECRET", "AZURE_WEBHOOK_HEADER"]);
546
593
  /**
@@ -564,7 +611,7 @@ export const CLI_SERVICE_KEYS = Object.freeze({
564
611
  gh: Object.freeze(["GH_TOKEN", "GITHUB_TOKEN", "GH_CONFIG_DIR", "GH_HOST", "XDG_CONFIG_HOME", "HOME"]),
565
612
  });
566
613
  const CLI_KEY_NAMES = [...new Set(Object.values(CLI_SERVICE_KEYS).flat())];
567
- export const SERVICE_ENV_KEYS = Object.freeze([...STACK_KEYS, "VALKEY_URL", VALKEY_SHARED_KEY, VALKEY_PASSWORD_KEY, VALKEY_PORT_KEY, "PI_PROVIDER", ...GITHUB_SERVICE_KEYS, "PI_JOBS_DIR", "PI_SANDBOX_DIR", "TMPDIR", "PI_WORKER_NAME", ...WORKER_SERVICE_KEYS, ...RECEIVER_SERVICE_KEYS, ...CLI_KEY_NAMES]);
614
+ export const SERVICE_ENV_KEYS = Object.freeze([...STACK_KEYS, "VALKEY_URL", VALKEY_SHARED_KEY, VALKEY_PASSWORD_KEY, VALKEY_PORT_KEY, "PI_PROVIDER", "PI_MODEL", ...GITHUB_SERVICE_KEYS, "PI_JOBS_DIR", "PI_SANDBOX_DIR", "TMPDIR", "PI_WORKER_NAME", ...WORKER_SERVICE_KEYS, ...RECEIVER_SERVICE_KEYS, ...CLI_KEY_NAMES]);
568
615
 
569
616
  /** Issue #471 (gate round 1): what a service manager gives every service itself, so a `.env` need not carry it. */
570
617
  const AMBIENT_SERVICE_KEYS = Object.freeze(["TMPDIR", "TEMP", "XDG_RUNTIME_DIR", "HOME"]);
@@ -633,7 +680,7 @@ export const STEERING_SERVICE_KEYS = Object.freeze({
633
680
  PI_JOB_IMAGE: "the image doctor inspects and runs (the canary, the gh probe, --live): the worker's `||` default and the one image rule its boot applies (image-ref.mjs: blank, padded, a leading dash, a control character), a refused value named and not used",
634
681
  GITHUB_AUTH_SOURCE: "whether doctor runs gh on this host: the worker's own three values",
635
682
  GITHUB_PAT_VAR: "which variable is the service's PAT: resolved and never printed; set by the file (or its PAT only in the file), the in-image gh probe is not run, since no program doctor starts is handed anything from .env",
636
- PI_GLOBAL_PI_DIR: "the directory `--fix` restages into and removes auth.json from (prompt tier, the command shown first): the worker's rule, unset or empty is off, set must exist",
683
+ PI_GLOBAL_PI_DIR: "the directory `--fix` restages into and removes auth.json from (prompt tier, the command shown first): the worker's rule, unset or empty is off, set must be an absolute path that exists",
637
684
  PI_TRIGGERS_FILE: "the file whose images, repos and folders doctor probes: a regular file only, parsed by the worker's own parser before any of its values is used",
638
685
  PI_SESSIONS_DIR: "the directory `--fix` creates: silent only for this shell's own value; a value from .env is offered at the prompt tier, shown first",
639
686
  PI_JOBS_DIR: "where --live makes its fixture: the jobs dir owner rule the worker applies at boot (#464)",
@@ -657,6 +704,9 @@ export const DOCTOR_SHELL_KEYS = Object.freeze({
657
704
  DOCKER_CONTENT_TRUST: "a docker CLI setting of this shell, named for the --live probes this shell's CLI runs",
658
705
  PI_PAUSE_WINDOWS_FILE: "read from .env by its own two-subject rule (ENV_FILE_READABLE_KEYS): the service judged from the file, this shell judged as itself",
659
706
  PI_SCOPED_LIMITS_FILE: "read from .env by its own two-subject rule (ENV_FILE_READABLE_KEYS)",
707
+ PI_PROJECTS_FILE: "read from .env by its own two-subject rule (ENV_FILE_READABLE_KEYS), #499",
708
+ PI_MODEL_ENDPOINTS_FILE: "read from .env by its own two-subject rule (ENV_FILE_READABLE_KEYS), #503",
709
+ PI_ENVELOPE_FILE: "read from .env by its own two-subject rule (ENV_FILE_READABLE_KEYS), #504",
660
710
  [VALKEY_SHARED_KEY]: "read from .env ONLY (#464); a value in this shell is named as ignored",
661
711
  });
662
712
 
@@ -947,7 +997,8 @@ export function envFileKeys(path, keys, { fileExists, readEnvFile, statFile = st
947
997
  }
948
998
 
949
999
  /**
950
- * The two files a worker LOADS AT BOOT, and the one place that knows what each is and how to ask.
1000
+ * The files a worker LOADS at boot, and the one place that knows what each is and how to ask. The model endpoints file
1001
+ * (issue #503) defaults to the deployment folder's copy where the other three are off when unset.
951
1002
  *
952
1003
  * `load` calls the worker's own loader (issue #384). Doctor used to carry its own parse for scoped limits
953
1004
  * and nothing at all for pause windows, so "will the worker start" had two answers and one silence. The
@@ -963,6 +1014,10 @@ const BOOT_FILES = Object.freeze([
963
1014
  unsetMeans: "the worker loads no windows at all",
964
1015
  unit: "window",
965
1016
  nothing: "quiet hours",
1017
+ fails: "REFUSES TO START",
1018
+ whenDeleted: "turns scoped pauses off",
1019
+ whenEmpty: "turns the worker off",
1020
+ emptyCost: "refuses the boot",
966
1021
  resolve: pauseWindowsFilePath,
967
1022
  load: (path, io) => loadPauseWindows({ pauseWindowsFile: path }, io),
968
1023
  }),
@@ -974,11 +1029,134 @@ const BOOT_FILES = Object.freeze([
974
1029
  unsetMeans: "the worker enforces no scoped limits at all",
975
1030
  unit: "limit",
976
1031
  nothing: "scoped limits",
1032
+ fails: "REFUSES TO START",
1033
+ whenDeleted: "turns scoped limits off",
1034
+ whenEmpty: "turns the worker off",
1035
+ emptyCost: "refuses the boot",
977
1036
  resolve: scopedLimitsFilePath,
978
1037
  load: (path, io) => loadScopedLimits({ scopedLimitsFile: path }, io),
979
1038
  }),
1039
+ // Issue #499. Off when unset, like the two above: no projects, and every run records `project: null`. The worker
1040
+ // loads it at boot (start.mjs, `loadProjects`), so a file that does not load and an empty value refuse the start.
1041
+ Object.freeze({
1042
+ key: "PI_PROJECTS_FILE",
1043
+ noun: "projects",
1044
+ scaffold: "projects.json",
1045
+ off: "projects are OFF (every run records no project)",
1046
+ unsetMeans: "the worker groups no run into a project",
1047
+ unit: "project",
1048
+ nothing: "projects",
1049
+ // No panel writes this file yet (the project tools come with part C of issue #499), so the unset fix line must
1050
+ // not say the panel reports a project it wrote as applied live.
1051
+ panelWrites: false,
1052
+ fails: "REFUSES TO START",
1053
+ whenDeleted: "turns projects off",
1054
+ whenEmpty: "turns the worker off",
1055
+ emptyCost: "refuses the boot",
1056
+ resolve: projectsFilePath,
1057
+ load: (path, io) => loadProjects({ projectsFile: path }, io),
1058
+ }),
1059
+ // Issue #503. Not off when unset: the worker then reads the scaffold in the deployment folder, so `defaultsToScaffold`
1060
+ // swaps the "exists but unset" warning for a load of that file. The worker loads it at boot exactly like the two
1061
+ // above (start.mjs, `loadModelEndpoints`), so a file that does not load, a named file that is missing and an empty
1062
+ // value all REFUSE THE START; a live edit that does not load keeps the last good declaration instead
1063
+ // (`model_endpoints_reload_invalid`), which is a running worker's posture and not the one predicted here. A valid
1064
+ // `{"version":1,"endpoints":[]}` declares none, and only a missing DEFAULT file means the same. The file decides the
1065
+ // proxy include, the slot leases at pickup and the keyless credential gate, so a start on a bad one would be a
1066
+ // deployment whose rules and bounds the operator believes in and does not have.
1067
+ Object.freeze({
1068
+ key: "PI_MODEL_ENDPOINTS_FILE",
1069
+ noun: "declared model endpoints",
1070
+ scaffold: MODEL_ENDPOINTS_FILE_NAME,
1071
+ defaultsToScaffold: true,
1072
+ unsetMeans: `the worker reads ${MODEL_ENDPOINTS_FILE_NAME} in the deployment folder`,
1073
+ unit: "endpoint",
1074
+ nothing: "model endpoints",
1075
+ fails: "REFUSES TO START",
1076
+ whenDeleted: `reads ${MODEL_ENDPOINTS_FILE_NAME} in the deployment folder instead`,
1077
+ whenEmpty: "turns the worker off",
1078
+ emptyCost: "refuses the boot",
1079
+ resolve: modelEndpointsFilePath,
1080
+ // The worker's own default when VALKEY_URL is unset, so both refuse an endpoint on 6379 alike.
1081
+ load: (path, io, env) => loadModelEndpoints({ modelEndpointsFile: path, valkeyUrl: env?.VALKEY_URL ?? DEFAULT_VALKEY_URL }, io),
1082
+ }),
1083
+ // Issue #504 part B. Off when unset: no envelope and no delegation, and every cap is what the operator's rows and
1084
+ // windows set. The worker loads it at boot (start.mjs, `loadEnvelopeChecked`) against the projects and scoped limits
1085
+ // it loaded, with the merged per-job cap, and refuses the start when it lies inside a job path, so this load asks the
1086
+ // same three questions through the same function (`loadEnvelopeAsTheWorker`).
1087
+ Object.freeze({
1088
+ key: "PI_ENVELOPE_FILE",
1089
+ noun: "the allocation envelope",
1090
+ scaffold: "envelope.json",
1091
+ off: "delegated allocation is OFF (every dollar cap is the operator's own)",
1092
+ unsetMeans: "no envelope governs any job and no priorities plan applies",
1093
+ unit: "envelope",
1094
+ nothing: "allocation envelope",
1095
+ // The admin writes this file (`dispatch_envelope_set`, issue #504 part C), but only at the path the key names: it
1096
+ // has no default path, so the fix line's "the admin panel defaults to this same file" sentence does not apply.
1097
+ panelWrites: false,
1098
+ fails: "REFUSES TO START",
1099
+ whenDeleted: "turns delegated allocation off",
1100
+ whenEmpty: "turns the worker off",
1101
+ emptyCost: "refuses the boot",
1102
+ resolve: envelopeFilePath,
1103
+ load: (path, io, env) => loadEnvelopeAsTheWorker(path, io, env),
1104
+ }),
980
1105
  ]);
981
1106
 
1107
+ /** A key's value when it is a non-empty string, else null: the boot reads an unset key as off. */
1108
+ function nonEmpty(value) {
1109
+ return typeof value === "string" && value.trim() !== "" ? value : null;
1110
+ }
1111
+
1112
+ /**
1113
+ * The envelope as the WORKER would load it (issue #504 part B): against the projects and scoped limits the service
1114
+ * names (each loaded by its own loader; one that does not load is its own BOOT_FILES line, so here it counts as none),
1115
+ * with the merged per-job cap, and with the containment check against the triggers file's cron folders and skills
1116
+ * dirs, the run roots and the global pi dir. Throws what the worker's boot would throw.
1117
+ */
1118
+ export function loadEnvelopeAsTheWorker(path, io = {}, env = {}) {
1119
+ return loadEnvelopeChecked({ envelopeFile: path }, { ...envelopeContextOf(env, io), jobPaths: envelopeJobPaths({ triggersFile: nonEmpty(env.PI_TRIGGERS_FILE), dispatchRunRoots: delimitedList(env.PI_DISPATCH_RUN_ROOTS), globalPiDir: nonEmpty(env.PI_GLOBAL_PI_DIR) }, io) }, { io });
1120
+ }
1121
+
1122
+ /** The projects, the scoped limits and the merged per-job cap (micro-dollars, or null) the envelope is judged against. */
1123
+ function envelopeContextOf(env, io = {}) {
1124
+ const quiet = (load) => {
1125
+ try {
1126
+ return load();
1127
+ } catch {
1128
+ return [];
1129
+ }
1130
+ };
1131
+ const projects = quiet(() => loadProjects({ projectsFile: nonEmpty(env.PI_PROJECTS_FILE) }, io));
1132
+ const limits = quiet(() => loadScopedLimits({ scopedLimitsFile: nonEmpty(env.PI_SCOPED_LIMITS_FILE) }, io));
1133
+ let maxCostMicros = null;
1134
+ try {
1135
+ const settings = deploymentSettingsOf(env, settingsFilePath(env), (p) => (io.existsSync ?? existsSync)(p));
1136
+ maxCostMicros = optionalUsdMicros(settings.maxCostUsd, "maxCostUsd");
1137
+ } catch {
1138
+ // A malformed cap is its own line; the envelope then reads as having none, which it refuses.
1139
+ }
1140
+ return { projects, limits, maxCostMicros };
1141
+ }
1142
+
1143
+ /**
1144
+ * Is anything at `path`, by `statFile`: ENOENT is absence, so a directory or an unreadable entry is judged. A stat that
1145
+ * cannot say what the entry is (no `isFile`: a seam answering only an owner) is no evidence of a file either.
1146
+ */
1147
+ function presentAt(statFile, path) {
1148
+ try {
1149
+ return typeof statFile(path)?.isFile === "function";
1150
+ } catch (err) {
1151
+ return err?.code !== "ENOENT";
1152
+ }
1153
+ }
1154
+
1155
+ /** The number of boot files in words, for the one sentence that counts them. */
1156
+ function countWord(n) {
1157
+ return ["zero", "one", "two", "three", "four", "five"][n] ?? String(n);
1158
+ }
1159
+
982
1160
  /**
983
1161
  * Would the worker load this path? The loader answers, with two guards doctor owes it.
984
1162
  *
@@ -1029,7 +1207,7 @@ function loadVerdict(spec, rawPath, cwd, io, platform = process.platform) {
1029
1207
  return { ok: false, reason: err?.code === "ENOENT" ? `${shown} does not exist` : `${shown} cannot be read: ${envValueShown(err?.message ?? String(err))}` };
1030
1208
  }
1031
1209
  try {
1032
- spec.load(path, io.loaderIo);
1210
+ spec.load(path, io.loaderIo, io.env);
1033
1211
  return { ok: true };
1034
1212
  } catch (err) {
1035
1213
  return { ok: false, reason: envValueShown(err?.message ?? String(err)) };
@@ -1078,7 +1256,7 @@ export async function collectChecks(shellVars, seams) {
1078
1256
  const valkeyUnread = service.unread.find((u) => u.key === "VALKEY_URL" && !u.shellSet);
1079
1257
  const unreadValkey = valkeyUnread ? readValkeyKeys(serviceEnvFile.text, { loader: serviceEnvFile.loader, path: serviceEnvFile.path }).error : null;
1080
1258
  if (unreadValkey) valkeyUnread.skip = true;
1081
- const valkeyUrl = env.VALKEY_URL ?? "redis://127.0.0.1:6379";
1259
+ const valkeyUrl = env.VALKEY_URL ?? DEFAULT_VALKEY_URL;
1082
1260
  // PR #478's gate: a VALKEY_URL path that names no database (`/abc`) crashed doctor with an unhandled rejection from
1083
1261
  // ioredis's SELECT. It is a ✗ of its own, the worker's refusal (exit 2), and nothing below contacts a Valkey.
1084
1262
  const valkeyDbProblem = unreadValkey ? null : valkeyUrlProblem(valkeyUrl);
@@ -1253,8 +1431,9 @@ export async function collectChecks(shellVars, seams) {
1253
1431
  // image checks just below, and `optingOut`/`requiring` colour the staged-packages lines further down.
1254
1432
  // `optingOut` counts the only value that withholds the staged set; `requiring` counts an explicit
1255
1433
  // run.packages: true, which arms nothing any more but is still an operator statement of intent.
1256
- const { requiring, waiting, waitProfiles, waitAfters, optingOut, resuming, replicating, instructing, commands, secreting, onceArmed, onceSpent, secretProfiles, localSecretFolders, secretNames, folders, images, imageRoutes, namedBackends, skillsDirs, forges, repositories, flows, parseError, path: triggersFilePath } = readTriggerFacts(env, fileExists, cwd, declaredWorkerName);
1434
+ const { requiring, waiting, listing, waitProfiles, waitAfters, optingOut, resuming, replicating, instructing, commands, secreting, onceArmed, onceSpent, secretProfiles, localSecretFolders, secretNames, folders, images, imageRoutes, namedBackends, skillsDirs, forges, repositories, flows, costCaps, modelRuns, parseError, path: triggersFilePath } = readTriggerFacts(env, fileExists, cwd, declaredWorkerName);
1257
1435
  const scopedLimitFacts = readScopedLimitFacts(env, fileExists);
1436
+ const pauseWindowFacts = readPauseWindowFacts(env, fileExists);
1258
1437
  // FIRST, and fail rather than warn: every check below this line reads counts that a parse failure
1259
1438
  // zeroed, so a green run here would be reporting on a file nobody could read. The receiver loads this
1260
1439
  // file unconditionally and refuses to start without it, which is the consequence worth naming.
@@ -1285,6 +1464,7 @@ export async function collectChecks(shellVars, seams) {
1285
1464
  fix: `fix ${triggersFilePath} so it loads (the message above names the entry and the reason), then re-run doctor -- every trigger-derived check below is skipped until it loads`,
1286
1465
  });
1287
1466
  }
1467
+ checks.push(...dollarChecks(env, costCaps, triggersSet !== undefined));
1288
1468
 
1289
1469
  // Only meaningful if docker itself responds; otherwise the image check is noise on top of a down daemon. Issue #433:
1290
1470
  // and never without `local`, where the podman section's line (the image in THIS ACCOUNT'S store) is the image check.
@@ -1298,7 +1478,11 @@ export async function collectChecks(shellVars, seams) {
1298
1478
  fix:
1299
1479
  imageRun.ended === "timeout"
1300
1480
  ? "restart Docker first: this asked the same daemon that did not answer above, so whether the image is present is unknown rather than false"
1301
- : "docker pull ghcr.io/edgehero/pi-job:latest && docker tag ghcr.io/edgehero/pi-job:latest pi-job:latest (or build image/Dockerfile)",
1481
+ : jobImage === "pi-job:latest"
1482
+ ? `${jobImageFix("docker", jobImage)} (or build image/Dockerfile)`
1483
+ : // Issue #523 (review): the image the worker runs, by the rule `up` follows, never ghcr's latest for an
1484
+ // overriding name (which would not make THEIR image exist) and never a pull of a short name.
1485
+ jobImageFix("docker", jobImage),
1302
1486
  // Prompt tier, and ONLY for the deployment default: a PI_JOB_IMAGE the operator overrode is a trust
1303
1487
  // choice this command cannot honestly satisfy (pulling ghcr's pi-job would not make THEIR image
1304
1488
  // exist), so an overridden name keeps the plain fix line -- the same never-tier reasoning as the
@@ -1458,7 +1642,7 @@ export async function collectChecks(shellVars, seams) {
1458
1642
  // is advisory and carries no fixAction (triggers content is the never tier). A deployment with no
1459
1643
  // command triggers adds no line at all, so its output is byte-identical.
1460
1644
  if (commands > 0) {
1461
- checks.push({ ok: true, label: `${commands} command trigger(s): a command is only verifiable in-container -- the runner refuses an unregistered one pre-spend (command-unregistered)` });
1645
+ checks.push({ ok: true, label: `${commands} command trigger(s): a command is only verifiable in-container -- the runner refuses an unregistered one before the prompt is sent (command-unregistered)` });
1462
1646
  }
1463
1647
 
1464
1648
  // Issue #41: every DISTINCT image a trigger names in run.image, minus the deployment default already
@@ -1573,6 +1757,32 @@ export async function collectChecks(shellVars, seams) {
1573
1757
  const readFactsOnce = () => (factsRead ??= makeDaemonFactsReader({ run: dockerRunVia(spawn, DAEMON_FACTS_TIMEOUT_MS) })());
1574
1758
  const egress = localUsed ? await egressChecks(env, { ...seams, readFactsOnce }, { dockerCode, imageCode, jobImage, endpoint }) : [];
1575
1759
  checks.push(...egress);
1760
+ // Issue #503: an allowlist naming a host alias, once for every venue (both mount this folder's allowlist). Read only with
1761
+ // the policy armed, which is when the file is a policy at all; a file that is not there or cannot be read says nothing.
1762
+ let allowlistArmed = false;
1763
+ try {
1764
+ allowlistArmed = egressArmed(env) === true;
1765
+ } catch {
1766
+ // A malformed PI_EGRESS: the .env check reports it.
1767
+ }
1768
+ if (allowlistArmed) {
1769
+ let aliases = [];
1770
+ try {
1771
+ aliases = allowlistHostAliases(readFileSync(join(seams.cwd, "egress-allowlist.conf"), "utf8"));
1772
+ } catch {
1773
+ // No allowlist in this folder.
1774
+ }
1775
+ for (const { alias, entry } of aliases) {
1776
+ // The entry as written, when it is not the alias itself, is quoted: it is the operator's own file, but still text.
1777
+ const named = entry.toLowerCase().replace(/^\./, "") === alias ? alias : `${quotedShown(entry)}, which admits ${alias}`;
1778
+ checks.push({
1779
+ ok: false,
1780
+ warn: true,
1781
+ label: `egress-allowlist.conf lists ${named}, which lets a job open a CONNECT to that host's port 443 and send plain HTTP to its port 80, and reaches no model server's port`,
1782
+ fix: `remove it, and declare the model server in model-endpoints.json instead, which opens a tunnel to exactly its host and port: docs/egress.md, "Local model servers"`,
1783
+ });
1784
+ }
1785
+ }
1576
1786
  if (facts) {
1577
1787
  let armed;
1578
1788
  try {
@@ -1733,6 +1943,9 @@ export async function collectChecks(shellVars, seams) {
1733
1943
  }
1734
1944
  }
1735
1945
 
1946
+ // Issue #508: with the egress policy on, a job reaches a forge only over https on port 443.
1947
+ checks.push(...forgeUrlEgressChecks(env, forges));
1948
+
1736
1949
  // Azure DevOps, when the triggers file names it (issue #80) -- same shape, mirrored from
1737
1950
  // receiver/src/config.mjs loadAzureConfig. AZURE_WEBHOOK_MODE gets its own line because it is
1738
1951
  // required-UNDEFAULTED: Azure offers no HMAC at all, so both modes are shared-secret compares, and
@@ -2027,7 +2240,7 @@ export async function collectChecks(shellVars, seams) {
2027
2240
  }
2028
2241
  if (token && ghProbeBin === "docker" && endpoint.local !== true) {
2029
2242
  // NOT RUN on a daemon that is not observed on this host (#278). The probe hands the operator's own gh
2030
- // token -- full scope and non-expiring by default -- to `docker run -e`, and on a redirected CLI that
2243
+ // token -- full scope and non-expiring by default -- to `docker run` (on stdin since #521), and on a redirected CLI that
2031
2244
  // token rides to another machine. A check must not do the thing the credentialTransit line warns
2032
2245
  // about. Only once there IS a token: app mode and an unset PAT never run the probe at all.
2033
2246
  checks.push({
@@ -2037,8 +2250,11 @@ export async function collectChecks(shellVars, seams) {
2037
2250
  fix: "point the docker CLI at this host and re-run doctor to check it",
2038
2251
  });
2039
2252
  } else if (token) {
2040
- // Value-less `-e` flags: the CLI forwards GH_TOKEN/GITHUB_TOKEN from the spawn env, so the
2041
- // token value never enters argv (visible in `ps`) and never reaches doctor's output.
2253
+ // On stdin (issue #521, `ghProbeArgs`): the token never enters argv (visible in `ps`), the container
2254
+ // create request (which Docker Desktop logs) or doctor's output, and doctor adds no copy of it to the
2255
+ // CLI's environment. That environment may already hold it (a PAT source reads it from this shell's
2256
+ // GITHUB_PAT), and that copy stays out of the container because docker forwards nothing `-e` does not
2257
+ // name and podman's probe carries `--env-host=false`.
2042
2258
  //
2043
2259
  // The CLI's own environment stays doctor's whole one on both runtimes, and that is deliberate rather than
2044
2260
  // left over (review round 1 asked): what keeps it OUT of the container is the pinned argv above, while
@@ -2046,16 +2262,25 @@ export async function collectChecks(shellVars, seams) {
2046
2262
  // variables) is what decides WHICH Podman and which store answer. The podman section read that service
2047
2263
  // with this same environment, so a trimmed one could send the token to a service nothing here checked.
2048
2264
  // An allowlist of what Podman needs was rejected for that reason: it is a list nothing derives.
2049
- const probe = await runCmdCapture(spawn, ghProbeBin, ghProbeArgs(jobImage, ghProbeBin), { env: { ...spawnEnv, GH_TOKEN: token, GITHUB_TOKEN: token } });
2265
+ const probe = await runCmdCapture(spawn, ghProbeBin, ghProbeArgs(jobImage, ghProbeBin), { env: spawnEnv, input: `${token}\n` });
2050
2266
  const where = ghProbeBin === "podman" ? "podman: " : "";
2267
+ // 126 and 127 are the runtime's and the shell's "could not run that program" (measured on docker 27.4: a
2268
+ // missing entrypoint and a missing exec target both exit 127, a non-executable one 126); `gh auth status`
2269
+ // itself exits 0, 1, 2, 4 or 8. Since the probe's entrypoint became `sh` (#521), an image without one
2270
+ // lands here, and naming it as an auth failure would send the operator to check egress for nothing.
2271
+ const cannotRun = probe.code === 126 || probe.code === 127;
2051
2272
  checks.push({
2052
2273
  ok: probe.code === 0,
2053
2274
  warn: true,
2054
2275
  label:
2055
2276
  probe.code === 0
2056
2277
  ? `${where}gh authenticates inside the job image (${jobImage})`
2057
- : `${where}gh cannot authenticate inside the job image (${jobImage})`,
2058
- fix: "check network egress from containers or rebuild/pull the job image -- jobs that use gh will fail mid-run",
2278
+ : cannotRun
2279
+ ? `${where}gh could not be run inside the job image (${jobImage}): it has no sh or no gh (exit ${probe.code})`
2280
+ : `${where}gh cannot authenticate inside the job image (${jobImage})`,
2281
+ fix: cannotRun
2282
+ ? "rebuild or pull the job image: every image built FROM this repo's image/Dockerfile has both -- jobs that use gh will fail mid-run"
2283
+ : "check network egress from containers or rebuild/pull the job image -- jobs that use gh will fail mid-run",
2059
2284
  });
2060
2285
  } else if (patFromFile) {
2061
2286
  // The service's PAT is in `.env`, which hands no program anything (PR #474's round cap): named, not run.
@@ -2174,6 +2399,11 @@ export async function collectChecks(shellVars, seams) {
2174
2399
  // Valkey doctor reads, a host row is another party's text, and a control byte in a name or zone must not reach the
2175
2400
  // terminal. The registry's own charset already refuses them at the source; this is the reader not relying on it.
2176
2401
  const peers = (fleet.hosts ?? []).map((h) => ({ ...h, name: printable(h.name), tz: h.tz ? printable(h.tz) : h.tz })).filter((h) => h.name !== workerNameOf(declaredWorkerName));
2402
+ // The applied split (issue #504 part B): one GET whenever this command may talk to the Valkey, so a single host with
2403
+ // no envelope that refuses every job is told why. `{ digest }`, `{ undecodable: true }` for a key that exists and does
2404
+ // not decode (the worker's EXISTS still counts it as governed), or null (no split, or no answer: nothing is said).
2405
+ const valkeyUsable = !(unreadValkey || valkeyRefused || valkeyDbProblem || valkeyAuthVerdict?.state === "dbrange");
2406
+ const appliedSplit = valkeyUsable ? await (seams.readAppliedSplit ?? defaultReadAppliedSplit)(valkeyTalkUrl) : null;
2177
2407
  // Read only when there is a peer to compare against. Every line below is gated on a peer existing, and
2178
2408
  // the SUBPROCESS has to be too: otherwise every `doctor` run on every single-host deployment spawns an
2179
2409
  // extra docker call whose answer nothing reads.
@@ -2244,10 +2474,42 @@ export async function collectChecks(shellVars, seams) {
2244
2474
  fix: "check that those workers are running and that the clocks agree -- a stale row is either a dead worker or a skewed clock, and both matter",
2245
2475
  });
2246
2476
  }
2477
+
2478
+ // Issue #501 part 6: the dollar counters are shared, the caps they are judged against are per host. This host's
2479
+ // fingerprint is computed here from the service's own settings, by the worker's function, never read back from
2480
+ // its registry row: doctor answers for the configuration, which a worker that has not restarted may not run yet.
2481
+ // The scoped-limits rows are the parsed file's, or none when it does not load (the worker refuses to boot then).
2482
+ const dollarsHere = deploymentSettingsOf(env, settingsFilePath(env, home), fileExists);
2483
+ let envListHere = null;
2484
+ try {
2485
+ envListHere = allowedModelsFrom(env);
2486
+ } catch {
2487
+ // A malformed list is its own line below; the worker refuses to boot on it.
2488
+ }
2489
+ const usdHere = usdFingerprint(dollarsHere, scopedLimitFacts.parseError === null ? scopedLimitFacts.limits : [], envListHere);
2490
+ checks.push(...(await fleetDollarChecks(usdHere, peers, { dollarKeysExist: () => (seams.dollarKeysExist ?? defaultDollarKeysExist)(valkeyTalkUrl) })));
2491
+ // Issue #499 part C: each host resolves its jobs' project from its OWN projects.json, while the project rows' counters
2492
+ // are shared. This host's `fpProjects` from the file the service names, against every peer's published one. SKIPPED
2493
+ // when this host's file does not load (PR #569's review): the BOOT_FILES line already fails on it, and comparing
2494
+ // "no projects" against healthy peers would send the operator to the wrong host.
2495
+ const projectFactsHere = readProjectFacts(env, fileExists);
2496
+ if (projectFactsHere.parseError === null) checks.push(...fleetProjectsChecks(projectsFingerprint(projectFactsHere.projects), peers));
2497
+ // Issue #504 part B: one applied split, judged on every host against its own envelope. A host whose envelope digest
2498
+ // differs refuses every governed job as `envelope-mismatch`. SKIPPED when this host's file does not load, for the
2499
+ // projects check's reason above.
2500
+ const envelopeHere = readEnvelopeFacts(env);
2501
+ if (envelopeHere.parseError === null && appliedSplit === null) checks.push(...fleetEnvelopeChecks(envelopeHere.envelope ? envelopeDigest(envelopeHere.envelope) : NO_ENVELOPE_FINGERPRINT, peers));
2247
2502
  } else if (fleet.unreachable) {
2248
2503
  // Said, rather than silently absent: "no peers" and "could not ask" are different facts.
2249
2504
  checks.push({ ok: true, label: `Fleet: could not read the host registry (${printable(fleet.unreachable)})` });
2250
2505
  }
2506
+ // Issue #504 part B: the APPLIED split names the envelope it was made for, and every host
2507
+ // whose envelope differs refuses its governed jobs. Read whenever this command may talk to the Valkey (above), from
2508
+ // the same Valkey the fleet was read from; with no split there, or no answer, nothing is said.
2509
+ if (appliedSplit !== null) {
2510
+ const envelopeHere = readEnvelopeFacts(env);
2511
+ if (envelopeHere.parseError === null) checks.push(...appliedSplitChecks(appliedSplit, envelopeHere.envelope ? envelopeDigest(envelopeHere.envelope) : NO_ENVELOPE_FINGERPRINT, workerNameOf(declaredWorkerName), peers));
2512
+ }
2251
2513
 
2252
2514
  // Which variable holds a provider's key is PI'S fact, asked of pi rather than copied (issue #286).
2253
2515
  // The copy this replaced had anthropic's two variables in the WRONG precedence order, invented
@@ -2268,7 +2530,41 @@ export async function collectChecks(shellVars, seams) {
2268
2530
  const keyFromFile = keyCandidates.some((name) => (env[name] ?? "") !== "") ? {} : keyRes.fromFile;
2269
2531
  // And auth.json where the SERVICE's worker reads it: the agent dir the file names, where this shell names none.
2270
2532
  const keyAgentDir = fileSays("PI_CODING_AGENT_DIR").length > 0 ? agentDirFrom(env) : agentDir;
2271
- const keyCheck = providerKeyCheck({ provider, env: { ...env, ...keyFromFile }, agentDir: keyAgentDir, oracle, nodeOk: checks[0]?.ok });
2533
+ // Issue #503: a provider pi does not know may be keyless, judged by the worker's own verdict on the declared endpoints
2534
+ // and the overlay models.json, read here as the service reads them. Read only for such a provider, and only when an
2535
+ // endpoint is declared, which is the worker's own rule (index.mjs reads the overlay only then).
2536
+ const unknownToPi = keyCandidates.length === 0 && typeof oracle?.piProviders === "function" && !oracle.piProviders().includes(provider);
2537
+ // Through doctor's own seams (the boot-file loader's io), and with the queue port the worker refuses: an endpoint on
2538
+ // it refuses the worker's boot, so it must not read as keyless here.
2539
+ const keylessIo = { existsSync: (p) => fileExists(p), readFileSync: readEnvFile ? (p, enc) => asText(readEnvFile(p), enc) : readFileSync };
2540
+ const keylessValkey = env.VALKEY_URL ?? DEFAULT_VALKEY_URL;
2541
+ const keylessEndpoints = unknownToPi ? (seams.declaredEndpoints ?? ((a) => declaredEndpointsIn({ ...a, fs: keylessIo })))({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform, valkeyUrl: keylessValkey }) : [];
2542
+ let keylessModels = null;
2543
+ let keylessUnreadable = null;
2544
+ // The worker's rule (config.mjs `resolveGlobalPiDir`, PR #553's review): only an absolute PI_GLOBAL_PI_DIR is an
2545
+ // overlay; a relative one refuses the worker's boot, and the overlay section below says so. Both reads take the
2546
+ // value as it is, so they read the same folder.
2547
+ if (keylessEndpoints.length > 0 && typeof env.PI_GLOBAL_PI_DIR === "string" && isAbsolute(env.PI_GLOBAL_PI_DIR)) {
2548
+ try {
2549
+ keylessModels = readOverlayModels(env.PI_GLOBAL_PI_DIR, keylessIo);
2550
+ } catch (error) {
2551
+ // The reader's one rule (PR #520 round 2): an errno it rethrows is not a verdict on the provider, so doctor
2552
+ // names the code rather than calling the provider unknown (retried or refused by `isTransientOverlayRead`).
2553
+ // Invalid JSON is the overlay check's line below; here it only means nothing is keyless.
2554
+ if (typeof error?.code === "string") keylessUnreadable = error.code;
2555
+ }
2556
+ }
2557
+ // Issue #587's review: the rename hint is judged on the overlay itself, read whenever there is one, endpoint or not.
2558
+ let hintModels = keylessModels;
2559
+ let hintUnread = false;
2560
+ if (unknownToPi && hintModels === null && Object.hasOwn(RENAMED_PROVIDERS, provider) && typeof env.PI_GLOBAL_PI_DIR === "string" && isAbsolute(env.PI_GLOBAL_PI_DIR)) {
2561
+ try {
2562
+ hintModels = readOverlayModels(env.PI_GLOBAL_PI_DIR, keylessIo);
2563
+ } catch {
2564
+ hintUnread = true;
2565
+ }
2566
+ }
2567
+ const keyCheck = providerKeyCheck({ provider, env: { ...env, ...keyFromFile }, agentDir: keyAgentDir, oracle, nodeOk: checks[0]?.ok, keyless: { endpoints: keylessEndpoints, models: keylessModels, unreadable: keylessUnreadable, hintModels, hintUnread } });
2272
2568
  const keyNamed = Object.keys(keyFromFile).filter((name) => keyCheck.label?.includes(`: ${name})`));
2273
2569
  // Issue #481 (PR #485 review round 2): a key found nowhere doctor can look is one an env-setup script may export, which
2274
2570
  // is the documented home for a provider key fetched from a secrets manager (docs/secrets.md).
@@ -2302,7 +2598,15 @@ export async function collectChecks(shellVars, seams) {
2302
2598
  // Global pi overlay (REQ-GLOBAL-PI-OVERLAY), only when configured. The overlay is mounted :ro into an
2303
2599
  // adversarial-input container, so the load-bearing checks are that it holds NO credential.
2304
2600
  const overlay = env.PI_GLOBAL_PI_DIR;
2305
- if (overlay) {
2601
+ if (overlay && !isAbsolute(overlay)) {
2602
+ // PR #553's review: the worker refuses to boot on it (config.mjs `resolveGlobalPiDir`), since a relative value is
2603
+ // resolved differently by the worker and the container runtime.
2604
+ checks.push({
2605
+ ok: false,
2606
+ label: `PI_GLOBAL_PI_DIR is ${JSON.stringify(overlay)}, which is not an absolute path, so the worker refuses to boot${fromFileNote(fileSays("PI_GLOBAL_PI_DIR"))}`,
2607
+ fix: "set PI_GLOBAL_PI_DIR to the overlay folder's absolute path; a relative value is resolved differently by the worker and the container runtime",
2608
+ });
2609
+ } else if (overlay) {
2306
2610
  const dirOk = fileExists(overlay);
2307
2611
  checks.push({ ok: dirOk, label: `Global overlay dir exists (${envValueShown(overlay)})${fromFileNote(fileSays("PI_GLOBAL_PI_DIR"))}`, fix: "run `pi-dispatch import-pi`, or fix PI_GLOBAL_PI_DIR" });
2308
2612
  if (dirOk) {
@@ -2324,21 +2628,165 @@ export async function collectChecks(shellVars, seams) {
2324
2628
  },
2325
2629
  });
2326
2630
  const modelsPath = join(overlay, "models.json");
2327
- let modelsOk = true;
2328
- let modelsFix = "";
2329
- if (fileExists(modelsPath)) {
2330
- try {
2331
- const leak = findLiteralSecret(JSON.parse(readFileSync(modelsPath, "utf8")));
2631
+ // Through the one reader (`readOverlayModels`, PR #520): an exists test answered false for a file under an
2632
+ // unreadable directory, so this line passed in silence on a file nobody had read. Absent is a pass (nothing to
2633
+ // leak); a transient errno (`isTransientOverlayRead`) is ⚠ naming the code, a read the worker retries once; any
2634
+ // other errno, EACCES among them, is ✗, since the job loads none of the file and the worker refuses every job
2635
+ // (issue #552); so is a models.json that is a link (PR #553's review) or a named pipe, socket or device, which
2636
+ // the reader judges from its `lstat` and never opens (issue #556); text that does not parse, or is not an
2637
+ // object, is ✗.
2638
+ let overlayModels = null;
2639
+ let modelsRead = null;
2640
+ try {
2641
+ overlayModels = readOverlayModels(overlay, { readFileSync: seams.readOverlayFile ?? ((p, enc) => readFileSync(p, enc)), ...(seams.lstatOverlayFile ? { lstatSync: seams.lstatOverlayFile } : {}) });
2642
+ } catch (error) {
2643
+ modelsRead = error;
2644
+ }
2645
+ if (modelsRead?.overlayLink === true) {
2646
+ // PR #553's review: the job's read-only mount does not resolve a link the way the host does.
2647
+ checks.push({
2648
+ ok: false,
2649
+ label: "Overlay models.json is a link, so every job is refused as model-unknown (overlay-link)",
2650
+ fix: `${OVERLAY_LINK_FIX}: ${modelsPath}; no job runs until then`,
2651
+ });
2652
+ } else if (modelsRead?.overlayNotAFile === true) {
2653
+ // Issue #556: judged by the reader's `lstat` and never opened, so doctor cannot hang on a FIFO with no writer.
2654
+ checks.push({
2655
+ ok: false,
2656
+ label: "Overlay models.json is not a regular file (a named pipe, socket or device), so every job is refused as model-unknown (overlay-not-a-file)",
2657
+ fix: `${OVERLAY_NOT_A_FILE_FIX}: ${modelsPath}; no job runs until then`,
2658
+ });
2659
+ } else if (modelsRead?.code === "EISDIR") {
2660
+ // Issue #539: pi fails to read a directory the same way, so it loads no models.json, and the worker refuses
2661
+ // every job (not retried: no retry turns a directory into a file).
2662
+ checks.push({
2663
+ ok: false,
2664
+ label: "Overlay models.json is a directory, so pi loads none of it and every job is refused as model-unknown (overlay-is-a-directory)",
2665
+ fix: `replace ${modelsPath} with a models.json file, or remove it; no job runs until then`,
2666
+ });
2667
+ } else if (isTransientOverlayRead(modelsRead?.code)) {
2668
+ checks.push({
2669
+ ok: false,
2670
+ warn: true,
2671
+ label: `Overlay models.json could not be read just now (${modelsRead.code}), so whether it is credential-free is not known; the worker retries each job once, then fails it`,
2672
+ fix: `check the disk or file handles behind ${modelsPath}, then re-run doctor`,
2673
+ });
2674
+ } else if (typeof modelsRead?.code === "string") {
2675
+ // Issue #552: the job reads the file through the read-only mount, and pi in the job loads none of it (the
2676
+ // existence check in image/runner/run-job.mjs, or pi's own read, fails), so a builtin provider the
2677
+ // file routes would go to its public endpoint. The worker refuses every job instead.
2678
+ const permission = modelsRead.code === "EACCES" || modelsRead.code === "EPERM";
2679
+ checks.push({
2680
+ ok: false,
2681
+ label: `Overlay models.json cannot be read by the worker (${modelsRead.code}), so a job loads none of it and every job is refused as model-unknown (overlay-unreadable)`,
2682
+ fix: permission ? `make ${modelsPath} and its folder readable by the account the worker runs as; every job is refused until then` : `check ${modelsPath} on the worker host (the read failed with ${modelsRead.code}); every job is refused until the worker can read it`,
2683
+ });
2684
+ } else {
2685
+ let modelsOk = true;
2686
+ let modelsFix = "";
2687
+ if (modelsRead !== null) {
2688
+ modelsOk = false;
2689
+ // Issue #539: pi then drops every entry, so a builtin model of a provider the file routes would run against
2690
+ // that provider's public endpoint. The worker refuses every job until the file is fixed, for either cause.
2691
+ const cause = /not valid JSON/.test(String(modelsRead?.message)) ? "is not valid JSON" : "does not match pi's models.json schema";
2692
+ modelsFix = `overlay models.json ${cause}, so pi loads none of it: every job is refused as model-unknown (overlay-unparseable) until the file is fixed, since pi would run even a builtin model against its provider's public endpoint instead of the route the file sets`;
2693
+ } else if (overlayModels !== null) {
2694
+ const leak = findLiteralSecret(overlayModels);
2332
2695
  if (leak) {
2333
2696
  modelsOk = false;
2334
2697
  modelsFix = `literal secret at ${leak} — move it to env/auth.json or a "$VAR" reference`;
2335
2698
  }
2336
- } catch {
2337
- modelsOk = false;
2338
- modelsFix = "overlay models.json is not valid JSON";
2699
+ }
2700
+ checks.push({ ok: modelsOk, label: "Overlay models.json is credential-free", fix: modelsFix });
2701
+ // Issue #539: an entry the file loads with but pi will not compose (the model gate's own rule,
2702
+ // `overlayProviderProblem`). pi drops that whole entry, so the worker refuses every job on the provider.
2703
+ const providers = overlayModels?.providers;
2704
+ for (const name of providers !== null && typeof providers === "object" && !Array.isArray(providers) ? Object.keys(providers) : []) {
2705
+ // Issue #587: an entry under an id pi renamed that lacks what a provider of its own needs (an api, a baseUrl
2706
+ // and models). It was an override of the builtin provider; under the new pi it is a custom provider with no
2707
+ // models, so it overrides nothing, and the builtin models it meant run with their own (empty) baseUrl.
2708
+ const entry = providers[name];
2709
+ if (Object.hasOwn(RENAMED_PROVIDERS, name) && entry !== null && typeof entry === "object" && !Array.isArray(entry) && (entry.api === undefined || entry.baseUrl === undefined || !Array.isArray(entry.models))) {
2710
+ const renamedTo = RENAMED_PROVIDERS[name];
2711
+ checks.push({
2712
+ ok: false,
2713
+ warn: true,
2714
+ label: `Overlay models.json entry ${JSON.stringify(name)} no longer overrides anything: pi 1.0.3 renamed that provider to ${JSON.stringify(renamedTo)}, so this entry is now a provider of its own, holding only what it declares itself, and the ${renamedTo} models do not get its settings`,
2715
+ fix: `rename the entry to ${JSON.stringify(renamedTo)} in ${modelsPath} (and every ${name}/ model reference in the triggers to ${renamedTo}/), then re-run doctor`,
2716
+ });
2717
+ }
2718
+ const problem = overlayProviderProblem(entry, name);
2719
+ if (problem === null) continue;
2720
+ checks.push({
2721
+ ok: false,
2722
+ label: `Overlay models.json entry ${JSON.stringify(name)} is one pi will not compose (${problem}), so pi drops all of it, its baseUrl and headers included`,
2723
+ fix: `every job that runs or lists a model of ${JSON.stringify(name)} is refused as model-unknown (overlay-provider-invalid): give each model an api and a baseUrl (its own or the provider's), a contextWindow and maxTokens above zero, and set a baseUrl beside oauth`,
2724
+ });
2725
+ }
2726
+ }
2727
+ // Issue #502's open question: pi's own loader against the worker's catalog, on the file as it is, whenever the reader
2728
+ // found one (a file the worker refuses is the first case worth comparing). `pi-model-loader.mjs` says why it is
2729
+ // the pi beside the worker rather than a container of the job image. Only a file that was READ: one the worker
2730
+ // refuses for its text (unparseable, or against pi's schema) is compared, while a link, an unreadable file or
2731
+ // anything else the reader refuses before reading (PR #553: overlay-link, overlay-unreadable; PR #557:
2732
+ // overlay-not-a-file) is not, since the job never loads it and the line above already says every job is refused.
2733
+ const refusedForText = modelsRead?.piDispatchConfig === true && typeof modelsRead.code !== "string" && modelsRead.overlayLink !== true && modelsRead.overlayNotAFile !== true && /^overlay models\.json (is not valid JSON|does not match)/.test(String(modelsRead.message));
2734
+ if (overlayModels !== null || refusedForText) {
2735
+ const catalog = await (seams.modelCatalog ?? defaultModelCatalog)();
2736
+ if (catalog) checks.push(...(await overlayLoaderParityChecks(modelsPath, { pi: await (seams.piModelLoader ?? defaultPiModelLoader)(), checkModelsKnown: catalog.checkModelsKnown })));
2737
+ }
2738
+ // Issue #503: a model whose baseUrl is localhost or a loopback literal can never be reached from a job, egress on
2739
+ // or off, because inside a job that address is the job's own container. Said whatever is declared: an overlay
2740
+ // pointed at localhost is the first thing an operator tries, and the fix is the declaration. Nothing when the file
2741
+ // is absent, does not parse, or is a file the job does not load (the line above names that): judged on the one
2742
+ // reader's result, so a models.json link is not read here around it.
2743
+ if (overlayModels !== null) {
2744
+ const loopback = overlayLoopbackModels(overlayModels);
2745
+ if (loopback.length > 0) {
2746
+ checks.push({
2747
+ ok: false,
2748
+ warn: true,
2749
+ label: `Overlay models.json points ${loopback.join(", ")} at a loopback address, which inside a job is the job's own container, so no job reaches that server`,
2750
+ fix: "serve the model on an address the egress proxy reaches, declare it in model-endpoints.json (host.docker.internal on Docker, host.containers.internal on Podman), and point the baseUrl there: docs/egress.md, \"Local model servers\"",
2751
+ });
2752
+ }
2753
+ // Issue #571: a model that costs money but asks for no usage in the stream. pi records each of its calls as
2754
+ // zeros, so the meter counts it `costUnreported` and every job on it settles at the floor (`≥` in the cost
2755
+ // views). Judged on the same reader's result: the overlay's own models, and the builtin models a provider-level
2756
+ // compat or a modelOverrides entry turns off, priced from the worker's catalog (none when it cannot load).
2757
+ const usageCatalog = await (seams.modelCatalog ?? defaultModelCatalog)();
2758
+ const unreported = unreportedUsageModels(overlayModels, { builtinModel: usageCatalog?.builtinModel, builtinChatModels: usageCatalog?.builtinChatModels });
2759
+ if (unreported.length > 0) {
2760
+ const SHOWN = 5;
2761
+ const named = unreported.slice(0, SHOWN).map((m) => `${quotedShown(m.provider)}/${quotedShown(m.modelId)}`).join(", ");
2762
+ const more = unreported.length > SHOWN ? ` and ${unreported.length - SHOWN} more` : "";
2763
+ checks.push({
2764
+ ok: false,
2765
+ warn: true,
2766
+ label: `Overlay models.json sets compat.supportsUsageInStreaming to false on ${named}${more} with a nonzero cost table, so those calls report no usage: each counts as costUnreported, and every job that calls one settles at the floor`,
2767
+ fix: "remove supportsUsageInStreaming: false if the server sends usage when asked (stream_options.include_usage), or set the model's cost to zeros if it is free: docs/costs.md, \"A call that reports no usage\"",
2768
+ });
2769
+ }
2770
+ // Issue #507: a priced model on a declared endpoint whose output cap would travel as max_completion_tokens,
2771
+ // which Ollama ignores. The runner's cost guard counts every such call unboundable, so a job under a dollar
2772
+ // cap is refused at its first call. The endpoints as the service reads them, through the keyless line's io.
2773
+ // A model a capped job may use is named by the cost-cap line below (`costCapFitChecks`), which runs on the
2774
+ // same catalog seam, so this line names only the rest: one line per model.
2775
+ const capEndpoints = (seams.declaredEndpoints ?? ((a) => declaredEndpointsIn({ ...a, fs: keylessIo })))({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform, valkeyUrl: keylessValkey });
2776
+ const capped = usageCatalog ? cappedModelNames(env, { runs: parseError ? [] : modelRuns, deployment: deploymentSettingsOf(env, settingsFilePath(env, home), fileExists) }) : new Set();
2777
+ const uncapped = ignoredOutputCapModels({ models: overlayModels, endpoints: capEndpoints, builtinModel: usageCatalog?.builtinModel, builtinChatModels: usageCatalog?.builtinChatModels }).filter((m) => !capped.has(`${m.provider}/${m.modelId}`));
2778
+ if (uncapped.length > 0) {
2779
+ const SHOWN = 5;
2780
+ const named = uncapped.slice(0, SHOWN).map((m) => `${quotedShown(m.provider)}/${quotedShown(m.modelId)}`).join(", ");
2781
+ const more = uncapped.length > SHOWN ? ` and ${uncapped.length - SHOWN} more` : "";
2782
+ checks.push({
2783
+ ok: false,
2784
+ warn: true,
2785
+ label: `Overlay models.json sends the output cap of ${named}${more} as max_completion_tokens to a declared model endpoint, which a local server may ignore (Ollama does), so under a dollar cap every call to it is refused as unboundable`,
2786
+ fix: "set \"compat\": { \"maxTokensField\": \"max_tokens\" } on the model or its provider in models.json, or set its cost to zeros if it is free: docs/egress.md, \"Local model servers\"",
2787
+ });
2339
2788
  }
2340
2789
  }
2341
- checks.push({ ok: modelsOk, label: "Overlay models.json is credential-free", fix: modelsFix });
2342
2790
  // Staged extensions load unless the operator opted out, so this pair reports what WILL run, not
2343
2791
  // what is switched on. The ⚠ sits on the loading case: it is the one where code the operator may
2344
2792
  // have staged months ago is executing against adversarial input right now. It stays a warning and
@@ -2599,6 +3047,77 @@ export async function collectChecks(shellVars, seams) {
2599
3047
  });
2600
3048
  }
2601
3049
 
3050
+ // Issue #502, allowed-model lists. The deployment list is judged unconditionally, `PI_WAIT_PROFILES`' rule: a
3051
+ // malformed or spaced value is a worker that will not boot. Its grammar is the worker's own (`allowedModelsFrom`).
3052
+ // The value doctor reads is the SERVICE's (`WORKER_SERVICE_KEYS`), so a spaced value in a `.env` a shell sources is
3053
+ // already named above as a line that loader reads differently, which is the case where the list silently vanishes.
3054
+ if (typeof env.PI_ALLOWED_MODELS === "string" && env.PI_ALLOWED_MODELS !== "") {
3055
+ let list = null;
3056
+ let why = null;
3057
+ try {
3058
+ list = allowedModelsFrom(env);
3059
+ } catch (err) {
3060
+ why = err?.message ?? "invalid";
3061
+ }
3062
+ checks.push(
3063
+ why === null
3064
+ ? { ok: true, label: `PI_ALLOWED_MODELS limits every job whose trigger names no run.models to ${list.length} model(s): ${list.join(", ")}` }
3065
+ : { ok: false, label: "PI_ALLOWED_MODELS is not a valid list", fix: `${why} -- the worker refuses to boot until this is fixed` },
3066
+ );
3067
+ }
3068
+ if (listing > 0) {
3069
+ // The version-floor disclosure, the run.waitFor line's twin and for its reason: doctor cannot see the
3070
+ // receiver's version from here, and the worker's own skew check refuses a job that arrives without its list.
3071
+ checks.push({
3072
+ ok: true,
3073
+ label: `${listing} trigger(s) name run.models, which needs a worker and a receiver that carry issue #502 (a service below that drops the list silently; the worker refuses such a job as trigger-skew rather than running it unrestricted, and only when it can read the triggers file itself)`,
3074
+ });
3075
+ }
3076
+ if (costCaps.length > 0) {
3077
+ // Issue #540: the run.models line's twin for run.maxCostUsd (#501), for its reason. A service below the floor
3078
+ // tolerated the key as unknown and dropped it, so the job would run under the deployment's cap, or none.
3079
+ checks.push({
3080
+ ok: true,
3081
+ label: `${costCaps.length} trigger(s) set run.maxCostUsd, which needs a worker and a receiver that carry issue #501 (a service below that drops the cap silently; the worker refuses such a job as trigger-skew rather than running it under the deployment's cap or none, and only when it can read the triggers file itself)`,
3082
+ });
3083
+ }
3084
+
3085
+ // Issues #501 and #502 (the round's doctor half): three lines about the models a job may use, asked of the worker's
3086
+ // own catalog (model-catalog.mjs, imported lazily: it loads pi-ai, which this module never does at load) with the
3087
+ // overlay models.json the worker reads and the deployment's settings resolved as a job resolves them. Nothing when
3088
+ // pi-ai does not load: the provider key line below already fails on that. A triggers file that did not load
3089
+ // contributes no trigger (its own line says so), and the deployment's default is still judged for its cap.
3090
+ {
3091
+ const catalog = await (seams.modelCatalog ?? defaultModelCatalog)();
3092
+ if (catalog) {
3093
+ const overlayDir = typeof env.PI_GLOBAL_PI_DIR === "string" && env.PI_GLOBAL_PI_DIR !== "" ? resolve(cwd, env.PI_GLOBAL_PI_DIR) : null;
3094
+ const readOverlayDoc = () => (overlayDir === null ? null : readOverlayModels(overlayDir, { readFileSync: seams.readOverlayFile ?? ((p, enc) => readFileSync(p, enc)), ...(seams.lstatOverlayFile ? { lstatSync: seams.lstatOverlayFile } : {}) }));
3095
+ let overlayDoc = null;
3096
+ try {
3097
+ overlayDoc = readOverlayDoc();
3098
+ } catch {
3099
+ // Unreadable or refused: the overlay lines say which. The checks below then judge builtin models only.
3100
+ }
3101
+ let envList = null;
3102
+ try {
3103
+ envList = allowedModelsFrom(env);
3104
+ } catch {
3105
+ // A malformed PI_ALLOWED_MODELS is its own line above (the worker refuses to boot on it).
3106
+ }
3107
+ const subjects = modelSubjects({ runs: parseError ? [] : modelRuns, deployment: deploymentSettingsOf(env, settingsFilePath(env, home), fileExists), envList });
3108
+ checks.push(...unknownModelChecks(subjects, catalog.checkModelsKnown, readOverlayDoc));
3109
+ checks.push(...costCapFitChecks(subjects, (ref) => boundModelOf(ref, { builtinModel: catalog.builtinModel, overlay: overlayDoc }), { unboundable: (ref) => outputUnboundable(outputCapView({ models: overlayDoc, provider: ref.provider, modelId: ref.id, builtinModel: catalog.builtinModel, builtinChatModels: catalog.builtinChatModels })) }));
3110
+ checks.push(
3111
+ ...listedProviderCredentialChecks(subjects, {
3112
+ candidatesOf: (name) => (typeof oracle?.providerKeyCandidates === "function" ? oracle.providerKeyCandidates(name) : []),
3113
+ forwarded: (env.PI_FORWARD_ENV ?? "").split(",").map((name) => name.trim()).filter((name) => name !== ""),
3114
+ env,
3115
+ overlay: overlayDoc,
3116
+ }),
3117
+ );
3118
+ }
3119
+ }
3120
+
2602
3121
  // REQ-TRIGGER-SECRETS. Only reported when a trigger actually binds one, on the run.resume block's
2603
3122
  // reasoning below: a deployment that uses no secrets should not be told about a variable it has no
2604
3123
  // reason to set.
@@ -2853,7 +3372,7 @@ export async function collectChecks(shellVars, seams) {
2853
3372
  // a deployment whose unit exits 1 in a restart loop is not a deployment that is merely unconfigured in
2854
3373
  // this shell. Where the file merely configures what this shell does not, the line still says which
2855
3374
  // process would honour it, and is still a warning.
2856
- const envFile = envFileKeys(join(cwd, ".env"), ["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE"], { fileExists, readEnvFile, platform });
3375
+ const envFile = envFileKeys(join(cwd, ".env"), BOOT_FILES.map((spec) => spec.key), { fileExists, readEnvFile, platform });
2857
3376
  // ONE RULE FOR BOTH BOOT FILES, and it asks the worker's own loader rather than a second opinion
2858
3377
  // (issue #384). Before this there were two hand-written copies of a shell-shaped check for
2859
3378
  // `PI_PAUSE_WINDOWS_FILE`, a THIRD rule for a scoped-limits file that does not parse, and no rule at all
@@ -2873,7 +3392,7 @@ export async function collectChecks(shellVars, seams) {
2873
3392
  // The IO the load verdict uses, injected like everything else this file touches: `statFile` for the
2874
3393
  // regular-file guard and the loaders' own two reads. A test drives a whole deployment through these
2875
3394
  // without a real file, which is how the fixtures below stay honest about content.
2876
- const seamsForLoad = { statFile: statSeam, loaderIo: { existsSync: (p) => fileExists(p), readFileSync: readEnvFile ? (p, enc) => asText(readEnvFile(p), enc) : readFileSync } };
3395
+ const seamsForLoad = { env, statFile: statSeam, loaderIo: { existsSync: (p) => fileExists(p), readFileSync: readEnvFile ? (p, enc) => asText(readEnvFile(p), enc) : readFileSync } };
2877
3396
  // ONCE PER FILE, not once per key (issue #396). This is a fact about the FILE -- one line the reader
2878
3397
  // cannot model -- and it was announced inside the per-key loop, so a `.env` with one such line produced
2879
3398
  // two near-identical warnings differing only in which key they named. The keys it prevents a verdict
@@ -2893,7 +3412,7 @@ export async function collectChecks(shellVars, seams) {
2893
3412
  // A SHAPE comes from systemd's own line structure (issue #447): the line is one systemd reads differently from
2894
3413
  // this command, and the table names it and what to change -- the same words `service install` refuses with.
2895
3414
  const shape = envFile.hazard.shape ? SYSTEMD_HAZARD_SHAPES[envFile.hazard.shape] : null;
2896
- const limit = "Other keys in the same file are affected too and are not checked here: this command reads only the two it names";
3415
+ const limit = `Other keys in the same file are affected too and are not checked here: this command reads only the ${countWord(BOOT_FILES.length)} it names`;
2897
3416
  // A file systemd will not LOAD is a service that does not start (measured on systemd 259), which is a failure
2898
3417
  // rather than a doubt about one reading (issue #447, gate round 1).
2899
3418
  const unloadable = envFile.hazard.shape === "nul" || envFile.hazard.shape === "invalid-utf8" || envFile.hazard.shape === "exec-too-large";
@@ -2951,10 +3470,10 @@ export async function collectChecks(shellVars, seams) {
2951
3470
  checks.push({
2952
3471
  ok: false,
2953
3472
  warn: envSetup !== null,
2954
- label: `${spec.key} is assigned an EMPTY value in ${join(cwd, ".env")}, which is not unset: ${loaderName} keeps it, the worker tries to load "" and REFUSES TO START${alsoExported === undefined ? "" : `, while ${otherName} would take ${envValueShown(alsoExported)}, so two deployments of this one file disagree`}`,
3473
+ label: `${spec.key} is assigned an EMPTY value in ${join(cwd, ".env")}, which is not unset: ${loaderName} keeps it, the worker tries to load "" and ${spec.fails}${alsoExported === undefined ? "" : `, while ${otherName} would take ${envValueShown(alsoExported)}, so two deployments of this one file disagree`}`,
2955
3474
  fix: envSetup !== null
2956
3475
  ? `${envSetup} runs after that file and may replace it, which is why this is a warning: if it does not, delete the ${spec.key} line, or give it the absolute path (${scaffolded})`
2957
- : `delete the ${spec.key} line from that .env, or give it a path: ${fixLineFor(spec.key, scaffolded)}. Deleting it turns ${spec.noun} off; an empty value turns the worker off`,
3476
+ : `delete the ${spec.key} line from that .env, or give it a path: ${fixLineFor(spec.key, scaffolded)}. Deleting it ${spec.whenDeleted}; an empty value ${spec.whenEmpty}`,
2958
3477
  });
2959
3478
  } else if (envFile.hazard == null && notPlainLine !== undefined) {
2960
3479
  // NAMED, NEVER QUOTED. The value is outside the grammar every loader reads the same way, so this
@@ -2983,7 +3502,7 @@ export async function collectChecks(shellVars, seams) {
2983
3502
  checks.push({
2984
3503
  ok: false,
2985
3504
  warn: true,
2986
- label: `${spec.key} is assigned TWICE in ${join(cwd, ".env")} with different values: ${loaderName} takes ${envValueShown(fileRaw)}, ${otherName} would take ${alsoExported === "" ? "an EMPTY value, which refuses the boot" : envValueShown(alsoExported)}`,
3505
+ label: `${spec.key} is assigned TWICE in ${join(cwd, ".env")} with different values: ${loaderName} takes ${envValueShown(fileRaw)}, ${otherName} would take ${alsoExported === "" ? `an EMPTY value, which ${spec.emptyCost}` : envValueShown(alsoExported)}`,
2987
3506
  fix: `keep one assignment. Which one is in force depends on how the worker starts, so two of them means two deployments of the same file disagree`,
2988
3507
  });
2989
3508
  }
@@ -3004,7 +3523,7 @@ export async function collectChecks(shellVars, seams) {
3004
3523
  : {
3005
3524
  ok: false,
3006
3525
  warn: envSetup !== null,
3007
- label: `${spec.key} is set in ${join(cwd, ".env")} (${envValueShown(fileRaw)}) to a file the worker cannot load, so a service started from it REFUSES TO START: ${verdict.reason}`,
3526
+ label: `${spec.key} is set in ${join(cwd, ".env")} (${envValueShown(fileRaw)}) to a file the worker cannot load, so a service started from it ${spec.fails}: ${verdict.reason}`,
3008
3527
  fix: envSetup !== null ? `${envSetup} runs after that file and may replace it; if it does not, fix ${envValueShown(fileRaw)} or point the key at a file that loads` : `fix ${envValueShown(fileRaw)}, or point ${spec.key} at a file that loads`,
3009
3528
  },
3010
3529
  );
@@ -3014,18 +3533,35 @@ export async function collectChecks(shellVars, seams) {
3014
3533
  if (typeof shellRaw === "string" && shellRaw.trim() === "") {
3015
3534
  checks.push({
3016
3535
  ok: false,
3017
- label: `${spec.key} is set to an EMPTY value in this shell, which is not unset: the worker keeps it, tries to load "" and REFUSES TO START`,
3018
- fix: `unset ${spec.key} in this shell (that turns ${spec.noun} off), or give it the absolute path: export ${fixLineFor(spec.key, scaffolded)}`,
3536
+ label: `${spec.key} is set to an EMPTY value in this shell, which is not unset: the worker keeps it, tries to load "" and ${spec.fails}`,
3537
+ fix: `unset ${spec.key} in this shell (that ${spec.whenDeleted}), or give it the absolute path: export ${fixLineFor(spec.key, scaffolded)}`,
3019
3538
  });
3020
3539
  } else if (typeof shellRaw === "string") {
3021
3540
  const verdict = loadVerdict(spec, shellRaw, cwd, seamsForLoad, platform);
3022
3541
  if (!verdict.ok) {
3023
3542
  checks.push({
3024
3543
  ok: false,
3025
- label: `${spec.key} is set in this shell to a file the worker cannot load, so it REFUSES TO START: ${verdict.reason}`,
3544
+ label: `${spec.key} is set in this shell to a file the worker cannot load, so it ${spec.fails}: ${verdict.reason}`,
3026
3545
  fix: `fix ${envValueShown(shellRaw)}, or point ${spec.key} at a file that loads`,
3027
3546
  });
3028
3547
  }
3548
+ } else if (spec.defaultsToScaffold) {
3549
+ // Issue #503: unset is the scaffold, so it is that file which is judged, when nothing in .env says otherwise and it
3550
+ // is there at all (a missing default file declares nothing, which is valid). Silent when it loads. Before the
3551
+ // unreadable-.env line, which asks whether the service is configured for a feature unset turns off: unset turns
3552
+ // nothing off here.
3553
+ // Asked of `statFile`, the seam the load itself uses, so "missing" is ENOENT from the same place and a missing
3554
+ // default stays silent however `fileExists` is seamed.
3555
+ if (fileRaw === undefined && onlyExported === undefined && alsoExported === undefined && notPlainLine === undefined && !blankInFile && envFile.hazard == null && presentAt(statSeam, scaffolded)) {
3556
+ const verdict = loadVerdict(spec, scaffolded, cwd, seamsForLoad, platform);
3557
+ if (!verdict.ok) {
3558
+ checks.push({
3559
+ ok: false,
3560
+ label: `${spec.key} is unset, so ${spec.unsetMeans}, and it does not load: the worker ${spec.fails}: ${verdict.reason}`,
3561
+ fix: `fix ${envValueShown(scaffolded)}, or empty its "endpoints" list`,
3562
+ });
3563
+ }
3564
+ }
3029
3565
  } else if (envFile.unreadable === true) {
3030
3566
  // COULD NOT READ, said as itself. The alternative -- the "unset" line below -- is a positive claim
3031
3567
  // about a key in a file nobody could open.
@@ -3044,11 +3580,17 @@ export async function collectChecks(shellVars, seams) {
3044
3580
  ok: false,
3045
3581
  warn: true,
3046
3582
  label: `${scaffolded} exists but ${spec.key} is unset -- the worker ignores it, so ${spec.off}`,
3047
- fix: `set ${fixLineFor(spec.key, scaffolded)} in .env and restart the worker -- unset means ${spec.unsetMeans}, while the admin panel defaults to this same file and reports each ${spec.unit} it writes as applied live; delete the file if this deployment has no ${spec.nothing}`,
3583
+ fix: `set ${fixLineFor(spec.key, scaffolded)} in .env and restart the worker -- unset means ${spec.unsetMeans}${spec.panelWrites === false ? "" : `, while the admin panel defaults to this same file and reports each ${spec.unit} it writes as applied live`}; delete the file if this deployment has no ${spec.nothing}`,
3048
3584
  });
3049
3585
  }
3050
3586
  }
3051
3587
 
3588
+ // A scope that can only ever be a folder (issue #242's dead-scope rule): a forge repo always contains "/" and never
3589
+ // begins "/", "./" or "../" or carries a backslash. A forge-qualified row (issue #498) names a repo by construction,
3590
+ // so it is never one. Shared by the dead-scope advisory and the bare-repo one below, so one row is never called a
3591
+ // dead folder by one line and a bare repo by the other.
3592
+ const folderOnly = (s) => scopeFormOf(s) !== "qualified" && (s.startsWith("/") || s.startsWith("./") || s.startsWith("../") || s.includes("\\") || !s.includes("/") || /^[A-Za-z]:/.test(s));
3593
+
3052
3594
  // The dead-scope advisory (issue #242), honest about what doctor can actually judge. A forge repo
3053
3595
  // always contains "/" and never begins "/", "./" or "../" or carries a backslash, so a scope in any
3054
3596
  // of THOSE shapes can only ever be a folder -- and a folder row that matches no trigger's canonical
@@ -3062,18 +3604,51 @@ export async function collectChecks(shellVars, seams) {
3062
3604
  // shape `render` prints the fix line too, which the old `ok: true` never did.
3063
3605
  if (scopedLimitFacts.parseError === null && scopedLimitFacts.limits.length > 0 && parseError === null && triggersFilePath !== null) {
3064
3606
  const folderSet = new Set(folders);
3065
- const folderOnly = (s) => s.startsWith("/") || s.startsWith("./") || s.startsWith("../") || s.includes("\\") || !s.includes("/") || /^[A-Za-z]:/.test(s);
3066
- const dead = scopedLimitFacts.limits.map((l) => l.scope).filter((s) => folderOnly(s) && !folderSet.has(s));
3607
+ // A model row (version 2) names a model, never a folder, so it is never "a folder no trigger runs in"; nor is a
3608
+ // project row (issue #499 part B), which names a project.
3609
+ const dead = scopedLimitFacts.limits.map((l) => l.scope).filter((s) => !isModelScope(s) && !isProjectScope(s) && folderOnly(s) && !folderSet.has(s));
3067
3610
  if (dead.length > 0) {
3068
3611
  checks.push({
3069
3612
  ok: false,
3070
3613
  warn: true,
3071
- label: `${dead.length} scoped limit(s) name a folder no trigger runs in (${dead.join(", ")}) -- the cap guards nothing; scopes match exactly (no globs, folders by resolved ABSOLUTE path), so check the spelling against triggers.json run.folder or delete the entry`,
3614
+ label: `${dead.length} scoped limit(s) name a folder no trigger runs in (${dead.join(", ")}) -- no trigger runs there, so unless a CLI or local job does, the cap guards nothing; scopes match exactly (no globs, folders by resolved ABSOLUTE path), so check the spelling against triggers.json run.folder or delete the entry`,
3072
3615
  fix: `edit ${scopedLimitFacts.path} by hand or via dispatch_limit_edit/_delete -- repo-shaped scopes are never flagged here, because a webhook job's repo comes from the delivery, which triggers.json cannot enumerate`,
3073
3616
  });
3074
3617
  }
3075
3618
  }
3076
3619
 
3620
+ // Issue #499 part B: a `project:<id>` row whose id is not in projects.json. The worker refuses to start on it
3621
+ // (start.mjs, `checkProjectRows`), so this is a FAILURE naming the row and the id. Skipped when either file does not
3622
+ // load: that is the BOOT_FILES line's, and a zeroed list here would name every project row as dangling.
3623
+ if (scopedLimitFacts.parseError === null && scopedLimitFacts.limits.length > 0) {
3624
+ const projectFacts = readProjectFacts(env, fileExists);
3625
+ if (projectFacts.parseError === null) checks.push(...projectRowChecks(scopedLimitFacts.limits, projectFacts.projects, scopedLimitFacts.path));
3626
+ }
3627
+
3628
+ // Issue #504 part B: the envelope's advisories, on the file the service names, when it loads (a file that does not is
3629
+ // the BOOT_FILES line's). Warnings only: the worker boots on both.
3630
+ const envelopeFactsHere = readEnvelopeFacts(env);
3631
+ if (envelopeFactsHere.envelope) checks.push(...envelopeChecks(envelopeFactsHere.envelope, envelopeFactsHere.projects, envelopeFactsHere.maxCostMicros));
3632
+
3633
+ // Issue #498: a BARE repo scope (`acme/web`) matches that repo on every forge, so with triggers on more than one forge
3634
+ // kind a bare row is one cap (one lease, one count) shared by GitHub's acme/web and Forgejo's, and a bare pause window
3635
+ // pauses both. That may be meant, so this is a WARNING naming the qualified spellings, never a failure: refusing bare
3636
+ // rows would stop existing workers from booting. Guarded like the dead-scope advisory, on readable triggers.
3637
+ if (parseError === null && triggersFilePath !== null && forges.length > 1) {
3638
+ const bareRows = scopedLimitFacts.parseError === null ? scopedLimitFacts.limits.map((l) => l.scope).filter((s) => !isModelScope(s) && !isProjectScope(s) && scopeFormOf(s) === "bare" && !folderOnly(s)) : [];
3639
+ const bareWindows = pauseWindowFacts.parseError === null ? [...new Set(pauseWindowFacts.windows.map((w) => w.scope).filter((s) => s !== "*" && scopeFormOf(s) === "bare" && !folderOnly(s)))] : [];
3640
+ const named = [...new Set([...bareRows, ...bareWindows])];
3641
+ if (named.length > 0) {
3642
+ const what = [bareRows.length > 0 ? `${bareRows.length} scoped limit(s)` : null, bareWindows.length > 0 ? `${bareWindows.length} pause window(s)` : null].filter(Boolean).join(" and ");
3643
+ checks.push({
3644
+ ok: false,
3645
+ warn: true,
3646
+ label: `${what} name a bare repo (${named.join(", ")}) while triggers run on ${forges.join(" and ")}: a bare scope matches that repo on EVERY forge, so one cap, lease or pause covers all of them`,
3647
+ fix: `if each forge should count on its own, write the scope forge-qualified: ${named.map((s) => forges.map((k) => `${k}:${s}`).join(" or ")).join("; ")} (a qualified row starts a new count, and a bare and a qualified row for one repo cannot both exist); keep the bare form if one shared cap is what you meant`,
3648
+ });
3649
+ }
3650
+ }
3651
+
3077
3652
  // Issue #464: the jobs dir is this account's to write. PI_JOBS_DIR, PI_SANDBOX_DIR and TMPDIR as the service reads them
3078
3653
  // (TMPDIR in .env is a documented way out of a squatted default root, so doctor judges the root the service will use).
3079
3654
  let dirRefused = false;
@@ -3192,6 +3767,37 @@ export async function collectChecks(shellVars, seams) {
3192
3767
  const refused = inRoot.length > 0 && !said ? accountRootRefusal(root, { uid, fs: seams.jobsDirFs, ownerName: (id) => ownerNameFromPasswd(seams.passwd, id), what: `this account's ${inRoot.map((st) => (st.key === "PI_LOGS_DIR" ? "run history" : "settings overlay")).join(" and ")} (no home directory, so the default is here)` }) : null;
3193
3768
  if (refused) checks.push({ ok: false, label: refused.label, fix: `${refused.fix}; or set ${inRoot.map((st) => st.key).join(" and ")} to a path this account owns` });
3194
3769
  }
3770
+ // Issue #501: the overlay itself, read by the worker's own reader. A present-but-invalid file refuses EVERY job
3771
+ // (`settings-overlay-invalid`) while the worker otherwise runs clean, and since #501 a file that loaded before can
3772
+ // become invalid on upgrade: a duplicate key, which used to take the last value, is now refused. The reason names
3773
+ // keys only, never values. Absent is the normal empty overlay and says nothing.
3774
+ if (fileExists(settingsFile)) {
3775
+ const overlay = readOverlay(settingsFile);
3776
+ if (overlay.invalid) {
3777
+ checks.push({
3778
+ ok: false,
3779
+ label: `settings overlay ${settingsFile} is invalid (${overlay.invalid}) -- the worker refuses every job as settings-overlay-invalid until it is fixed, and the panel refuses to write over it`,
3780
+ // The fix follows the reason: only a duplicate key gets the upgrade note, since only it loaded before #501.
3781
+ fix: /duplicate key/.test(overlay.invalid)
3782
+ ? "remove the repeated key from that file, keeping the value you mean (a duplicate key is refused since issue #501; it used to take the last value), or delete the file to start from an empty overlay"
3783
+ : "fix what the reason names in that file, or delete the file to start from an empty overlay",
3784
+ });
3785
+ } else {
3786
+ // Issue #501 (PR #542's review): a VALID overlay can still break the dollar invariant once merged over env
3787
+ // (a window in one, no maxCostUsd in either), which the worker checks per job and answers by refusing every
3788
+ // job. The env-only half is `dollarChecks`; this is the merged half, said only when env alone is fine.
3789
+ const merged = overlayDollarProblem(overlay.overlay, env);
3790
+ if (merged) checks.push({ ok: false, label: `settings overlay ${settingsFile}: ${merged} -- the worker refuses every job as settings-overlay-invalid`, fix: "set maxCostUsd in the overlay (or PI_MAX_COST_USD in .env), or remove the dollar window" });
3791
+ }
3792
+ }
3793
+ // PR #549's review: the scoped-limits dollar rows against the deployment's per-job cap, env and overlay merged
3794
+ // the worker's way (an overlay value wins; an invalid overlay leaves env).
3795
+ if (scopedLimitFacts.parseError === null && scopedLimitFacts.limits.length > 0) {
3796
+ const overlay = fileExists(settingsFile) ? readOverlay(settingsFile) : { overlay: {} };
3797
+ const fromOverlay = overlay.invalid ? undefined : overlay.overlay?.maxCostUsd;
3798
+ const envCap = env.PI_MAX_COST_USD === undefined || env.PI_MAX_COST_USD === "" ? undefined : env.PI_MAX_COST_USD;
3799
+ checks.push(...scopedDollarRowChecks(scopedLimitFacts.limits, fromOverlay ?? envCap, scopedLimitFacts.path));
3800
+ }
3195
3801
  if (noHome) {
3196
3802
  checks.push({
3197
3803
  ok: false,
@@ -3433,7 +4039,7 @@ async function defaultProviderOracle() {
3433
4039
  * Never carries a fixAction (the never tier): doctor cannot know which provider an operator meant, and
3434
4040
  * never mints a credential.
3435
4041
  */
3436
- function providerKeyCheck({ provider, env, agentDir, oracle, nodeOk }) {
4042
+ function providerKeyCheck({ provider, env, agentDir, oracle, nodeOk, keyless = null }) {
3437
4043
  if (!oracle?.providerKeyCandidates) {
3438
4044
  // pi did not load: below-floor Node, or a tree with no dependencies installed.
3439
4045
  //
@@ -3459,7 +4065,7 @@ function providerKeyCheck({ provider, env, agentDir, oracle, nodeOk }) {
3459
4065
  };
3460
4066
  }
3461
4067
  const candidates = oracle.providerKeyCandidates(provider);
3462
- if (candidates.length === 0) return noKeyVariableCheck(provider, oracle);
4068
+ if (candidates.length === 0) return noKeyVariableCheck(provider, oracle, keyless);
3463
4069
 
3464
4070
  // Presence by PI'S truthiness, not a stricter one. This used to trim, and trimming made doctor pick a
3465
4071
  // DIFFERENT variable than the worker will: with a whitespace `ANTHROPIC_OAUTH_TOKEN` beside a real
@@ -3622,7 +4228,7 @@ function usablePiLoginKey(agentDir, provider) {
3622
4228
  * the distinction `findEnvKeys`'s single `undefined` cannot make, and the reason issue #286 needed a
3623
4229
  * second question at all.
3624
4230
  */
3625
- function noKeyVariableCheck(provider, oracle) {
4231
+ function noKeyVariableCheck(provider, oracle, keyless = null) {
3626
4232
  if (oracle.piProviders().includes(provider)) {
3627
4233
  // `amazon-bedrock` wants AWS credentials or a profile, `openai-codex` an OAuth login. Both are
3628
4234
  // credential SOURCES the closed container env has no door for, so buildContainerEnv refuses every
@@ -3637,14 +4243,37 @@ function noKeyVariableCheck(provider, oracle) {
3637
4243
  // `<PROVIDER>_API_KEY`. For the case that motivated this -- PI_PROVIDER=gemini -- GEMINI_API_KEY is
3638
4244
  // `google`'s variable, so the answer is exact. Nothing edit-distance-based would find it (gemini and
3639
4245
  // google differ by five characters), which is why this matches on the VARIABLE, not on the name.
4246
+ // Issue #503: the worker's own keyless verdict (model-endpoints.mjs), on the same predicate and in the same order as
4247
+ // the credential gate (no key variable, not in pi's catalog), so this line and the worker cannot disagree.
4248
+ const endpoints = Array.isArray(keyless?.endpoints) ? keyless.endpoints : [];
4249
+ if (endpoints.length > 0 && typeof keyless?.unreadable === "string") {
4250
+ // Issue #552: only a transient errno is retried; any other refuses every job at the model gate, before this one.
4251
+ const retried = isTransientOverlayRead(keyless.unreadable);
4252
+ return {
4253
+ ok: false,
4254
+ warn: retried,
4255
+ label: `Provider key: could not read models.json (${keyless.unreadable}), so whether ${JSON.stringify(provider)} is keyless is not known; ${retried ? "the worker retries such a job once, then fails it" : "the worker refuses every job until it can read it (overlay-unreadable)"}`,
4256
+ fix: "make the overlay's models.json readable by the account the worker runs as, then re-run doctor",
4257
+ };
4258
+ }
4259
+ const verdict = endpoints.length > 0 ? keylessVerdict({ models: keyless.models, provider, endpoints }) : { keyless: false, why: null };
4260
+ if (verdict.keyless) {
4261
+ return { ok: true, label: `Provider key: none needed (${provider} is keyless: served by declared endpoint ${verdict.endpoints.join(", ")})` };
4262
+ }
3640
4263
  const wanted = `${provider.toUpperCase().replace(/[^A-Z0-9]/g, "_")}_API_KEY`;
3641
4264
  const owner = oracle.piProviders().find((id) => oracle.providerKeyCandidates(id).includes(wanted));
4265
+ // Why a custom provider the overlay defines is not keyless, when it is one: the one fact the operator has to change.
4266
+ const why = verdict.why ? `; it is not keyless because ${verdict.why}` : "";
4267
+ // Issue #587: an id pi renamed (azure-openai-responses is `azure` since pi 1.0.3), named, unless the overlay declares it.
4268
+ const renamed = providerRenameHint(provider, { models: keyless?.hintModels ?? keyless?.models ?? null, overlayUnread: keyless?.hintUnread === true, piProviders: oracle.piProviders() });
3642
4269
  return {
3643
4270
  ok: false,
3644
- label: `PI_PROVIDER is ${JSON.stringify(provider)}, which is not a provider pi has`,
3645
- fix: owner
3646
- ? `set PI_PROVIDER=${owner}, the provider pi reads ${wanted} from -- or unset it for the default \`anthropic\``
3647
- : `set PI_PROVIDER to a provider id pi has, or unset it for the default \`anthropic\`: ${oracle.piProviders().join(", ")}`,
4271
+ label: `PI_PROVIDER is ${JSON.stringify(provider)}, which is not a provider pi has${why}${renamed ? `;${renamed}` : ""}`,
4272
+ fix: renamed
4273
+ ? `set PI_PROVIDER=${RENAMED_PROVIDERS[provider]}, and rename the provider in every trigger's model and allowed-models entries and in models.json the same way (docs/triggers.md)`
4274
+ : owner
4275
+ ? `set PI_PROVIDER=${owner}, the provider pi reads ${wanted} from -- or unset it for the default \`anthropic\`; or, ${KEYLESS_HOW}`
4276
+ : `set PI_PROVIDER to a provider id pi has, or unset it for the default \`anthropic\`: ${oracle.piProviders().join(", ")}; or, ${KEYLESS_HOW}`,
3648
4277
  };
3649
4278
  }
3650
4279
 
@@ -3848,102 +4477,887 @@ function readScopedLimitFacts(env, fileExists) {
3848
4477
  }
3849
4478
 
3850
4479
  /**
3851
- * How a line names its trigger: cron entries by their id, id-less webhook entries by raw file position (the admin's
3852
- * trigger:<index> identity). One function, because the flow lines and the venue lines must name a trigger alike.
4480
+ * The projects facts (issue #499 part B): the parsed projects when PI_PROJECTS_FILE is set, `[]` when it is unset (no
4481
+ * projects, so every project row dangles), or a parse error when it is set and does not load (the BOOT_FILES line's).
3853
4482
  */
3854
- function triggerLabel(t, index) {
3855
- return t.on.type === "cron" ? `cron "${t.on.id}"` : `${t.on.type} trigger #${index}`;
4483
+ function readProjectFacts(env, fileExists) {
4484
+ const path = env.PI_PROJECTS_FILE;
4485
+ if (typeof path !== "string" || path.trim() === "") return { projects: [], parseError: null };
4486
+ try {
4487
+ if (!fileExists(path) || !statSync(path).isFile()) return { projects: [], parseError: "unreadable" };
4488
+ return { projects: loadProjects({ projectsFile: path }, { readFileSync, existsSync: fileExists }), parseError: null };
4489
+ } catch (e) {
4490
+ return { projects: [], parseError: e?.message ?? String(e) };
4491
+ }
3856
4492
  }
3857
4493
 
3858
- function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
3859
- const none = { requiring: 0, waiting: 0, waitProfiles: [], waitAfters: [], optingOut: 0, resuming: 0, replicating: 0, instructing: 0, commands: 0, secreting: 0, onceArmed: 0, onceSpent: 0, secretProfiles: [], localSecretFolders: [], secretNames: [], folders: [], images: [], imageRoutes: [], namedBackends: [], skillsDirs: [], forges: [], repositories: [], flows: [], parseError: null, path: null };
4494
+ /**
4495
+ * The envelope facts (issue #504 part B): the envelope as the worker would load it, the projects and the per-job cap it
4496
+ * was judged against, or a parse error when `PI_ENVELOPE_FILE` is set and does not load (the BOOT_FILES line's).
4497
+ * `envelope` is null when the key is unset.
4498
+ */
4499
+ function readEnvelopeFacts(env) {
4500
+ const path = nonEmpty(env.PI_ENVELOPE_FILE);
4501
+ if (path === null) return { envelope: null, projects: [], maxCostMicros: null, parseError: null };
4502
+ const io = { readFileSync, existsSync };
3860
4503
  try {
3861
- // Unset falls back to ./triggers.json in cwd, MIRRORING the receiver's own default
3862
- // (receiver/src/config.mjs) -- the two must read the same file, or doctor preflights a deployment
3863
- // the receiver will not boot. An absent file still means "no triggers at all", exactly as before.
3864
- const path = triggersPath(env, cwd);
3865
- if (!fileExists(path)) return none;
3866
- // The same regular-file guard, for the same reason: this read is unbounded too, and a triggers path
3867
- // naming a FIFO hangs the command with no output and no timeout that can reach it.
3868
- if (!statSync(path).isFile()) return { ...none, parseError: `triggers file is not a regular file: ${path}`, path };
3869
- const text = readFileSync(path, "utf8");
3870
- const triggers = parseTriggers(text, path);
3871
- // The one-shot facts are counted from the RAW entries, not the parsed records, because the
3872
- // validator collapses a disarmed entry to a sentinel that carries neither `once` nor
3873
- // `disarmed` -- exactly so nothing can match it -- which also erases it from every parsed
3874
- // count above. Doctor is the surface that must still SEE the spent entry: "why did nothing
3875
- // fire" is answered by a spent row, and only the raw file still holds it. Safe unguarded:
3876
- // parseTriggers just accepted this same text, so JSON.parse cannot throw here.
3877
- const rawEntries = JSON.parse(text)?.triggers ?? [];
3878
- // Issue #433 review rounds 2 and 3: whether THIS HOST'S WORKER schedules a cron trigger, for the unblessed-venue
3879
- // line, by the worker's two conditions and nothing else. (a) The worker schedules cron only from a PI_TRIGGERS_FILE
3880
- // it was given (`config.triggersFile` is null without one, and `loadSchedules` then returns []), while doctor
3881
- // falls back to ./triggers.json for everything else it reads; so without the variable no cron trigger is judged.
3882
- // (b) Its placement, by the worker's own predicate (`cronPlacement`): judged unless it is `"elsewhere"`, another
3883
- // machine's folder on a fleet. A single host's absent folder is `"refused"`, the worker's boot refusal, and is
3884
- // still judged. Round 2 ran the whole `loadSchedules` instead and caught its throw as "judge everything", which let
3885
- // one served trigger's bad skillsDir bring back the false line for another machine's trigger: a predicate per
3886
- // trigger cannot be derailed by a sibling.
3887
- // The name collectChecks resolved (issue #464), never this shell's alone.
3888
- const fleet = Boolean(declaredWorkerName);
3889
- const cronScheduledHere = (t) => env.PI_TRIGGERS_FILE !== undefined && cronPlacement(t.run, { existsSync: fileExists, fleet }) !== "elsewhere";
3890
- return {
3891
- onceArmed: rawEntries.filter((t) => t?.on?.once === true && t.on.disarmed === undefined).length,
3892
- onceSpent: rawEntries.filter((t) => t?.on?.disarmed !== undefined).length,
3893
- requiring: triggers.filter((t) => t.run.packages === true).length,
3894
- resuming: triggers.filter((t) => t.run.resume === true).length,
3895
- // REQ-PER-TRIGGER-INSTRUCTION. Counted beside `resuming` for the same reason: it is a per-trigger
3896
- // choice that changes what every job of it is told, and an operator should see it before it fires.
3897
- instructing: triggers.filter((t) => typeof t.run.instructions === "string").length,
3898
- // REQ-REPLICA-RUNS. `> 1` rather than `!== undefined` because the loader already refuses anything
3899
- // else -- this counts triggers that will actually multiply spend, which is the only reason to say so.
3900
- replicating: triggers.filter((t) => t.run.replicas > 1).length,
3901
- // run.command triggers (issue #189), counted for the one advisory line below. The `flows`
3902
- // tuple list already filters to `typeof f.flow === "string"`, so a command trigger drops out
3903
- // of the flow-tier probes naturally -- no exclusion needed there.
3904
- commands: triggers.filter((t) => typeof t.run.command === "string").length,
3905
- // REQ-TRIGGER-SECRETS. Counted beside `instructing` for its reason: a per-trigger choice that
3906
- // changes what every job of it can reach, and one that lives only in triggers.json.
3907
- secreting: triggers.filter((t) => t.run.secrets !== undefined).length,
3908
- // The distinct profile NAMES the file selects, deduped like `images`/`skillsDirs`: the checks below
3909
- // cost a stat each, and two triggers naming one profile are one question. `default` is substituted
3910
- // for an absent field so the table answers what the worker will actually look up.
3911
- secretProfiles: [...new Set(triggers.filter((t) => t.run.secrets !== undefined).map((t) => t.run.secretsProfile ?? "default"))].sort(),
3912
- // LOCAL triggers that bind secrets, by folder. A local job's /workspace IS this folder, bind-mounted
3913
- // read-write with no clone, so a credential an agent writes into .env lands in the operator's real
3914
- // repository rather than a temp dir that gets swept. Deduped for skillsDirs' reason.
3915
- localSecretFolders: [...new Set(triggers.filter((t) => t.run.secrets !== undefined && t.run.kind === "local" && typeof t.run.folder === "string").map((t) => t.run.folder))].sort(),
3916
- // Issue #309. The distinct variable NAMES the file binds, deduped like the profiles above. The
3917
- // pre-spend gate refuses a name pi reads for the job's provider, and unlike the version that gate
3918
- // replaced, that question no longer needs host state to answer -- so doctor can answer it at setup
3919
- // rather than leaving the operator to meet it as a public refusal on a live job.
3920
- secretNames: [...new Set(triggers.filter((t) => t.run.secrets !== undefined).flatMap((t) => Object.keys(t.run.secrets)))].sort(),
3921
- // Issue #242: every local run.folder, CANONICALIZED the way the scoped-limits matcher
3922
- // canonicalizes a job's folder (one derivation -- canonicalScope, never re-spelled here), so
3923
- // the unreferenced-scope advisory compares like with like across spelling variants.
3924
- folders: [...new Set(triggers.filter((t) => t.run.kind === "local" && typeof t.run.folder === "string").map((t) => canonicalScope({ kind: "local", folder: t.run.folder })))].sort(),
3925
- // Issue #230. How many triggers hold their jobs, and the distinct profile NAMES they select --
3926
- // deduped like `secretProfiles` and for its reason: each name costs a lookup, and two triggers
3927
- // waiting on one profile are one question.
3928
- waiting: triggers.filter((t) => Array.isArray(t.run.waitFor) && t.run.waitFor.length > 0).length,
3929
- // The `after` instants as WRITTEN, deduped. Not parsed here: `readTriggerFacts` is a fact reader and
3930
- // the ceiling it is measured against is env, which belongs at the check. Two triggers naming one
3931
- // instant are one finding, and the raw string is what the operator has to go and edit.
3932
- waitAfters: [...new Set(triggers.flatMap((t) => (Array.isArray(t.run.waitFor) ? t.run.waitFor : [])).map((c) => c?.after).filter((v) => typeof v === "string"))].sort(),
3933
- waitProfiles: [
3934
- ...new Set(
3935
- triggers
3936
- .filter((t) => Array.isArray(t.run.waitFor))
3937
- .flatMap((t) => t.run.waitFor.map((c) => c?.profile).filter((n) => typeof n === "string")),
3938
- ),
3939
- ].sort(),
3940
- optingOut: triggers.filter((t) => t.run.packages === false).length,
3941
- images: [...new Set(triggers.map((t) => t.run.image).filter((i) => typeof i === "string"))].sort(),
3942
- // Issue #433: each image with the venue its trigger names (undefined: the deployment default), so the image is
3943
- // asked of the runtime its jobs start on. The worker's own shape (`{ backend }`), for `resolveBackendName`.
3944
- imageRoutes: triggers.filter((t) => typeof t.run.image === "string").map((t) => ({ image: t.run.image, backend: t.run.backend })),
3945
- // Issue #433 review round 1: every trigger that NAMES a venue, with the label the lines below name it by, so
3946
- // doctor can say which of them this deployment does not bless (the worker refuses each of their jobs).
4504
+ const context = envelopeContextOf(env, io);
4505
+ return { envelope: loadEnvelopeAsTheWorker(path, io, env), projects: context.projects, maxCostMicros: context.maxCostMicros, parseError: null };
4506
+ } catch (e) {
4507
+ return { envelope: null, projects: [], maxCostMicros: null, parseError: e?.message ?? String(e) };
4508
+ }
4509
+ }
4510
+
4511
+ /**
4512
+ * The envelope's advisories (issue #504 part B, INT-ENVELOPE-FILE-CONTRACT), WARNINGS the worker boots on:
4513
+ * - a floor above 0 and below the per-job cost cap: every governed job reserves its per-job cap against its share, so a
4514
+ * floor that small admits no job of its own (it still counts toward the total);
4515
+ * - a project in projects.json that the envelope does not name: its jobs count in `_other`'s share.
4516
+ * Plus one line of facts: the digest (what `fpEnvelope` and `alloc:envelope:expected` hold), the window, the total and
4517
+ * whether delegation is on. Ids and amounts only.
4518
+ */
4519
+ export function envelopeChecks(envelope, projects, maxCostMicros) {
4520
+ const checks = [{ ok: true, label: `Allocation envelope ${envelopeDigest(envelope)}: ${formatMicros(envelope.totalMicros)} a ${envelope.window}, ${Object.keys(envelope.floors).length} entries, delegation ${envelope.delegation?.enabled ? `ON (writers ${envelope.delegation.writers.join(", ")}, step ${envelope.delegation.maxStepPct}%, interval ${envelope.delegation.minIntervalHours}h)` : "OFF"}` }];
4521
+ const low = Object.entries(envelope.floors).filter(([, floor]) => floor > 0 && Number.isSafeInteger(maxCostMicros) && floor < maxCostMicros);
4522
+ if (low.length > 0) {
4523
+ checks.push({
4524
+ ok: false,
4525
+ warn: true,
4526
+ label: `envelope floor(s) ${low.map(([id, floor]) => `${id} (${formatMicros(floor)})`).join(", ")} are below the per-job cost cap (${formatMicros(maxCostMicros)}): each governed job reserves its whole cap against its share, so a floor that small admits no job of its own`,
4527
+ fix: "raise the floor to at least the per-job cap (PI_MAX_COST_USD), lower the cap, or set the floor to 0 if the project needs no guaranteed share",
4528
+ });
4529
+ }
4530
+ const absent = (Array.isArray(projects) ? projects : []).map((p) => p?.id).filter((id) => typeof id === "string" && id !== OTHER && envelope.floors[id] === undefined);
4531
+ if (absent.length > 0) {
4532
+ checks.push({
4533
+ ok: false,
4534
+ warn: true,
4535
+ label: `project(s) ${absent.join(", ")} are in projects.json and not in the envelope, so their jobs count in ${OTHER}'s share of the split`,
4536
+ fix: "add each to the envelope's floorsUsd (0 is a floor) if it should have a share of its own; leave it out if counting it with the work in no project is what you meant",
4537
+ });
4538
+ }
4539
+ return checks;
4540
+ }
4541
+
4542
+ /**
4543
+ * Issue #504 part B: do this host's envelope and its peers' agree? `mine` is this host's `fpEnvelope` as doctor computes
4544
+ * it from the service's envelope file (`envelopeDigest`, or `none`), `peers` the registry rows of every OTHER host.
4545
+ * WARNINGS only, the fleet block's rule, but the consequence is named: a host whose digest is not the applied split's
4546
+ * refuses every governed job as `envelope-mismatch`.
4547
+ *
4548
+ * - A peer whose `fpEnvelope` differs: the hosts judge one shared split against two envelopes, so one side refuses.
4549
+ * - A peer with no `fpEnvelope`: a worker from before the envelope, which enforces no split at all. Said only when an
4550
+ * envelope is in use somewhere, so a fleet without one hears nothing new on upgrade.
4551
+ * Hosts are named, never a value: the registry carries a digest.
4552
+ */
4553
+ export function fleetEnvelopeChecks(mine, peers) {
4554
+ const checks = [];
4555
+ const opinions = peers.filter((h) => typeof h.fpEnvelope === "string" && h.fpEnvelope !== "");
4556
+ const differing = opinions.filter((h) => h.fpEnvelope !== mine);
4557
+ if (differing.length > 0) {
4558
+ checks.push({
4559
+ ok: false,
4560
+ warn: true,
4561
+ label: `Hosts disagree about the allocation envelope: ${differing.map((h) => h.name).join(", ")} ${differing.length === 1 ? "has" : "have"} ${differing.every((h) => h.fpEnvelope === NO_ENVELOPE_FINGERPRINT) ? "no envelope" : "an envelope with other numbers"}, and this host ${mine === NO_ENVELOPE_FINGERPRINT ? "has none" : `has ${mine}`}, so the hosts whose envelope is not the applied split's refuse every governed job as envelope-mismatch`,
4562
+ fix: "copy the same envelope.json to every host (PI_ENVELOPE_FILE), or unset it on every host and DEL alloc:plan alloc:envelope:expected; to make a hand edit the fleet's, copy it to every host and SET alloc:envelope:expected to its digest (doctor prints it)",
4563
+ });
4564
+ }
4565
+ const silent = peers.filter((h) => typeof h.fpEnvelope !== "string" || h.fpEnvelope === "");
4566
+ const inUse = mine !== NO_ENVELOPE_FINGERPRINT || opinions.some((h) => h.fpEnvelope !== NO_ENVELOPE_FINGERPRINT);
4567
+ if (silent.length > 0 && inUse) {
4568
+ checks.push({
4569
+ ok: false,
4570
+ warn: true,
4571
+ label: `${silent.map((h) => h.name).join(", ")} ${silent.length === 1 ? "publishes" : "publish"} no envelope digest, so ${silent.length === 1 ? "it enforces" : "they enforce"} no allocation split while the rest of the fleet does`,
4572
+ fix: "upgrade every worker on this Valkey to the same release: a worker from before the envelope reserves against the operator's caps alone",
4573
+ });
4574
+ }
4575
+ return checks;
4576
+ }
4577
+
4578
+ /**
4579
+ * The applied split against every host (issue #504 part B). `applied` is `{ digest }`, the
4580
+ * envelope digest `alloc:plan` was made for; `mine` this host's (`none` without an envelope); `peers` the registry rows.
4581
+ * One line of facts naming the hosts that match it, and a FAILURE for this host and for each peer that does not: such a
4582
+ * host refuses every governed job as `envelope-mismatch` (a host with no envelope in a governed fleet included).
4583
+ */
4584
+ export function appliedSplitChecks(applied, mine, myName, peers) {
4585
+ const TURN_OFF = "remove PI_ENVELOPE_FILE from every host, then `valkey-cli DEL alloc:plan alloc:envelope:expected`";
4586
+ // Named as the peer lines name theirs (issue #507): on a fleet each host's doctor says "this host", and the line read
4587
+ // beside another host's output did not say which host refuses.
4588
+ const me = typeof myName === "string" && myName !== "" ? `this host (${myName})` : "this host";
4589
+ const noEnvelopeHere = { ok: false, label: `${me} has no envelope while the fleet has an applied budget split (alloc:plan), so it refuses every job as envelope-mismatch`, fix: `install the fleet's envelope here (PI_ENVELOPE_FILE), or turn delegation off for the whole fleet: ${TURN_OFF}` };
4590
+ if (applied.undecodable) {
4591
+ // The key exists and is no split this build can read: a host with an envelope replaces it with the neutral split at
4592
+ // its next job, while a host without one counts it as governed and refuses.
4593
+ const checks = [{ ok: false, warn: true, label: "alloc:plan exists but is not a budget split this build can read; a host with an envelope replaces it with the neutral split at its next job", fix: `if delegation is meant to be off: ${TURN_OFF}` }];
4594
+ if (mine === NO_ENVELOPE_FINGERPRINT) checks.push(noEnvelopeHere);
4595
+ return checks;
4596
+ }
4597
+ const d = applied.digest;
4598
+ const matching = [...(mine === d ? [myName] : []), ...peers.filter((h) => h.fpEnvelope === d).map((h) => h.name)].sort();
4599
+ // Green only while some host matches: a split no host carries the envelope of refuses every governed job everywhere.
4600
+ const checks = [
4601
+ matching.length > 0
4602
+ ? { ok: true, label: `Applied budget split (alloc:plan) was made for envelope ${d}: ${matching.join(", ")} ${matching.length === 1 ? "matches" : "match"} it` }
4603
+ : { ok: false, warn: true, label: `Applied budget split (alloc:plan) was made for envelope ${d}, and no host matches it`, fix: `copy that envelope to the hosts, or make another one the fleet's (copy it to every host, then \`valkey-cli SET alloc:envelope:expected <its digest>\`), or turn delegation off: ${TURN_OFF}` },
4604
+ ];
4605
+ if (mine !== d) {
4606
+ checks.push(
4607
+ mine === NO_ENVELOPE_FINGERPRINT
4608
+ ? noEnvelopeHere
4609
+ : { ok: false, label: `${me} carries envelope ${mine}, not the one the applied budget split was made for (${d}), so it refuses every governed job as envelope-mismatch`, fix: `copy the fleet's envelope here; or, to make this one the fleet's, copy it to every host and run \`valkey-cli SET alloc:envelope:expected ${mine}\`` },
4610
+ );
4611
+ }
4612
+ const off = peers.filter((h) => typeof h.fpEnvelope === "string" && h.fpEnvelope !== "" && h.fpEnvelope !== d);
4613
+ if (off.length > 0) {
4614
+ checks.push({
4615
+ ok: false,
4616
+ label: `${off.map((h) => h.name).join(", ")} ${off.length === 1 ? "carries" : "carry"} ${off.every((h) => h.fpEnvelope === NO_ENVELOPE_FINGERPRINT) ? "no envelope" : "another envelope"}, not the one the applied budget split was made for (${d}), so ${off.length === 1 ? "it refuses" : "they refuse"} every governed job as envelope-mismatch`,
4617
+ fix: `copy the fleet's envelope to those hosts (PI_ENVELOPE_FILE), or turn delegation off for the whole fleet: ${TURN_OFF}`,
4618
+ });
4619
+ }
4620
+ return checks;
4621
+ }
4622
+
4623
+ /**
4624
+ * The applied split from Valkey (`alloc:plan`): `{ digest }`, `{ undecodable: true }` when the key exists and holds no
4625
+ * readable split, or null with no key or no answer.
4626
+ */
4627
+ export async function defaultReadAppliedSplit(url) {
4628
+ try {
4629
+ const { makeRedisClient } = await import("./connection.mjs");
4630
+ const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
4631
+ client.on("error", () => {});
4632
+ try {
4633
+ await client.connect();
4634
+ const text = await client.get(ALLOC_PLAN_KEY);
4635
+ if (text === null || text === undefined) return null;
4636
+ let digest = null;
4637
+ try {
4638
+ digest = JSON.parse(text)?.envelopeDigest;
4639
+ } catch {}
4640
+ return typeof digest === "string" && /^[0-9a-f]{16}$/.test(digest) ? { digest } : { undecodable: true };
4641
+ } finally {
4642
+ client.disconnect();
4643
+ }
4644
+ } catch {
4645
+ return null;
4646
+ }
4647
+ }
4648
+
4649
+ /**
4650
+ * A FAILURE per scoped-limits file that holds a `project:<id>` row whose id is not a project in `projects` (issue #499
4651
+ * part B): the worker refuses to start on it, and a live reload that would create one is kept out. Rows are named by
4652
+ * index and their scope (`project:<id>`, an operator id, never a path or a name).
4653
+ */
4654
+ export function projectRowChecks(limits, projects, path) {
4655
+ const dangling = danglingProjectRows(limits, projects);
4656
+ if (dangling.length === 0) return [];
4657
+ return [
4658
+ {
4659
+ ok: false,
4660
+ label: `scoped limit(s) ${dangling.map((d) => `#${d.index} (project:${d.id})`).join(", ")} in ${path} name a project that is not in the projects file -- the worker refuses to start`,
4661
+ fix: "add the project to the file PI_PROJECTS_FILE names (set the key if it is unset), or remove the row",
4662
+ },
4663
+ ];
4664
+ }
4665
+
4666
+ /**
4667
+ * The pause-windows facts (issue #498): the parsed windows when PI_PAUSE_WINDOWS_FILE is set, else none. A file that
4668
+ * does not load is the BOOT_FILES check's line, so here it only silences the advisories that read the windows.
4669
+ */
4670
+ function readPauseWindowFacts(env, fileExists) {
4671
+ const path = env.PI_PAUSE_WINDOWS_FILE;
4672
+ if (typeof path !== "string" || path.trim() === "") return { windows: [], parseError: null };
4673
+ try {
4674
+ if (!fileExists(path) || !statSync(path).isFile()) return { windows: [], parseError: "unreadable" };
4675
+ return { windows: loadPauseWindows({ pauseWindowsFile: path }, { readFileSync, existsSync: fileExists }), parseError: null };
4676
+ } catch (e) {
4677
+ return { windows: [], parseError: e?.message ?? String(e) };
4678
+ }
4679
+ }
4680
+
4681
+ /** A written scope's form (`parseScopeString`), or null for one it refuses: a fact reader classifies, never throws. */
4682
+ function scopeFormOf(scope) {
4683
+ try {
4684
+ return parseScopeString(scope).type;
4685
+ } catch {
4686
+ return null;
4687
+ }
4688
+ }
4689
+
4690
+ /**
4691
+ * The dollar settings (issue #501), read from env with the worker's own parser (`money.mjs`), so doctor and
4692
+ * the boot refusal cannot disagree. Silent when nothing is set. Each setting that does not parse, a window
4693
+ * without the per-job cap, and every trigger
4694
+ * whose `run.maxCostUsd` is above `PI_MAX_COST_USD`: a FAILURE when the worker loads the triggers file
4695
+ * (`workerReadsTriggers`, it refuses to start on it), a warning otherwise (the job still runs under the
4696
+ * smaller cap, so the higher one is a value that reads as allowed and is not).
4697
+ */
4698
+ /**
4699
+ * The merged-values half of the dollar invariant (issue #501, PR #542's review): `overlay` over `env`, the worker's
4700
+ * own merge, checked by the worker's own rule. Returns the reason, or null when it holds, or when env ALONE already
4701
+ * breaks it (that is `dollarChecks`' line, and the worker refuses to start on it).
4702
+ */
4703
+ export function overlayDollarProblem(overlay, env) {
4704
+ const fromEnv = {};
4705
+ for (const key of DOLLAR_SETTING_KEYS) {
4706
+ const raw = env?.[DOLLAR_ENV_NAMES[key]];
4707
+ if (raw !== undefined && raw !== "") fromEnv[key] = raw;
4708
+ }
4709
+ if (checkDollarInvariant(fromEnv) !== null) return null;
4710
+ const merged = { ...fromEnv };
4711
+ for (const key of DOLLAR_SETTING_KEYS) if (overlay?.[key] !== undefined && overlay[key] !== null) merged[key] = overlay[key];
4712
+ return checkDollarInvariant(merged)?.invalid ?? null;
4713
+ }
4714
+
4715
+ /**
4716
+ * The scoped-limits dollar rows (issues #501 part 5, #502 part 6) against the deployment's per-job cap, WARNINGS only
4717
+ * (PR #549's review). With no per-job cap, a dollar row refuses every job it applies to as `config-refused` unless the
4718
+ * job's trigger sets `run.maxCostUsd`. With one, a row window BELOW it refuses every job it applies to, every time
4719
+ * (a job reserves its whole cap), until one of the two changes. Rows are named by index and kind, and only the caps
4720
+ * are compared, never a scope string (a folder scope is a host path). `maxCostUsd` is the merged value or undefined;
4721
+ * one that does not parse is `dollarChecks`' line, so this says nothing more.
4722
+ */
4723
+ export function scopedDollarRowChecks(limits, maxCostUsd, path) {
4724
+ const checks = [];
4725
+ const label = (r) => `#${r.index} (${r.kind === "model" ? "a model row" : r.kind === "project" ? "a project row" : "a repo or folder row"})`;
4726
+ const missing = dollarRowsWithoutCap(limits, maxCostUsd);
4727
+ if (missing.length > 0) {
4728
+ checks.push({
4729
+ ok: false,
4730
+ warn: true,
4731
+ label: `scoped limit(s) ${missing.map(label).join(", ")} in ${path} set a dollar window, but the deployment has no per-job cost cap (maxCostUsd) -- every job such a row applies to is refused as config-refused unless its trigger sets run.maxCostUsd`,
4732
+ fix: "set PI_MAX_COST_USD in .env (or maxCostUsd in the overlay), or give each trigger that reaches the row a run.maxCostUsd",
4733
+ });
4734
+ }
4735
+ let below = [];
4736
+ try {
4737
+ below = dollarRowsBelowJobCap(limits, maxCostUsd);
4738
+ } catch {
4739
+ return checks;
4740
+ }
4741
+ if (below.length > 0) {
4742
+ checks.push({
4743
+ ok: false,
4744
+ warn: true,
4745
+ label: `scoped limit window(s) ${below.map((r) => `${label(r)} ${r.window}`).join(", ")} in ${path} are below the per-job cost cap (maxCostUsd) -- a job reserves its whole cap, so every job that reaches such a window is refused dollar-cap, every time`,
4746
+ fix: "raise the window to at least maxCostUsd, or lower maxCostUsd (a trigger's smaller run.maxCostUsd also fits)",
4747
+ });
4748
+ }
4749
+ return checks;
4750
+ }
4751
+
4752
+ export function dollarChecks(env, costCaps, workerReadsTriggers) {
4753
+ const envName = DOLLAR_ENV_NAMES;
4754
+ const checks = [];
4755
+ const values = {};
4756
+ for (const key of DOLLAR_SETTING_KEYS) {
4757
+ const raw = env[envName[key]];
4758
+ if (raw === undefined || raw === "") continue;
4759
+ try {
4760
+ optionalUsdMicros(raw, envName[key]);
4761
+ values[key] = raw;
4762
+ } catch (error) {
4763
+ checks.push({ ok: false, label: `${error.message} -- the worker refuses to start`, fix: `set ${envName[key]} to a plain dollar amount such as 2.50, or remove it` });
4764
+ }
4765
+ }
4766
+ const broken = checkDollarInvariant(values);
4767
+ if (broken) {
4768
+ const window = DOLLAR_SETTING_KEYS.find((key) => key !== "maxCostUsd" && values[key] !== undefined);
4769
+ checks.push({ ok: false, label: `${envName[window]} is set without PI_MAX_COST_USD -- the worker refuses to start`, fix: "a dollar window reserves each job's per-job cap, so set PI_MAX_COST_USD too" });
4770
+ }
4771
+ if (values.maxCostUsd !== undefined) {
4772
+ const deployment = parseUsdMicros(values.maxCostUsd, "PI_MAX_COST_USD");
4773
+ for (const cap of costCaps.filter((c) => parseUsdMicros(c.maxCostUsd, "run.maxCostUsd") > deployment)) {
4774
+ checks.push({
4775
+ ok: false,
4776
+ ...(workerReadsTriggers ? {} : { warn: true }),
4777
+ label: workerReadsTriggers
4778
+ ? `${cap.label}: run.maxCostUsd is above PI_MAX_COST_USD -- the worker refuses to start (a trigger can only narrow the per-job cost cap)`
4779
+ : `${cap.label}: run.maxCostUsd is above PI_MAX_COST_USD -- its jobs run under PI_MAX_COST_USD anyway (a trigger can only narrow the per-job cost cap)`,
4780
+ fix: "lower that trigger's run.maxCostUsd to PI_MAX_COST_USD or below, or remove it",
4781
+ });
4782
+ }
4783
+ }
4784
+ return checks;
4785
+ }
4786
+
4787
+ /**
4788
+ * Issue #501 part 6: do this host's dollar caps match its peers'? `mine` is this host's `fpUsd` as doctor computes it
4789
+ * from the service's settings (`deploymentSettingsOf`, `usdFingerprint`), `peers` the registry rows of every OTHER
4790
+ * host. WARNINGS only, never a failure, the fleet block's rule: this command runs on one machine and must not refuse
4791
+ * a deployment for a condition that machine cannot fix.
4792
+ *
4793
+ * - A peer whose `fpUsd` differs: the dollar counters are shared, so the host with the larger cap admits a job the
4794
+ * other would refuse, and each host's view of "full" is its own.
4795
+ * - A peer with no `fpUsd` (or an empty one): it runs a worker from before hosts published one, so whether it holds
4796
+ * the same caps is unknown. Said only when dollar caps are in use somewhere: this host's fingerprint, or any
4797
+ * peer's, is not the empty one, or else a dollar counter (`budget:usd:*`) exists on this Valkey
4798
+ * (`dollarKeysExist`, asked only then). The counters are the one trace a capped host that publishes nothing
4799
+ * leaves (PR #551's review: an old capped host beside a new uncapped one was silent). A fleet that never used a
4800
+ * dollar setting hears nothing new on upgrade.
4801
+ *
4802
+ * Hosts are named, never their caps: the registry carries a digest, so "different" is all a reader can know.
4803
+ */
4804
+ export async function fleetDollarChecks(mine, peers, { dollarKeysExist = async () => false } = {}) {
4805
+ const checks = [];
4806
+ const opinions = peers.filter((h) => typeof h.fpUsd === "string" && h.fpUsd !== "");
4807
+ const differing = opinions.filter((h) => h.fpUsd !== mine);
4808
+ if (differing.length > 0) {
4809
+ checks.push({
4810
+ ok: false,
4811
+ warn: true,
4812
+ label: `Hosts disagree about the dollar caps: ${differing.map((h) => h.name).join(", ")} ${differing.length === 1 ? "judges" : "judge"} the shared dollar counters against a different per-job cap, dollar windows, scoped-limits dollar rows or PI_ALLOWED_MODELS than this host's settings`,
4813
+ fix: "set PI_MAX_COST_USD, PI_DAILY_COST_USD, PI_WEEKLY_COST_USD, PI_MONTHLY_COST_USD, the overlay's dollar keys, the scoped-limits file's dollar rows and PI_ALLOWED_MODELS (it picks the model rows a job without its own list reserves in) alike on every host: the counters are shared, so a host with a larger cap admits a job another would refuse. An env change needs a restart; an overlay or scoped-limits edit shows within one beat",
4814
+ });
4815
+ }
4816
+ const silent = peers.filter((h) => typeof h.fpUsd !== "string" || h.fpUsd === "");
4817
+ if (silent.length === 0) return checks;
4818
+ let inUse = mine !== EMPTY_USD_FINGERPRINT || opinions.some((h) => h.fpUsd !== EMPTY_USD_FINGERPRINT);
4819
+ let byCounters = false;
4820
+ if (!inUse) {
4821
+ byCounters = (await Promise.resolve().then(dollarKeysExist).catch(() => false)) === true;
4822
+ inUse = byCounters;
4823
+ }
4824
+ if (inUse) {
4825
+ checks.push({
4826
+ ok: false,
4827
+ warn: true,
4828
+ label: `${silent.map((h) => h.name).join(", ")} ${silent.length === 1 ? "publishes no fingerprint of its" : "publish no fingerprint of their"} dollar caps, so whether ${silent.length === 1 ? "it holds" : "they hold"} the same caps as this host is unknown${byCounters ? " (no host that publishes one sets a dollar cap, but dollar counters exist on this Valkey, so some host reserved dollars recently)" : ""}`,
4829
+ fix: "upgrade every worker on this Valkey to the same release: a worker from before hosts compared their dollar caps may judge the shared dollar counters against other caps, or reserve no dollars at all",
4830
+ });
4831
+ }
4832
+ return checks;
4833
+ }
4834
+
4835
+ /**
4836
+ * Issue #499 part C: do this host's projects match its peers'? `mine` is this host's `fpProjects` as doctor computes it
4837
+ * from the service's projects file (`projectsFingerprint`), `peers` the registry rows of every OTHER host. WARNINGS
4838
+ * only, the fleet block's rule.
4839
+ *
4840
+ * - A peer whose `fpProjects` differs: each host resolves a job's project from its own copy, so one repo can be in
4841
+ * two projects, or in a project on one host and in none on another, depending on which host ran the job. Its runs
4842
+ * are then recorded under two ids and counted against two project rows (or none), while the counters are shared.
4843
+ * - A peer with no `fpProjects` (or an empty one): it runs a worker from before hosts published one. Said only when
4844
+ * projects are in use somewhere (this host's fingerprint, or any peer's, is not the one of no projects), so a
4845
+ * fleet that never used a projects file hears nothing new on upgrade.
4846
+ *
4847
+ * Hosts are named, never a project's members or name: the registry carries a digest, so "different" is all a reader
4848
+ * can know.
4849
+ */
4850
+ export function fleetProjectsChecks(mine, peers) {
4851
+ const checks = [];
4852
+ const opinions = peers.filter((h) => typeof h.fpProjects === "string" && h.fpProjects !== "");
4853
+ const differing = opinions.filter((h) => h.fpProjects !== mine);
4854
+ if (differing.length > 0) {
4855
+ checks.push({
4856
+ ok: false,
4857
+ warn: true,
4858
+ label: `Hosts disagree about the projects: ${differing.map((h) => h.name).join(", ")} ${differing.length === 1 ? "reads" : "read"} a projects.json with other ids or members than this host's, so a run is recorded under, and counted against, a different project depending on which host ran it`,
4859
+ fix: "copy the same projects.json to every host (PI_PROJECTS_FILE): the project rows' counters are shared, and each host decides a job's project from its own copy. An edit shows within one beat",
4860
+ });
4861
+ }
4862
+ const silent = peers.filter((h) => typeof h.fpProjects !== "string" || h.fpProjects === "");
4863
+ const inUse = mine !== EMPTY_PROJECTS_FINGERPRINT || opinions.some((h) => h.fpProjects !== EMPTY_PROJECTS_FINGERPRINT);
4864
+ if (silent.length > 0 && inUse) {
4865
+ checks.push({
4866
+ ok: false,
4867
+ warn: true,
4868
+ label: `${silent.map((h) => h.name).join(", ")} ${silent.length === 1 ? "publishes no fingerprint of its" : "publish no fingerprint of their"} projects, so whether ${silent.length === 1 ? "it reads" : "they read"} the same projects.json as this host is unknown`,
4869
+ fix: "upgrade every worker on this Valkey to the same release: a worker from before hosts compared their projects records no project and counts no project row",
4870
+ });
4871
+ }
4872
+ return checks;
4873
+ }
4874
+
4875
+ /**
4876
+ * The runner's own input overhead (`BOUND_OVERHEAD_TOKENS`, image/runner/src/usage-meter.mjs), which its bound adds to
4877
+ * every call. Copied, because the worker package does not ship the runner; `doctor.test.mjs` holds the two equal.
4878
+ */
4879
+ export const FIRST_CALL_OVERHEAD_TOKENS = 8192;
4880
+ /**
4881
+ * The least a job's FIRST call carries before its task, in bytes: pi 0.99.1's default system prompt (1,753 bytes) and
4882
+ * the schemas of its four default tools (2,712 bytes) serialise to 4,465 bytes, and PR #542's lab measured 10,667 for
4883
+ * a short task. 4,096 is below both, so the bound below stays a lower bound for a job that adds nothing.
4884
+ */
4885
+ export const FIRST_CALL_MIN_CONTEXT_BYTES = 4096;
4886
+
4887
+ /**
4888
+ * A LOWER BOUND, in integer micro-dollars, on what the runner's cost guard (`callCostBound`) reserves for a job's
4889
+ * first call on `model`, or null when this cannot say. The guard refuses a call when its bound would pass the cap, so
4890
+ * a per-job cap below this number can never admit a call on the model: with the main model, the job makes no call at
4891
+ * all (PR #542's lab: the default model's first call was bounded at $1.03 under a $1 cap).
4892
+ *
4893
+ * The guard's bound is (request bytes + 8,192) x the dearest input rate + the output limit x the dearest output rate,
4894
+ * times a service-tier multiplier, over the model's table, its tiers and its fallbacks. This takes the parts of that
4895
+ * which cannot be smaller: the model's own table (not its tiers or fallbacks, which only raise the bound), the output
4896
+ * limit `model.maxTokens` (pi sends none of its own, so the guard uses the model's), an input of the overhead plus
4897
+ * `FIRST_CALL_MIN_CONTEXT_BYTES`, and the runner's own service-tier multiplier (`serviceTierMultiplier`). The input
4898
+ * rate is the guard's: the highest of input, cache read and cache write (not the 1h write, which needs long
4899
+ * retention). Null for a table or limit it cannot read.
4900
+ */
4901
+ export function firstCallFloorMicros(model) {
4902
+ const cost = model?.cost;
4903
+ const usable = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0;
4904
+ if (!cost || !["input", "output", "cacheRead", "cacheWrite"].every((field) => usable(cost[field]))) return null;
4905
+ const maxTokens = model.maxTokens;
4906
+ if (!(typeof maxTokens === "number" && Number.isFinite(maxTokens) && maxTokens > 0)) return null;
4907
+ const inputRate = Math.max(cost.input, cost.cacheRead, cost.cacheWrite);
4908
+ return Math.ceil(((FIRST_CALL_OVERHEAD_TOKENS + FIRST_CALL_MIN_CONTEXT_BYTES) * inputRate + maxTokens * cost.output) * serviceTierMultiplier(model));
4909
+ }
4910
+
4911
+ /**
4912
+ * The runner's service-tier multiplier (`callCostBound` step 7, image/runner/src/usage-meter.mjs), copied like the
4913
+ * overhead above: pi multiplies a settled cost by the tier the RESPONSE reports on these two apis, which can be the
4914
+ * account's default rather than the request's, so the runner always bounds at the dearest tier. Leaving it out
4915
+ * (PR #551's review) let doctor stay silent on a cap the runner refuses every call under: openai/gpt-5.5 under $5.
4916
+ * `fleet-dollars.test.mjs` derives the runner's multiplier from its own bound for every priced api and holds this
4917
+ * equal to it.
4918
+ */
4919
+ export function serviceTierMultiplier(model) {
4920
+ if (model?.api !== "openai-responses" && model?.api !== "openai-codex-responses") return 1;
4921
+ return model.id === "gpt-5.5" ? 2.5 : 2;
4922
+ }
4923
+
4924
+ /**
4925
+ * The model object the floor above is computed from: the builtin catalog's (`builtinModel`, model-catalog.mjs), or a
4926
+ * CUSTOM model the overlay `models.json` defines (its own `cost` and `maxTokens`). Null, so the model is not judged,
4927
+ * when the overlay redefines or overrides a builtin model: pi composes the two, and this does not copy that rule.
4928
+ */
4929
+ export function boundModelOf({ provider, id }, { builtinModel, overlay = null }) {
4930
+ const providers = overlay?.providers;
4931
+ const entry = providers !== null && typeof providers === "object" && !Array.isArray(providers) && Object.hasOwn(providers, provider) ? providers[provider] : null;
4932
+ const builtin = builtinModel(provider, id);
4933
+ if (entry !== null && typeof entry === "object") {
4934
+ const overrides = entry.modelOverrides;
4935
+ if (overrides !== null && typeof overrides === "object" && Object.hasOwn(overrides, id)) return null;
4936
+ const definition = Array.isArray(entry.models) ? entry.models.find((m) => m !== null && typeof m === "object" && m.id === id) : undefined;
4937
+ if (definition !== undefined) return builtin ? null : definition;
4938
+ }
4939
+ return builtin;
4940
+ }
4941
+
4942
+ /**
4943
+ * Who runs on which models under which cap (issues #501 and #502), as the worker resolves a job: the deployment's own
4944
+ * default, then every trigger. `deployment` is `{ provider, model, maxCostUsd }` (overlay over env,
4945
+ * `deploymentSettingsOf`); `runs` the trigger facts; `envList` the parsed `PI_ALLOWED_MODELS`, or null. Each subject
4946
+ * is `{ label, main, list, cap, secretNames, named }`: `main` and `list` are `{ provider, id }`, `list` is the
4947
+ * trigger's `run.models` or else the env list, `cap` the effective per-job cap in micro-dollars (the smaller of the
4948
+ * trigger's and the deployment's, `effectiveCostCapMicros`) or null, and `named` says the trigger names a provider, a
4949
+ * model or a list of its own.
4950
+ */
4951
+ export function modelSubjects({ runs = [], deployment, envList = null }) {
4952
+ const listOf = (list) => (Array.isArray(list) ? list.map(splitModelEntry).filter((ref) => ref !== null).map((ref) => ({ provider: ref.provider, id: ref.model })) : []);
4953
+ const capOf = (triggerValue) => {
4954
+ try {
4955
+ return effectiveCostCapMicros(triggerValue, deployment.maxCostUsd);
4956
+ } catch {
4957
+ return null; // a deployment cap that does not parse is `dollarChecks`' line
4958
+ }
4959
+ };
4960
+ const subjects = [{ label: "the deployment default", main: { provider: deployment.provider, id: deployment.model }, list: listOf(envList), cap: capOf(undefined), secretNames: [], named: false, deployment: true }];
4961
+ for (const run of runs) {
4962
+ subjects.push({
4963
+ label: run.label,
4964
+ main: { provider: run.provider ?? deployment.provider, id: run.model ?? deployment.model },
4965
+ list: listOf(run.models ?? envList),
4966
+ cap: capOf(run.maxCostUsd),
4967
+ secretNames: run.secretNames ?? [],
4968
+ named: run.provider !== undefined || run.model !== undefined || run.models !== undefined,
4969
+ });
4970
+ }
4971
+ return subjects;
4972
+ }
4973
+
4974
+ /**
4975
+ * The `provider/id` of every model a job under a per-job dollar cap may use (the main model and the list, as
4976
+ * `modelSubjects` resolves them), for the overlay's output-cap line to leave to `costCapFitChecks` (issue #507). Empty
4977
+ * when the settings cannot be read.
4978
+ */
4979
+ export function cappedModelNames(env, { runs, deployment }) {
4980
+ let envList = null;
4981
+ try {
4982
+ envList = allowedModelsFrom(env);
4983
+ } catch {
4984
+ // A malformed PI_ALLOWED_MODELS is its own line.
4985
+ }
4986
+ const names = new Set();
4987
+ for (const s of modelSubjects({ runs, deployment, envList })) {
4988
+ if (s.cap === null || s.cap === undefined) continue;
4989
+ for (const ref of [s.main, ...s.list]) names.add(`${ref.provider}/${ref.id}`);
4990
+ }
4991
+ return names;
4992
+ }
4993
+
4994
+ /**
4995
+ * Issue #501's open question, answered with a warning: a per-job cap below one full-output call of the main model,
4996
+ * or of a listed model, is a cap the runner can never admit that call under (`firstCallFloorMicros`). One line,
4997
+ * grouped by model and cap so a deployment default every trigger inherits is named once.
4998
+ *
4999
+ * Issue #507: `unboundable(ref)` says the runner cannot bound the model's output at all (`outputUnboundable`,
5000
+ * output-cap.mjs: openai-completions to a server off the trusted hosts without `compat.maxTokensField: "max_tokens"`),
5001
+ * so every call to it is refused under any cap. Such a model gets a line of its own, grouped by model, and no floor.
5002
+ */
5003
+ export function costCapFitChecks(subjects, modelOf, { unboundable = () => false } = {}) {
5004
+ const groups = new Map();
5005
+ const unbounded = new Map();
5006
+ for (const s of subjects) {
5007
+ if (s.cap === null || s.cap === undefined) continue;
5008
+ const seen = new Set();
5009
+ for (const ref of [s.main, ...s.list]) {
5010
+ const name = `${ref.provider}/${ref.id}`;
5011
+ if (seen.has(name)) continue;
5012
+ seen.add(name);
5013
+ if (unboundable(ref)) {
5014
+ if (!unbounded.has(name)) unbounded.set(name, { main: false, labels: [] });
5015
+ const u = unbounded.get(name);
5016
+ u.main ||= ref === s.main;
5017
+ u.labels.push(s.label);
5018
+ continue;
5019
+ }
5020
+ const floor = firstCallFloorMicros(modelOf(ref));
5021
+ if (floor === null || s.cap >= floor) continue;
5022
+ const main = ref === s.main;
5023
+ const key = `${name}\u0000${s.cap}\u0000${main}`;
5024
+ if (!groups.has(key)) groups.set(key, { name, floor, cap: s.cap, main, labels: [] });
5025
+ groups.get(key).labels.push(s.label);
5026
+ }
5027
+ }
5028
+ const out = [];
5029
+ if (unbounded.size > 0) {
5030
+ const items = [...unbounded].map(([name, u]) => `${printable(name)}${u.main ? " (the main model, so such a job makes no call at all)" : ""} (${u.labels.join(", ")})`);
5031
+ out.push({
5032
+ ok: false,
5033
+ warn: true,
5034
+ label: `A job under a per-job cost cap may use a model whose output cap travels as max_completion_tokens to a server that may ignore it (one outside pi's own hosted providers), so the runner counts every call to it unboundable and refuses it under the cap: ${items.join("; ")}`,
5035
+ fix: "set \"compat\": { \"maxTokensField\": \"max_tokens\" } on the model or its provider in models.json, or set its cost to zeros if it is free: docs/egress.md, \"Local model servers\"",
5036
+ });
5037
+ }
5038
+ if (groups.size === 0) return out;
5039
+ const items = [...groups.values()].map((g) => `${printable(g.name)}${g.main ? " (the main model, so such a job makes no call at all)" : ""} needs at least $${formatMicros(g.floor)} a call under a $${formatMicros(g.cap)} cap (${g.labels.join(", ")})`);
5040
+ return [
5041
+ ...out,
5042
+ {
5043
+ ok: false,
5044
+ warn: true,
5045
+ label: `The per-job cost cap is below one full-output call of a model a job may use, so the runner refuses every call to that model before it is sent: ${items.join("; ")}`,
5046
+ fix: "raise PI_MAX_COST_USD (or the trigger's run.maxCostUsd) above the amount named, or use a model with a smaller output limit. The amount is a lower bound: the runner's own bound adds the whole request, so a cap just above it can still refuse a call",
5047
+ },
5048
+ ];
5049
+ }
5050
+
5051
+ /**
5052
+ * Issue #502 part 2: a trigger whose model, or a model on its list, the worker's free gate would refuse, asked of the
5053
+ * worker's own gate (`checkModelsKnown`, model-catalog.mjs) with the overlay `models.json` it reads. Every trigger kind,
5054
+ * cron included, and only triggers that name a provider, a model or a list of their own: a trigger that names none
5055
+ * runs on the deployment's settings, which get one line of their own (the deployment subject). An overlay that cannot be read just now
5056
+ * (`unavailable`) says nothing: the worker retries such a job.
5057
+ */
5058
+ export function unknownModelChecks(subjects, checkModelsKnown, readOverlay) {
5059
+ const unknown = [];
5060
+ const fallbacks = [];
5061
+ const checks = [];
5062
+ // The deployment's own default model and PI_ALLOWED_MODELS (PR #551's review): a typo there refuses every job
5063
+ // whose trigger names neither, so it gets a line of its own, worded for the settings that hold it.
5064
+ for (const s of subjects.filter((x) => x.deployment === true)) {
5065
+ const verdict = checkModelsKnown([{ ...s.main, main: true }, ...s.list], { readOverlay });
5066
+ const ref = verdict?.unknown ?? verdict?.fallbackUnlisted;
5067
+ if (!ref) continue;
5068
+ // A broken overlay is the overlay lines' to name (they say every job is refused); blaming .env here would send
5069
+ // the operator to the wrong file (PR #551's review, round 2).
5070
+ if (verdict.unknown && typeof verdict.why === "string" && verdict.why.startsWith("overlay-")) continue;
5071
+ checks.push({
5072
+ ok: false,
5073
+ warn: true,
5074
+ label: verdict.unknown
5075
+ ? `The deployment's default model (PI_PROVIDER/PI_MODEL, or the panel's provider/model) or a model on PI_ALLOWED_MODELS is one this deployment does not know: ${printable(`${ref.provider}/${ref.id}`)} (${verdict.why}), so every job whose trigger names no model of its own is refused before it starts (model-unknown)`
5076
+ : `A model on PI_ALLOWED_MODELS declares server-side fallbacks the list does not name: ${printable(`${ref.provider}/${ref.id}`)}, so every job whose trigger names no list of its own is refused before it starts (model-not-allowed)`,
5077
+ fix: verdict.unknown ? "fix the id in .env (or the panel's model setting): the worker knows pi's builtin catalog at its pin and the models the overlay models.json declares" : "add that model's fallback models to PI_ALLOWED_MODELS, under the same provider, or remove it",
5078
+ });
5079
+ }
5080
+ for (const s of subjects) {
5081
+ if (!s.named) continue;
5082
+ const verdict = checkModelsKnown([{ ...s.main, main: true }, ...s.list], { readOverlay });
5083
+ if (verdict?.unknown) unknown.push(`${s.label}: ${printable(`${verdict.unknown.provider}/${verdict.unknown.id}`)} (${verdict.why})`);
5084
+ else if (verdict?.fallbackUnlisted) fallbacks.push(`${s.label}: ${printable(`${verdict.fallbackUnlisted.provider}/${verdict.fallbackUnlisted.id}`)}`);
5085
+ }
5086
+ if (unknown.length > 0) {
5087
+ checks.push({
5088
+ ok: false,
5089
+ warn: true,
5090
+ label: `${unknown.length} trigger(s) name a model this deployment does not know, so every job of them is refused before it starts (model-unknown): ${unknown.join("; ")}`,
5091
+ fix: "fix the provider or model id: the worker knows pi's builtin catalog at its pin and the models the overlay models.json declares. not-in-catalog is a typo or a model only an extension defines (declare it in the overlay models.json); an overlay reason names the file as the problem",
5092
+ });
5093
+ }
5094
+ if (fallbacks.length > 0) {
5095
+ checks.push({
5096
+ ok: false,
5097
+ warn: true,
5098
+ label: `${fallbacks.length} trigger(s) list a model whose server-side fallbacks are not on the list, so every job of them is refused before it starts (model-not-allowed): ${fallbacks.join("; ")}`,
5099
+ fix: "list that model's fallback models too, under the same provider, or remove it from the list",
5100
+ });
5101
+ }
5102
+ return checks;
5103
+ }
5104
+
5105
+ /**
5106
+ * Issue #502's open question, answered with a warning: only the main provider's key is resolved for a job; any other
5107
+ * provider on its list needs a key that arrives by `run.secrets` or `PI_FORWARD_ENV`. A listed provider with neither
5108
+ * starts the job and then fails the first call to it, after the budget is reserved. The names looked for are pi's
5109
+ * own (`candidatesOf`, the provider oracle), or the variable the overlay's `apiKey` for that provider references.
5110
+ * Said nothing about a provider it cannot judge: no names at all (Bedrock reads the AWS chain), the keyless marker
5111
+ * (#503), or an `apiKey` that is not a variable reference.
5112
+ */
5113
+ export function listedProviderCredentialChecks(subjects, { candidatesOf, forwarded = [], env = {}, overlay = null }) {
5114
+ const providers = overlay?.providers;
5115
+ const namesFor = (provider) => {
5116
+ const entry = providers !== null && typeof providers === "object" && !Array.isArray(providers) && Object.hasOwn(providers, provider) ? providers[provider] : null;
5117
+ const apiKey = entry !== null && typeof entry === "object" ? entry.apiKey : undefined;
5118
+ if (typeof apiKey === "string" && apiKey !== "") {
5119
+ if (apiKey === KEYLESS_API_KEY) return [];
5120
+ const ref = /^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/.exec(apiKey);
5121
+ return ref ? [ref[1]] : [];
5122
+ }
5123
+ return candidatesOf(provider) ?? [];
5124
+ };
5125
+ const missing = new Map();
5126
+ for (const s of subjects) {
5127
+ for (const provider of new Set(s.list.map((ref) => ref.provider))) {
5128
+ if (provider === s.main.provider) continue;
5129
+ const names = namesFor(provider);
5130
+ if (names.length === 0) continue;
5131
+ if (names.some((n) => s.secretNames.includes(n) || (forwarded.includes(n) && typeof env[n] === "string" && env[n] !== ""))) continue;
5132
+ if (!missing.has(provider)) missing.set(provider, { names, labels: [] });
5133
+ missing.get(provider).labels.push(s.label);
5134
+ }
5135
+ }
5136
+ if (missing.size === 0) return [];
5137
+ const items = [...missing.entries()].map(([provider, m]) => `${printable(provider)} (${m.labels.join(", ")}; looked for ${m.names.join(" or ")})`);
5138
+ return [
5139
+ {
5140
+ ok: false,
5141
+ warn: true,
5142
+ label: `A provider on an allowed-model list, other than the job's main one, has no credential this deployment hands its jobs, so a call to it fails inside the container, after the job started: ${items.join("; ")}`,
5143
+ fix: "only the main provider's key is resolved for a job. Bind another provider's key with run.secrets on the trigger, or name it in PI_FORWARD_ENV and set it in .env; or take the provider off the list",
5144
+ },
5145
+ ];
5146
+ }
5147
+
5148
+ /**
5149
+ * The deployment's provider, model and dollar settings as the worker resolves them for a job: the overlay over env
5150
+ * (`resolveSettings`), falling back to env when the overlay is invalid or the merged dollar values break the
5151
+ * invariant, which is the fallback `start.mjs` takes for its slot count and its `fpUsd`. Values as written; nothing
5152
+ * here throws.
5153
+ */
5154
+ export function deploymentSettingsOf(env, settingsFile, fileExists) {
5155
+ const fromEnv = { provider: env.PI_PROVIDER ?? DEFAULT_PROVIDER, model: env.PI_MODEL ?? DEFAULT_MODEL };
5156
+ for (const key of DOLLAR_SETTING_KEYS) {
5157
+ const raw = env[DOLLAR_ENV_NAMES[key]];
5158
+ fromEnv[key] = raw === undefined || raw === "" ? null : raw;
5159
+ }
5160
+ let read;
5161
+ try {
5162
+ read = fileExists(settingsFile) ? readOverlay(settingsFile) : { overlay: {} };
5163
+ } catch {
5164
+ return fromEnv;
5165
+ }
5166
+ const resolved = resolveSettings(fromEnv, read);
5167
+ if (resolved.invalid) return fromEnv;
5168
+ return Object.fromEntries(Object.keys(fromEnv).map((key) => [key, resolved[key] ?? null]));
5169
+ }
5170
+
5171
+ /** The worker's model catalog (model-catalog.mjs), imported lazily for `defaultProviderOracle`'s reason, or null. */
5172
+ async function defaultModelCatalog() {
5173
+ try {
5174
+ return await import("./model-catalog.mjs");
5175
+ } catch {
5176
+ return null;
5177
+ }
5178
+ }
5179
+
5180
+ /** pi's own model loader (pi-model-loader.mjs), imported lazily like the catalog, or null. */
5181
+ async function defaultPiModelLoader() {
5182
+ try {
5183
+ const { loadPiModelLoader } = await import("./pi-model-loader.mjs");
5184
+ return await loadPiModelLoader();
5185
+ } catch {
5186
+ return null;
5187
+ }
5188
+ }
5189
+
5190
+ /** The `provider/id` pairs a models.json text declares, read leniently: a file pi drops still names ids worth asking about. */
5191
+ function declaredOverlayModels(text) {
5192
+ // Read as pi reads text (BOM, comments and trailing commas stripped; PR #551's review: plain JSON.parse found no
5193
+ // model in a JSONC overlay), without pi's schema, so a file pi refuses still names ids worth asking about.
5194
+ let doc = parseModelsJson(text).value;
5195
+ if (doc === undefined || doc === null) {
5196
+ try {
5197
+ doc = JSON.parse(stripJsonComments(stripBom(text)));
5198
+ } catch {
5199
+ return [];
5200
+ }
5201
+ }
5202
+ const out = [];
5203
+ const providers = doc?.providers;
5204
+ if (providers === null || typeof providers !== "object" || Array.isArray(providers)) return out;
5205
+ for (const [provider, entry] of Object.entries(providers)) {
5206
+ if (entry === null || typeof entry !== "object") continue;
5207
+ for (const m of Array.isArray(entry.models) ? entry.models : []) if (typeof m?.id === "string" && m.id !== "") out.push([provider, m.id]);
5208
+ }
5209
+ return out;
5210
+ }
5211
+
5212
+ /**
5213
+ * Issue #502's open question: does pi's own loader read the overlay `models.json` as the worker's catalog does? The
5214
+ * worker admits or refuses a job by its copy of pi's rules (`parseModelsJson`, `checkModelsKnown`); pi decides what
5215
+ * runs. A disagreement is a model the worker calls known that pi lacks (a paid job that exits 2) or the reverse (a
5216
+ * working model refused), and either is a defect in the copy or a pi beside the worker that is not the image's. Asked
5217
+ * per file (loads or not) and per declared model. `pi` is `defaultPiModelLoader`'s answer, or null.
5218
+ */
5219
+ export async function overlayLoaderParityChecks(modelsPath, { pi, checkModelsKnown, readText = (p) => readFileSync(p, "utf8") }) {
5220
+ if (pi === null || pi === undefined) {
5221
+ return [{ ok: true, label: "Overlay models.json was not compared with pi's own loader: pi-coding-agent is not installed beside the worker (a checkout of this repository has it after npm ci)" }];
5222
+ }
5223
+ // Only the pinned pi speaks for the job image (PR #551's review): a global layout can put any pi beside the worker.
5224
+ if (typeof pi.pinned !== "string" || pi.version !== pi.pinned) {
5225
+ return [{ ok: true, label: `Overlay models.json was not compared with pi's own loader: pi ${printable(String(pi.version))} beside the worker is not the worker's pinned pi-ai ${typeof pi.pinned === "string" ? printable(pi.pinned) : "(pin unknown)"}` }];
5226
+ }
5227
+ let text;
5228
+ let theirs;
5229
+ try {
5230
+ text = readText(modelsPath);
5231
+ theirs = await pi.read(modelsPath);
5232
+ } catch (error) {
5233
+ return [{ ok: false, warn: true, label: `Overlay models.json could not be compared with pi ${printable(String(pi.version))}'s own loader (${printable(String(error?.code ?? error?.message ?? "error"))})`, fix: "re-run doctor; if it persists, report it with the error" }];
5234
+ }
5235
+ const parsed = parseModelsJson(text);
5236
+ const ours = parsed.error === undefined;
5237
+ const readOverlay = () => {
5238
+ if (parsed.error !== undefined) throw Object.assign(new Error(parsed.error), { piDispatchConfig: true });
5239
+ return parsed.value;
5240
+ };
5241
+ const items = [];
5242
+ if (theirs.loads !== ours) items.push(`pi ${theirs.loads ? "loads the file and the worker refuses it" : "drops the file and the worker reads it"}`);
5243
+ const declared = declaredOverlayModels(text);
5244
+ for (const [provider, id] of declared) {
5245
+ const piHas = theirs.has(provider, id) === true;
5246
+ const workerHas = checkModelsKnown([{ provider, id, main: true }], { readOverlay })?.ok === true;
5247
+ if (piHas !== workerHas) items.push(`${printable(`${provider}/${id}`)}: pi ${piHas ? "has it" : "lacks it"}, the worker calls it ${workerHas ? "known" : "unknown"}`);
5248
+ }
5249
+ if (items.length > 0) {
5250
+ return [
5251
+ {
5252
+ ok: false,
5253
+ warn: true,
5254
+ label: `pi ${printable(String(pi.version))}'s own loader and the worker's model catalog disagree about overlay models.json: ${items.join("; ")}`,
5255
+ fix: "the worker gates jobs by its own reading of this file and pi decides what runs, so a model the worker knows and pi lacks is a paid job that exits 2, and the reverse is a working model refused. Report it with the file's shape (not its keys); until then, avoid the construct named",
5256
+ },
5257
+ ];
5258
+ }
5259
+ return [{ ok: true, label: `Overlay models.json reads the same in pi ${printable(String(pi.version))}'s own loader as in the worker's model catalog (${declared.length} declared model(s))` }];
5260
+ }
5261
+
5262
+ /**
5263
+ * How a line names its trigger: cron entries by their id, id-less webhook entries by raw file position (the admin's
5264
+ * trigger:<index> identity). One function, because the flow lines and the venue lines must name a trigger alike.
5265
+ */
5266
+ function triggerLabel(t, index) {
5267
+ return t.on.type === "cron" ? `cron "${t.on.id}"` : `${t.on.type} trigger #${index}`;
5268
+ }
5269
+
5270
+ function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
5271
+ const none = { requiring: 0, waiting: 0, listing: 0, waitProfiles: [], waitAfters: [], optingOut: 0, resuming: 0, replicating: 0, instructing: 0, commands: 0, secreting: 0, onceArmed: 0, onceSpent: 0, secretProfiles: [], localSecretFolders: [], secretNames: [], folders: [], images: [], imageRoutes: [], namedBackends: [], skillsDirs: [], forges: [], repositories: [], flows: [], costCaps: [], modelRuns: [], parseError: null, path: null };
5272
+ try {
5273
+ // Unset falls back to ./triggers.json in cwd, MIRRORING the receiver's own default
5274
+ // (receiver/src/config.mjs) -- the two must read the same file, or doctor preflights a deployment
5275
+ // the receiver will not boot. An absent file still means "no triggers at all", exactly as before.
5276
+ const path = triggersPath(env, cwd);
5277
+ if (!fileExists(path)) return none;
5278
+ // The same regular-file guard, for the same reason: this read is unbounded too, and a triggers path
5279
+ // naming a FIFO hangs the command with no output and no timeout that can reach it.
5280
+ if (!statSync(path).isFile()) return { ...none, parseError: `triggers file is not a regular file: ${path}`, path };
5281
+ const text = readFileSync(path, "utf8");
5282
+ const triggers = parseTriggers(text, path);
5283
+ // The one-shot facts are counted from the RAW entries, not the parsed records, because the
5284
+ // validator collapses a disarmed entry to a sentinel that carries neither `once` nor
5285
+ // `disarmed` -- exactly so nothing can match it -- which also erases it from every parsed
5286
+ // count above. Doctor is the surface that must still SEE the spent entry: "why did nothing
5287
+ // fire" is answered by a spent row, and only the raw file still holds it. Safe unguarded:
5288
+ // parseTriggers just accepted this same text, so JSON.parse cannot throw here.
5289
+ const rawEntries = JSON.parse(text)?.triggers ?? [];
5290
+ // Issue #433 review rounds 2 and 3: whether THIS HOST'S WORKER schedules a cron trigger, for the unblessed-venue
5291
+ // line, by the worker's two conditions and nothing else. (a) The worker schedules cron only from a PI_TRIGGERS_FILE
5292
+ // it was given (`config.triggersFile` is null without one, and `loadSchedules` then returns []), while doctor
5293
+ // falls back to ./triggers.json for everything else it reads; so without the variable no cron trigger is judged.
5294
+ // (b) Its placement, by the worker's own predicate (`cronPlacement`): judged unless it is `"elsewhere"`, another
5295
+ // machine's folder on a fleet. A single host's absent folder is `"refused"`, the worker's boot refusal, and is
5296
+ // still judged. Round 2 ran the whole `loadSchedules` instead and caught its throw as "judge everything", which let
5297
+ // one served trigger's bad skillsDir bring back the false line for another machine's trigger: a predicate per
5298
+ // trigger cannot be derailed by a sibling.
5299
+ // The name collectChecks resolved (issue #464), never this shell's alone.
5300
+ const fleet = Boolean(declaredWorkerName);
5301
+ const cronScheduledHere = (t) => env.PI_TRIGGERS_FILE !== undefined && cronPlacement(t.run, { existsSync: fileExists, fleet }) !== "elsewhere";
5302
+ return {
5303
+ onceArmed: rawEntries.filter((t) => t?.on?.once === true && t.on.disarmed === undefined).length,
5304
+ onceSpent: rawEntries.filter((t) => t?.on?.disarmed !== undefined).length,
5305
+ requiring: triggers.filter((t) => t.run.packages === true).length,
5306
+ resuming: triggers.filter((t) => t.run.resume === true).length,
5307
+ // REQ-PER-TRIGGER-INSTRUCTION. Counted beside `resuming` for the same reason: it is a per-trigger
5308
+ // choice that changes what every job of it is told, and an operator should see it before it fires.
5309
+ instructing: triggers.filter((t) => typeof t.run.instructions === "string").length,
5310
+ // REQ-REPLICA-RUNS. `> 1` rather than `!== undefined` because the loader already refuses anything
5311
+ // else -- this counts triggers that will actually multiply spend, which is the only reason to say so.
5312
+ replicating: triggers.filter((t) => t.run.replicas > 1).length,
5313
+ // run.command triggers (issue #189), counted for the one advisory line below. The `flows`
5314
+ // tuple list already filters to `typeof f.flow === "string"`, so a command trigger drops out
5315
+ // of the flow-tier probes naturally -- no exclusion needed there.
5316
+ commands: triggers.filter((t) => typeof t.run.command === "string").length,
5317
+ // REQ-TRIGGER-SECRETS. Counted beside `instructing` for its reason: a per-trigger choice that
5318
+ // changes what every job of it can reach, and one that lives only in triggers.json.
5319
+ secreting: triggers.filter((t) => t.run.secrets !== undefined).length,
5320
+ // The distinct profile NAMES the file selects, deduped like `images`/`skillsDirs`: the checks below
5321
+ // cost a stat each, and two triggers naming one profile are one question. `default` is substituted
5322
+ // for an absent field so the table answers what the worker will actually look up.
5323
+ secretProfiles: [...new Set(triggers.filter((t) => t.run.secrets !== undefined).map((t) => t.run.secretsProfile ?? "default"))].sort(),
5324
+ // LOCAL triggers that bind secrets, by folder. A local job's /workspace IS this folder, bind-mounted
5325
+ // read-write with no clone, so a credential an agent writes into .env lands in the operator's real
5326
+ // repository rather than a temp dir that gets swept. Deduped for skillsDirs' reason.
5327
+ localSecretFolders: [...new Set(triggers.filter((t) => t.run.secrets !== undefined && t.run.kind === "local" && typeof t.run.folder === "string").map((t) => t.run.folder))].sort(),
5328
+ // Issue #309. The distinct variable NAMES the file binds, deduped like the profiles above. The
5329
+ // pre-spend gate refuses a name pi reads for the job's provider, and unlike the version that gate
5330
+ // replaced, that question no longer needs host state to answer -- so doctor can answer it at setup
5331
+ // rather than leaving the operator to meet it as a public refusal on a live job.
5332
+ secretNames: [...new Set(triggers.filter((t) => t.run.secrets !== undefined).flatMap((t) => Object.keys(t.run.secrets)))].sort(),
5333
+ // Issue #242: every local run.folder, CANONICALIZED the way the scoped-limits matcher
5334
+ // canonicalizes a job's folder (one derivation -- canonicalScope, never re-spelled here), so
5335
+ // the unreferenced-scope advisory compares like with like across spelling variants.
5336
+ folders: [...new Set(triggers.filter((t) => t.run.kind === "local" && typeof t.run.folder === "string").map((t) => canonicalScope({ kind: "local", folder: t.run.folder })))].sort(),
5337
+ // Issue #230. How many triggers hold their jobs, and the distinct profile NAMES they select --
5338
+ // deduped like `secretProfiles` and for its reason: each name costs a lookup, and two triggers
5339
+ // waiting on one profile are one question.
5340
+ waiting: triggers.filter((t) => Array.isArray(t.run.waitFor) && t.run.waitFor.length > 0).length,
5341
+ // Issue #502: how many triggers name an allowed-model list, for the version-floor line.
5342
+ listing: triggers.filter((t) => Array.isArray(t.run.models) && t.run.models.length > 0).length,
5343
+ // The `after` instants as WRITTEN, deduped. Not parsed here: `readTriggerFacts` is a fact reader and
5344
+ // the ceiling it is measured against is env, which belongs at the check. Two triggers naming one
5345
+ // instant are one finding, and the raw string is what the operator has to go and edit.
5346
+ waitAfters: [...new Set(triggers.flatMap((t) => (Array.isArray(t.run.waitFor) ? t.run.waitFor : [])).map((c) => c?.after).filter((v) => typeof v === "string"))].sort(),
5347
+ waitProfiles: [
5348
+ ...new Set(
5349
+ triggers
5350
+ .filter((t) => Array.isArray(t.run.waitFor))
5351
+ .flatMap((t) => t.run.waitFor.map((c) => c?.profile).filter((n) => typeof n === "string")),
5352
+ ),
5353
+ ].sort(),
5354
+ optingOut: triggers.filter((t) => t.run.packages === false).length,
5355
+ images: [...new Set(triggers.map((t) => t.run.image).filter((i) => typeof i === "string"))].sort(),
5356
+ // Issue #433: each image with the venue its trigger names (undefined: the deployment default), so the image is
5357
+ // asked of the runtime its jobs start on. The worker's own shape (`{ backend }`), for `resolveBackendName`.
5358
+ imageRoutes: triggers.filter((t) => typeof t.run.image === "string").map((t) => ({ image: t.run.image, backend: t.run.backend })),
5359
+ // Issue #433 review round 1: every trigger that NAMES a venue, with the label the lines below name it by, so
5360
+ // doctor can say which of them this deployment does not bless (the worker refuses each of their jobs).
3947
5361
  namedBackends: triggers
3948
5362
  .map((t, index) => ({ label: triggerLabel(t, index), backend: t.run.backend, served: t.on.type !== "cron" || cronScheduledHere(t) }))
3949
5363
  .filter((r) => typeof r.backend === "string"),
@@ -3975,6 +5389,20 @@ function readTriggerFacts(env, fileExists, cwd, declaredWorkerName) {
3975
5389
  packages: t.run.packages !== false,
3976
5390
  }))
3977
5391
  .filter((f) => typeof f.flow === "string"),
5392
+ // Issue #501: every trigger that sets run.maxCostUsd, with the label the cap check names it by. The value
5393
+ // already parsed (the loader validated it), so the check below only compares.
5394
+ costCaps: triggers.map((t, index) => ({ label: triggerLabel(t, index), maxCostUsd: t.run.maxCostUsd })).filter((r) => r.maxCostUsd !== undefined && r.maxCostUsd !== null),
5395
+ // Issues #501 and #502 (PR 9 of that round): every trigger's model choice, list, cap and bound secret NAMES, for the
5396
+ // unknown-model, cap-fit and listed-provider credential lines. Every kind, cron included, whatever host serves it:
5397
+ // a model the worker does not know is refused wherever the job runs.
5398
+ modelRuns: triggers.map((t, index) => ({
5399
+ label: triggerLabel(t, index),
5400
+ provider: typeof t.run.provider === "string" ? t.run.provider : undefined,
5401
+ model: typeof t.run.model === "string" ? t.run.model : undefined,
5402
+ models: Array.isArray(t.run.models) ? t.run.models : undefined,
5403
+ maxCostUsd: t.run.maxCostUsd ?? undefined,
5404
+ secretNames: t.run.secrets !== undefined && t.run.secrets !== null ? Object.keys(t.run.secrets) : [],
5405
+ })),
3978
5406
  // Explicit on the success path too (issue #242): the dead-scope advisory distinguishes
3979
5407
  // "facts read clean" (path set, no error) from the zeroed `none` -- an implicit undefined
3980
5408
  // here made that test silently false for every deployment.
@@ -4190,11 +5618,15 @@ export async function sweepStaleCanaryNetworks({ run, pid, isAlive, endpoint, bi
4190
5618
  // gets added to the producer and not to the reaper.
4191
5619
  //
4192
5620
  // ASYMMETRY ON PURPOSE, recorded because it looks like an oversight three characters apart: the OWNER is
4193
- // escaped and the slugs are interpolated raw. `CANARY_PROBE_SLUGS` is a frozen literal of two plain words
5621
+ // escaped and the slugs are interpolated raw. `CANARY_PROBE_SLUGS` is a frozen literal of three plain words
4194
5622
  // three lines below, so today there is nothing to escape and escaping it would say the set is untrusted
4195
5623
  // when it is this file's own. It is here so that whoever adds a slug with a `.` or a `-` in it sees the
4196
5624
  // obligation: a metacharacter there widens what this `rm -f` matches (issue #360, item 6).
4197
- const probeOf = (owner) => new RegExp(`^${EGRESS_CANARY_PROBE_PREFIX}(?:${CANARY_PROBE_SLUGS.join("|")})-${owner.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`);
5625
+ //
5626
+ // The model endpoint probes (issue #503) are matched the same way, their slugs from `ENDPOINT_PROBE_SLUGS` and the id
5627
+ // by the parser's own rule (`MODEL_ENDPOINT_ID_RE`, its anchors dropped), never `\\S+`: a name that matches is one
5628
+ // this file builds, and the class cannot drift from what the parser accepts.
5629
+ const probeOf = (owner) => new RegExp(`^(?:${EGRESS_CANARY_PROBE_PREFIX}(?:${CANARY_PROBE_SLUGS.join("|")})|${EGRESS_ENDPOINT_PROBE_PREFIX}(?:${ENDPOINT_PROBE_SLUGS.join("|")})-${ENDPOINT_ID_CLASS})-${owner.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`);
4198
5630
  const checks = [];
4199
5631
  for (const name of String(listed.stdout ?? "").split("\n").map((n) => n.trim()).filter(Boolean)) {
4200
5632
  const m = shape.exec(name);
@@ -4306,8 +5738,55 @@ export async function sweepStaleCanaryNetworks({ run, pid, isAlive, endpoint, bi
4306
5738
  return checks;
4307
5739
  }
4308
5740
 
4309
- /** The canary's two probe directions. ONE list: the loop names its containers from it and the sweep matches on it. */
4310
- export const CANARY_PROBE_SLUGS = Object.freeze(["provider", "unlisted"]);
5741
+ /**
5742
+ * Issue #508: with the egress policy on, a forge must be served over https on port 443, the one shape a job's git reaches
5743
+ * it by. git never uses the job's proxy for an `http://` remote: the job gets the proxy variables in UPPERCASE only, and
5744
+ * libcurl ignores `HTTP_PROXY` for `http://` (measured in the job image, git 2.39.5: `git ls-remote http://...` went to
5745
+ * DNS, and the proxy saw nothing), so an http:// forge fails behind the `--internal` network on ANY port, 80 included.
5746
+ * An `https://` remote is a CONNECT, which the proxy refuses to any port but 443 (a rule older than #508). The clone
5747
+ * itself runs on the host before the container exists, so it is the job's git push and fetch that fail, and its API
5748
+ * calls too unless the forge is on port 80: glab and tea (Go) do use `HTTP_PROXY` for `http://`, and the proxy passes
5749
+ * plain HTTP to a listed host on port 80 (round 3 of #508's review). Any other scheme reaches nothing. One ⚠
5750
+ * per such URL, for a forge the triggers file names; nothing with the policy off or unreadable (the .env check reports
5751
+ * a malformed PI_EGRESS).
5752
+ */
5753
+ export function forgeUrlEgressChecks(env, forges = []) {
5754
+ let armed;
5755
+ try {
5756
+ armed = egressArmed(env);
5757
+ } catch {
5758
+ return [];
5759
+ }
5760
+ if (armed !== true) return [];
5761
+ const checks = [];
5762
+ for (const [forge, key] of [["gitlab", "GITLAB_URL"], ["forgejo", "FORGEJO_URL"]]) {
5763
+ if (!forges.includes(forge) || typeof env[key] !== "string") continue;
5764
+ let url;
5765
+ try {
5766
+ url = new URL(env[key].trim());
5767
+ } catch {
5768
+ continue;
5769
+ }
5770
+ // The WHATWG parser drops a default port, so an empty port on https: is 443.
5771
+ if (url.protocol === "https:" && url.port === "") continue;
5772
+ const why =
5773
+ url.protocol === "https:"
5774
+ ? `https:// on port ${url.port}, and the egress proxy refuses a CONNECT to any port but 443, so with the egress policy on every job's push, fetch and API call to it fails`
5775
+ : url.protocol === "http:"
5776
+ ? `http://, so with the egress policy on a job's git push and fetch fail (git never sends http:// through the proxy)${url.port === "" ? "" : `, and on port ${url.port}, not 80, its API calls fail too`}`
5777
+ : "not an https:// URL, so with the egress policy on a job cannot reach it";
5778
+ checks.push({
5779
+ ok: false,
5780
+ warn: true,
5781
+ label: `${key} (${urlShown(env[key])}) is ${why} (issue #508)`,
5782
+ fix: `serve the forge over https:// on port 443 and point ${key} there, or set PI_EGRESS=0 if you accept jobs without the policy (docs/egress.md)`,
5783
+ });
5784
+ }
5785
+ return checks;
5786
+ }
5787
+
5788
+ /** The canary's three probes. ONE list: the loop names its containers from it and the sweep matches on it. */
5789
+ export const CANARY_PROBE_SLUGS = Object.freeze(["provider", "unlisted", "plainhttp"]);
4311
5790
 
4312
5791
  /** Where the job image's runner keeps the module its own provider call goes through (issue #427). */
4313
5792
  export const EGRESS_CANARY_RUNNER_MODULE = "/app/image/runner/src/env-proxy.mjs";
@@ -4331,6 +5810,21 @@ export function egressCanaryScript(url) {
4331
5810
  return `import(${JSON.stringify(EGRESS_CANARY_RUNNER_MODULE)}).then(m=>m.loadPiThenRestore(),e=>{console.log("error",e.code??e.message);process.exit(e.code==="ERR_MODULE_NOT_FOUND"?${EGRESS_CANARY_STALE_RUNNER}:5)}).then(()=>fetch(${JSON.stringify(url)},{method:"POST"}).then(r=>{console.log("reached",r.status);process.exit(0)},e=>{console.log("blocked",e.cause?.code??e.message);process.exit(3)}),e=>{console.log("error",e.message);process.exit(5)})`;
4332
5811
  }
4333
5812
 
5813
+ /**
5814
+ * The third probe's in-container script (issue #508): ONE plain forward request, `GET <absolute url>`, written straight
5815
+ * to `proxyUrl`, which the caller passes as `egressProxyUrl(proxy)`: the value both venues' probe argv put in
5816
+ * HTTPS_PROXY. Passed rather than read from the container's environment because this file's text is pinned to read no
5817
+ * environment by name outside the resolver (issue #471), and a script string is text. Raw `node:http` with `agent: false`, and NOT the
5818
+ * runner's route (`loadPiThenRestore`, `fetch`): those tunnel every origin, so they would send a CONNECT, which is
5819
+ * the path the other two probes read, never the plain one a curl or a package manager takes. Exit 3 only for squid's
5820
+ * own refusal (403 with an `X-Squid-Error` of ERR_ACCESS_DENIED), 0 for any other answer (the request went on: a 400
5821
+ * from a TLS port spoken to in clear, or squid's 502 when the far end hung up, both mean the proxy let it through),
5822
+ * and 5 for a proxy that could not be spoken to or never answered, which is no reading at all.
5823
+ */
5824
+ export function egressCanaryPlainScript(url, { proxyUrl, timeoutMs = 15_000 } = {}) {
5825
+ return `import("node:http").then(h=>{let p,u;try{p=new URL(${JSON.stringify(String(proxyUrl))});u=new URL(${JSON.stringify(url)})}catch(e){console.log("error",e.message);process.exit(5)}const q=h.request({host:p.hostname,port:p.port||80,method:"GET",path:u.href,headers:{Host:u.host},agent:false,timeout:${Number(timeoutMs)}},r=>{const x=String(r.headers["x-squid-error"]??"");r.resume();if(r.statusCode===403&&x.startsWith("ERR_ACCESS_DENIED")){console.log("blocked",r.statusCode,x);process.exit(3)}console.log("reached",r.statusCode,x);process.exit(0)});q.on("timeout",()=>{console.log("error","timeout");process.exit(5)});q.on("error",e=>{console.log("error",e.code??e.message);process.exit(5)});q.end()},e=>{console.log("error",e.message);process.exit(5)})`;
5826
+ }
5827
+
4334
5828
  /**
4335
5829
  * The bound on one canary docker step. Shorter than the 30 s the PROBES get, because these are `network`
4336
5830
  * calls that either answer at once or are wedged, and the teardown must not be the slow part of a doctor run.
@@ -4442,7 +5936,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4442
5936
  name: "deploy/egress-proxy.conf in this folder",
4443
5937
  seams,
4444
5938
  skip: () => proxyFileIsDirectory(join(seams.cwd, "deploy/egress-proxy.conf")),
4445
- refresh: `\`pi-dispatch up\` from this folder offers to replace it with the package's copy, keeping this one as a backup, and then to restart the proxy, since squid reads its rules only at start`,
5939
+ refresh: `\`pi-dispatch up\` from this folder offers to replace it with the package's copy, keeping this one as a backup, and then to restart the proxy, since squid reads its rules only at start; a proxy made before #503, which lacks the model-endpoints.conf mount the new rules need, is REPLACED in the same step instead, asked once`,
4446
5940
  });
4447
5941
  if (stale) checks.push(stale);
4448
5942
  }
@@ -4505,23 +5999,54 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4505
5999
  // know which folder the service uses), and only where every bind source resolves on this host; otherwise that half
4506
6000
  // is said to be unknown, never stale.
4507
6001
  const inFolder = ["egress-allowlist.conf", "deploy/egress-proxy.conf"].every((f) => proxyFilesExist(join(seams.cwd, f)));
4508
- // A DIRECTORY at either path (PR #488's review): docker bind-mounts it where squid reads a file, and the proxy cannot
4509
- // start from it. Said for the shipped proxy, whose run and compose file mount this folder's two paths.
6002
+ // A DIRECTORY at any of its paths (PR #488's review): docker bind-mounts it where squid reads a file, and the proxy
6003
+ // cannot start from it. Said for the shipped proxy, whose run and compose file mount this folder's paths. The model
6004
+ // endpoints' include (issue #503) is one of them: a directory there parses as NO rules with no warning (measured on
6005
+ // Docker Desktop), and `egress render` refuses to write it.
4510
6006
  if (!custom) {
4511
- for (const f of ["egress-allowlist.conf", "deploy/egress-proxy.conf"]) {
6007
+ for (const f of ["egress-allowlist.conf", "deploy/egress-proxy.conf", MODEL_ENDPOINTS_INCLUDE_NAME]) {
4512
6008
  if (proxyFileIsDirectory(join(seams.cwd, f))) {
4513
6009
  checks.push({ ok: false, label: `${f} in this folder is a directory, not a file`, fix: `the egress proxy mounts it where squid reads a file, so it cannot start from this folder: remove the directory, then \`pi-dispatch init\` writes the file (create-only)` });
4514
6010
  }
4515
6011
  }
4516
6012
  }
4517
- const judged = !custom && parsed ? shippedProxyDrift(parsed, { cwd: seams.cwd, platform: seams.platform ?? process.platform, realpath: (p) => realpathSync(p), compareMounts: inFolder }) : { drift: [], unknown: null };
6013
+ // Issue #503's governing rule: what the third mount and the include file cost is decided by whether the rules this
6014
+ // proxy runs (the folder's deploy/egress-proxy.conf) include the file. `includeNeeds` is a seam for tests.
6015
+ const { includeNeeds = ({ env: e, cwd, platform }) => ({ rulesInclude: rulesFileIncludes(join(cwd, "deploy/egress-proxy.conf"), { readFileSync }), endpointsDeclared: endpointsDeclaredIn({ env: e, cwd, fs: { readFileSync, existsSync }, platform }) }) } = seams;
6016
+ const needs = !custom && inFolder ? includeNeeds({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform }) : { rulesInclude: false, endpointsDeclared: false };
6017
+ if (!custom && inFolder) {
6018
+ // MISSING in a deployment folder: ✗ when the rules include it, since squid then refuses to start (measured); ⚠
6019
+ // otherwise, since a proxy on rules from before #503 runs without it, and the rules refresh needs it.
6020
+ if (!proxyFilesExist(join(seams.cwd, MODEL_ENDPOINTS_INCLUDE_NAME))) {
6021
+ checks.push(
6022
+ needs.rulesInclude
6023
+ ? { ok: false, label: `${MODEL_ENDPOINTS_INCLUDE_NAME} is not in this folder`, fix: `the egress proxy mounts it and its rules include it, and squid will not start without it: \`pi-dispatch init\` writes it (create-only), then \`pi-dispatch up\` starts the proxy` }
6024
+ : { ok: false, warn: true, label: `${MODEL_ENDPOINTS_INCLUDE_NAME} is not in this folder`, fix: "the proxy's rules here predate #503 and run without it, but the next rules refresh includes it and the proxy will not start without it then: `pi-dispatch init` writes it (create-only)" },
6025
+ );
6026
+ }
6027
+ // Endpoints declared under rules that predate #503: the one line up, doctor and egress render share.
6028
+ if (needs.endpointsDeclared && !needs.rulesInclude) checks.push({ ok: false, warn: true, label: rulesPredateEndpointsLine("docker"), fix: "a reload or a proxy replace changes nothing here: the rules themselves must include the file, and the refresh `pi-dispatch up` offers replaces the proxy with them" });
6029
+ }
6030
+ // Issue #503: the declared model endpoints, read as the service reads them. None declared is no line and no container,
6031
+ // so a deployment without them prints what it always did. Under rules that predate the include the line just above is
6032
+ // the whole story (a probe would only fail it again), so nothing more is said there. A proxy PI_EGRESS_PROXY names
6033
+ // must include the file itself, and is read like the shipped one.
6034
+ const { declaredEndpoints = ({ env: e, cwd, platform }) => declaredEndpointsIn({ env: e, cwd, fs: { readFileSync, existsSync }, platform }), hostAddresses = () => lanIPv4Addresses(networkInterfaces()) } = seams;
6035
+ const declared = declaredEndpoints({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform });
6036
+ const endpoints = declared.length > 0 && !(!custom && inFolder && !needs.rulesInclude) ? declared : [];
6037
+ if (endpoints.length > 0) {
6038
+ const runtime = routeRuntimeFromFacts(seams.readFactsOnce ? await seams.readFactsOnce() : null, seams.platform ?? process.platform);
6039
+ const addresses = hostAddresses();
6040
+ checks.push(...endpointRouteChecks({ runtime: runtime && addresses ? { ...runtime, hostAddresses: addresses } : runtime, endpoints, prefix: "", proof: "The endpoint probes below are the proof, once the proxy runs and the job image is here." }));
6041
+ }
6042
+ const judged = !custom && parsed ? shippedProxyDrift(parsed, { cwd: seams.cwd, platform: seams.platform ?? process.platform, realpath: (p) => realpathSync(p), compareMounts: inFolder, rulesInclude: needs.rulesInclude }) : { drift: [], unknown: null };
4518
6043
  // What `up` does with it, told the way `up` decides it (PR #456's final check): a proxy stale by its image, entrypoint
4519
6044
  // or command is offered for replacement whatever its mounts; one stale on its mounts alone is not while one of its own
4520
6045
  // two mounts is unknown, so the fix names the commands rather than an offer that `up` would not make.
4521
6046
  const mountsOnly = judged.unknown !== null && judged.drift.length > 0 && shippedProxyDrift(parsed, { cwd: seams.cwd, compareMounts: false }).drift.length === 0;
4522
6047
  const replaceFix = mountsOnly
4523
- ? `check its mounts (\`docker inspect --format '{{json .Mounts}}' ${proxy}\`), then \`docker rm -f ${proxy}\` and \`pi-dispatch up\` from the deployment folder replace it with the shipped one; \`up\` does not offer to on its own while one of its mounts cannot be compared here`
4524
- : `\`pi-dispatch up\` from the deployment folder offers to replace it with the shipped one (docker rm -f ${proxy}, then the shipped run)`;
6048
+ ? `check its mounts (\`docker inspect --format '{{json .Mounts}}' ${proxy}\`), then \`docker rm -f -v ${proxy}\` and \`pi-dispatch up\` from the deployment folder replace it with the shipped one; \`up\` does not offer to on its own while one of its mounts cannot be compared here`
6049
+ : `\`pi-dispatch up\` from the deployment folder offers to replace it with the shipped one (docker rm -f -v ${proxy}, then the shipped run)`;
4525
6050
  // The worker's simple rule (egress.mjs): paused, exited, dead and created (`STOPPED_PROXY_STATES`) refuse every job,
4526
6051
  // so ✗; any other state is one the worker retries through (restarting, stopping, removing, ...), so ⚠, and the fix
4527
6052
  // says jobs wait rather than fail. `restarting` is a crash loop that fails every job (one retry, then failed), so ✗;
@@ -4549,7 +6074,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4549
6074
  : custom
4550
6075
  ? `PI_EGRESS_PROXY names your own proxy, which neither compose nor \`pi-dispatch up\` starts: ${state.code === 0 ? `\`docker ${parsed?.status === "paused" ? "unpause" : "start"} ${proxy}\`` : `create and start ${proxy} yourself`} -- the egress policy refuses every job pre-spend while it is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)`
4551
6076
  : judged.drift.length > 0
4552
- ? `it is not this deployment's either (${judged.drift.join("; ")}): ${replaceFix} -- the egress policy refuses every job pre-spend while it is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)`
6077
+ ? `${judged.outOfDate ? "it is this deployment's proxy but out of date" : "it is not this deployment's either"} (${judged.drift.join("; ")}): ${replaceFix} -- the egress policy refuses every job pre-spend while it is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)`
4553
6078
  : parsed?.status === "paused"
4554
6079
  ? `docker unpause ${proxy} -- as \`pi-dispatch up\` offers; the egress policy refuses every job pre-spend while it is paused, which costs no budget but runs nothing (PI_EGRESS=0 opts out)`
4555
6080
  : "`pi-dispatch up` from the deployment folder starts it -- the egress policy refuses every job pre-spend while this is down, which costs no budget but runs nothing (PI_EGRESS=0 opts out)",
@@ -4563,7 +6088,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4563
6088
  if (judged.drift.length > 0) {
4564
6089
  checks.push({
4565
6090
  ok: false,
4566
- label: `Egress proxy is running but is not this deployment's (${proxy}): ${judged.drift.join("; ")}`,
6091
+ label: judged.outOfDate ? `Egress proxy is this deployment's proxy but out of date (${proxy}): ${judged.drift.join("; ")}` : `Egress proxy is running but is not this deployment's (${proxy}): ${judged.drift.join("; ")}`,
4567
6092
  fix: `${replaceFix}; until then every job's egress runs through a policy this deployment did not ship`,
4568
6093
  proxyState: proxyCheck.proxyState,
4569
6094
  });
@@ -4573,7 +6098,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4573
6098
  ok: false,
4574
6099
  warn: true,
4575
6100
  label: `Egress proxy's mounts could not be compared on this host (${proxy}): ${judged.unknown}`,
4576
- fix: "its image, entrypoint and command were compared; check its two mounts by hand (`docker inspect --format '{{json .Mounts}}' " + proxy + "`): /etc/squid/squid.conf must be this deployment's deploy/egress-proxy.conf and /etc/pi-dispatch/allowlist.conf its egress-allowlist.conf",
6101
+ fix: "its image, entrypoint and command were compared; check its mounts by hand (`docker inspect --format '{{json .Mounts}}' " + proxy + "`): /etc/squid/squid.conf must be this deployment's deploy/egress-proxy.conf, /etc/pi-dispatch/allowlist.conf its egress-allowlist.conf, and /etc/pi-dispatch/model-endpoints.conf its model-endpoints.conf",
4577
6102
  });
4578
6103
  }
4579
6104
  }
@@ -4592,7 +6117,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4592
6117
  }
4593
6118
 
4594
6119
  // The end-to-end probe, and the only place in this codebase that proves the policy rather than
4595
- // inspecting it. Two containers, on a throwaway network built exactly like a job's, gated on the image
6120
+ // inspecting it. Three containers, on a throwaway network built exactly like a job's, gated on the image
4596
6121
  // being present because it uses the job image's own node and runner module -- which is the point: it proves the
4597
6122
  // operator's OWN image routes a request through the proxy the way its runner does (pi loaded, then the runner's
4598
6123
  // restore), the property a stale image would silently lack and the one whose absence turns the whole policy into an
@@ -4601,12 +6126,21 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4601
6126
  // Credential-free by construction: `api.anthropic.com` answers 401 to an unauthenticated request, so
4602
6127
  // reaching the provider and being refused for the key proves the entire path and costs nothing. That is
4603
6128
  // docs/egress.md's own method, promoted from prose to a check.
6129
+ // Issue #503: the include as the running proxy sees it. Not for a shipped proxy already judged stale above: its mounts
6130
+ // are that line's to name, and a two-mount proxy has no include to read.
6131
+ if (endpoints.length > 0 && !(!custom && judged.drift.length > 0)) {
6132
+ const inside = await runCmdCapture(spawn, "docker", ["exec", proxy, "cat", ENDPOINTS_INCLUDE_IN_PROXY], { stdoutOnly: true, timeoutMs: CANARY_STEP_TIMEOUT_MS });
6133
+ // The file the proxy ACTUALLY mounts there, by its bind source from the inspect above, for the shipped proxy and
6134
+ // an operator's alike: never this folder's, which may not be the one mounted (gate round 2).
6135
+ const mounted = readIncludeBindSource(parsed?.mounts);
6136
+ checks.push(endpointsIncludeCheck({ answer: inside, endpoints, bin: "docker", proxy, prefix: "", mounted, recreate: custom ? `recreate ${proxy} so it mounts the file anew` : `\`docker rm -f -v ${proxy}\`, then \`pi-dispatch up\` starts it on the file anew (a restart cuts running jobs' tunnels)` }));
6137
+ }
4604
6138
  if (imageCode !== 0) return checks;
4605
6139
  // Issue #431: the canary itself is `runEgressCanary`, shared with the podman venue's `--live` and the conformance
4606
6140
  // script. docker's probes keep `runCmdCapture` (its 30 s bound, stderr merged, SIGTERM at the bound) rather than the
4607
6141
  // bounded step runner, so docker's spawns, and what a probe that never launched leaves for the teardown, are exactly
4608
6142
  // what they were: pinned in doctor.test.mjs against the output and argv captured before the move.
4609
- const canary = await runEgressCanary({ run: liveRunVia(spawn), probeRun: (args) => runCmdCapture(spawn, "docker", args), proxy, image: jobImage, pid, gate });
6143
+ const canary = await runEgressCanary({ run: liveRunVia(spawn), probeRun: (args) => runCmdCapture(spawn, "docker", args), proxy, image: jobImage, pid, gate, endpoints });
4610
6144
  // Through a stale proxy the canary reads THAT proxy's policy (exec round 3): a failure is the proxy's to fix, never
4611
6145
  // this deployment's allowlist, which it may not even mount.
4612
6146
  checks.push(...(staleRunning ? canary.checks.map((c) => (c.ok ? c : { ...c, fix: `the canary ran through ${proxy}, which is not this deployment's proxy (above), so this says nothing about egress-allowlist.conf: ${replaceFix}, then re-run doctor` })) : canary.checks));
@@ -4619,6 +6153,10 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4619
6153
  * docker's is what it always was: a plain `docker run` on the canary network with the two proxy variables, which is
4620
6154
  * pinned byte for byte and not widened here, because moving it is its own change with its own review.
4621
6155
  *
6156
+ * `script` is what the container runs, the runner's route (`egressCanaryScript`) unless a probe says otherwise: the
6157
+ * plain HTTP probe (issue #508) passes `egressCanaryPlainScript`, and its argv differs from the others in nothing else.
6158
+ * A model endpoint's probe (issue #503) passes its own `name` and `httpProxy`, which adds the job's HTTP_PROXY on docker.
6159
+ *
4622
6160
  * podman's is a JOB's, built by the podman venue's own builder (`podmanArgsFromSpec` over `containerSpec`, the path
4623
6161
  * `buildPodmanRunArgs` takes), never a hand-rolled argv: `ISOLATION_FLAGS`, the job's memory and cpu bounds,
4624
6162
  * the job user this host decides as `--user`, `--userns=keep-id` and `PODMAN_PINNED_FLAGS`, and a job's own egress
@@ -4635,8 +6173,7 @@ async function egressChecks(env, seams, { dockerCode, imageCode, jobImage, endpo
4635
6173
  * then replaced by none; `CANARY_NO_WORKSPACE` is a path nothing creates, so if a later edit ever kept the mount, the
4636
6174
  * run would fail on a missing source rather than bind a real directory.
4637
6175
  */
4638
- export function egressCanaryProbeArgs({ bin = "docker", slug, pid, network, proxy, image, url, user = null }) {
4639
- const name = egressCanaryProbe(slug, pid);
6176
+ export function egressCanaryProbeArgs({ bin = "docker", slug, pid, network, proxy, image, url, user = null, script = egressCanaryScript(url), name = egressCanaryProbe(slug, pid), httpProxy = false }) {
4640
6177
  if (bin !== "podman") {
4641
6178
  return [
4642
6179
  "run",
@@ -4651,20 +6188,23 @@ export function egressCanaryProbeArgs({ bin = "docker", slug, pid, network, prox
4651
6188
  `--network=${network}`,
4652
6189
  "-e",
4653
6190
  `HTTPS_PROXY=http://${proxy}:3128`,
6191
+ // Only for a model endpoint's probe (issue #503), whose URL is `http://`: the runner's dispatcher sends that to
6192
+ // HTTP_PROXY, as a job's does. The canary's own three argv are unchanged.
6193
+ ...(httpProxy ? ["-e", `HTTP_PROXY=http://${proxy}:3128`] : []),
4654
6194
  "-e",
4655
6195
  "NODE_USE_ENV_PROXY=1",
4656
6196
  "--entrypoint",
4657
6197
  "node",
4658
6198
  image,
4659
6199
  "-e",
4660
- egressCanaryScript(url),
6200
+ script,
4661
6201
  ];
4662
6202
  }
4663
6203
  // HOME as a podman job gets it (`resolvePodmanImageUser` always answers CONTAINER_HOME): under keep-id the job user's
4664
6204
  // passwd entry otherwise names this host's home path, which does not exist in the image, and the canary loads pi as
4665
6205
  // that user. No credential rides along: the canary proves the route, and a 401 from the provider is its success.
4666
6206
  const { mounts: _placeholder, ...spec } = containerSpec({ image, name, env: { HOME: CONTAINER_HOME, ...egressEnv({ proxy, armed: true }) }, workspace: CANARY_NO_WORKSPACE, network, user, userns: "keep-id", extraFlags: ["--entrypoint", "node"] });
4667
- return [...podmanArgsFromSpec({ ...spec, mounts: [] }), "-e", egressCanaryScript(url)];
6207
+ return [...podmanArgsFromSpec({ ...spec, mounts: [] }), "-e", script];
4668
6208
  }
4669
6209
 
4670
6210
  /** The workspace `containerSpec` requires and the canary never mounts (see `egressCanaryProbeArgs`). */
@@ -4672,10 +6212,11 @@ const CANARY_NO_WORKSPACE = "/nonexistent/pi-dispatch-egress-canary-mounts-nothi
4672
6212
 
4673
6213
  /**
4674
6214
  * The egress canary (REQ-EGRESS-ALLOWLIST, issue #431): a throwaway `--internal` network built like a job's, the proxy
4675
- * attached, two probe containers running `egressCanaryScript` (the runner's own route to the network, #427), one that
4676
- * must reach the provider and one that must not reach an unlisted host, and a teardown that removes everything it made
4677
- * or names what it could not. Returns `{ checks, results }`: the lines to print, and the `[{ property, want, reached }]`
4678
- * readings `egressVerdict` folds into `--live`.
6215
+ * attached, three probe containers, and a teardown that removes everything it made or names what it could not. Two run
6216
+ * `egressCanaryScript` (the runner's own route to the network, #427): one must reach the provider and one must not reach
6217
+ * an unlisted host. The third runs `egressCanaryPlainScript` and must not get plain HTTP through to a listed host on a
6218
+ * port other than 80 (issue #508). Returns `{ checks, results }`: the lines to print, and the
6219
+ * `[{ property, want, reached, probe }]` readings `egressVerdict` folds into `--live`, `probe` being the slug.
4679
6220
  *
4680
6221
  * ONE canary for every caller: docker's `doctor` (`egressChecks`), the podman venue's `doctor --live`
4681
6222
  * (`podmanLiveChecks`) and `.github/scripts/podman-conformance.mjs`. The conformance script used to carry a canary of
@@ -4688,8 +6229,11 @@ const CANARY_NO_WORKSPACE = "/nonexistent/pi-dispatch-egress-canary-mounts-nothi
4688
6229
  * measured), which a 30 s bound would read as a probe that did not run. `user` is the job user, required on podman,
4689
6230
  * whose builder refuses a keep-id argv without one. What it does NOT check is the proxy: every caller has read the
4690
6231
  * proxy's state on its own runtime first, and runs this only when it is up and the job image is present.
6232
+ *
6233
+ * `endpoints` (issue #503) are the declared model endpoints to prove on the same network after the three
6234
+ * (`runEndpointProbes`), `[]` by default, so the conformance script and a deployment with none run exactly the three.
4691
6235
  */
4692
- export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = process.pid, user = null, probeRun = null, gate = makeDetachGate((args, opts) => run(args, { timeoutMs: opts?.timeoutMs ?? CANARY_STEP_TIMEOUT_MS }), { bin }) }) {
6236
+ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid = process.pid, user = null, probeRun = null, endpoints = [], gate = makeDetachGate((args, opts) => run(args, { timeoutMs: opts?.timeoutMs ?? CANARY_STEP_TIMEOUT_MS }), { bin }) }) {
4693
6237
  const venue = canaryVenueFor(bin);
4694
6238
  const probe = probeRun ?? ((args) => run(args, { timeoutMs: bin === "podman" ? PODMAN_FIRST_START_TIMEOUT_MS : RUN_TIMEOUTS.cmd }));
4695
6239
  const checks = [];
@@ -4702,6 +6246,8 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid =
4702
6246
  // Probe containers this run may have left BEHIND its CLI, see the `code === null` branch below.
4703
6247
  const unfinished = [];
4704
6248
  let created = false;
6249
+ // The canary stopped on an image with no runner module: the endpoint probes take the same route, so they are not run.
6250
+ let staleRunner = false;
4705
6251
  // FIRST, before anything exists (issue #452, gate round 3): this canary's own teardown detaches the running proxy, which
4706
6252
  // on a rootless Podman 4.x without a holding keeper cuts its route out, through `podman`, `podman-docker` or the Docker
4707
6253
  // API alike (measured). So the detach gate every teardown goes through is asked up front, and a refusal runs nothing:
@@ -4756,11 +6302,17 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid =
4756
6302
  // which no proxy can reach, so a proxy allowing every host still read as denying this one (measured: an
4757
6303
  // allow-all squid answered 503 for it and let `example.com` through). `example.com` is reserved for documentation
4758
6304
  // (RFC 2606), answers everywhere, and is contacted only when the proxy lets the request out, which is the finding.
4759
- for (const [slug, host, url, want] of [
6305
+ //
6306
+ // The THIRD probe (issue #508) asks the proxy for plain HTTP to a LISTED host on a port other than 80, as a client
6307
+ // that forwards rather than tunnels would (npm undici's EnvHttpProxyAgent without proxyTunnel, a package manager). Port 443 of the provider because it is a
6308
+ // port that host certainly listens on, so a proxy that lets it through gets an answer (a 400 for clear text on a
6309
+ // TLS port) rather than a timeout, and no second host has to be listed for the canary.
6310
+ for (const [slug, host, url, want, script] of [
4760
6311
  [CANARY_PROBE_SLUGS[0], "the provider", "https://api.anthropic.com/v1/messages", true],
4761
6312
  [CANARY_PROBE_SLUGS[1], "an unlisted host", "https://example.com/", false],
6313
+ [CANARY_PROBE_SLUGS[2], "plain HTTP to a listed host off port 80", "http://api.anthropic.com:443/", false, egressCanaryPlainScript("http://api.anthropic.com:443/", { proxyUrl: egressProxyUrl(proxy) })],
4762
6314
  ]) {
4763
- const answer = await probe(egressCanaryProbeArgs({ bin, slug, pid, network: net, proxy, image, url, user }));
6315
+ const answer = await probe(egressCanaryProbeArgs({ bin, slug, pid, network: net, proxy, image, url, user, ...(script ? { script } : {}) }));
4764
6316
  // The script exits 0 (reached) or 3 (blocked). Anything else is the container not running it -- a name clash,
4765
6317
  // the image, the daemon -- which is no reading at all, and must not pass for a deny.
4766
6318
  // `code === null` is the ONE case where a container may still be RUNNING under a name we chose: the
@@ -4772,7 +6324,7 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid =
4772
6324
  // nothing of ours to remove. Harmless either way today (`docker rm -f <missing>` exits 0, measured),
4773
6325
  // and written out because the conflation it removes is the one this item is about.
4774
6326
  if (answer.code === null && answer.ended !== "error") unfinished.push(egressCanaryProbe(slug, pid));
4775
- // Said ONCE and the second probe not run: the unlisted host's "blocked" would come from the internal network
6327
+ // Said ONCE and the later probes not run: the unlisted host's "blocked" would come from the internal network
4776
6328
  // refusing a runner that never tried the proxy, which reads exactly like a policy that denies it.
4777
6329
  if (answer.code === EGRESS_CANARY_STALE_RUNNER) {
4778
6330
  checks.push({
@@ -4780,8 +6332,9 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid =
4780
6332
  warn: true,
4781
6333
  label: `${venue.prefix}Egress policy: not proved, because the job image could not find ${EGRESS_CANARY_RUNNER_MODULE} (or an import of it): its runner predates issue #427, whose provider call goes around the proxy so that with egress armed every job fails at its first turn, or the image is not built from this project's`,
4782
6334
  fix: `use a job image built after issue #427 (ghcr.io/edgehero/pi-job:latest, or rebuild yours FROM it), or set PI_EGRESS=0 until you can`,
4783
- readBack: { property: "egress", want, reached: null },
6335
+ readBack: { property: "egress", want, reached: null, probe: slug },
4784
6336
  });
6337
+ staleRunner = true;
4785
6338
  break;
4786
6339
  }
4787
6340
  if (answer.code !== 0 && answer.code !== 3) {
@@ -4790,17 +6343,31 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid =
4790
6343
  warn: true,
4791
6344
  label: `${venue.prefix}Egress policy probe for ${host} did not run (${answer.code === null ? `${bin} run did not finish` : `${bin} run exited ${answer.code}`})`,
4792
6345
  fix: "re-run doctor; if it persists, run the job image by hand to see why a container on this network will not start",
4793
- readBack: { property: "egress", want, reached: null },
6346
+ readBack: { property: "egress", want, reached: null, probe: slug },
4794
6347
  });
4795
6348
  continue;
4796
6349
  }
4797
6350
  const reached = answer.code === 0;
6351
+ if (slug === CANARY_PROBE_SLUGS[2]) {
6352
+ checks.push({
6353
+ ok: reached === want,
6354
+ warn: true,
6355
+ readBack: { property: "egress", want, reached, probe: slug },
6356
+ label: `${venue.prefix}${
6357
+ reached === want
6358
+ ? "Egress policy refuses plain HTTP to a listed host off port 80 (api.anthropic.com:443 without a tunnel; only a CONNECT reaches 443)"
6359
+ : "Egress policy lets plain HTTP through to api.anthropic.com on port 443, so a client that does not tunnel reaches every port of a listed host"
6360
+ }`,
6361
+ fix: `the proxy runs rules from before issue #508: restart it on the current egress-proxy.conf (\`pi-dispatch up\`; on the podman venue \`pi-dispatch service install --force\`; for a proxy you started by hand, update the egress-proxy.conf it mounts, then \`${bin} restart ${proxy}\`); a copy you keep by hand must add "http_access deny !Safe_ports !CONNECT" right before "http_access allow allowed"`,
6362
+ });
6363
+ continue;
6364
+ }
4798
6365
  checks.push({
4799
6366
  ok: reached === want,
4800
6367
  warn: true,
4801
6368
  // NOT rendered: what `doctor --live` folds into its egress read-back (live-probes.mjs), so the canary is
4802
6369
  // run once and read twice rather than a second canary built beside it.
4803
- readBack: { property: "egress", want, reached },
6370
+ readBack: { property: "egress", want, reached, probe: slug },
4804
6371
  label: `${venue.prefix}${
4805
6372
  reached === want
4806
6373
  ? want
@@ -4815,6 +6382,15 @@ export async function runEgressCanary({ run, bin = "docker", proxy, image, pid =
4815
6382
  : `check egress-allowlist.conf: a rule wider than you meant (a bare domain where you wanted a subdomain) lets a job reach hosts you did not list`,
4816
6383
  });
4817
6384
  }
6385
+ // Issue #503: each declared model endpoint, on this same network, after the canary's own three. Its lines carry no
6386
+ // `readBack`, so `egressVerdict` reads exactly the three readings it always did.
6387
+ if (endpoints.length > 0) {
6388
+ if (staleRunner) {
6389
+ checks.push({ ok: false, warn: true, label: `${venue.prefix}Model endpoints: not probed, because the job image has no runner module (above), and their probes take the runner's route`, fix: "use a job image built after issue #427, then re-run doctor" });
6390
+ } else {
6391
+ checks.push(...(await runEndpointProbes({ probe, bin, pid, network: net, proxy, image, user, endpoints, venue, unfinished })));
6392
+ }
6393
+ }
4818
6394
  } finally {
4819
6395
  // The probes FIRST, by the name carrying this pid, then the network: a member still attached is exactly
4820
6396
  // why the old `network rm` failed, and it failed silently. `rm -f` of a name that is not there is docker
@@ -4852,6 +6428,407 @@ function readingsOf(checks) {
4852
6428
  return checks.filter((c) => c.readBack?.property === "egress").map((c) => c.readBack);
4853
6429
  }
4854
6430
 
6431
+ /**
6432
+ * The probes doctor runs per declared model endpoint (issue #503, REQ-EGRESS-ALLOWLIST), in this order, and ONE list:
6433
+ * the loop names its containers from it and the dead-pid sweep matches on it, as `CANARY_PROBE_SLUGS` is shared. Not
6434
+ * folded into `CANARY_PROBE_SLUGS`, whose three readings are what `egressVerdict` reads; these never reach it.
6435
+ * - `models`: `GET http://<host>:<port>/v1/models` by the runner's route, which tunnels it, as a job's provider call
6436
+ * goes. Must answer 200: Ollama, llama-server, vLLM and LM Studio all serve that path.
6437
+ * - `nextport`: the same, to the nearest port above that no declaration for that host uses and that the allowlist's
6438
+ * own rules do not open (`undeclaredPortNear`). Must get the proxy's 403 on the CONNECT, which proves the
6439
+ * endpoint's rule is port-exact.
6440
+ * - `plain`: a plain forward `GET` to the declared port. Must get squid's 403, which proves the include adds a tunnel
6441
+ * and nothing else.
6442
+ */
6443
+ export const ENDPOINT_PROBE_SLUGS = Object.freeze(["models", "nextport", "plain"]);
6444
+
6445
+ /** The parser's id rule without its anchors, for the sweep's pattern. */
6446
+ const ENDPOINT_ID_CLASS = MODEL_ENDPOINT_ID_RE.source.replace(/^\^/, "").replace(/\$$/, "");
6447
+
6448
+ /** How long an endpoint probe waits for its answer inside the container: a server that takes the tunnel and never answers. */
6449
+ const ENDPOINT_PROBE_TIMEOUT_MS = 15_000;
6450
+
6451
+ /** Where the proxy reads the include, in every venue's mount (compose, the Quadlet unit, the hand-started recipe). */
6452
+ export const ENDPOINTS_INCLUDE_IN_PROXY = "/etc/pi-dispatch/model-endpoints.conf";
6453
+
6454
+ /**
6455
+ * The tunnelled endpoint probes' in-container script: the runner's route (`loadPiThenRestore`), then one `GET`. It is
6456
+ * `egressCanaryScript` with two differences. It sends a GET, the method a model list is read with. And it reports the
6457
+ * PROXY's answer to the CONNECT when there is one: a refused or failed tunnel reaches the caller only as an error whose
6458
+ * cause chain carries undici's `Proxy response (<status>) !== 200 when HTTP Tunneling` (measured on 2026-09-30, M9:
6459
+ * a job sees squid's 503, "allowed but nothing answered", and its 403, "not declared", alike as "Connection error"). So
6460
+ * the script walks the chain for that text, and doctor shows the status. The text is pinned at the 0.99.1 pin's undici
6461
+ * 8.10.2 (`lib/dispatcher/proxy-agent.js`) by a test that runs this script against a fake proxy.
6462
+ *
6463
+ * Prints ONE line: `reached <status>` (exit 0), `tunnel <status>` (exit 3), `blocked <code>` (exit 6: no answer, and no
6464
+ * proxy status, as for a request that went around the proxy or timed out), `error ...` (exit 5, or 4 for an image with
6465
+ * no runner module).
6466
+ */
6467
+ export function egressEndpointScript(url, { timeoutMs = ENDPOINT_PROBE_TIMEOUT_MS } = {}) {
6468
+ return `import(${JSON.stringify(EGRESS_CANARY_RUNNER_MODULE)}).then(m=>m.loadPiThenRestore(),e=>{console.log("error",e.code??e.message);process.exit(e.code==="ERR_MODULE_NOT_FOUND"?${EGRESS_CANARY_STALE_RUNNER}:5)}).then(()=>fetch(${JSON.stringify(url)},{signal:AbortSignal.timeout(${Number(timeoutMs)})}).then(r=>{console.log("reached",r.status);process.exit(0)},e=>{let c=e,s=null;for(let i=0;c&&i<8&&s===null;i++){const m=/Proxy response \\((\\d{3})\\) !== 200 when HTTP Tunneling/.exec(String(c.message??""));if(m)s=m[1];c=c.cause}if(s!==null){console.log("tunnel",s);process.exit(3)}console.log("blocked",e.cause?.code??e.name??"error");process.exit(6)}),e=>{console.log("error",e.message);process.exit(5)})`;
6469
+ }
6470
+
6471
+ /**
6472
+ * What one endpoint probe printed, as `{ kind, status, detail }`, or null. Read from the LAST line of that shape, since
6473
+ * a runtime may print its own lines first. `detail` is kept only when it is a short code word: the plain probe's is the
6474
+ * `X-Squid-Error` header, which a server that answered instead of the proxy writes, and a terminal is not handed that.
6475
+ */
6476
+ function endpointProbeReading(answer) {
6477
+ const lines = String(answer?.output ?? answer?.stdout ?? "").split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
6478
+ for (let i = lines.length - 1; i >= 0; i--) {
6479
+ const m = /^(reached|tunnel|blocked|error)(?: (\d{3}))?(?: (\S+))?/.exec(lines[i]);
6480
+ if (m) return { kind: m[1], status: m[2] ? Number(m[2]) : null, detail: m[3] && /^[A-Za-z_]{1,40}$/.test(m[3]) ? m[3] : "" };
6481
+ }
6482
+ return null;
6483
+ }
6484
+
6485
+ /** `host:port` as a URL writes it (an IPv6 host is stored in brackets already). */
6486
+ function endpointAddress(host, port) {
6487
+ return `${host}:${port}`;
6488
+ }
6489
+
6490
+ /** Endpoints in id order, the order the include is rendered in. */
6491
+ function endpointsById(endpoints) {
6492
+ return [...endpoints].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
6493
+ }
6494
+
6495
+ /** The exits each probe's script ends a READING with; any other exit is a container that did not run it. */
6496
+ const ENDPOINT_PROBE_READ_EXITS = Object.freeze({ models: [0, 3, 6], nextport: [0, 3, 6], plain: [0, 3] });
6497
+
6498
+ /**
6499
+ * One endpoint probe's line. Every line names the endpoint id and shows the proxy's status, and every one is warn-tier,
6500
+ * as the canary's are: each needs the network to answer. What passes, and nothing else:
6501
+ * - `models`: the server answered 200 through the tunnel;
6502
+ * - `nextport`: the proxy refused the TUNNEL with 403. A 503 there means the proxy let the CONNECT through and found
6503
+ * nothing listening, so the rule is not port-exact; a server's own 403 through a tunnel is the same finding;
6504
+ * - `plain`: a 403 carrying squid's `X-Squid-Error: ERR_ACCESS_DENIED`. A bare 403 may be the server's own answer to a
6505
+ * request the proxy forwarded.
6506
+ */
6507
+ function endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next }) {
6508
+ const head = `${venue.prefix}Model endpoint ${endpoint.id}`;
6509
+ const at = endpointAddress(endpoint.host, endpoint.port);
6510
+ const reading = endpointProbeReading(answer);
6511
+ const tag = { id: endpoint.id, probe: slug, kind: reading?.kind ?? null, status: reading?.status ?? null };
6512
+ if (!ENDPOINT_PROBE_READ_EXITS[slug].includes(answer?.code) || !reading || reading.kind === "error") {
6513
+ return {
6514
+ ok: false,
6515
+ warn: true,
6516
+ label: `${head}: the ${slug} probe did not run (${answer?.code === null || answer?.code === undefined ? `${bin} run did not finish` : `${bin} run exited ${answer.code}`})`,
6517
+ fix: "re-run doctor; if it persists, run the job image by hand to see why a container on this network will not start",
6518
+ endpointProbe: tag,
6519
+ };
6520
+ }
6521
+ const reload = reloadCommand(bin, proxy);
6522
+ if (slug === "models") {
6523
+ if (reading.kind === "reached" && reading.status === 200) return { ok: true, warn: true, label: `${head} answers through the proxy (GET http://${at}/v1/models through a CONNECT tunnel: 200)`, endpointProbe: tag };
6524
+ // The SERVER refused the request for want of a key (vLLM --api-key, a gateway in front): the route and the rule are
6525
+ // proved, since only an open tunnel carries the server's own answer. Said apart from a fault, still warn-tier.
6526
+ if (reading.kind === "reached" && (reading.status === 401 || reading.status === 403)) {
6527
+ return {
6528
+ ok: false,
6529
+ warn: true,
6530
+ label: `${head}: the route through the proxy works (the tunnel to ${at} opened), and the server wants a key (GET /v1/models answered ${reading.status})`,
6531
+ fix: "declare it without \"keyless\" and give its provider its key, as for any provider that needs one; nothing about the egress proxy needs changing",
6532
+ endpointProbe: tag,
6533
+ };
6534
+ }
6535
+ // A server that took the connection and never answered is not one to start: say what was seen.
6536
+ const timedOut = reading.kind === "blocked" && reading.detail === "TimeoutError";
6537
+ const why =
6538
+ reading.kind === "tunnel"
6539
+ ? reading.status === 503
6540
+ ? `the proxy allowed the tunnel to ${at} and answered 503, so nothing answered there`
6541
+ : reading.status === 403
6542
+ ? `the proxy refused the tunnel to ${at} with 403, so the rules it runs do not allow this endpoint`
6543
+ : `the proxy answered ${reading.status} to the CONNECT to ${at}`
6544
+ : reading.kind === "reached"
6545
+ ? `the tunnel to ${at} was let through, but GET /v1/models answered ${reading.status}, not 200`
6546
+ : timedOut
6547
+ ? `it accepted the connection but did not answer /v1/models within ${ENDPOINT_PROBE_TIMEOUT_MS / 1000} s`
6548
+ : `no answer came back through the proxy (${reading.detail || "no reason given"})`;
6549
+ const fix =
6550
+ reading.kind === "tunnel" && reading.status === 403
6551
+ ? `\`pi-dispatch egress render\` in the deployment folder, then \`${reload}\`; the include line above says whether the proxy holds the declared rules`
6552
+ : reading.kind === "reached"
6553
+ ? "check that this port is the model server's and that it serves the OpenAI-compatible API under /v1 (Ollama, llama-server, vLLM and LM Studio do)"
6554
+ : timedOut
6555
+ ? "the server is there but stuck or busy (a model still loading, every slot taken): check its own log, then re-run doctor"
6556
+ : bin === "podman"
6557
+ ? `start the model server, bound to this host's LAN address or 0.0.0.0 (never 127.0.0.1 alone), and declare a server on this host as host.containers.internal (docs/egress.md, "Local model servers"). A job meets this as "Connection error"`
6558
+ : `start the model server, and check it listens where the proxy reaches it (docs/egress.md, "Local model servers"; on Docker Engine that is 172.17.0.1 or 0.0.0.0, never 127.0.0.1 alone). A job meets this as "Connection error"`;
6559
+ return { ok: false, warn: true, label: `${head} does NOT answer: ${why}`, fix, endpointProbe: tag };
6560
+ }
6561
+ const wider = `the proxy's rules are wider than model-endpoints.conf renders: compare the include inside it (\`${bin} exec ${proxy} cat ${ENDPOINTS_INCLUDE_IN_PROXY}\`) with \`pi-dispatch egress render\`'s, check egress-allowlist.conf does not list ${endpoint.host}, then \`${reload}\``;
6562
+ if (slug === "nextport") {
6563
+ const nextAt = endpointAddress(endpoint.host, next);
6564
+ if (reading.kind === "tunnel" && reading.status === 403) return { ok: true, warn: true, label: `${head}'s rule is port-exact (a CONNECT to ${nextAt}, a port nobody declared, got the proxy's 403)`, endpointProbe: tag };
6565
+ const what =
6566
+ reading.kind === "tunnel"
6567
+ ? reading.status === 503
6568
+ ? `the proxy let a CONNECT to ${nextAt}, a port nobody declared, through and answered 503 (nothing listens there)`
6569
+ : `the proxy answered ${reading.status} to a CONNECT to ${nextAt}, not its 403`
6570
+ : reading.kind === "reached"
6571
+ ? `a CONNECT to ${nextAt}, a port nobody declared, was let through, and something there answered ${reading.status}`
6572
+ : `a CONNECT to ${nextAt} got no answer from the proxy (${reading.detail || "no reason given"}), so the refusal was not seen`;
6573
+ return { ok: false, warn: true, label: `${head}'s rule is NOT shown to be port-exact: ${what}`, fix: wider, endpointProbe: tag };
6574
+ }
6575
+ if (reading.status === 403 && reading.detail.startsWith("ERR_ACCESS_DENIED")) return { ok: true, warn: true, label: `${head} admits no plain forward request (GET http://${at}/v1/models without a tunnel got the proxy's 403 ${reading.detail})`, endpointProbe: tag };
6576
+ return {
6577
+ ok: false,
6578
+ warn: true,
6579
+ label: `${head} admits a plain forward request: GET http://${at}/v1/models without a tunnel got ${reading.status ?? "no status"}${reading.detail ? ` ${reading.detail}` : ""}, not the proxy's 403 ERR_ACCESS_DENIED`,
6580
+ fix: wider,
6581
+ endpointProbe: tag,
6582
+ };
6583
+ }
6584
+
6585
+ /**
6586
+ * The port the `nextport` probe asks for (issue #503): the first one above the declared port that no declaration for the
6587
+ * same host uses, wrapping below the declared port past 65535. Never 443 or 80, which the allowlist admits for a listed
6588
+ * host (a CONNECT to 443, plain HTTP to 80): a host that is also listed would pass there by that rule, and the line would
6589
+ * blame the endpoint's. Skipped whether or not the host is listed, which costs nothing and reads no file. The port
6590
+ * itself is shown on the line. `null` only when every other port of the host is declared, which no real file does.
6591
+ */
6592
+ export function undeclaredPortNear(endpoint, endpoints) {
6593
+ const taken = new Set((endpoints ?? []).filter((e) => e.host === endpoint.host).map((e) => e.port));
6594
+ const free = (p) => p !== endpoint.port && !taken.has(p) && p !== 443 && p !== 80;
6595
+ for (let p = endpoint.port + 1; p <= 65535; p++) if (free(p)) return p;
6596
+ for (let p = endpoint.port - 1; p >= 1; p--) if (free(p)) return p;
6597
+ return null;
6598
+ }
6599
+
6600
+ /**
6601
+ * The three probes for each declared endpoint (issue #503), in id order, on the canary's network and through the
6602
+ * canary's own runner, so the teardown that removes the canary's probes removes these too (`unfinished` is the
6603
+ * canary's list). Each container is built by `egressCanaryProbeArgs`, a job's shape, with the job's plain-HTTP proxy
6604
+ * variable as well on docker: the runner's dispatcher sends an `http://` origin to HTTP_PROXY, which docker's canary
6605
+ * argv does not otherwise carry (podman's is a job's environment and has it).
6606
+ */
6607
+ async function runEndpointProbes({ probe, bin, pid, network, proxy, image, user, endpoints, venue, unfinished }) {
6608
+ const checks = [];
6609
+ for (const endpoint of endpointsById(endpoints)) {
6610
+ const next = undeclaredPortNear(endpoint, endpoints);
6611
+ const models = `http://${endpointAddress(endpoint.host, endpoint.port)}/v1/models`;
6612
+ const nextUrl = `http://${endpointAddress(endpoint.host, next)}/v1/models`;
6613
+ const runs = [
6614
+ [ENDPOINT_PROBE_SLUGS[0], models, egressEndpointScript(models)],
6615
+ ...(next === null ? [] : [[ENDPOINT_PROBE_SLUGS[1], nextUrl, egressEndpointScript(nextUrl)]]),
6616
+ [ENDPOINT_PROBE_SLUGS[2], models, egressCanaryPlainScript(models, { proxyUrl: egressProxyUrl(proxy) })],
6617
+ ];
6618
+ for (const [slug, url, script] of runs) {
6619
+ const name = egressEndpointProbe(slug, endpoint.id, pid);
6620
+ const answer = await probe(egressCanaryProbeArgs({ bin, slug, name, pid, network, proxy, image, url, user, script, httpProxy: true }));
6621
+ if (answer?.code === null && answer.ended !== "error") unfinished.push(name);
6622
+ checks.push(endpointProbeCheck({ slug, endpoint, answer, venue, bin, proxy, next }));
6623
+ }
6624
+ }
6625
+ return checks;
6626
+ }
6627
+
6628
+ /**
6629
+ * The include as the RUNNING proxy sees it, against what the declaration renders (issue #503). Read inside the
6630
+ * container, never from this folder's file: a single-file bind mount holds the inode, so a file replaced by a rename
6631
+ * leaves the container on the OLD one, and a reload then silently loads the old rules (measured on 2026-09-30, M3, on
6632
+ * Docker and Podman on Linux). `answer` is the `<bin> exec <proxy> cat` capture; `recreate` is the venue's way to start
6633
+ * the proxy on the file anew.
6634
+ */
6635
+ function endpointsIncludeCheck({ answer, endpoints, bin, proxy, prefix, recreate, mounted = null }) {
6636
+ const read = `${bin} exec ${proxy} cat ${ENDPOINTS_INCLUDE_IN_PROXY}`;
6637
+ if (answer?.code !== 0) {
6638
+ return {
6639
+ ok: false,
6640
+ warn: true,
6641
+ label: `${prefix}Model endpoints: the include inside the running proxy could not be read (\`${read}\` ${answer?.code === null || answer?.code === undefined ? "did not finish" : `exited ${answer.code}`})`,
6642
+ fix: `the proxy must mount this deployment's model-endpoints.conf at ${ENDPOINTS_INCLUDE_IN_PROXY}, as the shipped proxy does; run the command yourself to see why it fails`,
6643
+ };
6644
+ }
6645
+ const ids = endpointsById(endpoints).map((e) => e.id).join(", ");
6646
+ // The file the proxy mounts, by its bind source, against what the container holds, FIRST (issue #503, gate rounds 1
6647
+ // and 2): a file replaced by a rename on the host leaves the container on the old inode, so the two differ, and no
6648
+ // render or reload reaches the proxy until it is started on the file anew. Without this, a hand-replaced file whose
6649
+ // JSON did not change read ✓. `mounted` is null when there is no such bind or it cannot be read here: no compare.
6650
+ if (mounted && mounted.text !== String(answer.output)) {
6651
+ return {
6652
+ ok: false,
6653
+ label: `${prefix}Model endpoints: the proxy is mounted on a replaced file: the file it mounts (${mounted.path}) differs from what ${proxy} reads at ${ENDPOINTS_INCLUDE_IN_PROXY}, so it was replaced (a rename, an editor's save) rather than written in place, and no reload reaches the proxy`,
6654
+ fix: `recreate the proxy so it mounts the file anew: ${recreate}. From then on, change the file only with \`pi-dispatch egress render\`, which writes in place`,
6655
+ };
6656
+ }
6657
+ if (String(answer.output) === renderEndpointsInclude(endpoints)) {
6658
+ return { ok: true, label: `${prefix}Model endpoints: the include inside the running proxy matches model-endpoints.json (${ids})` };
6659
+ }
6660
+ return {
6661
+ ok: false,
6662
+ label: `${prefix}Model endpoints: the include inside the running proxy (${ENDPOINTS_INCLUDE_IN_PROXY} in ${proxy}) does not match model-endpoints.json (${ids}), so a reload would not load the declared rules`,
6663
+ fix: `\`pi-dispatch egress render\` in the deployment folder, then \`${reloadCommand(bin, proxy)}\`. If the render says the file already matches and this line stays, the proxy holds an earlier copy of a file that was replaced rather than written in place, which a reload never sees: ${recreate}`,
6664
+ };
6665
+ }
6666
+
6667
+ /**
6668
+ * The text of the host file a proxy bind-mounts at the include's path (issue #503, gate round 2), from its inspected
6669
+ * `[{ type, source, destination }]`, or null: no such bind, or a source this host cannot read (Docker Desktop reports the
6670
+ * real macOS path, so it reads there), and then the replaced-file compare is skipped rather than guessed.
6671
+ */
6672
+ function readIncludeBindSource(mounts) {
6673
+ const bind = (Array.isArray(mounts) ? mounts : []).find((m) => m?.type === "bind" && m.destination === ENDPOINTS_INCLUDE_IN_PROXY);
6674
+ if (!bind || typeof bind.source !== "string" || !isAbsolute(bind.source)) return null;
6675
+ const text = readTextOrNull(bind.source);
6676
+ return text === null ? null : { path: bind.source, text };
6677
+ }
6678
+
6679
+ /** `{{json .Mounts}}` as `[{ type, source, destination }]`, the shape `parseProxyState` gives docker's; `[]` when unreadable. */
6680
+ function mountsFromJson(output) {
6681
+ try {
6682
+ const list = JSON.parse(String(output ?? "").trim());
6683
+ return (Array.isArray(list) ? list : []).filter((m) => m && typeof m === "object").map((m) => ({ type: String(m.Type ?? ""), source: String(m.Source ?? ""), destination: String(m.Destination ?? "") }));
6684
+ } catch {
6685
+ return [];
6686
+ }
6687
+ }
6688
+
6689
+ /** A regular file's text, or null for anything else or any failure: a compare that has nothing to compare says nothing. */
6690
+ function readTextOrNull(path) {
6691
+ try {
6692
+ return statSync(path).isFile() ? readFileSync(path, "utf8") : null;
6693
+ } catch {
6694
+ return null;
6695
+ }
6696
+ }
6697
+
6698
+ /**
6699
+ * The route from the egress proxy to each endpoint's host on THIS runtime (issue #503), from the measured table
6700
+ * (`hostRouteFor`, `HOST_ROUTES` in backends.mjs). Only a REFUTED route fails: the proxy cannot reach that host here,
6701
+ * whatever is listening. A measured route is a ✓ saying what it needs. One that is not measured on this runtime, or is
6702
+ * another machine's ordinary outbound route, is said as information and NOT warned about: nearly every runtime version
6703
+ * is unmeasured, so a ⚠ there would be on every run, and the endpoint probe is what proves the route.
6704
+ */
6705
+ function endpointRouteChecks({ runtime, endpoints, prefix, proof }) {
6706
+ return endpointsById(endpoints).map((endpoint) => {
6707
+ const { status, sentence } = hostRouteFor(runtime, endpoint.host);
6708
+ const head = `${prefix}Model endpoint ${endpoint.id} (${endpointAddress(endpoint.host, endpoint.port)})`;
6709
+ if (status === HOST_ROUTE_REFUTED) {
6710
+ return { ok: false, label: `${head} has no route from the egress proxy on this runtime: ${sentence}`, fix: "declare the host this runtime reaches (docs/backends.md lists the measured routes per venue: host.docker.internal on Docker, host.containers.internal on Podman), then `pi-dispatch egress render` and the reload it prints" };
6711
+ }
6712
+ if (status === HOST_ROUTE_WORKS) return { ok: true, label: `${head}: ${sentence}` };
6713
+ // Another machine: an ordinary route out, said as such, never prefixed "not measured" before a sentence that names
6714
+ // where it WAS measured.
6715
+ if (status === HOST_ROUTE_LAN) return { ok: true, label: `${head}: ${sentence} ${proof}` };
6716
+ // A rootless Podman that did not say which helper it runs: the table's rows are per helper, so none applies. Said as
6717
+ // what this Podman did not report, never as a fault of the host or the endpoint.
6718
+ if (runtime?.backend === "podman" && runtime.rootless === true && typeof runtime.helper !== "string") {
6719
+ return { ok: true, label: `${head}: the rootless network helper was not reported by this Podman, so the route is not judged here. ${proof}` };
6720
+ }
6721
+ return { ok: true, label: `${head}: the route from the proxy is not measured here. ${sentence} ${proof}` };
6722
+ });
6723
+ }
6724
+
6725
+ /**
6726
+ * The runtime `hostRouteFor` reads, from the `docker info` answer doctor already holds (`makeDaemonFactsReader`), or
6727
+ * null. Docker Desktop is the daemon that calls its OS `Docker Desktop`, as the job-user line reads it; rootful Podman
6728
+ * through its Docker API is `podman`. Nothing else is asked: a field doctor does not have stays missing and the route
6729
+ * reads unmeasured.
6730
+ */
6731
+ function routeRuntimeFromFacts(answer, platform) {
6732
+ const facts = answer?.answered === true ? answer.facts : null;
6733
+ if (!facts) return null;
6734
+ const version = facts.serverVersion ?? "";
6735
+ if (facts.podman === true) return { backend: "podman", version, ...(typeof facts.rootless === "boolean" ? { rootless: facts.rootless } : {}) };
6736
+ const desktop = facts.os === "Docker Desktop";
6737
+ return { backend: "docker", version, desktop, ...(desktop ? { os: platform } : {}) };
6738
+ }
6739
+
6740
+ /**
6741
+ * The rootless network helper this account is running (`slirp4netns` or `pasta`), from `observeRootlessNetns`, or null
6742
+ * when none runs, the read fails, or two kinds run at once. Podman 4.9.3's `podman info` names no helper, so this is how
6743
+ * its slirp4netns is known there; with no bridge container running there is no helper to see, and the route reads
6744
+ * unjudged rather than guessed from the version.
6745
+ */
6746
+ function runningNetnsHelper({ fs, euid, runRoot }) {
6747
+ if (!fs || !Number.isInteger(euid)) return null;
6748
+ try {
6749
+ const seen = observeRootlessNetns({ fs, euid, runRoot });
6750
+ const kinds = [...new Set((seen.helpers ?? []).map((h) => h.kind))];
6751
+ return kinds.length === 1 ? kinds[0] : null;
6752
+ } catch {
6753
+ return null;
6754
+ }
6755
+ }
6756
+
6757
+ /** The interface names whose addresses are a runtime's own bridges or a VM's, never this host's LAN address. */
6758
+ const NOT_LAN_INTERFACE = /^(?:docker|br-|virbr|veth|cni|podman|flannel|cali|vmnet|vboxnet|utun|bridge|lima)/;
6759
+
6760
+ /**
6761
+ * This host's own IPv4 LAN addresses for `hostRouteFor` (`hostAddresses`), or undefined for none: plain dotted quads,
6762
+ * not loopback, not link-local, and not on a container bridge (docker0, a compose `br-`, podman's), whose address a
6763
+ * declaration naming it would reach by another route than the own-address row was measured on.
6764
+ */
6765
+ export function lanIPv4Addresses(interfaces) {
6766
+ const found = [];
6767
+ for (const [name, list] of Object.entries(interfaces ?? {})) {
6768
+ if (NOT_LAN_INTERFACE.test(name) || !Array.isArray(list)) continue;
6769
+ for (const a of list) {
6770
+ if ((a?.family !== "IPv4" && a?.family !== 4) || a.internal === true || typeof a.address !== "string") continue;
6771
+ if (!/^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/.test(a.address) || a.address.startsWith("169.254.") || a.address.startsWith("127.")) continue;
6772
+ if (!found.includes(a.address)) found.push(a.address);
6773
+ }
6774
+ }
6775
+ return found.length > 0 ? found : undefined;
6776
+ }
6777
+
6778
+ /** Whether a host as `baseUrlTarget` gives it is loopback or host-local from inside a job (`PROXY_LOCAL_ADDRESSES`). */
6779
+ function loopbackTarget(host) {
6780
+ if (host.startsWith("[") && host.endsWith("]")) return isProxyLocalHost("ipv6", host.slice(1, -1));
6781
+ if (/^\d{1,3}(?:\.\d{1,3}){3}$/.test(host)) return isProxyLocalHost("ipv4", host);
6782
+ return isProxyLocalHost("name", host);
6783
+ }
6784
+
6785
+ /**
6786
+ * The overlay models.json's models whose effective baseUrl (the model's own, else its provider's) names localhost or
6787
+ * a loopback literal (issue #503), as `provider/model (host:port)` strings. Inside a job that address is the job's own
6788
+ * container, egress on or off, so no job reaches the server. A provider that lists no models is named by itself.
6789
+ */
6790
+ export function overlayLoopbackModels(models) {
6791
+ const providers = models?.providers;
6792
+ if (providers === null || typeof providers !== "object" || Array.isArray(providers)) return [];
6793
+ const found = [];
6794
+ for (const [name, entry] of Object.entries(providers)) {
6795
+ if (entry === null || typeof entry !== "object" || Array.isArray(entry)) continue;
6796
+ const listed = Array.isArray(entry.models) ? entry.models.filter((m) => m !== null && typeof m === "object" && typeof m.id === "string") : [];
6797
+ const check = (baseUrl, who) => {
6798
+ const target = baseUrlTarget(baseUrl);
6799
+ if (target && loopbackTarget(target.host)) found.push(`${who} (${target.host}:${target.port})`);
6800
+ };
6801
+ if (listed.length === 0) check(entry.baseUrl, quotedShown(name));
6802
+ for (const m of listed) check(typeof m.baseUrl === "string" ? m.baseUrl : entry.baseUrl, `${quotedShown(name)}/${quotedShown(m.id)}`);
6803
+ }
6804
+ return found;
6805
+ }
6806
+
6807
+ /** The two host aliases a model server on this machine is declared by, never listed in the allowlist. */
6808
+ const HOST_ALIASES = Object.freeze(["host.docker.internal", "host.containers.internal"]);
6809
+
6810
+ /**
6811
+ * Host aliases the allowlist admits (issue #503), from its text, as `[{ alias, entry }]` with the first entry that admits
6812
+ * each: squid's `dstdomain` file, one entry per word, `#` comments, and an entry with a leading dot admitting that domain
6813
+ * and every name under it, so `.internal` and `.docker.internal` admit `host.docker.internal` as surely as the name
6814
+ * itself. Admitting one opens that host's port 443 by CONNECT and its port 80 by plain HTTP, and no model server's port,
6815
+ * which is the confusion this names.
6816
+ */
6817
+ export function allowlistHostAliases(text) {
6818
+ const found = new Map();
6819
+ for (const line of String(text ?? "").split(/\r?\n/)) {
6820
+ for (const word of line.replace(/#.*/, "").trim().split(/\s+/)) {
6821
+ const entry = word.toLowerCase();
6822
+ if (entry === "") continue;
6823
+ for (const alias of HOST_ALIASES) {
6824
+ const admits = entry.startsWith(".") ? alias === entry.slice(1) || alias.endsWith(entry) : alias === entry;
6825
+ if (admits && !found.has(alias)) found.set(alias, word);
6826
+ }
6827
+ }
6828
+ }
6829
+ return HOST_ALIASES.filter((a) => found.has(a)).map((alias) => ({ alias, entry: found.get(alias) }));
6830
+ }
6831
+
4855
6832
  /**
4856
6833
  * The worker and receiver services installed for THIS folder, as `[{ which, source, setup }]` (`setup` null for a unit
4857
6834
  * rendered without `--env-setup`): an installed unit whose WorkingDirectory is this folder (systemd, launchd), or the
@@ -5219,15 +7196,23 @@ function buildScriptsOf(packageDir, fileExists) {
5219
7196
  function runCmdCapture(spawn, cmd, args, opts = {}) {
5220
7197
  // `stdoutOnly` for a caller that PARSES the answer (issue #345): Podman's docker emulation prints a banner on
5221
7198
  // stderr ("Emulate Docker CLI using podman ..."), which a merged capture puts in front of the value.
5222
- const { timeoutMs = 30000, stdoutOnly = false } = opts;
7199
+ // `input` for a caller that hands the child a secret (issue #521): written to its stdin and closed, so the value is in
7200
+ // neither argv nor any environment. Without it stdin stays ignored, as every other caller wants.
7201
+ const { timeoutMs = 30000, stdoutOnly = false, input } = opts;
5223
7202
  return new Promise((resolve) => {
5224
7203
  let child;
5225
7204
  try {
5226
- child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"], ...(opts.env ? { env: opts.env } : {}), ...(opts.cwd ? { cwd: opts.cwd } : {}) });
7205
+ child = spawn(cmd, args, { stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"], ...(opts.env ? { env: opts.env } : {}), ...(opts.cwd ? { cwd: opts.cwd } : {}) });
5227
7206
  } catch {
5228
7207
  resolve({ code: null, output: "" });
5229
7208
  return;
5230
7209
  }
7210
+ if (input !== undefined) {
7211
+ // A child that exits before reading (a missing image, a CLI that refuses the argv) closes the pipe under the
7212
+ // write: EPIPE is that child's answer, which its exit code already carries, not doctor's crash.
7213
+ child.stdin?.on("error", () => {});
7214
+ child.stdin?.end(input);
7215
+ }
5231
7216
  let output = "";
5232
7217
  let done = false;
5233
7218
  const finish = (code) => {
@@ -5302,6 +7287,63 @@ async function defaultReadHosts(url) {
5302
7287
  }
5303
7288
  }
5304
7289
 
7290
+ /**
7291
+ * Has any host reserved dollars on this Valkey recently? For `fleetDollarChecks`, asked only when a peer publishes no
7292
+ * `fpUsd` and nothing else says dollar caps are in use. BEST EFFORT, and it says so wherever it is described:
7293
+ * 1. EXISTS on the deployment's current day, week and month keys (`budget:usd:YYYY-MM-DD`, `:w:<Monday>`,
7294
+ * `:m:YYYY-MM`, `budget.mjs`'s key functions at `now`), the counters any host with a dollar window writes;
7295
+ * 2. else a bounded SCAN for `budget:usd:*` (at most ten passes of COUNT 1000), for scope and model windows.
7296
+ * The whole read is bounded by `deadlineMs`, so a large keyspace or a hung server answers "not seen". Any fault is
7297
+ * false: the answer can add a warning, never remove one. A host whose only dollar setting is the per-job cap writes no
7298
+ * counter at all, so it is not found this way (PR #551's review, round 2).
7299
+ */
7300
+ export async function dollarKeysExistWith(client, { now = () => new Date(), deadlineMs = 2000 } = {}) {
7301
+ let timer;
7302
+ const deadline = new Promise((resolve) => {
7303
+ // NOT unref'd, `host-registry.mjs`'s reason: an unref'd timer does not fire when the hung call is the last thing
7304
+ // holding the loop, which is the case it is for. It is cleared as soon as the answer is in.
7305
+ timer = setTimeout(() => resolve(false), deadlineMs);
7306
+ });
7307
+ const ask = (async () => {
7308
+ const at = now();
7309
+ const current = [dayKey(at, DOLLAR_KEY_PREFIX), weekKey(at, DOLLAR_KEY_PREFIX), monthKey(at, DOLLAR_KEY_PREFIX)];
7310
+ if (Number(await client.exists(...current)) > 0) return true;
7311
+ let cursor = "0";
7312
+ for (let pass = 0; pass < 10; pass++) {
7313
+ const [next, keys] = await client.scan(cursor, "MATCH", `${DOLLAR_KEY_PREFIX}:*`, "COUNT", 1000);
7314
+ if (Array.isArray(keys) && keys.length > 0) return true;
7315
+ cursor = String(next);
7316
+ if (cursor === "0") return false;
7317
+ }
7318
+ return false;
7319
+ })();
7320
+ ask.catch(() => {});
7321
+ try {
7322
+ return (await Promise.race([ask, deadline])) === true;
7323
+ } catch {
7324
+ return false;
7325
+ } finally {
7326
+ clearTimeout(timer);
7327
+ }
7328
+ }
7329
+
7330
+ /** `dollarKeysExistWith` over a fail-fast client on `url`, always disconnected. */
7331
+ export async function defaultDollarKeysExist(url) {
7332
+ try {
7333
+ const { makeRedisClient } = await import("./connection.mjs");
7334
+ const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
7335
+ client.on("error", () => {});
7336
+ try {
7337
+ await client.connect();
7338
+ return await dollarKeysExistWith(client);
7339
+ } finally {
7340
+ client.disconnect();
7341
+ }
7342
+ } catch {
7343
+ return false;
7344
+ }
7345
+ }
7346
+
5305
7347
  async function defaultProbeValkey(url) {
5306
7348
  const { makeRedisClient } = await import("./connection.mjs");
5307
7349
  const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
@@ -5866,7 +7908,8 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
5866
7908
  checks.push({
5867
7909
  ok: false,
5868
7910
  label: `podman: job image is not in this account's Podman store (${jobImage})`,
5869
- fix: `pull or load it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: ${jobImage === "pi-job:latest" ? PODMAN_JOB_IMAGE_PULL : `podman pull ${jobImage}`} -- jobs run with --pull=never, so the worker never fetches it`,
7911
+ // A name no pull is offered for (issue #523, round 2) is built or loaded, never pulled.
7912
+ fix: `${jobImage === "pi-job:latest" || pullOffered(jobImage) ? "pull or load" : "build or load"} it AS THE WORKER'S ACCOUNT, since rootless Podman keeps one image store per account: ${jobImageFix("podman", jobImage)} -- jobs run with --pull=never, so the worker never fetches it`,
5870
7913
  ...(!localUsed && jobImage === "pi-job:latest"
5871
7914
  ? {
5872
7915
  fixAction: {
@@ -5920,6 +7963,10 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
5920
7963
  let proxyRunning = null;
5921
7964
  // Issue #458: why `--live` must not tear a network down under the proxy here, or null where nothing stops it.
5922
7965
  let keeperBlocked = null;
7966
+ // Issue #503: the declared model endpoints `--live` probes on this venue, `[]` for none; and whether the rules the
7967
+ // proxy runs include their file (only the shipped proxy's account copy is read; another proxy must include it).
7968
+ let endpoints = [];
7969
+ let endpointRulesInclude = true;
5923
7970
  if (armed !== false) {
5924
7971
  // `.State.Status` (issue #453): only `running` carries traffic; Podman reports paused and between-restarts states
5925
7972
  // with their own words (measured on 4.9.3 and 5.8.1).
@@ -5952,7 +7999,7 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
5952
7999
  // an image copies its layers (27 to 32 s measured), and `--live` is where this venue already starts containers
5953
8000
  // built like a job's.
5954
8001
  if (proxyRunning && seams.live !== true) {
5955
- checks.push({ ok: false, warn: true, label: "podman: the egress allowlist is read back by `pi-dispatch doctor --live` on this venue, not by this run", fix: "run `pi-dispatch doctor --live`: its egress canary runs two containers built like a podman job's (the job user, --userns=keep-id, the venue's pinned flags) on a job-shaped --internal network under this account's Podman, one that must reach the provider through the proxy and one that must not reach an unlisted host" });
8002
+ checks.push({ ok: false, warn: true, label: "podman: the egress allowlist is read back by `pi-dispatch doctor --live` on this venue, not by this run", fix: "run `pi-dispatch doctor --live`: its egress canary runs three containers built like a podman job's (the job user, --userns=keep-id, the venue's pinned flags) on a job-shaped --internal network under this account's Podman, one that must reach the provider through the proxy, one that must not reach an unlisted host, and one that must not get plain HTTP through to a listed host off port 80" });
5956
8003
  }
5957
8004
  // Issue #484: the account-owned copy of the rules the shipped unit mounts, against the installed package's. `service
5958
8005
  // install` compares it on every run and replaces a differing one only under --force, and `up` installs nothing
@@ -5966,6 +8013,40 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
5966
8013
  refresh: "`pi-dispatch service install` lists it among what differs and `pi-dispatch service install --force` replaces it and restarts the proxy (squid reads its rules only at start); --force also replaces every other item that list names",
5967
8014
  });
5968
8015
  if (stale) checks.push(stale);
8016
+ // Issue #503: endpoints declared while the account copy predates the include, said as docker's venue says it.
8017
+ let rulesInclude = false;
8018
+ try {
8019
+ rulesInclude = rulesIncludeEndpoints(String((seams.readProxyConf ?? ((p) => readFileSync(p, "utf8")))(proxyConfCopyPath(home))));
8020
+ } catch {
8021
+ // No copy yet: `service install` writes the current rules, include and all.
8022
+ rulesInclude = true;
8023
+ }
8024
+ const declared = (seams.includeNeeds ? seams.includeNeeds({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform }).endpointsDeclared : endpointsDeclaredIn({ env, cwd: seams.cwd, fs: { readFileSync, existsSync }, platform: seams.platform ?? process.platform }));
8025
+ if (declared && !rulesInclude) checks.push({ ok: false, warn: true, label: `podman: ${rulesPredateEndpointsLine("podman")}`, fix: "a reload changes nothing here: the rules themselves must include the file, and `service install --force` writes them with the unit that mounts it" });
8026
+ endpointRulesInclude = rulesInclude;
8027
+ }
8028
+ // Issue #503: the declared model endpoints on this venue, as docker's section reads them. The route rows and the
8029
+ // include read here; the three probes per endpoint run with `--live`, beside the canary they share a network with.
8030
+ // None declared, or rules that predate the include (the line above says it all), is nothing more.
8031
+ const { declaredEndpoints = ({ env: e, cwd, platform: p }) => declaredEndpointsIn({ env: e, cwd, fs: { readFileSync, existsSync }, platform: p }), hostAddresses = () => lanIPv4Addresses(networkInterfaces()) } = seams;
8032
+ const declaredList = declaredEndpoints({ env, cwd: seams.cwd, platform: seams.platform ?? process.platform });
8033
+ if (declaredList.length > 0 && endpointRulesInclude) {
8034
+ endpoints = declaredList;
8035
+ // The helper as Podman 5 names it in `podman info` (`rootlessNetworkCmd`), else as this account's running rootless
8036
+ // network shows it (`observeRootlessNetns`, the record or the 4.x argv the worker's widening check reads), else
8037
+ // not known, which the route row says as such.
8038
+ const helper = info?.rootless === true ? (info.rootlessNetworkCmd ?? runningNetnsHelper({ fs: observationFs, euid: ids.euid, runRoot: info.runRoot })) : null;
8039
+ const runtime = info ? { backend: "podman", version: info.version ?? "", ...(typeof info.rootless === "boolean" ? { rootless: info.rootless } : {}), ...(helper ? { helper } : {}) } : null;
8040
+ const addresses = hostAddresses();
8041
+ checks.push(...endpointRouteChecks({ runtime: runtime && addresses ? { ...runtime, hostAddresses: addresses } : runtime, endpoints, prefix: "podman: ", proof: seams.live === true ? "The endpoint probes in this --live run are the proof." : "`pi-dispatch doctor --live` probes it, which is the proof." }));
8042
+ if (proxyRunning) {
8043
+ const inside = await runCmdCapture(spawn, "podman", ["exec", proxy, "cat", ENDPOINTS_INCLUDE_IN_PROXY], { stdoutOnly: true, timeoutMs: CANARY_STEP_TIMEOUT_MS });
8044
+ // The file the proxy actually mounts there, from its own inspect (the Quadlet unit mounts the deployment folder's,
8045
+ // which need not be the folder doctor runs in). Podman gives the source as a plain host path (4.9.3, 5.8.1).
8046
+ const mounts = await runCmdCapture(spawn, "podman", ["inspect", "--format", "{{json .Mounts}}", proxy], { stdoutOnly: true, timeoutMs: CANARY_STEP_TIMEOUT_MS });
8047
+ const mounted = mounts.code === 0 ? readIncludeBindSource(mountsFromJson(mounts.output)) : null;
8048
+ checks.push(endpointsIncludeCheck({ answer: inside, endpoints, bin: "podman", proxy, prefix: "podman: ", mounted, recreate: proxy === DEFAULT_EGRESS_PROXY ? "`systemctl --user restart pi-dispatch-egress-proxy.service` starts a new container on the file (a restart cuts running jobs' tunnels)" : `recreate ${proxy} so it mounts the file anew` }));
8049
+ }
5969
8050
  }
5970
8051
  const keeper = await netnsKeeperCheck(spawn, info?.version, { proxy, now: typeof seams.wallClock === "function" ? seams.wallClock : Date.now });
5971
8052
  keeperBlocked = keeper.keeperBlocked ?? null;
@@ -5974,7 +8055,7 @@ export async function podmanChecks(env, seams, { jobImage, jobImageNote = "" })
5974
8055
  if (keeperNetwork) checks.push(keeperNetwork);
5975
8056
  }
5976
8057
 
5977
- forLive = { run: true, user, relabel, info, imagePresent, egress: { armed, proxy, proxyRunning, keeperBlocked } };
8058
+ forLive = { run: true, user, relabel, info, imagePresent, egress: { armed, proxy, proxyRunning, keeperBlocked, endpoints } };
5978
8059
  return { checks, observed, relabel, imagesReadable, imageDigest, forLive };
5979
8060
  }
5980
8061
 
@@ -6473,8 +8554,12 @@ async function podmanEgressCanary({ podman, readInfo, run, image, pid, isAlive,
6473
8554
  if (podman.egress.proxyRunning !== true || podman.imagePresent !== true) return { checks, results: [] };
6474
8555
  // Announced, as `runLiveProbes` announces its own containers: these start before it, and a cold first keep-id start
6475
8556
  // can take half a minute each, which is a long silence on an operator's terminal.
6476
- announce(`starting ${CANARY_PROBE_SLUGS.map((slug) => egressCanaryProbe(slug, pid)).join(" and ")} from ${image} (as the job user ${podman.user}) on the --internal network ${egressCanaryNetwork(pid)}, with ${podman.egress.proxy} attached, to read the egress allowlist back; both are removed when the canary ends`);
6477
- const canary = await runEgressCanary({ run, bin: "podman", proxy: podman.egress.proxy, image, pid, user: podman.user, gate });
8557
+ const probes = CANARY_PROBE_SLUGS.map((slug) => egressCanaryProbe(slug, pid));
8558
+ // Issue #503: three more per declared model endpoint, said in a second sentence so the canary's own stays as it was.
8559
+ const endpoints = podman.egress.endpoints ?? [];
8560
+ const more = endpoints.length > 0 ? `; then three per declared model endpoint (${endpointsById(endpoints).map((e) => e.id).join(", ")}), named ${EGRESS_ENDPOINT_PROBE_PREFIX}<probe>-<id>-${pid}, removed the same way` : "";
8561
+ announce(`starting ${probes.slice(0, -1).join(", ")} and ${probes.at(-1)} from ${image} (as the job user ${podman.user}) on the --internal network ${egressCanaryNetwork(pid)}, with ${podman.egress.proxy} attached, to read the egress allowlist back; all three are removed when the canary ends${more}`);
8562
+ const canary = await runEgressCanary({ run, bin: "podman", proxy: podman.egress.proxy, image, pid, user: podman.user, gate, endpoints });
6478
8563
  return { checks: [...checks, ...canary.checks], results: canary.results };
6479
8564
  }
6480
8565