@edgehero/pi-dispatch 2.1.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/.env.example +41 -5
  2. package/README.md +11 -5
  3. package/deploy/docker-compose.yml +12 -0
  4. package/deploy/egress-proxy.conf +28 -3
  5. package/deploy/pi-dispatch-egress-proxy.container +8 -2
  6. package/package.json +8 -1
  7. package/src/allocation.mjs +731 -0
  8. package/src/backends.mjs +243 -0
  9. package/src/budget.mjs +40 -4
  10. package/src/cli.mjs +222 -11
  11. package/src/config.mjs +126 -5
  12. package/src/daemon-facts.mjs +3 -0
  13. package/src/deployment-venue.mjs +1 -0
  14. package/src/doctor.mjs +2261 -203
  15. package/src/dollar-budget.mjs +373 -0
  16. package/src/dollar-fingerprint.mjs +83 -0
  17. package/src/egress-cli.mjs +316 -0
  18. package/src/egress-proxy-state.mjs +35 -5
  19. package/src/egress.mjs +12 -0
  20. package/src/env-allowlist.mjs +107 -6
  21. package/src/env-file.mjs +194 -25
  22. package/src/envelope.mjs +413 -0
  23. package/src/exit-code.mjs +22 -0
  24. package/src/fleet-lease.mjs +85 -25
  25. package/src/get-token.mjs +16 -5
  26. package/src/git-dirty.mjs +67 -0
  27. package/src/github-app-setup.mjs +6 -3
  28. package/src/github-host.mjs +5 -3
  29. package/src/identity.mjs +2 -1
  30. package/src/image-preflight.mjs +98 -24
  31. package/src/image-ref.mjs +37 -0
  32. package/src/import-pi.mjs +4 -2
  33. package/src/index.mjs +407 -62
  34. package/src/init.mjs +18 -0
  35. package/src/job-id.mjs +26 -3
  36. package/src/live-probes.mjs +24 -9
  37. package/src/model-catalog.mjs +297 -0
  38. package/src/model-endpoints.mjs +649 -0
  39. package/src/model-ref.mjs +151 -0
  40. package/src/models-json.mjs +262 -0
  41. package/src/money.mjs +144 -0
  42. package/src/octokit-log.mjs +65 -0
  43. package/src/outbox-plan.mjs +218 -0
  44. package/src/outbox.mjs +29 -9
  45. package/src/output-cap.mjs +157 -0
  46. package/src/pause-windows.mjs +81 -2
  47. package/src/pi-model-loader.mjs +77 -0
  48. package/src/podman-stack.mjs +16 -3
  49. package/src/portfolio-snapshot.mjs +304 -0
  50. package/src/prepare-local.mjs +247 -12
  51. package/src/prepare.mjs +35 -3
  52. package/src/priorities.mjs +569 -0
  53. package/src/processor.mjs +599 -170
  54. package/src/project-id.mjs +17 -0
  55. package/src/projects.mjs +238 -0
  56. package/src/provider-steering.mjs +179 -65
  57. package/src/queue.mjs +111 -6
  58. package/src/reserved-env.mjs +30 -0
  59. package/src/run-container.mjs +59 -5
  60. package/src/run-history.mjs +379 -24
  61. package/src/run-mirror.mjs +30 -0
  62. package/src/runtime-settings.mjs +104 -9
  63. package/src/schedules.mjs +33 -1
  64. package/src/scoped-limits.mjs +447 -27
  65. package/src/service.mjs +15 -4
  66. package/src/session-store.mjs +131 -6
  67. package/src/start.mjs +528 -40
  68. package/src/triggers-file.mjs +65 -4
  69. package/src/triggers.mjs +135 -7
  70. package/src/up.mjs +308 -34
  71. package/src/valkey-endpoint.mjs +3 -2
package/src/up.mjs CHANGED
@@ -8,8 +8,10 @@
8
8
  * Doctrine this module must never drift from:
9
9
  * - init's never-clobber is contractual: up always calls runInit, and it always leaves existing
10
10
  * files (and an existing WEBHOOK_SECRET value) untouched — see env-file.mjs.
11
- * - up only ever pulls the repo's OWN default image (ghcr.io/edgehero/pi-job:latest, re-tagged
12
- * pi-job:latest). NEVER a trigger-named run.image: those are operator-declared and doctor's
11
+ * - up only ever pulls the image the worker will run as its deployment default, and only when it is
12
+ * absent (issue #523): the repo's own ghcr.io/edgehero/pi-job:latest, re-tagged pi-job:latest, when
13
+ * PI_JOB_IMAGE is unset, and otherwise exactly the image PI_JOB_IMAGE names, with no tag.
14
+ * NEVER a trigger-named run.image: those are operator-declared and doctor's
13
15
  * presence check covers them — a setup convenience must not become "pull whatever the triggers
14
16
  * file happens to name" (the same reasoning as jobs running with --pull=never).
15
17
  * - no secrets printed: the generated WEBHOOK_SECRET is announced, never echoed.
@@ -30,7 +32,7 @@
30
32
  */
31
33
  import { randomBytes } from "node:crypto";
32
34
  import { spawn as nodeSpawn } from "node:child_process";
33
- import { chmodSync, existsSync, lstatSync, readFileSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs";
35
+ import { existsSync, lstatSync, readFileSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs";
34
36
  import { mkdirSync, readFileSync as readPackageFile } from "node:fs";
35
37
  import { connect as netConnect } from "node:net";
36
38
  import { lookup as dnsLookup } from "node:dns/promises";
@@ -39,21 +41,31 @@ import { dirname, join, posix, resolve, win32 } from "node:path";
39
41
  import { fileURLToPath } from "node:url";
40
42
  import { PODMAN_BOOT_REFUSING_CAUSES, makePodmanInfoReader, decidePodmanJobUser, podmanJobUserRefusal } from "./backend-podman.mjs";
41
43
  import { venuesOf } from "./backends.mjs";
42
- import { logsDirPath, settingsFilePath } from "./config.mjs";
44
+ import { jobImageFrom, logsDirPath, settingsFilePath } from "./config.mjs";
45
+ import { jobImageFix, pullOffered, registryQualified } from "./image-ref.mjs";
43
46
  import { DEFAULT_EGRESS_PROXY, egressArmed as egressArmedFn, egressProxyName } from "./egress.mjs";
44
- import { envKeyIsBlank, envValueShown, readEnvAssignments, updateEnvFile } from "./env-file.mjs";
47
+ import { ENV_WRITER_FS, ENV_VALUE_UNWRITABLE, envKeyIsBlank, envValueShown, readEnvAssignments, updateEnvFile } from "./env-file.mjs";
45
48
  import { COMPOSE_FILE, COMPOSE_VALKEY_OVERRIDE, OWNER_MARKER_KEY, VALKEY_PASSWORD_KEY, VALKEY_PORT_KEY, OWNER_CHECK_CONTAINER, OWNER_CHECK_EXEC, VALKEY_VOLUME_RECORD, adoptVolumeQuestion, composeArgs, ownerCheckAnswer, readVolumeRecord, valkeyOwnerCheckArgs, volumeRecordMatches, volumeRecordText, composeProjectName, foreignContainerSentence, foreignMarkerRefusal, foreignVolumeLabelRefusal, foreignVolumeRefusal, foreignVolumeUsers, isLoopbackHost, newValkeyPassword, unadoptedVolumeRefusal, valkeyContainerOwner, valkeyDockerRunArgs, valkeyPasswordDecision, valkeyPortEnvDecision, valkeyVolumeCreateArgs, valkeyVolumeOwner } from "./valkey-auth.mjs";
46
49
  import { deploymentValkeyEnv, deploymentVenueEnv } from "./deployment-venue.mjs";
50
+ import { resolveServiceEnv, serviceEnvFileOf, serviceEnvLoader } from "./service-env.mjs";
47
51
  import { PACKAGED_EGRESS_PROXY_CONF, judgeProxyConfCopy, packageCopyName, readPackagedProxyConf, replaceProxyConfCopy } from "./egress-conf-copy.mjs";
48
- import { EGRESS_PROXY_IMAGE, PROXY_STATE_FORMAT, jobNetworksOf, parseProxyState, shippedProxyDrift } from "./egress-proxy-state.mjs";
49
- import { DEFAULT_VALKEY_PORT, NETNS_KEEPER, NETNS_KEEPER_FORMAT, QUADLET_FILES, STACK_KEYS, applyStack, decideValkey, describeRollBack, passwdNameFrom, readSubuidRanges, rollBackWrites, valkeySharedOn, VALKEY_SHARED_KEY, readValkeyKeys, valkeyTarget, judgeNetnsKeeper, keeperUnderRunningProxyHint, managerEnvRefusal, describeAction, foreignContainerRefusal, foreignContainers, lingerNote, planStack, readLinger, readStackKeys, stackComponents, unknownContainerRefusal, userBusRefusal } from "./podman-stack.mjs";
52
+ import { EGRESS_PROXY_IMAGE, MODEL_ENDPOINTS_TARGET, PROXY_STATE_FORMAT, jobNetworksOf, parseProxyState, rulesIncludeEndpoints, shippedProxyDrift } from "./egress-proxy-state.mjs";
53
+ import { endpointsDeclaredIn, rulesFileIncludes, rulesPredateEndpointsLine } from "./egress-cli.mjs";
54
+ import { MODEL_ENDPOINTS_INCLUDE_NAME } from "./model-endpoints.mjs";
55
+ import { DEFAULT_VALKEY_PORT, proxyConfCopyPath, NETNS_KEEPER, NETNS_KEEPER_FORMAT, QUADLET_FILES, STACK_KEYS, applyStack, decideValkey, describeRollBack, passwdNameFrom, readSubuidRanges, rollBackWrites, valkeySharedOn, VALKEY_SHARED_KEY, readValkeyKeys, valkeyTarget, judgeNetnsKeeper, keeperUnderRunningProxyHint, managerEnvRefusal, describeAction, foreignContainerRefusal, foreignContainers, lingerNote, planStack, readLinger, readStackKeys, stackComponents, unknownContainerRefusal, userBusRefusal } from "./podman-stack.mjs";
56
+
57
+ /** This command's default fs seam: its own calls, and every call the `.env` writer makes (`ENV_WRITER_FS`, issue #522). */
58
+ export const UP_FS = { existsSync, lstatSync, ...ENV_WRITER_FS };
50
59
 
51
60
  // The shipped Quadlet templates, module-relative like service.mjs's: worker/deploy in a checkout, <pkg>/deploy under npm.
52
61
  const TEMPLATES_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "..", "deploy");
53
62
 
54
- // The one image up may ever fetch, and the local name jobs run under. Literal on purpose (not
55
- // env.PI_JOB_IMAGE): an operator who pointed PI_JOB_IMAGE elsewhere has outgrown the quickstart, and
56
- // up pulling an arbitrary configured name would break the only-our-own-image doctrine above.
63
+ // The default image up fetches, and the local name jobs run under when PI_JOB_IMAGE is unset: the worker's own default
64
+ // (`jobImageFrom`, `PI_JOB_IMAGE || "pi-job:latest"`) is a local-only tag with no registry behind it, so the re-tag is
65
+ // what makes the pulled image the one the worker runs. It is load-bearing for that default ONLY. Issue #523: up used to
66
+ // check and pull these two whatever PI_JOB_IMAGE said, so a deployment running ghcr.io/edgehero/pi-job:2.1.0, present
67
+ // on the host, was handed a 3 GB pull of an image its worker never runs. An image PI_JOB_IMAGE names is now checked
68
+ // under that name and, when absent, pulled under that name (`overrideImageStep`), and these two are never touched.
57
69
  const UPSTREAM_IMAGE = "ghcr.io/edgehero/pi-job:latest";
58
70
  const LOCAL_IMAGE = "pi-job:latest";
59
71
  const PULL_ARGS = ["pull", UPSTREAM_IMAGE];
@@ -74,8 +86,8 @@ const VALKEY_STOP_ARGS = ["stop", "pi-dispatch-valkey"];
74
86
  const VALKEY_RM_ARGS = ["rm", "pi-dispatch-valkey"];
75
87
 
76
88
  // deploy/docker-compose.yml's `egress` profile, reproduced as one docker run (REQ-EGRESS-ALLOWLIST):
77
- // same digest-pinned image, same two mounts, same explicit container name, same restart policy, on the
78
- // same upstream network. Written out here for the same reason VALKEY_RUN_ARGS is -- an operator who runs
89
+ // same digest-pinned image, same three mounts, same host entry, same explicit container name, same restart
90
+ // policy, on the same upstream network. Written out here for the same reason VALKEY_RUN_ARGS is -- an operator who runs
79
91
  // `up` and one who runs compose must end up with the same component, and two ways of starting one thing
80
92
  // is two places for it to drift.
81
93
  //
@@ -86,9 +98,13 @@ const EGRESS_NETWORK_ARGS = ["network", "create", "pi-dispatch-egress-out"];
86
98
  // the argv below (issue #453; the reasons are at step e2).
87
99
  const EGRESS_START_ARGS = ["start", "pi-dispatch-egress-proxy"];
88
100
  const EGRESS_UNPAUSE_ARGS = ["unpause", "pi-dispatch-egress-proxy"];
89
- const EGRESS_RM_ARGS = ["rm", "-f", "pi-dispatch-egress-proxy"];
101
+ // `-v` (issue #503's review): a replaced proxy would otherwise leave the image's two anonymous squid volumes behind, one
102
+ // pair per replace. They hold squid's logs and an unused cache, nothing a later proxy reads.
103
+ const EGRESS_RM_ARGS = ["rm", "-f", "-v", "pi-dispatch-egress-proxy"];
90
104
  // Issue #484: how a running proxy reads refreshed rules (see the refresh step in `runUp` for why a restart).
91
105
  const EGRESS_RESTART_ARGS = ["restart", "pi-dispatch-egress-proxy"];
106
+ // The deployment folder's files the shipped proxy mounts, each checked before a start or a recreate.
107
+ const PROXY_FOLDER_FILES = Object.freeze(["egress-allowlist.conf", "deploy/egress-proxy.conf", MODEL_ENDPOINTS_INCLUDE_NAME]);
92
108
  const EGRESS_RUN_ARGS = [
93
109
  "run",
94
110
  "-d",
@@ -105,6 +121,15 @@ const EGRESS_RUN_ARGS = [
105
121
  "./deploy/egress-proxy.conf:/etc/squid/squid.conf:ro,z",
106
122
  "-v",
107
123
  "./egress-allowlist.conf:/etc/pi-dispatch/allowlist.conf:ro,z",
124
+ // Issue #503: the declared model endpoints' rules, which the rules `include`. `init` scaffolds the file, and squid
125
+ // will not start without it. `egress render` rewrites it in place and prints the reload.
126
+ "-v",
127
+ `./${MODEL_ENDPOINTS_INCLUDE_NAME}:/etc/pi-dispatch/model-endpoints.conf:ro,z`,
128
+ // How the proxy reaches a model server on this host (issue #503): on Docker Engine the name does not exist without
129
+ // it, and host-gateway is the bridge gateway. Docker Desktop resolves the name already and host-gateway there is the
130
+ // same host address, so it is on both, as in the compose file. The PROXY only: no job gets it.
131
+ "--add-host",
132
+ "host.docker.internal:host-gateway",
108
133
  EGRESS_PROXY_IMAGE,
109
134
  ];
110
135
 
@@ -137,7 +162,7 @@ export async function runUp(argv = [], deps = {}) {
137
162
  // regular file. It calls it optionally, so leaving it out here made that repair dead code in the
138
163
  // only production caller, and the test that covered it attached the method to its own fake.
139
164
  // `lstatSync` for the rules refresh (issue #484), which follows no symlink.
140
- fs = { existsSync, lstatSync, readFileSync, writeFileSync, renameSync, statSync, chmodSync, realpathSync, unlinkSync },
165
+ fs = UP_FS,
141
166
  probeTcp = defaultProbeTcp,
142
167
  // Issue #468: whether the Valkey on 127.0.0.1:6379 answers a client that sends NO password ("ok" when it does),
143
168
  // through the project's one connection function. Real only where the TCP probe is: a test that stands in for the
@@ -238,6 +263,12 @@ export async function runUp(argv = [], deps = {}) {
238
263
  else
239
264
  out("pi-dispatch up — one pass over the quickstart; every docker action asks first (--yes accepts)\n\n");
240
265
 
266
+ // Issue #523: the image the worker will run as its deployment default, resolved as the worker resolves it, from where
267
+ // the service reads it. Both image steps below check (and offer to pull) this image and no other.
268
+ const jobImage = deploymentJobImage({ env, fs, envPath: join(cwd, ".env"), platform });
269
+ if (jobImage.skip) out(`⚠ ${jobImage.skip}\n`);
270
+ if (jobImage.note) out(`⚠ ${jobImage.note}\n`);
271
+
241
272
  if (dockerUsed) {
242
273
  // (a) docker binary + daemon, before anything is offered: every mutation below runs through the
243
274
  // docker CLI, so with the daemon down the prompts would only collect consent for failures.
@@ -251,7 +282,9 @@ export async function runUp(argv = [], deps = {}) {
251
282
  out("✓ Docker daemon reachable\n");
252
283
 
253
284
  // (b) the default job image. Presence first, so the happy path re-run prompts for nothing.
254
- if (await runCmd(spawn, "docker", ["image", "inspect", LOCAL_IMAGE]) === 0) {
285
+ if (jobImage.skip) summary.push(["job image", "not checked: which image the worker runs is unknown (see above)"]);
286
+ else if (!jobImage.isDefault) await overrideImageStep({ bin: "docker", jobImage, spawn, out, yes, prompt, summary });
287
+ else if (await runCmd(spawn, "docker", ["image", "inspect", LOCAL_IMAGE]) === 0) {
255
288
  out(`✓ Job image present (${LOCAL_IMAGE})\n`);
256
289
  summary.push(["job image", `already present (${LOCAL_IMAGE})`]);
257
290
  } else {
@@ -292,7 +325,7 @@ export async function runUp(argv = [], deps = {}) {
292
325
  out(" the docker steps above are unaffected; the podman steps below are skipped\n");
293
326
  } else {
294
327
  podmanReady = true;
295
- await podmanImageStep({ spawn, out, yes, prompt, summary });
328
+ await podmanImageStep({ jobImage, spawn, out, yes, prompt, summary });
296
329
  }
297
330
  }
298
331
 
@@ -327,7 +360,9 @@ export async function runUp(argv = [], deps = {}) {
327
360
  // `loadPauseWindows`/`loadScopedLimits` unconditionally at boot, which throw on a path that does not
328
361
  // exist: the worker does not ignore the feature, it refuses to start. `PI_LOGS_DIR` and
329
362
  // `PI_SETTINGS_FILE` use `||` and fall back to the account default; `WEBHOOK_SECRET` reads as absent.
330
- const EMPTY_REFUSES_BOOT = new Set(["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE"]);
363
+ // Issue #504 part B: PI_ENVELOPE_FILE refuses the boot on an empty value too, and joins this set ONLY: up never writes
364
+ // it (unset means no delegation, the safe default), so it is said below only when the operator's line is blank.
365
+ const EMPTY_REFUSES_BOOT = new Set(["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE", "PI_PROJECTS_FILE", "PI_ENVELOPE_FILE"]);
331
366
  const emptyNote = (key) =>
332
367
  EMPTY_REFUSES_BOOT.has(key)
333
368
  ? `left untouched: the line is there and its value is EMPTY, which is not the same as no line -- a shell that sources this file exports it as "", the worker keeps it and REFUSES TO BOOT. up never clobbers a key an operator wrote, so fill it in or delete the line`
@@ -384,13 +419,13 @@ export async function runUp(argv = [], deps = {}) {
384
419
  // `.env.example`). Same never-clobber discipline as WEBHOOK_SECRET, at key granularity: a value the
385
420
  // operator set survives untouched.
386
421
  //
387
- // TWO get the deployment folder and TWO get the RESOLVED ACCOUNT DEFAULT, and the split is the
388
- // sharpest edge in this change rather than an inconsistency. `pause-windows.json` and
389
- // `scoped-limits.json` are scaffolded by `init` into this folder, and the panel defaults to this
390
- // folder, so pointing the worker here is what makes the three agree. `PI_LOGS_DIR` and
422
+ // THREE get the deployment folder and TWO get the RESOLVED ACCOUNT DEFAULT, and the split is the
423
+ // sharpest edge in this change rather than an inconsistency. `pause-windows.json`,
424
+ // `scoped-limits.json` and (issue #499) `projects.json` are scaffolded by `init` into this folder, and the
425
+ // panel defaults to this folder, so pointing the worker here is what makes the three agree. `PI_LOGS_DIR` and
391
426
  // `PI_SETTINGS_FILE` are different in kind: `makeLogReaper` unlinks EVERY `.log` and `.json` in
392
427
  // `PI_LOGS_DIR` past the window with no name shape and no ownership check, so a deployment folder
393
- // there would eat `triggers.json`, `pause-windows.json`, `scoped-limits.json` and
428
+ // there would eat `triggers.json`, `pause-windows.json`, `scoped-limits.json`, `projects.json` and
394
429
  // `subscriptions.json` thirty days in, silently, and the worker would then run nothing while
395
430
  // reporting success. `<deployment>/logs` is no better: `service.mjs` creates exactly that directory
396
431
  // at install time and the plist puts `worker.out.log` in it. So these two get what
@@ -400,6 +435,7 @@ export async function runUp(argv = [], deps = {}) {
400
435
  for (const [key, raw, durable] of [
401
436
  ["PI_PAUSE_WINDOWS_FILE", join(cwd, "pause-windows.json"), false],
402
437
  ["PI_SCOPED_LIMITS_FILE", join(cwd, "scoped-limits.json"), false],
438
+ ["PI_PROJECTS_FILE", join(cwd, "projects.json"), false],
403
439
  ["PI_LOGS_DIR", logsDirPathFn(env), true],
404
440
  ["PI_SETTINGS_FILE", settingsFilePathFn(env), true],
405
441
  ]) {
@@ -416,8 +452,8 @@ export async function runUp(argv = [], deps = {}) {
416
452
  //
417
453
  // RELATIVE is the sharper of the two and it is not hypothetical: `PI_LOGS_DIR=.` resolves against
418
454
  // the unit's `WorkingDirectory`, which IS the deployment folder, so the retention sweep then
419
- // deletes `triggers.json`, `pause-windows.json`, `scoped-limits.json` and `subscriptions.json`
420
- // thirty days in. All three deploy templates document this key as absolute for exactly that
455
+ // deletes `triggers.json`, `pause-windows.json`, `scoped-limits.json`, `projects.json` and
456
+ // `subscriptions.json` thirty days in. All three deploy templates document this key as absolute for exactly that
421
457
  // reason. INSIDE THIS FOLDER is the same harm reached with an absolute path.
422
458
  //
423
459
  // The refusals live here and not in the resolver, because the resolver is right for the worker:
@@ -451,8 +487,13 @@ export async function runUp(argv = [], deps = {}) {
451
487
  try {
452
488
  ({ changed } = updateEnvFile(envPath, key, value, { fs, platform }));
453
489
  } catch (err) {
490
+ // Advice about THE VALUE only where the value is what was refused (issue #522). Every other refusal of the writer
491
+ // is about the file and ends in its own fix; this sentence once followed all of them, so a gid mismatch was told
492
+ // to move the deployment somewhere "without that character in its path". Where the value came from decides the
493
+ // advice: the two folder keys are this folder's own paths, the two durable ones the account default or this shell.
494
+ const valueAdvice = err?.code !== ENV_VALUE_UNWRITABLE ? "" : durable ? ". Set it by hand, to a path without that character" : ". Set it by hand, or move the deployment somewhere without that character in its path";
454
495
  out(`✗ ${key} could not be written: ${err?.message}\n`);
455
- summary.push([key, `NOT written: ${err?.message}. Set it by hand, or move the deployment somewhere without that character in its path`]);
496
+ summary.push([key, `NOT written: ${err?.message}${valueAdvice}`]);
456
497
  continue;
457
498
  }
458
499
  if (changed) {
@@ -463,9 +504,11 @@ export async function runUp(argv = [], deps = {}) {
463
504
  summary.push([key, writtenButEmpty(key) ? emptyNote(key) : "already set — left untouched"]);
464
505
  }
465
506
  }
507
+ // Never written, never defaulted: only a blank line the operator left is worth a word, since it refuses the boot.
508
+ if (writtenButEmpty("PI_ENVELOPE_FILE")) summary.push(["PI_ENVELOPE_FILE", emptyNote("PI_ENVELOPE_FILE")]);
466
509
  } else {
467
510
  summary.push(["WEBHOOK_SECRET", "no .env here — skipped (set it wherever your env lives)"]);
468
- for (const key of ["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE", "PI_LOGS_DIR", "PI_SETTINGS_FILE"]) {
511
+ for (const key of ["PI_PAUSE_WINDOWS_FILE", "PI_SCOPED_LIMITS_FILE", "PI_PROJECTS_FILE", "PI_LOGS_DIR", "PI_SETTINGS_FILE"]) {
469
512
  summary.push([key, "no .env here — skipped (set it wherever your env lives)"]);
470
513
  }
471
514
  }
@@ -761,7 +804,36 @@ export async function runUp(argv = [], deps = {}) {
761
804
  // (e1b) the folder's copy of the proxy's rules (issue #484), before the proxy step so that a proxy started, replaced
762
805
  // or restarted below reads the refreshed file. Only where the shipped docker proxy mounts it: a proxy PI_EGRESS_PROXY
763
806
  // names is the operator's, and the podman venue mounts an account-owned copy `service install` writes.
764
- const confRefreshed = dockerUsed && egressArmedFn(venueEnv) && proxyName === DEFAULT_EGRESS_PROXY ? await proxyConfRefreshStep({ fs, cwd, out, prompt, summary, readPackagedConf, now, yes }) : false;
807
+ //
808
+ // Issue #503's governing rule: the folder's rules include model-endpoints.conf only together with a proxy that mounts
809
+ // it. So when the refresh brings the include and a proxy without the third mount exists, the refresh and that proxy's
810
+ // replacement are ONE step, asked once (`coupled` below): declined, or not possible, nothing is written; a replace that
811
+ // fails puts the old rules back. Otherwise the proxy's next restart would exit on the missing include (measured).
812
+ const coupled = {
813
+ // The proxy as it is now, read only when a refresh is about to be offered.
814
+ inspect: async () => {
815
+ const inspect = await runCmdQuery(spawn, "docker", ["inspect", PROXY_STATE_FORMAT, DEFAULT_EGRESS_PROXY]);
816
+ return inspect.code === 0 ? parseProxyState(inspect.stdout) : null;
817
+ },
818
+ // Why the replace could not happen from here, or null: a file the run mounts is missing, or a mount of it cannot
819
+ // be compared on this host while nothing else shows it stale (up never removes a proxy on that ground alone).
820
+ blocked: (state) => {
821
+ const missing = PROXY_FOLDER_FILES.find((f) => f !== "deploy/egress-proxy.conf" && (!fs.existsSync(join(cwd, f)) || pathIsDirectory(fs, join(cwd, f))));
822
+ if (missing) return `${missing} is not a file in this folder, so the proxy could not be started with it`;
823
+ const judged = shippedProxyDrift(state, { cwd, platform, realpath: (p) => (typeof fs.realpathSync === "function" ? fs.realpathSync(p) : p) });
824
+ const configStale = shippedProxyDrift(state, { cwd, compareMounts: false }).drift.length > 0;
825
+ if (judged.unknown && !configStale) return `its mounts could not be compared on this host (${judged.unknown}), and up does not remove a proxy it cannot show is this folder's`;
826
+ return null;
827
+ },
828
+ lines: async (network) => [`docker ${EGRESS_RM_ARGS.join(" ")}`, ...(await egressNetworkLines(spawn, network)), `docker ${quoteArgs(EGRESS_RUN_ARGS)}`],
829
+ replace: async (network) => {
830
+ await runStreamed(spawn, "docker", EGRESS_RM_ARGS, out);
831
+ await ensureEgressNetwork(spawn, out, network);
832
+ return (await runStreamed(spawn, "docker", EGRESS_RUN_ARGS, out)) === 0;
833
+ },
834
+ handled: false,
835
+ };
836
+ const confRefreshed = dockerUsed && egressArmedFn(venueEnv) && proxyName === DEFAULT_EGRESS_PROXY ? await proxyConfRefreshStep({ fs, cwd, out, prompt, summary, readPackagedConf, now, yes, coupled }) : false;
765
837
  if (dockerUsed && egressArmedFn(venueEnv) && proxyName !== DEFAULT_EGRESS_PROXY) {
766
838
  // PI_EGRESS_PROXY names the operator's own proxy (issue #430). This step used to look for, and offer to start, the
767
839
  // shipped name whatever that key said, so it could report a proxy present that no job attaches to, or start one
@@ -783,6 +855,8 @@ export async function runUp(argv = [], deps = {}) {
783
855
  out(`\n✗ PI_EGRESS_PROXY names ${proxyName}, and docker has no container of that name. up starts only the shipped ${DEFAULT_EGRESS_PROXY}, so start ${proxyName} yourself\n`);
784
856
  summary.push(["egress", `${proxyName} (PI_EGRESS_PROXY) not found; every job is refused pre-spend until it runs`]);
785
857
  }
858
+ } else if (dockerUsed && egressArmedFn(venueEnv) && coupled.handled) {
859
+ // The refresh step replaced the proxy itself, or wrote nothing and said why (above): nothing more to judge here.
786
860
  } else if (dockerUsed && egressArmedFn(venueEnv)) {
787
861
  // RUNNING and CURRENT, from one inspect (issue #453; `egress-proxy-state.mjs` says why each). Running is
788
862
  // `.State.Status` "running". Current is the pinned image with its own entrypoint and command and THIS folder's two
@@ -790,14 +864,25 @@ export async function runUp(argv = [], deps = {}) {
790
864
  // whether it is running or stopped. Mounts whose sources do not resolve on this host are UNKNOWN, never stale.
791
865
  const inspect = await runCmdQuery(spawn, "docker", ["inspect", PROXY_STATE_FORMAT, DEFAULT_EGRESS_PROXY]);
792
866
  const state = inspect.code === 0 ? parseProxyState(inspect.stdout) : null;
793
- const judged = state ? shippedProxyDrift(state, { cwd, platform, realpath: (p) => (typeof fs.realpathSync === "function" ? fs.realpathSync(p) : p) }) : { drift: [], unknown: null };
867
+ // Issue #503: a proxy without the third mount is current until the include is needed (`shippedProxyDrift`). Read
868
+ // AFTER the rules refresh above, so a refreshed copy that includes the file makes a two-mount proxy stale here and
869
+ // it is replaced rather than restarted into squid's missing-include FATAL.
870
+ const rulesInclude = rulesFileIncludes(join(cwd, "deploy/egress-proxy.conf"), fs);
871
+ const judged = state ? shippedProxyDrift(state, { cwd, platform, realpath: (p) => (typeof fs.realpathSync === "function" ? fs.realpathSync(p) : p), rulesInclude }) : { drift: [], unknown: null };
872
+ // Endpoints declared under rules that predate #503: not drift (a replace would cut tunnels and change nothing), the
873
+ // rules refresh's to fix, said in the one line up, doctor and egress render share.
874
+ if (!rulesInclude && endpointsDeclaredIn({ env, cwd, fs, platform })) {
875
+ out(`\n⚠ ${rulesPredateEndpointsLine("docker")}\n`);
876
+ summary.push(["model endpoints", "unreachable: deploy/egress-proxy.conf predates #503; accept the rules refresh"]);
877
+ }
794
878
  const drift = judged.drift;
795
879
  // The files a start or a recreate mounts. Both, not the allowlist alone: a missing egress-proxy.conf is bind-mounted
796
880
  // as a directory the runtime creates in its place (measured, Docker Engine 29.8.1: the host path became a root-owned
797
881
  // directory, and the create failed "not a directory: Are you trying to mount a directory onto a file", exit 127).
798
882
  // A DIRECTORY at either path counts as missing too (PR #488's review): it is mounted where squid reads a file.
799
883
  const fileProblem = (f) => (!fs.existsSync(join(cwd, f)) ? "is not here" : pathIsDirectory(fs, join(cwd, f)) ? "here is a directory, not a file" : null);
800
- const missingFile = ["egress-allowlist.conf", "deploy/egress-proxy.conf"].find((f) => fileProblem(f) !== null);
884
+ // Issue #503: and the model endpoints' include, which the rules name; squid will not start without it (measured).
885
+ const missingFile = PROXY_FOLDER_FILES.find((f) => fileProblem(f) !== null);
801
886
  const missingSaid = missingFile ? `${missingFile} ${fileProblem(missingFile)}` : "";
802
887
  const noFile = missingFile ? (fileProblem(missingFile) === "is not here" ? `no ${missingFile} in this folder` : `${missingFile} in this folder is a directory`) : "";
803
888
  // Whether the proxy's network is missing, filled by `egressNetworkLines` when an offer is about to be shown.
@@ -872,7 +957,8 @@ export async function runUp(argv = [], deps = {}) {
872
957
  // STALE, running or not: never started as it is. Its policy is not this deployment's, so the offer is to replace
873
958
  // it with the shipped one, shown line for line. A running one may be carrying jobs: its job and sandbox networks
874
959
  // are named, since removing it cuts each of them off mid-run.
875
- out(`\n✗ ${DEFAULT_EGRESS_PROXY} exists (${state.status}) but is not this deployment's proxy: ${drift.join("; ")}\n`);
960
+ // Issue #503 (gate): a proxy of THIS folder made before #503 is said as what it is, out of date, not a stranger.
961
+ out(judged.outOfDate ? `\n✗ ${DEFAULT_EGRESS_PROXY} exists (${state.status}) and is this deployment's proxy but out of date: ${drift.join("; ")}\n` : `\n✗ ${DEFAULT_EGRESS_PROXY} exists (${state.status}) but is not this deployment's proxy: ${drift.join("; ")}\n`);
876
962
  const attached = jobNetworksOf(state);
877
963
  // Never removed on its mounts while one of its two own mounts is unknown (round-cap re-review): the other findings
878
964
  // are said. Its image, entrypoint or command is ground enough (`configStale` above).
@@ -901,8 +987,8 @@ export async function runUp(argv = [], deps = {}) {
901
987
  summary.push(["egress", "replaced the stale pi-dispatch-egress-proxy with the shipped one"]);
902
988
  }
903
989
  } else {
904
- out(`skipped: until it is replaced, jobs use a proxy whose policy is not this deployment's (\`docker ${EGRESS_RM_ARGS.join(" ")}\`, then \`pi-dispatch up\`)\n`);
905
- summary.push(["egress", "stale proxy left as it is (declined): its policy is not this deployment's"]);
990
+ out(judged.outOfDate ? `skipped: it runs until its next restart, which exits on the missing include (\`docker ${EGRESS_RM_ARGS.join(" ")}\`, then \`pi-dispatch up\`)\n` : `skipped: until it is replaced, jobs use a proxy whose policy is not this deployment's (\`docker ${EGRESS_RM_ARGS.join(" ")}\`, then \`pi-dispatch up\`)\n`);
991
+ summary.push(["egress", judged.outOfDate ? "out-of-date proxy left as it is (declined): its next restart exits on the missing include" : "stale proxy left as it is (declined): its policy is not this deployment's"]);
906
992
  }
907
993
  } else if (missingFile) {
908
994
  out(`\n✗ the egress policy is on but ${missingSaid}, so no proxy is started without it\n`);
@@ -1061,8 +1147,120 @@ async function podmanGate({ readPodmanInfo, platform, euid, egid, out }) {
1061
1147
  return { ok: false, why };
1062
1148
  }
1063
1149
 
1150
+ /**
1151
+ * The job image the worker will run as its deployment default (issue #523), as `up` must judge it:
1152
+ * `{ image, isDefault, from, skip }`. The worker's own rule (`jobImageFrom`: `PI_JOB_IMAGE || "pi-job:latest"`, then
1153
+ * the one image rule), over the value the service reads (`resolveServiceEnv`, doctor's resolver): this shell's where
1154
+ * it sets the key, else the deployment `.env`'s, read with the service's own loader. Where `up` cannot tell which image
1155
+ * that is (the shell and the file disagree, the file's line or the file itself is one the loader reads differently,
1156
+ * or the worker refuses the value at boot), `skip` says why and nothing is checked or pulled: a guess here is exactly
1157
+ * the 3 GB pull of an image the worker never runs that this exists to stop. Doctor, below, names each of those cases.
1158
+ */
1159
+ export function deploymentJobImage({ env, fs, envPath, platform }) {
1160
+ let file = null;
1161
+ let unreadable = false;
1162
+ if (fs.existsSync(envPath)) {
1163
+ try {
1164
+ // Bytes, as the service's loader meets them (`serviceEnvFileOf` judges what systemd refuses to load).
1165
+ file = serviceEnvFileOf(fs.readFileSync(envPath), envPath, serviceEnvLoader(platform));
1166
+ } catch {
1167
+ // Unreadable is not unset: the service may well read a PI_JOB_IMAGE this account cannot.
1168
+ unreadable = true;
1169
+ if (typeof env.PI_JOB_IMAGE !== "string") return { skip: `${envPath} could not be read, so up cannot tell which job image the worker runs (PI_JOB_IMAGE): it checks and pulls none` };
1170
+ }
1171
+ }
1172
+ // A file the service's loader reads differently somewhere, which spells the key, gives no value (round 2): where this
1173
+ // shell sets the key `resolveServiceEnv` records nothing for it, so the shell's value was credited to `.env` and a
1174
+ // different value on a line after the hazard went unseen. Cannot tell, whoever sets it.
1175
+ if (file?.hazard && file.text.includes("PI_JOB_IMAGE")) {
1176
+ return { skip: `${envPath} has a line in it that the service's loader may read differently from this reader (line ${file.hazard.line}), so up cannot tell which job image the worker runs: it checks and pulls none. Doctor below names the line` };
1177
+ }
1178
+ const read = resolveServiceEnv({ env, file, keys: ["PI_JOB_IMAGE"] });
1179
+ // Compared as the worker READS them (review), as `deploymentVenueEnv` compares the venue keys: an empty value and
1180
+ // `pi-job:latest` are both the default, and refusing them sent an operator to reconcile two values that agree.
1181
+ const [disagreement] = read.disagreements.filter((d) => jobImageMeaning(d.shell) !== jobImageMeaning(d.file));
1182
+ if (disagreement) {
1183
+ return { skip: `PI_JOB_IMAGE is ${JSON.stringify(disagreement.shell)} in this shell and ${JSON.stringify(disagreement.file)} in ${envPath}, so up cannot tell which job image the worker runs: it checks and pulls none. Make them agree (the service runs the file's), then re-run` };
1184
+ }
1185
+ // `hazardSkipped` is answered above, before the resolver, whoever sets the key.
1186
+ if (read.unread.length > 0) {
1187
+ return { skip: `${envPath} has a line in it that the service's loader may read differently from this reader, so up cannot tell which job image the worker runs: it checks and pulls none. Doctor below names the line` };
1188
+ }
1189
+ let image;
1190
+ try {
1191
+ image = jobImageFrom(read.env);
1192
+ } catch (err) {
1193
+ return { skip: `${err.message}: the worker refuses to boot on it, so up checks and pulls no job image. Fix PI_JOB_IMAGE, then re-run` };
1194
+ }
1195
+ const shellSet = typeof env.PI_JOB_IMAGE === "string" && env.PI_JOB_IMAGE !== "";
1196
+ // The file sets no PI_JOB_IMAGE of its own: there is none, or `resolveServiceEnv` found the key in this shell only.
1197
+ const fileLacks = !unreadable && (file === null || read.shellOnly.includes("PI_JOB_IMAGE"));
1198
+ const from = shellSet && (fileLacks || unreadable) ? "this shell" : Object.hasOwn(read.fromFile, "PI_JOB_IMAGE") || shellSet ? envPath : null;
1199
+ // Said, never silent (review): a worker started by hand from this shell runs this image, an installed service does
1200
+ // not, since it reads `.env` (where init writes PI_JOB_IMAGE=pi-job:latest), so the image up readies may not be its.
1201
+ const note = shellSet && fileLacks ? `PI_JOB_IMAGE comes from this shell only: a worker started from this shell runs ${image}, but an installed service reads ${envPath}, not this shell (init writes PI_JOB_IMAGE=pi-job:latest there), so put it there if the service should run it` : null;
1202
+ return { image, isDefault: image === LOCAL_IMAGE, from, note, skip: null };
1203
+ }
1204
+
1205
+ /** What a PI_JOB_IMAGE value means to the worker (`jobImageFrom`), for comparing two spellings of it; a refused value keeps its raw spelling. */
1206
+ function jobImageMeaning(value) {
1207
+ try {
1208
+ return `image:${jobImageFrom({ PI_JOB_IMAGE: value })}`;
1209
+ } catch {
1210
+ return `raw:${value}`;
1211
+ }
1212
+ }
1213
+
1214
+ /**
1215
+ * The image an overriding PI_JOB_IMAGE names (issue #523), into the store `bin` reads: present is left alone, absent is
1216
+ * offered as ONE pull of exactly that name when it names its registry (a short name is never pulled). No tag: the worker runs the name as configured, so `pi-job:latest` means
1217
+ * nothing to it, and re-pointing that tag would change what a later unset PI_JOB_IMAGE runs. The name is the operator's
1218
+ * own configuration, shown in full before consent, and jobs still run with --pull=never.
1219
+ */
1220
+ async function overrideImageStep({ bin, jobImage, spawn, out, yes, prompt, summary }) {
1221
+ const { image, from } = jobImage;
1222
+ const podman = bin === "podman";
1223
+ const row = podman ? "podman job image" : "job image";
1224
+ const where = podman ? "in this account's Podman store" : "on this host";
1225
+ const named = `PI_JOB_IMAGE from ${from}`;
1226
+ const pullArgs = ["pull", image];
1227
+ if ((await runCmd(spawn, bin, podman ? ["image", "exists", image] : ["image", "inspect", image])) === 0) {
1228
+ out(`✓ Job image present ${podman ? "in this account's Podman store " : ""}(${image}, ${named})\n`);
1229
+ summary.push([row, `already present (${image})`]);
1230
+ return;
1231
+ }
1232
+ // A SHORT name is never pulled (issue #523, review): with no registry host the runtime resolves it on Docker Hub (or
1233
+ // podman's search registries), where anyone can publish `pi-job:2.1.0` or `my-job:dev`, and under --yes that would
1234
+ // fetch and later run a stranger's image unprompted. Such a name is almost always one built or tagged here, so up
1235
+ // says how to provide it and pulls nothing; doctor below still fails on the missing image. A `localhost/` name
1236
+ // (round 2) is a locally built one with no registry behind it, so it is never offered as a pull either.
1237
+ if (!pullOffered(image)) {
1238
+ out(`✗ The job image ${named} names (${image}) is not ${where}, and up does not pull it: ${jobImageFix(bin, image)}\n`);
1239
+ summary.push([row, `${image} is not present: not pulled, since ${registryQualified(image) ? "it is a locally built name" : "it names no registry host"} (build or tag it here)`]);
1240
+ return;
1241
+ }
1242
+ const accepted = await consent(`The job image ${named} names (${image}) is not ${where}. up would run:`, [`${bin} ${pullArgs.join(" ")}`], { yes, out, prompt });
1243
+ if (!accepted) {
1244
+ out("skipped: pull it later with the command above\n");
1245
+ summary.push([row, `skipped (declined): ${image} is not present, and jobs run with --pull=never, so nothing fetches it later`]);
1246
+ } else if ((await runStreamed(spawn, bin, pullArgs, out)) !== 0) {
1247
+ out(`✗ ${bin} pull failed: continuing; doctor below will re-check the image\n`);
1248
+ summary.push([row, `pull of ${image} FAILED: re-run \`pi-dispatch up\`, or pull it by hand`]);
1249
+ } else {
1250
+ out(`✓ pulled ${image}${podman ? " into this account's Podman store" : ""}\n`);
1251
+ summary.push([row, `pulled ${image}`]);
1252
+ }
1253
+ }
1254
+
1064
1255
  /** The default job image into this account's Podman store, mirroring the docker step (b) line for line. */
1065
- async function podmanImageStep({ spawn, out, yes, prompt, summary }) {
1256
+ async function podmanImageStep({ jobImage, spawn, out, yes, prompt, summary }) {
1257
+ // The same rule as docker's step (issue #523): only the image the worker runs, under its own name when PI_JOB_IMAGE
1258
+ // names one. The podman venue reads the same key (`jobImageFrom`), so its store needs exactly that image too.
1259
+ if (jobImage.skip) {
1260
+ summary.push(["podman job image", "not checked: which image the worker runs is unknown (see above)"]);
1261
+ return;
1262
+ }
1263
+ if (!jobImage.isDefault) return overrideImageStep({ bin: "podman", jobImage, spawn, out, yes, prompt, summary });
1066
1264
  if ((await runCmd(spawn, "podman", PODMAN_EXISTS_ARGS)) === 0) {
1067
1265
  out(`✓ Job image present in this account's Podman store (${LOCAL_IMAGE})\n`);
1068
1266
  summary.push(["podman job image", `already present (${LOCAL_IMAGE})`]);
@@ -1173,6 +1371,12 @@ async function podmanStackStep({ env, valkeyKeys = {}, state = {}, venues, spawn
1173
1371
  }
1174
1372
  const components = stackComponents({ venues, env, includeValkey, armed, valkeyPort, valkeyPassword });
1175
1373
  for (const note of components.notes) out(`\n⚠ ${note}\n`);
1374
+ // Issue #503: endpoints declared while the account's rules copy predates the include, in the one line up, doctor and
1375
+ // egress render share. No copy yet is not that state: the unit's install writes the current rules.
1376
+ if (components.proxy && fs.existsSync(proxyConfCopyPath(home)) && !rulesFileIncludes(proxyConfCopyPath(home), fs) && endpointsDeclaredIn({ env, cwd, fs, platform })) {
1377
+ out(`\n⚠ ${rulesPredateEndpointsLine("podman")}\n`);
1378
+ summary.push(["model endpoints", "unreachable: the podman proxy's rules predate #503; `pi-dispatch service install --force`"]);
1379
+ }
1176
1380
  let keeperRestart = false;
1177
1381
  if (components.proxy) {
1178
1382
  // RUNNING, read off the output: `podman inspect` exits 0 for an EXITED container too and prints `false`
@@ -1190,6 +1394,11 @@ async function podmanStackStep({ env, valkeyKeys = {}, state = {}, venues, spawn
1190
1394
  out(`\n✗ the egress policy is on but egress-allowlist.conf ${pathIsDirectory(fs, join(cwd, "egress-allowlist.conf")) ? "here is a directory, not a file" : "is not here"}, not starting a proxy with no allowlist\n`);
1191
1395
  summary.push(["egress (podman)", "skipped: no egress-allowlist.conf in this folder; run `pi-dispatch init` here, then `up` again"]);
1192
1396
  components.proxy = false;
1397
+ } else if (!fs.existsSync(join(cwd, MODEL_ENDPOINTS_INCLUDE_NAME)) || pathIsDirectory(fs, join(cwd, MODEL_ENDPOINTS_INCLUDE_NAME))) {
1398
+ // Issue #503: the rules include it, and squid will not start without it.
1399
+ out(`\n✗ the egress policy is on but ${MODEL_ENDPOINTS_INCLUDE_NAME} ${pathIsDirectory(fs, join(cwd, MODEL_ENDPOINTS_INCLUDE_NAME)) ? "here is a directory, not a file" : "is not here"}, and the proxy's rules include it, so no proxy is started without it\n`);
1400
+ summary.push(["egress (podman)", `skipped: no ${MODEL_ENDPOINTS_INCLUDE_NAME} in this folder; run \`pi-dispatch init\` here, then \`up\` again`]);
1401
+ components.proxy = false;
1193
1402
  }
1194
1403
  }
1195
1404
  if (components.keeper) {
@@ -1345,7 +1554,7 @@ export function valkeyContainerPublishes(line, port) {
1345
1554
  * bytes are kept beside it either way (`replaceProxyConfCopy`), since an edit lost to a wrong "y" is still the
1346
1555
  * operator's.
1347
1556
  */
1348
- async function proxyConfRefreshStep({ fs, cwd, out, prompt, summary, readPackagedConf, now, yes }) {
1557
+ async function proxyConfRefreshStep({ fs, cwd, out, prompt, summary, readPackagedConf, now, yes, coupled = null }) {
1349
1558
  const rel = "deploy/egress-proxy.conf";
1350
1559
  const path = join(cwd, rel);
1351
1560
  // A directory there is the proxy step's to say (it starts nothing on one), so it is not read as a copy here.
@@ -1360,6 +1569,14 @@ async function proxyConfRefreshStep({ fs, cwd, out, prompt, summary, readPackage
1360
1569
  }
1361
1570
  out(`\n⚠ ${rel} differs from ${packageCopyName()}: ${judged.summary}. An upgrade does not rewrite it, so it is either an older version's rules or an edit of your own (\`diff ${rel} ${PACKAGED_EGRESS_PROXY_CONF}\` shows which)\n`);
1362
1571
  if (yes) out("--yes does not cover replacing a file that may hold your own edits: answer below\n");
1572
+ // Issue #503's governing rule: new rules that include model-endpoints.conf go in only together with a proxy that
1573
+ // mounts it. A proxy of the shipped name without that mount is replaced in the same step, asked once.
1574
+ if (coupled && rulesIncludeEndpoints(judged.packaged) && !rulesFileIncludes(path, fs)) {
1575
+ const state = await coupled.inspect();
1576
+ if (state && !state.mounts.some((m) => m.type === "bind" && m.destination === MODEL_ENDPOINTS_TARGET)) {
1577
+ return coupledRefreshStep({ fs, path, rel, judged, state, coupled, out, prompt, summary, now });
1578
+ }
1579
+ }
1363
1580
  const accepted = await consent(
1364
1581
  `up would replace it with ${packageCopyName()}, keeping yours beside it:`,
1365
1582
  [`cp ${rel} ${rel}.bak-<timestamp>`, `write ${PACKAGED_EGRESS_PROXY_CONF}'s content beside ${rel}, then rename it over ${rel}`],
@@ -1382,6 +1599,63 @@ async function proxyConfRefreshStep({ fs, cwd, out, prompt, summary, readPackage
1382
1599
  return true;
1383
1600
  }
1384
1601
 
1602
+ /**
1603
+ * Issue #503: the rules refresh and the replacement of a proxy that lacks the model endpoints' mount, as ONE step. The
1604
+ * new rules include a file that proxy does not mount, so on their own they would make its next start exit (measured:
1605
+ * `docker start` of such a proxy exited 1 on squid's missing-include FATAL). Asked once, never under `--yes` (the
1606
+ * refresh never is). Declined, or not possible from here, nothing is written. A replace that fails puts the old rules
1607
+ * back from the backup it just wrote, so the folder never holds rules no proxy can start on. Returns true when the
1608
+ * rules were replaced. `coupled.handled` is set ONLY when the replace succeeded: declined, blocked or failed, the old
1609
+ * rules are in place and the proxy step below judges the proxy as it always does (start a stopped one, unpause a
1610
+ * paused one, offer a stale one's replace, name declared endpoints the old rules cannot reach).
1611
+ */
1612
+ async function coupledRefreshStep({ fs, path, rel, judged, state, coupled, out, prompt, summary, now }) {
1613
+ const why = "the new rules include model-endpoints.conf, which this proxy does not mount, so they go in only together with a proxy that mounts it";
1614
+ const blocked = coupled.blocked(state);
1615
+ if (blocked) {
1616
+ out(`not replaced: ${why}, and up cannot replace the proxy here: ${blocked}. ${rel} and the proxy are left as they are. To take the new rules, remove the proxy with \`docker ${EGRESS_RM_ARGS.join(" ")}\` (fix a missing file first with \`pi-dispatch init\`), then run \`pi-dispatch up\` again, which starts the shipped proxy on them\n`);
1617
+ summary.push(["egress rules", `${rel} left as it is: ${blocked}`]);
1618
+ return false;
1619
+ }
1620
+ const attached = jobNetworksOf(state);
1621
+ const network = { missing: false };
1622
+ const lines = [`cp ${rel} ${rel}.bak-<timestamp>`, `write ${PACKAGED_EGRESS_PROXY_CONF}'s content beside ${rel}, then rename it over ${rel}`, ...(await coupled.lines(network))];
1623
+ const accepted = await consent(
1624
+ `up would replace ${rel} with ${packageCopyName()}, keeping yours beside it, and replace the proxy (${state.status}) with the shipped one in the same step: ${why}${attached.length > 0 ? `. It is attached to ${attached.join(", ")}: removing it cuts those jobs off from their egress mid-run, so stop the worker first (and let running jobs finish)` : ""}:`,
1625
+ lines,
1626
+ { yes: false, out, prompt },
1627
+ );
1628
+ if (!accepted) {
1629
+ out(`skipped: ${rel} and the proxy are left as they are; \`pi-dispatch up\` offers this again\n`);
1630
+ summary.push(["egress rules", `${rel} differs from ${packageCopyName()}, left as it is with its proxy (declined)`]);
1631
+ return false;
1632
+ }
1633
+ const done = replaceProxyConfCopy({ path, text: judged.packaged, fs, now });
1634
+ if (!done.ok) {
1635
+ out(`✗ not replaced: ${done.reason}. The proxy is left as it is\n`);
1636
+ summary.push(["egress rules", `NOT replaced: ${done.reason}`]);
1637
+ return false;
1638
+ }
1639
+ const backup = `${rel}${done.backup.slice(path.length)}`;
1640
+ if (await coupled.replace(network)) {
1641
+ coupled.handled = true;
1642
+ out(`✓ replaced ${rel} with ${packageCopyName()} (yours is kept as ${backup}) and the proxy with the shipped one, which mounts model-endpoints.conf\n`);
1643
+ summary.push(["egress rules", `${rel} replaced with ${packageCopyName()} (the old one is ${backup})`]);
1644
+ summary.push(["egress", "replaced pi-dispatch-egress-proxy with the shipped one, on the refreshed rules"]);
1645
+ return true;
1646
+ }
1647
+ let restored = false;
1648
+ try {
1649
+ fs.renameSync(done.backup, path);
1650
+ restored = true;
1651
+ } catch {
1652
+ // Said below: the new rules stay, and the backup is where it was.
1653
+ }
1654
+ out(`✗ could not start the shipped proxy; ${restored ? `${rel} is put back as it was (from ${backup}), since no proxy that mounts model-endpoints.conf runs` : `and ${rel} could not be put back: \`mv ${backup} ${rel}\` does it`}. Continuing; doctor below will re-check it\n`);
1655
+ summary.push(["egress", `replace FAILED after the rules refresh; ${restored ? `${rel} put back as it was` : `${rel} NOT put back (${backup} holds the old one)`}`]);
1656
+ return false;
1657
+ }
1658
+
1385
1659
  /**
1386
1660
  * Show the exact commands, then ask. Printing happens with or without `--yes`: consent is what the
1387
1661
  * flag waives, never visibility — every host mutation is on screen before it runs. The prompt
@@ -30,6 +30,7 @@ import { readFileSync } from "node:fs";
30
30
  import { networkInterfaces, userInfo } from "node:os";
31
31
  import { join } from "node:path";
32
32
  import { venuesOf } from "./backends.mjs";
33
+ import { DEFAULT_VALKEY_URL } from "./config.mjs";
33
34
  import { VALKEY_SHARED_KEY, judgeValkeyListeners, passwdNameFrom, pinnedValkeyUrl, probeTcpAddress, readStackKeys, readSubuidRanges, readValkeyKeys, valkeySharedOn, valkeySchemeOf } from "./podman-stack.mjs";
34
35
  import { VALKEY_PASSWORD_KEY, isLoopbackHost } from "./valkey-auth.mjs";
35
36
 
@@ -184,8 +185,8 @@ export function valkeyDbRangeSentence(url, db, databases) {
184
185
  return `VALKEY_URL ${urlShown(url)} names database ${db}, which that Valkey does not have: ${has}. No client uses another database in its place; name one it has, or raise \`databases\` in that Valkey's configuration`;
185
186
  }
186
187
 
187
- /** The default VALKEY_URL, the worker's own (config.mjs). */
188
- export const DEFAULT_VALKEY_URL = "redis://127.0.0.1:6379";
188
+ /** The default VALKEY_URL, the worker's own, defined once in config.mjs. */
189
+ export { DEFAULT_VALKEY_URL };
189
190
 
190
191
  /**
191
192
  * The VALKEY_URL a command run from a deployment folder uses (PR #475's review), by the same rule as the password it