@edgehero/pi-dispatch 1.10.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/.env.example +300 -148
  2. package/README.md +50 -0
  3. package/deploy/com.pi-dispatch.worker.plist +9 -3
  4. package/deploy/docker-compose.yml +49 -16
  5. package/deploy/egress-proxy.conf +32 -2
  6. package/deploy/nssm-install.cmd +12 -6
  7. package/deploy/pi-dispatch-egress-out.network +10 -0
  8. package/deploy/pi-dispatch-egress-proxy.container +50 -0
  9. package/deploy/pi-dispatch-netns-keeper.container +80 -0
  10. package/deploy/pi-dispatch-netns-keeper.network +18 -0
  11. package/deploy/pi-dispatch-valkey.container +51 -0
  12. package/deploy/pi-dispatch-valkey.network +16 -0
  13. package/deploy/receiver.service +6 -0
  14. package/deploy/worker-env-wrapper.cmd +11 -0
  15. package/deploy/worker-env-wrapper.sh +60 -34
  16. package/deploy/worker.service +18 -8
  17. package/package.json +14 -4
  18. package/src/azure-host.mjs +19 -0
  19. package/src/azure-identity.mjs +18 -2
  20. package/src/backend-conformance.mjs +71 -18
  21. package/src/backend-local.mjs +637 -21
  22. package/src/backend-podman.mjs +1168 -0
  23. package/src/backend-registry.mjs +86 -3
  24. package/src/backends.mjs +489 -37
  25. package/src/branch.mjs +7 -2
  26. package/src/cancel-cli.mjs +174 -0
  27. package/src/cancel-state.mjs +125 -0
  28. package/src/cli.mjs +188 -90
  29. package/src/config.mjs +503 -43
  30. package/src/connection.mjs +374 -8
  31. package/src/container-spec.mjs +102 -7
  32. package/src/daemon-facts.mjs +167 -0
  33. package/src/deployment-venue.mjs +158 -0
  34. package/src/docker-run.mjs +146 -15
  35. package/src/doctor.mjs +4701 -414
  36. package/src/egress-conf-copy.mjs +166 -0
  37. package/src/egress-proxy-state.mjs +151 -0
  38. package/src/egress.mjs +455 -25
  39. package/src/entry.mjs +27 -0
  40. package/src/env-allowlist.mjs +222 -40
  41. package/src/env-file.mjs +1869 -33
  42. package/src/exit-code.mjs +15 -0
  43. package/src/flow-gate.mjs +5 -3
  44. package/src/forgejo-host.mjs +19 -0
  45. package/src/forgejo-identity.mjs +21 -2
  46. package/src/get-token.mjs +67 -18
  47. package/src/git-dirty.mjs +9 -1
  48. package/src/git-hardening.mjs +33 -0
  49. package/src/github-app-setup.mjs +29 -12
  50. package/src/github-prompt.mjs +4 -1
  51. package/src/gitlab-host.mjs +19 -0
  52. package/src/gitlab-identity.mjs +19 -2
  53. package/src/host-registry.mjs +32 -5
  54. package/src/identity.mjs +29 -4
  55. package/src/image-preflight.mjs +46 -11
  56. package/src/image-ref.mjs +21 -0
  57. package/src/index.mjs +387 -17
  58. package/src/init.mjs +197 -38
  59. package/src/job-user.mjs +252 -0
  60. package/src/json-duplicates.mjs +204 -0
  61. package/src/live-probes.mjs +1020 -0
  62. package/src/materialize.mjs +4 -11
  63. package/src/netns-keeper.mjs +264 -0
  64. package/src/on-failure.mjs +119 -0
  65. package/src/outbox.mjs +7 -0
  66. package/src/podman-stack.mjs +1304 -0
  67. package/src/prepare-github.mjs +6 -6
  68. package/src/prepare-local.mjs +51 -17
  69. package/src/prepare.mjs +27 -6
  70. package/src/processor.mjs +505 -26
  71. package/src/provider-key.mjs +41 -0
  72. package/src/provider-steering.mjs +144 -0
  73. package/src/queue.mjs +35 -8
  74. package/src/redact.mjs +84 -0
  75. package/src/reserved-env.mjs +7 -3
  76. package/src/retention-sweep.mjs +178 -0
  77. package/src/run-container.mjs +181 -14
  78. package/src/run-history.mjs +105 -16
  79. package/src/runtime-observations.mjs +1152 -0
  80. package/src/runtime-settings.mjs +13 -8
  81. package/src/sandbox-cli.mjs +100 -95
  82. package/src/sandbox-store.mjs +612 -45
  83. package/src/sandbox.mjs +1459 -37
  84. package/src/schedules.mjs +16 -3
  85. package/src/secret-profiles.mjs +2 -1
  86. package/src/secrets.mjs +23 -6
  87. package/src/service-env.mjs +247 -0
  88. package/src/service.mjs +618 -28
  89. package/src/session-store.mjs +678 -53
  90. package/src/start.mjs +1395 -268
  91. package/src/transient.mjs +240 -0
  92. package/src/triggers-file.mjs +71 -15
  93. package/src/triggers.mjs +176 -19
  94. package/src/up.mjs +1399 -85
  95. package/src/valkey-auth.mjs +529 -0
  96. package/src/valkey-endpoint.mjs +367 -0
  97. package/src/watch-closer.mjs +158 -0
@@ -1,9 +1,11 @@
1
1
  import {
2
2
  copyFileSync,
3
+ fstatSync,
3
4
  lstatSync,
4
5
  mkdirSync,
5
6
  openSync,
6
7
  closeSync,
8
+ readSync,
7
9
  readFileSync,
8
10
  readdirSync,
9
11
  renameSync,
@@ -11,8 +13,12 @@ import {
11
13
  unlinkSync,
12
14
  writeFileSync,
13
15
  } from "node:fs";
16
+ import { randomBytes } from "node:crypto";
14
17
  import { join } from "node:path";
18
+ import { scrubCredentials } from "./redact.mjs";
15
19
  import { sessionKeyFor } from "./session-key.mjs";
20
+ import { resolveBackendName } from "./backend-registry.mjs";
21
+ import { UNATTRIBUTED_BACKEND } from "./backends.mjs";
16
22
 
17
23
  /**
18
24
  * session-store.mjs -- the host side of a resumable session (INT-SESSION-STORE-CONTRACT).
@@ -72,7 +78,75 @@ const RESUME_CHAIN_FILE = "resume-chain";
72
78
  * agent-influenced, since the transcript itself is agent-written.
73
79
  */
74
80
  const CONTEXT_FILE = "context";
75
- /** Both sidecar formats are a handful of bytes. Generous, and still nowhere near a job's wall clock. */
81
+ /**
82
+ * The venue whose container produced the transcript beside it (issue #277): a backend name, one line.
83
+ *
84
+ * A SIDECAR, NOT KEY MATERIAL. `DES-SESSION-KEY-IS-DERIVED-NOT-INDEXED` makes the key a pure function of
85
+ * (kind, repo, ref), and a venue in the key would fork a trigger moved between venues into a second lineage
86
+ * that nothing ever sweeps. As a sidecar it GATES the one lineage instead: the move costs one cold start.
87
+ *
88
+ * ABSENT READS AS `UNATTRIBUTED_BACKEND`, never as the deployment default. Every transcript written before this
89
+ * stamp existed ran on `local`, and the default is a setting that can move under it.
90
+ */
91
+ const VENUE_FILE = "venue";
92
+ /**
93
+ * What the venue stamp reads while a promotion is swapping the transcript under it. It matches no venue:
94
+ * parentheses are outside the charset every backend name is validated against (a test pins every name this
95
+ * build knows to it), so a key left holding this cold-starts on every venue until a promotion completes.
96
+ */
97
+ const VENUE_PENDING = "(pending)";
98
+
99
+ /**
100
+ * Per-promotion suffix for the transcript's in-flight copy, paired with the pid.
101
+ *
102
+ * The tmp name USED to be a fixed `<canonical>.incoming`, which was self-cleaning and harmless while the
103
+ * per-key lock guaranteed one writer. The stale-lock takeover concedes a window with two, and a SHARED tmp
104
+ * path voids exactly the atomicity the swap exists for: B's unlink removes A's in-flight copy, B's copy
105
+ * replaces it, and whichever renames second can put the other's half-written file at the canonical path --
106
+ * an agent handed a truncated conversation with no gate able to see it. `triggers-file.mjs` learned this
107
+ * on `triggers.json` and its `tmpPathFor` says the same thing.
108
+ *
109
+ * RANDOM BYTES, not just a pid and a counter, and the pid is why: two workers can share one. Two
110
+ * containers where node is pid 1, or two hosts on one shared `PI_SESSIONS_DIR` -- which is the very
111
+ * `OQ-031` shape the takeover's own skew analysis invokes -- produce byte-identical `<canonical>.1.0.incoming`
112
+ * names, measured, and a shared destination does tear (two processes copying 4 MiB to one path mixed both
113
+ * writers' bytes in 1 sample of 110). A pid-and-counter name separates two promotions inside ONE process,
114
+ * which is the one case that cannot happen, and leaves the case that can. The random half is what actually
115
+ * separates writers; the pid and counter stay because they make a straggler attributable.
116
+ *
117
+ * The cost is that a crash between the copy and the rename leaves a uniquely named straggler instead of one
118
+ * the next promotion overwrites; the reaper's recursive sweep of the key takes it with everything else --
119
+ * except under `PI_SESSIONS_TTL_DAYS=0`, where that sweep does not run at all, so stragglers accumulate
120
+ * there where the old shared name was self-cleaning. The same TTL-0 caveat the lock's takeover carries.
121
+ *
122
+ * The SIDECAR temps keep the shared name deliberately, and the asymmetry is the point rather than an
123
+ * oversight: a sidecar's worst case under a concurrent writer is a missing or half-written sidecar, and
124
+ * every one of those reads back as `null` and produces a cold start, which this store already treats as
125
+ * the safe outcome. The transcript's worst case is a corrupt transcript, which it does not.
126
+ */
127
+ let tmpSeq = 0;
128
+
129
+ /**
130
+ * A promotion lock older than this is a crashed writer's, not a live one's.
131
+ *
132
+ * `triggers-file.mjs` holds the same idiom at ten seconds; this is 360 times that, because the work under
133
+ * the two locks is not comparable. A trigger write is a read, one mutate and two syscalls. A
134
+ * promotion is a COPY of the container's transcript -- up to `PI_SESSION_MAX_BYTES`, 8 MiB by default --
135
+ * plus a rename and four small sidecar writes.
136
+ *
137
+ * AN ASSERTION ABOUT STORAGE, NOT A DERIVATION, and said plainly because `PI_SESSION_MAX_BYTES=0` is a
138
+ * supported setting and there is then no configured bound to derive from. An hour covers roughly three
139
+ * gigabytes at a megabyte a second, which is a slower store than anything this project will meet.
140
+ *
141
+ * WHAT MAKES A GENEROUS NUMBER THE RIGHT TRADE IS THE ASYMMETRY. Too short steals a LIVE writer's lock,
142
+ * and with the release-by-path residual below that degrades into the lock being functionally absent. Too
143
+ * long only delays recovery on a key nobody is watching, at one extra cold start per job until it passes.
144
+ *
145
+ * A CONSTANT RATHER THAN A KNOB, on triggers-file's precedent: the one value an operator would reach for
146
+ * is zero, and zero here means no lock at all.
147
+ */
148
+ const LOCK_STALE_MS = 3_600_000;
149
+ /** Every sidecar format is a handful of bytes. Generous, and still nowhere near a job's wall clock. */
76
150
  const SIDECAR_MAX_BYTES = 4096;
77
151
  /**
78
152
  * The host-effective provider and model as one token, or null when the job names neither.
@@ -93,6 +167,15 @@ function modelIdentity(job) {
93
167
  return /^[a-z0-9][a-z0-9._:/-]{0,127}$/.test(id) ? id : null;
94
168
  }
95
169
 
170
+ /**
171
+ * A venue name as the store handles it: a non-empty string, else `null`. A promotion stamps `String(venue)`,
172
+ * and `String(undefined)` is the word `undefined`, which is a valid-looking name; normalising first means a
173
+ * session with no resolvable venue is stamped with nothing rather than with that.
174
+ */
175
+ function normaliseVenue(venue) {
176
+ return typeof venue === "string" && venue !== "" ? venue : null;
177
+ }
178
+
96
179
  /**
97
180
  * Read-path outcomes. Every one is a named cold start rather than a bare `false`: a feature that fails
98
181
  * open is otherwise indistinguishable from a feature nobody switched on, which is precisely how "we
@@ -107,9 +190,12 @@ export function makeSessionStore({
107
190
  maxAgeDays = 0,
108
191
  maxResumeChain = 0,
109
192
  maxContextPct = null,
193
+ // The deployment's default venue, `PI_BACKENDS[0]`, for a job that names none (#277). `null` is a
194
+ // dependency-injection seam, and a job whose venue cannot be resolved never resumes.
195
+ defaultBackend = null,
110
196
  log = () => {},
111
197
  now = () => Date.now(),
112
- fs = { copyFileSync, lstatSync, mkdirSync, openSync, closeSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync },
198
+ fs = { copyFileSync, fstatSync, lstatSync, mkdirSync, openSync, closeSync, readSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync },
113
199
  }) {
114
200
  /**
115
201
  * Stage this job's /session directory and decide whether it resumes.
@@ -121,6 +207,8 @@ export function makeSessionStore({
121
207
  * /session mount at all (unarmed, or no key) -- which is byte-identical to a pre-feature job.
122
208
  */
123
209
  function resolveSession(job, { jobDir, resolved = {}, piVersion = null } = {}) {
210
+ // Declared outside the try so a fault can take back what this call staged (below).
211
+ let hostDir = null;
124
212
  try {
125
213
  // Unreachable in a wired worker, and deliberately kept: resolveSession is only ever called for a
126
214
  // job that armed run.resume (prepare-github.mjs), and processor.mjs refuses exactly that job
@@ -139,13 +227,91 @@ export function makeSessionStore({
139
227
  // count is 78% of a 32k window and 2.5% of a 1M one. Carried on the session object rather than
140
228
  // read again at promote time, so the reading is stamped with the model that produced it.
141
229
  const modelId = modelIdentity(job);
142
- const hostDir = join(jobDir, "session");
230
+ // The venue this job will run in, resolved exactly as the registry dispatches it (#277). Carried on
231
+ // the session like `modelId`, so the promotion stamps the venue that produced the transcript rather
232
+ // than re-deriving one later.
233
+ const venue = normaliseVenue(resolveBackendName(job, defaultBackend));
234
+ hostDir = join(jobDir, "session");
143
235
  const staged = join(hostDir, SESSION_FILE_NAME);
144
236
  fs.mkdirSync(hostDir, { recursive: true, mode: 0o700 });
145
237
 
146
- const verdict = readCanonical(key, piVersion, modelId);
238
+ // The identity is split off the verdict rather than carried on it: this return is spread onto the
239
+ // session object the processor holds for the WHOLE run, and an inode number is bookkeeping for the
240
+ // next few lines, not state a promotion an hour later should be able to read.
241
+ const { ident: judged = null, dirIdent: judgedDir = null, ...judgedVerdict } = readCanonical(key, piVersion, modelId, venue);
242
+ let verdict = judgedVerdict;
147
243
  if (verdict.resume) {
148
- fs.copyFileSync(canonicalFile(key), staged);
244
+ // THE BYTES COME OFF THE DESCRIPTOR THAT WAS JUDGED, never off the path a second time.
245
+ //
246
+ // The gates above ran by PATH, and so did this copy until issue #375's gate round: it was
247
+ // `copyFileSync(canonicalFile(key), staged)` followed by re-checks that also re-resolved the path.
248
+ // Every one of those resolutions is a separate walk of the same name, so an attacker who is a
249
+ // symlink DURING the copy and the original directory again before the re-check matched all three
250
+ // arms while the bytes came from somewhere else entirely. Measured on the pre-fix code by a plain
251
+ // second process doing rename/symlink/rename in a loop: 195 of 757 successful resumes in the
252
+ // review round's own harness, and 23 of 654 in a 90-second run of this file's, staged an
253
+ // attacker's transcript and reported `resumed`. The same shape one level down -- the
254
+ // A, B, A round trip on `current.jsonl` itself -- is what this contract used to state as a
255
+ // residual the identity re-check could only shrink.
256
+ //
257
+ // One `open` answers both. The descriptor is bound to the inode at the instant it is taken, so
258
+ // `fstat` on it cannot be lied to by a later swap, and bytes read from it cannot come from a file
259
+ // that replaced the name afterwards. What a swap can still do is make this refuse: the identity
260
+ // will not match what the gates judged, and the job cold-starts, which is the safe direction.
261
+ //
262
+ // Streamed in bounded chunks rather than read whole: `PI_SESSION_MAX_BYTES` defaults to 8 MiB but
263
+ // `0` means NO CAP, and buffering an uncapped transcript into memory to copy it would be a new
264
+ // way to hurt a host that the old `copyFileSync` never had.
265
+ let fd = null;
266
+ try {
267
+ fd = fs.openSync(canonicalFile(key), "r");
268
+ const opened = identityOf(fs.fstatSync(fd));
269
+ // TWO CHECKS, and neither subsumes the other. The gate round proved that the hard way, twice.
270
+ //
271
+ // The DESCRIPTOR check says the bytes about to be staged come off the inode this open took,
272
+ // and nothing can re-point it afterwards. It is what closes the A, B, A across the copy.
273
+ //
274
+ // The BY-PATH check after the copy says the name still resolves to the file the GATES judged.
275
+ // It is what the descriptor cannot say: every gate above ran by path, so a link planted after
276
+ // the first of those lstats has the whole ladder judge the attacker's files, and then the
277
+ // descriptor agrees with a `judged` that is already the attacker's. Dropping this check for
278
+ // one round reintroduced exactly that (measured: 2 attacker transcripts staged in 16,594
279
+ // resumes, where the by-path version staged none), and it is also what keeps the venue arm
280
+ // reachable: a promotion that wrote its `(pending)` sentinel and has not yet swapped moves no
281
+ // identity at all, so an identity-gated ladder never asks about it.
282
+ const swapped = () => {
283
+ const dirNow = inspectKeyDir(key);
284
+ return dirNow.reason === "key-not-a-directory"
285
+ ? "key-not-a-directory"
286
+ : dirNow.ident !== judgedDir
287
+ ? "transcript-replaced" // another REAL directory at the name: the gates judged its files
288
+ : readVenue(key) !== venue
289
+ ? "venue-changed"
290
+ : readIdentity(canonicalFile(key)) !== judged
291
+ ? "transcript-replaced"
292
+ : null;
293
+ };
294
+ // VENUE FIRST inside `swapped()`, as it has been since #277: a cross-venue promotion trips
295
+ // both and `venue-changed` names WHY where `transcript-replaced` only says something moved.
296
+ // The directory arm is ahead of both (issue #375), because a name that is not a directory is
297
+ // neither a venue move nor a swap.
298
+ const raced = opened !== judged ? (swapped() ?? "transcript-replaced") : null;
299
+ if (raced !== null) {
300
+ // A 0-byte staged file, the shape every cold start gets. If emptying fails, the catch
301
+ // below removes the staged directory rather than leave a copy behind.
302
+ fs.writeFileSync(staged, "");
303
+ verdict = COLD(raced);
304
+ } else {
305
+ copyFromDescriptor(fd, staged, verdict.bytes);
306
+ const after = swapped();
307
+ if (after !== null) {
308
+ fs.writeFileSync(staged, "");
309
+ verdict = COLD(after);
310
+ }
311
+ }
312
+ } finally {
313
+ if (fd !== null) fs.closeSync(fd);
314
+ }
149
315
  } else {
150
316
  // A 0-BYTE FILE, not an absent one. pi's setSessionFile then takes its empty-file branch and
151
317
  // writes its own header at this exact path, which marks the manager flushed -- so _persist
@@ -154,10 +320,22 @@ export function makeSessionStore({
154
320
  fs.writeFileSync(staged, "");
155
321
  }
156
322
  log("session_resolved", { key, resume: verdict.resume, reason: verdict.reason });
157
- return { hostDir, key, modelId, ...verdict };
323
+ return { hostDir, key, modelId, venue, ...verdict };
158
324
  } catch (err) {
159
325
  // A history fault must never fail the prepare that asked.
160
326
  log("session_store_failed", { phase: "resolve", reason: err?.message });
327
+ // `null` means NO MOUNT AND NOTHING WRITTEN, so make the second half true, best effort. A fault after
328
+ // the copy -- a partial copy, or the venue re-check failing to empty a transcript another venue just
329
+ // promoted (#277) -- would otherwise leave that file under the job dir, which is mounted `/job:ro`:
330
+ // no `/session`, but the transcript readable at `/job/session/current.jsonl` all the same.
331
+ if (hostDir !== null) {
332
+ try {
333
+ fs.rmSync(hostDir, { recursive: true, force: true });
334
+ } catch {
335
+ // If this fails too, the copy stays under the job dir, which the container mounts before
336
+ // teardown removes it. Two consecutive disk faults on one path; nothing further to try.
337
+ }
338
+ }
161
339
  return null;
162
340
  }
163
341
  }
@@ -170,6 +348,88 @@ export function makeSessionStore({
170
348
  * one key is a real shape (REQ-QUEUE-BURST-NO-DROP), and last-write-wins there would interleave two
171
349
  * agents' turns into one transcript.
172
350
  */
351
+ /**
352
+ * Take the per-key promotion lock, with ONE stale takeover. Returns `{ fd }`, or `{ locked: true }` when
353
+ * a LIVE writer holds it. Throws only for non-EEXIST failures, which is the doctrine this call site
354
+ * already carried: a read-only directory, a full disk or a vanished store all fail to create the lock
355
+ * too, and reporting those as `locked` sends an operator hunting a stuck file that does not exist.
356
+ *
357
+ * WITHOUT THIS, A PROCESS KILLED INSIDE THE LOCK WEDGED THE KEY FOREVER (issue #336). Later promotions
358
+ * reported `locked` until the reaper swept the key, and the reaper keys on the TRANSCRIPT's mtime: a key
359
+ * whose first promotion died before any transcript landed had none to key on, and with
360
+ * `PI_SESSIONS_TTL_DAYS=0` the reaper does not run at all. A takeover works in both of those, which is
361
+ * why it is the primary fix and the reaper's own repair is the secondary one.
362
+ *
363
+ * `lstatSync`, never `statSync`. The injected `fs` deliberately carries no `statSync`, this module's
364
+ * whole doctrine is lstat-in-a-key-directory, and a DANGLING link planted at this name throws ENOENT
365
+ * under `stat` on every attempt, which is the same wedge in a different coat.
366
+ *
367
+ * TWO DIFFERENT CLOCKS, and the comparison is between them: `now()` is this process's, `mtimeMs` is the
368
+ * FILESYSTEM's, which on a network mount is a server's. `OQ-031` already records two hosts sharing one
369
+ * working tree as a live hazard, and a shared `PI_SESSIONS_DIR` is exactly that shape. Both signs of the
370
+ * skew are bad, and differently. A server more than `LOCK_STALE_MS` BEHIND makes every live lock read as
371
+ * stale, so the takeover fires on every attempt and the conceded double-take window stops being rare;
372
+ * worse, `releaseLock` unlinks by PATH rather than by fd, so once A's lock is stolen A's release deletes
373
+ * B's. A server AHEAD makes the difference negative, so a genuinely crashed writer's lock is NEVER
374
+ * swept -- which is precisely today's behaviour, so under that skew this degrades to the status quo
375
+ * rather than to something worse. Nothing here closes either; `triggers-file.mjs` states the same pair.
376
+ */
377
+ function takeLock(dir, key) {
378
+ const lock = join(dir, LOCK_FILE);
379
+ let sweptAgeMs = null;
380
+ for (let attempt = 0; attempt < 2; attempt++) {
381
+ try {
382
+ const fd = fs.openSync(lock, "wx"); // exclusive create IS the lock; no daemon, no lease
383
+ // Logged only AFTER the retake create SUCCEEDED. The unlink alone proves nothing, since a rival
384
+ // sweeper can win the recreate race, and a takeover line for a lock we did not get would send an
385
+ // operator reading a history that never happened.
386
+ if (sweptAgeMs !== null) log("session_lock_stale_taken", { key, ageMs: sweptAgeMs });
387
+ return { fd };
388
+ } catch (err) {
389
+ if (err?.code !== "EEXIST") throw err;
390
+ let mtimeMs;
391
+ try {
392
+ mtimeMs = fs.lstatSync(lock).mtimeMs;
393
+ } catch {
394
+ // Released between our open and our stat: the next create answers.
395
+ continue;
396
+ }
397
+ if (now() - mtimeMs <= LOCK_STALE_MS) return { locked: true };
398
+ try {
399
+ fs.unlinkSync(lock);
400
+ } catch {
401
+ // Someone else swept it first; the retry create answers who won.
402
+ }
403
+ sweptAgeMs = Math.round(now() - mtimeMs);
404
+ }
405
+ }
406
+ return { locked: true };
407
+ }
408
+
409
+ /**
410
+ * Make the key directory, or answer that the name is not one (issue #375). The write-side twin of
411
+ * `inspectKeyDir`, separate because it CREATES: the read path must never make a directory for a key it is
412
+ * only asking about.
413
+ *
414
+ * An lstat that fails with anything but ENOENT is re-thrown, which the outer catch reports as
415
+ * `promote-failed`: a disk fault is not evidence about the shape of the name, and reporting it as one
416
+ * would send an operator hunting a symlink that does not exist. That is `takeLock`'s doctrine one step
417
+ * earlier.
418
+ */
419
+ function ensureKeyDir(dir) {
420
+ let st = null;
421
+ try {
422
+ st = fs.lstatSync(dir);
423
+ } catch (err) {
424
+ if (err?.code !== "ENOENT") throw err;
425
+ }
426
+ if (st === null) {
427
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
428
+ st = fs.lstatSync(dir); // the second lstat: a link planted between the two is caught here
429
+ }
430
+ return st.isDirectory() ? `${st.dev}:${st.ino}` : null;
431
+ }
432
+
173
433
  function promoteSession(session, { piVersion = null, context = null } = {}) {
174
434
  // The second DI-seam backstop, and unreachable for the same reason as the `!sessionsDir` return
175
435
  // above: sessionKeyFor is total and binary (null, or 32 hex chars), so resolveSession returns null
@@ -187,38 +447,150 @@ export function makeSessionStore({
187
447
  }
188
448
 
189
449
  const dir = keyDir(session.key);
190
- fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
450
+ // LSTAT BEFORE CREATE, and again after (issue #375). Recursive `mkdir` succeeds silently on an
451
+ // existing link to a directory, so asking it first and checking afterwards would accept exactly the
452
+ // shape this refuses; asking first means nothing already at the name is ever handed to `mkdir` at all,
453
+ // and the outcome does not depend on how `mkdir` treats a link, a file or a dangling link. Placed
454
+ // BEFORE `takeLock`, so a refused promotion never creates a lock file through a link either.
455
+ //
456
+ // The entry is NOT removed. It is not ours: this store creates key directories and nothing else, so
457
+ // whatever is sitting there was put there by an operator or by an attacker, and a sweep that deletes
458
+ // either is a worse answer than a refusal an operator can read. The key stays cold until they clear
459
+ // it, which is the loud direction.
460
+ const madeDir = ensureKeyDir(dir);
461
+ if (madeDir === null) {
462
+ log("session_promote_skipped", { key: session.key, reason: "key-not-a-directory" });
463
+ return { promoted: false, reason: "key-not-a-directory" };
464
+ }
465
+ // Is the key directory still the one `ensureKeyDir` made or found? SHAPE ALONE IS NOT ENOUGH, and
466
+ // the gate round proved it: a swap to another REAL directory passed a shape-only re-check and the
467
+ // promotion wrote the transcript and all four sidecars into it while reporting `promoted: true`.
468
+ const sameDir = () => inspectKeyDir(session.key).ident === madeDir;
191
469
 
192
470
  const lock = join(dir, LOCK_FILE);
193
- let fd;
194
- try {
195
- fd = fs.openSync(lock, "wx"); // exclusive create IS the lock; no daemon, no lease
196
- } catch (err) {
197
- // EEXIST is the only failure that MEANS locked. A read-only directory, a full disk or a
198
- // vanished store all failed to create the lock too, and reporting those as `locked` sends an
199
- // operator looking for a stuck lock file that does not exist. Anything else falls through to
200
- // the outer catch and reports `promote-failed`, which is what actually happened.
201
- if (err?.code !== "EEXIST") throw err;
471
+ // `locked` still means a LIVE writer and nothing else. A lock older than any plausible promotion is
472
+ // taken over rather than believed (see `takeLock`); anything that is not EEXIST still falls through
473
+ // to the outer catch and reports `promote-failed`, which is what actually happened.
474
+ const taken = takeLock(dir, session.key);
475
+ if (taken.locked) {
202
476
  log("session_promote_skipped", { key: session.key, reason: "locked" });
203
477
  return { promoted: false, reason: "locked" };
204
478
  }
479
+ const fd = taken.fd;
205
480
  try {
206
- // Atomic swap: a reader either sees the old file or the new one, never a half-written one.
207
- const tmp = `${canonicalFile(session.key)}.incoming`;
208
- try {
209
- // `copyFileSync` follows a link at the DESTINATION, so a link planted at this name would
210
- // receive the whole transcript and leave the canonical path pointing at it. The key
211
- // directory's name is derived rather than random, so the path is precomputable by anyone
212
- // who knows the repository and the branch; unlinking removes the link, never its target.
213
- fs.unlinkSync(tmp);
214
- } catch {
215
- // Absent is the desired state.
481
+ // RE-CHECKED UNDER THE LOCK (issue #375's gate round). `ensureKeyDir` ran before the lock, so a
482
+ // link swapped in between the two would take every write below it outside the store: measured on
483
+ // the first draft, the transcript and all four sidecars landed in the attacker's directory and
484
+ // the promotion still reported `promoted: true`. Asking again here closes that segment, and
485
+ // costs one lstat on a path already in the page cache.
486
+ //
487
+ // It does NOT close the whole window, and the residual is stated rather than implied: a swap
488
+ // after THIS check still redirects the writes, because Node exposes no `openat`/`renameat` and
489
+ // every write below therefore resolves the name again. What the re-check after the swap does is
490
+ // make that case reportable rather than silent.
491
+ //
492
+ // What the refusal costs where the swap landed after the lock was taken: the lock file is in the
493
+ // REAL key and `releaseLock` unlinks by path, which now resolves elsewhere, so the real key keeps
494
+ // a lock nobody holds and the next promotions report `locked` until the staleness takeover sweeps
495
+ // it an hour later. That is the same unlink-by-path residual `takeLock` already states for a
496
+ // takeover, reached by a different route, and it is the honest cost of refusing here rather than
497
+ // writing into whatever the name now points at.
498
+ if (!sameDir()) {
499
+ log("session_promote_skipped", { key: session.key, reason: "key-not-a-directory" });
500
+ return { promoted: false, reason: "key-not-a-directory" };
216
501
  }
502
+ // Atomic swap: a reader either sees the old file or the new one, never a half-written one.
503
+ // PER WRITER (see `tmpSeq`), not the fixed `.incoming` this shared before the lock could be taken
504
+ // over. The random half is what separates two writers; the pid and counter are there to make a
505
+ // straggler attributable, and neither is a writer identity on its own. The removal below stays
506
+ // regardless: a name that is hard to guess is not a name that cannot be guessed.
507
+ const tmp = `${canonicalFile(session.key)}.${process.pid}.${tmpSeq++}.${randomBytes(6).toString("hex")}.incoming`;
508
+ // `copyFileSync` follows a link at the DESTINATION, so anything left at this name would receive
509
+ // the whole transcript. `removeTemp` takes every shape; see its own comment for why one call
510
+ // could not.
511
+ removeTemp(tmp);
217
512
  fs.copyFileSync(staged, tmp);
513
+ // THE VENUE STAMP IS INVALIDATED BEFORE THE SWAP, and fatally (#277). Removing a stamp does not
514
+ // invalidate it, because an absent stamp reads as `local`; stamping only after the rename would
515
+ // leave a window, and a failed post-swap write a permanent state, in which one venue's transcript
516
+ // sits under another venue's stamp and the next job there resumes it. So every promotion first
517
+ // writes a stamp no venue matches. If that write fails, nothing has been swapped and the outer
518
+ // catch reports `promote-failed`; if the process dies after it, the key cold-starts everywhere
519
+ // until a promotion completes. A process KILLED inside this lock also leaks the lock itself, as it
520
+ // always has -- but it no longer wedges the key in the shapes that used to be permanent: the next
521
+ // promotion takes a lock older than `LOCK_STALE_MS` over, and the reaper reaches a key with no
522
+ // transcript through the directory's own mtime (issue #336). A lock whose mtime is in the FUTURE is
523
+ // still never stale, so under `PI_SESSIONS_TTL_DAYS=0` such a key stays wedged. Unconditional rather than only
524
+ // on a venue change, so the decision needs no read of the stamp under the lock and there is no
525
+ // branch to get wrong.
526
+ replaceSidecar(dir, VENUE_FILE, VENUE_PENDING);
527
+ // The temp's identity, read BEFORE the rename, is what makes the A, B, A reportable (issue #390).
528
+ // `rename` preserves the inode, so after a rename that landed where it was meant to, the canonical
529
+ // name holds THIS file. If the directory was swapped for the rename and swapped back afterwards,
530
+ // the canonical name resolves into the real key again and holds its OLD transcript, or nothing --
531
+ // a different inode either way. That is a fact the shape re-check below cannot see, because by
532
+ // then the shape is right again.
533
+ const tmpIdent = readIdentity(tmp);
218
534
  fs.renameSync(tmp, canonicalFile(session.key));
219
- fs.writeFileSync(join(dir, PI_VERSION_FILE), String(piVersion ?? ""));
220
- // The two sidecars, immediately after the swap and under the same lock. NOT part of the swap
221
- // itself, which is one rename and cannot be widened: what the lock buys them is that no
535
+ // DETECTION for the segment prevention cannot reach (issue #375's gate round). A link swapped in
536
+ // after the check above takes this rename and every sidecar write with it, and the old code then
537
+ // reported `promoted: true` for a transcript that landed outside the store: the operator's record
538
+ // said the next run would resume work that is not there. This cannot undo the write, the bytes
539
+ // being already wherever the name pointed, and it catches such a swap only while it is STILL
540
+ // STANDING here.
541
+ if (!sameDir()) {
542
+ log("session_promote_skipped", { key: session.key, reason: "key-not-a-directory" });
543
+ return { promoted: false, reason: "key-not-a-directory" };
544
+ }
545
+ // AND THE A, B, A, which the check above cannot see because by then the shape is right again
546
+ // (issue #390). Measured with a deterministic probe before this existed: `{"promoted":true,
547
+ // "reason":"promoted"}` with the transcript in the attacker's directory and the real key holding
548
+ // only `pi-version`, `resume-chain` and `venue` -- the next job on that key cold-starts as
549
+ // `absent` while the record says the work was promoted, which is the silent no-op `CLAUDE.md`
550
+ // calls the worst outcome available. A 45 second unsynchronised live race did not hit the exact
551
+ // ordering (16,591 promotions, 3 refusals, 0 lies), so this is a deterministic-window finding
552
+ // rather than a frequent one, and its precondition is the one the whole store concedes: write
553
+ // access to `PI_SESSIONS_DIR`.
554
+ //
555
+ // DETECTION, NOT PREVENTION, like its neighbour: the bytes are already wherever the name pointed
556
+ // and nothing here can recall them. What it buys is that the RECORD stops claiming otherwise.
557
+ // A shape-only comparison would not do it -- the round that added the checks above proved a swap
558
+ // to another REAL directory passes one -- so this compares the inode the rename preserved.
559
+ if (tmpIdent === null || readIdentity(canonicalFile(session.key)) !== tmpIdent) {
560
+ log("session_promote_skipped", { key: session.key, reason: "transcript-diverted" });
561
+ return { promoted: false, reason: "transcript-diverted" };
562
+ }
563
+ // The real stamp FIRST of the post-swap writes, and the ordering is the contract. It is the only one
564
+ // whose absence MISATTRIBUTES rather than cold-starts: between the rename and this write the key reads
565
+ // `(pending)` and cold-starts on every venue, so every write placed ahead of it lengthens that window,
566
+ // and `pi-version`, the chain counter and the context reading all degrade to a cold start instead.
567
+ // (It used to be ordered ahead of the pi-version write because THAT write could throw. It no longer
568
+ // can, and the ordering that survives is this one.) A session with no venue (a DI seam) leaves the
569
+ // sentinel on purpose: no stamp is better than a guessed one.
570
+ const venue = normaliseVenue(session.venue);
571
+ if (venue !== null) writeSidecar(dir, VENUE_FILE, session.key, venue);
572
+ // THROUGH `writeSidecar` like every other sidecar (issue #336). It was the one plain write left, and
573
+ // it carried both halves of that exception: `writeFileSync` FOLLOWS a link, so a symlink planted at
574
+ // this name -- at a path anyone who knows the repository and the branch can compute -- turned a
575
+ // promotion into a truncating write of the link's target with the pi version as its payload; and it
576
+ // sat inside the try, so a failure AFTER the swap reported `promote-failed` for a promotion that had
577
+ // landed.
578
+ //
579
+ // A FAILED STAMP IS SAFE IN BOTH DIRECTIONS, which is why this one needs no sentinel of its own and
580
+ // the venue stamp does. With no prior stamp the next read gets `null` and cold-starts. With one, it
581
+ // survives beside the new transcript -- and it is still TRUE, because this job resumed only if that
582
+ // stamp already matched its own image's pi, and a job runs one image. The remaining case, a job that
583
+ // cold-started under a new pi and then failed this write, leaves the OLD version beside the new
584
+ // transcript, and the next job on that image reads a mismatch and cold-starts. Stale-but-true, or a
585
+ // cold start. Never a transcript resumed under a version that did not write it.
586
+ //
587
+ // `String(piVersion ?? "")` is load-bearing and must not be simplified to skipping the write: an empty
588
+ // file is refused by `readSidecar`'s own size check and reads as `null`, so a promotion that knows no
589
+ // version INVALIDATES the stamp. Skipping instead would leave a PREVIOUS version beside a transcript
590
+ // written by an unknown pi, and the next matching job would resume it.
591
+ writeSidecar(dir, PI_VERSION_FILE, session.key, String(piVersion ?? ""));
592
+ // The chain and context sidecars, immediately after the swap and under the same lock. NOT part of
593
+ // the swap itself, which is one rename and cannot be widened: what the lock buys them is that no
222
594
  // other job can interleave, and what the ordering buys them is that they never describe a
223
595
  // transcript older than the one now in place.
224
596
  //
@@ -234,7 +606,8 @@ export function makeSessionStore({
234
606
  try {
235
607
  fs.unlinkSync(lock);
236
608
  } catch {
237
- // A leaked lock file wedges this key until the reaper sweeps it. Logged, never thrown.
609
+ // A lock left behind here is taken over by the next promotion once it is stale (`takeLock`).
610
+ // Logged, never thrown: the promotion itself succeeded.
238
611
  log("session_lock_stuck", { key: session.key });
239
612
  }
240
613
  }
@@ -273,21 +646,59 @@ export function makeSessionStore({
273
646
  * cold start when it will in fact resume. The bookkeeping loss is logged and the truth is kept.
274
647
  */
275
648
  function writeSidecar(dir, name, key, value) {
276
- const file = join(dir, name);
277
- const tmp = `${file}.incoming`;
278
649
  try {
279
- try {
280
- fs.unlinkSync(tmp);
281
- } catch {
282
- // Absent is the desired state.
283
- }
284
- fs.writeFileSync(tmp, value);
285
- fs.renameSync(tmp, file);
650
+ replaceSidecar(dir, name, value);
286
651
  } catch (err) {
287
652
  log("session_sidecar_failed", { key, file: name, reason: err?.message });
288
653
  }
289
654
  }
290
655
 
656
+ /**
657
+ * Remove whatever is at a temp path, whatever SHAPE it has, before anything writes there.
658
+ *
659
+ * ONE RULE BECAUSE CHOOSING BETWEEN THE TWO CALLS WAS GOT WRONG TWICE, in opposite directions.
660
+ * `unlinkSync` alone cannot remove a DIRECTORY planted at the name, and at the venue sentinel -- the one
661
+ * sidecar write that is fatal -- that wedged every promotion on the key forever. `rmSync` alone does not
662
+ * remove a DANGLING SYMLINK: it resolves the path, finds nothing, and with `force` reports success while
663
+ * leaving the link (measured, and it is the same measurement the reaper's removal note records further down
664
+ * this file -- measured there for the reaper, and not carried here until it had cost a round). The next write then follows the surviving link and creates a file at its target, which
665
+ * is the write-through-a-link hole this whole series exists to close, and if that target is unreachable
666
+ * the write throws and the key is wedged again.
667
+ *
668
+ * So: `unlinkSync` first, which takes a file or a link INCLUDING a dangling one; then `rmSync` for the
669
+ * one shape it cannot take. A directory is removed with its subtree, which is bounded to a temp name
670
+ * inside the key directory and is the point rather than a side effect: nothing may be left at that name.
671
+ */
672
+ function removeTemp(path) {
673
+ try {
674
+ fs.unlinkSync(path);
675
+ return;
676
+ } catch (err) {
677
+ if (err?.code === "ENOENT") return; // absent is the desired state
678
+ }
679
+ try {
680
+ fs.rmSync(path, { recursive: true, force: true });
681
+ } catch {
682
+ // Nothing further to try HERE, and the honest bound is worth stating: if both calls fail, anything
683
+ // left at the name is still there and the write that follows may go THROUGH it. The only way both
684
+ // fail is a key directory this process cannot write, and `takeLock`'s own exclusive create fails
685
+ // first in that state, so a promotion never reaches here with it -- which is why this is a bound
686
+ // rather than a hole.
687
+ }
688
+ }
689
+
690
+ /**
691
+ * `writeSidecar`'s link-safe temp-and-rename, and it THROWS. For the one write whose failure must stop a
692
+ * promotion rather than be logged past it: the venue sentinel, which runs before the swap (#277).
693
+ */
694
+ function replaceSidecar(dir, name, value) {
695
+ const file = join(dir, name);
696
+ const tmp = `${file}.incoming`;
697
+ removeTemp(tmp);
698
+ fs.writeFileSync(tmp, value);
699
+ fs.renameSync(tmp, file);
700
+ }
701
+
291
702
  /**
292
703
  * The context sidecar, whose three cases are all different.
293
704
  *
@@ -326,20 +737,138 @@ export function makeSessionStore({
326
737
  return join(keyDir(key), SESSION_FILE_NAME);
327
738
  }
328
739
 
740
+ /**
741
+ * Copy an already-open transcript to `dest`, in bounded chunks, and never more than `bytes`.
742
+ *
743
+ * The source is a DESCRIPTOR rather than a path, which is the whole point (issue #375): the bytes a job
744
+ * resumes must come off the same inode the gates judged, and any second resolution of the name is a second
745
+ * chance for something else to be standing there. `copyFileSync` cannot take one, so this is the copy.
746
+ *
747
+ * BOUNDED BY THE JUDGED SIZE, not by what the file turns out to hold. `inspectFile` capped the size
748
+ * against `PI_SESSION_MAX_BYTES` and the record reports that number, and a transcript still being written
749
+ * grows between the two: measured on the first draft of this copy, a file growing under it staged 2,460,306
750
+ * bytes against a 1,000,000 cap while the record said 306. Reading exactly what was judged makes the cap
751
+ * and the record true again. A SHORT read is refused rather than truncated silently, because a transcript
752
+ * cut off mid-line is a transcript pi will reject anyway. What REFUSES either case is the by-path identity
753
+ * re-check in `resolveSession`, since a transcript that grew or shrank has an identity that moved; this
754
+ * bound is what stops an unbounded append from being read at all, and it is deliberately not the thing
755
+ * that decides the verdict. A short read therefore needs no flag of its own: the re-check has it.
756
+ *
757
+ * 64 KiB because it is one buffer per resume and a transcript is usually a few hundred KiB; the loop is
758
+ * what keeps an uncapped `PI_SESSION_MAX_BYTES` from becoming a memory bound.
759
+ *
760
+ * THREE EQUIVALENT MUTANTS LIVE IN THIS FUNCTION, recorded so the next reader does not re-derive them
761
+ * (issue #391), because the lines around them ARE pinned and the difference is not visible by reading:
762
+ *
763
+ * - the buffer SIZING, `Math.min(64 * 1024, Math.max(1, bytes))` to a bare `64 * 1024`. The per-read
764
+ * clamp in the loop (`Math.min(buf.length, bytes - copied)`) is what bounds the read, so the sizing
765
+ * only decides how much memory a small file allocates.
766
+ * - the LOOP CONDITION, `copied < bytes` to `copied <= bytes`, for the same reason.
767
+ * - `Math.max(1, bytes)` to `bytes`, which is dead defensive code: `inspectFile` refuses a 0-byte
768
+ * transcript as `absent`, so `bytes === 0` never reaches this function at all. That unreachability
769
+ * is the thing a next reader is most likely to re-derive, which is why it is written down.
770
+ *
771
+ * All three were run against the whole suite and produce byte-identical staged output across transcript
772
+ * sizes either side of 64 KiB. What is NOT equivalent, and IS pinned by a descriptor count: the close
773
+ * below, the one in `resolveSession` that opened the descriptor this is handed, and the LOCK's close in
774
+ * `promoteSession` -- each of which leaks one descriptor per resume or per promotion in a process
775
+ * designed to run for weeks.
776
+ */
777
+ function copyFromDescriptor(fd, dest, bytes) {
778
+ const buf = Buffer.allocUnsafe(Math.min(64 * 1024, Math.max(1, bytes)));
779
+ let out = null;
780
+ let copied = 0;
781
+ try {
782
+ out = fs.openSync(dest, "w");
783
+ while (copied < bytes) {
784
+ const read = fs.readSync(fd, buf, 0, Math.min(buf.length, bytes - copied), null);
785
+ if (read === 0) break;
786
+ fs.writeFileSync(out, buf.subarray(0, read));
787
+ copied += read;
788
+ }
789
+ } finally {
790
+ if (out !== null) fs.closeSync(out);
791
+ }
792
+ }
793
+
794
+ /**
795
+ * Is the key's own name a real directory? `inspectFile`'s rule, one level up (issue #375).
796
+ *
797
+ * Every other read and write in here is an `lstat` on a name INSIDE the key directory, which is exactly
798
+ * what a key directory that is ITSELF a symlink defeats: the link is the directory, so each of those
799
+ * lstats resolves through it and every guarantee in this file lands wherever it points. Measured before
800
+ * the fix: a link planted at the derived path made `resolveSession` return `resumed` from a transcript
801
+ * outside the store, and `promoteSession` write `current.jsonl`, `pi-version`, `venue` and `resume-chain`
802
+ * through it. `mkdirSync(dir, { recursive: true })` succeeds on an existing link to a directory, so
803
+ * nothing on the write path noticed either.
804
+ *
805
+ * THE FINAL COMPONENT ONLY, deliberately. `PI_SESSIONS_DIR` and its ancestors may legitimately be links:
806
+ * macOS's own temp root is `/var -> private/var`, and moving the whole store behind a link is a supported
807
+ * layout. A `realpath` or an ancestor walk would refuse those, and would refuse every test fixture on
808
+ * this platform. The name this checks is the DERIVED one, which is what an attacker who knows the
809
+ * repository and the branch can precompute, and which nothing but this module should be creating.
810
+ *
811
+ * It answers the SHAPE only. It carried a `dev:ino` identity for the post-copy re-check in the first draft
812
+ * of this change, and the gate round removed the need for it: the staged bytes now come off one descriptor
813
+ * (see `resolveSession`), so a directory swapped under a job cannot change what that job reads, and there
814
+ * is nothing left for a directory identity to protect.
815
+ */
816
+ function inspectKeyDir(key) {
817
+ let st;
818
+ try {
819
+ st = fs.lstatSync(keyDir(key));
820
+ } catch {
821
+ // EVERY failure reads as `absent`, not only ENOENT, and that is this edge's posture rather than an
822
+ // oversight: `inspectFile` one level down has always done the same, and a read that cannot ask costs a
823
+ // job its history and nothing else. Its write-side twin `ensureKeyDir` takes the opposite trade
824
+ // deliberately, because a promotion that cannot ask must not report the key's SHAPE as the reason.
825
+ // The cost, stated: an unreadable store cold-starts silently, indistinguishable from a fresh key.
826
+ return { ok: false, reason: "absent" };
827
+ }
828
+ if (!st.isDirectory()) return { ok: false, reason: "key-not-a-directory" };
829
+ // `dev:ino` and NOTHING else. The file rule's `dev:ino:size:mtime` is wrong for a directory: both move
830
+ // on every entry a promotion creates inside the key (64 -> 96 -> 1376 bytes on APFS, measured), so a
831
+ // re-check built on it reports a race on every quiet run. BOTH edges compare this identity: the write
832
+ // edge against what `ensureKeyDir` made or found, the read edge against what the gates were computed
833
+ // from. Comparing the SHAPE alone on either edge lets a swap for another REAL directory through, which
834
+ // is how it was found on the write edge and, one round later, on this one.
835
+ return { ok: true, ident: `${st.dev}:${st.ino}` };
836
+ }
837
+
329
838
  /** The read path, gate by gate. The FIRST miss wins and names itself. */
330
- function readCanonical(key, piVersion, modelId) {
839
+ function readCanonical(key, piVersion, modelId, venue) {
840
+ // FIRST, because every gate below it is an lstat inside this directory and none of them can see that
841
+ // the directory is not one. A regular file or a link to one read as `absent` before this arm existed,
842
+ // through the ENOTDIR their inner lstat threw; they are now named for what they are.
843
+ const dirCheck = inspectKeyDir(key);
844
+ if (!dirCheck.ok) return COLD(dirCheck.reason);
845
+
846
+
331
847
  const file = canonicalFile(key);
332
848
  const check = inspectFile(file);
333
849
  if (!check.ok) return COLD(check.reason);
334
850
 
335
851
  if (ttlDays > 0 && now() - check.mtimeMs > ttlDays * 86400000) return COLD("expired");
336
852
 
853
+ // The VENUE the transcript was written in (#277). Placed HERE, after the arms a venue move cannot cause
854
+ // -- a transcript that is absent, not a regular file, too large or past its TTL is all of those on every
855
+ // venue -- and AHEAD of both pi-version arms, because a venue move CAN cause those: image preflight is
856
+ // dispatched per venue, so another venue can report another pi or none. The first miss names itself,
857
+ // and before this arm a venue move named itself as a version change. Without it, a trigger moved
858
+ // between venues staged the old venue's transcript into the new venue's container with nothing refusing.
859
+ //
860
+ // FAILS CLOSED: a job whose venue cannot be resolved (a DI seam) never resumes, the pi-version gate's
861
+ // polarity. The token also covers a stamp left pending by an interrupted promotion and a stamp that is
862
+ // present but unreadable -- a misnaming stated rather than hidden, as `pi-version-changed` already does
863
+ // for an unreadable version stamp.
864
+ if (venue === null || readVenue(key) !== venue) return COLD("venue-changed");
865
+
337
866
  // A transcript outlives the pi that wrote it, and pi's own docs record what then breaks: an older
338
867
  // session's stored tool-call arguments may no longer match the current tool schema. We cannot
339
868
  // repair that mid-run, so a version change is a cold start rather than a mid-run failure. An
340
869
  // image that declares no version never resumes -- the safe direction, never "assume it matches".
341
870
  if (piVersion === null) return COLD("pi-version-changed");
342
- // Through the same guarded read as the two sidecars below it. This one predates them and was the
871
+ // Through the same guarded read as the other sidecars. This one predates them and was the
343
872
  // one unguarded read left in the key directory; a symlink here would have decided a gate on the
344
873
  // contents of some other file entirely.
345
874
  const stamped = readSidecar(key, PI_VERSION_FILE);
@@ -425,7 +954,7 @@ export function makeSessionStore({
425
954
  if (!Number.isFinite(started)) return COLD("conversation-too-old");
426
955
  if (now() - started > maxAgeDays * 86400000) return COLD("conversation-too-old");
427
956
  }
428
- return { resume: true, reason: "resumed", bytes: check.bytes };
957
+ return { resume: true, reason: "resumed", bytes: check.bytes, ident: check.ident, dirIdent: dirCheck.ident };
429
958
  }
430
959
 
431
960
  /**
@@ -453,6 +982,25 @@ export function makeSessionStore({
453
982
  }
454
983
  }
455
984
 
985
+ /**
986
+ * The venue stamped beside a key's transcript (#277).
987
+ *
988
+ * ABSENT and UNREADABLE are different answers, and conflating them either way is a defect. An absent stamp
989
+ * is a transcript from before venues were recorded, which ran on `local`. A stamp that exists but cannot
990
+ * be read -- not a regular file, empty, oversized, unreadable -- matches nothing (`null`), so the key
991
+ * cold-starts. Absence is decided by `lstat` and ENOENT ALONE: `stat` or `existsSync` would follow a
992
+ * DANGLING symlink planted at this name, report it absent, and resume a transcript as `local` on the
993
+ * strength of a link that points nowhere. Any other `lstat` failure is not absence either.
994
+ */
995
+ function readVenue(key) {
996
+ try {
997
+ fs.lstatSync(join(keyDir(key), VENUE_FILE));
998
+ } catch (err) {
999
+ return err?.code === "ENOENT" ? UNATTRIBUTED_BACKEND : null;
1000
+ }
1001
+ return readSidecar(key, VENUE_FILE);
1002
+ }
1003
+
456
1004
  /**
457
1005
  * The consecutive-delivery counter for a key, or 0 when there is not a readable one. Never throws and
458
1006
  * never guesses: a missing, empty, corrupt or negative counter is 0, so the only way to be refused by
@@ -493,6 +1041,33 @@ export function makeSessionStore({
493
1041
  * (`fs.readFile` off the clone following a symlink into a worker-host file). The repo's own habit is
494
1042
  * the wrong one here: makeLogReaper uses statSync, which follows.
495
1043
  */
1044
+ /**
1045
+ * The canonical file's IDENTITY as one comparable token: device, inode, size, mtime.
1046
+ *
1047
+ * ONE STRING RATHER THAN FOUR FIELDS, so the comparison is one `!==` and there is no partial compare to
1048
+ * get wrong. `ino` is the field that does the work: a promotion renames a freshly created `.incoming`
1049
+ * file into place, so the inode behind the canonical path is a DIFFERENT one after every completed swap
1050
+ * -- including the A, B, A round trip whose venue stamp matches again, which is why this catches what the
1051
+ * venue re-check cannot. `dev` is what makes `ino` meaningful, since an inode number is unique per
1052
+ * device. `size` and `mtimeMs` narrow the inode-REUSE residual and nothing else: the swapped-away inode
1053
+ * is freed by its own rename and its number may be handed straight back to the next `.incoming` file.
1054
+ */
1055
+ function identityOf(st) {
1056
+ return `${st.dev}:${st.ino}:${st.size}:${st.mtimeMs}`;
1057
+ }
1058
+
1059
+ /**
1060
+ * The identity now at a path, or `null` when there is nothing readable there -- which compares unequal to
1061
+ * every real identity, so a transcript that VANISHED between the gate and the re-check cold-starts too.
1062
+ */
1063
+ function readIdentity(file) {
1064
+ try {
1065
+ return identityOf(fs.lstatSync(file));
1066
+ } catch {
1067
+ return null;
1068
+ }
1069
+ }
1070
+
496
1071
  function inspectFile(file) {
497
1072
  let st;
498
1073
  try {
@@ -503,17 +1078,20 @@ export function makeSessionStore({
503
1078
  if (!st.isFile()) return { ok: false, reason: "not-a-regular-file" };
504
1079
  if (st.size === 0) return { ok: false, reason: "absent" }; // a staged-but-unwritten transcript
505
1080
  if (maxBytes > 0 && st.size > maxBytes) return { ok: false, reason: "too-large" };
506
- return { ok: true, bytes: st.size, mtimeMs: st.mtimeMs };
1081
+ return { ok: true, bytes: st.size, mtimeMs: st.mtimeMs, ident: identityOf(st) };
507
1082
  }
508
1083
 
509
1084
  /**
510
- * Boot-time sweep. A SIBLING of makeLogReaper rather than a widening of it: that one's `.log`/`.json`
511
- * filter and logsDir scope are a documented contract, and these files have a different retention
512
- * policy and a different PII class. Same never-throws shape, same `0 = keep forever` sentinel.
1085
+ * The DISK sweep: at boot, and on the retention timer thereafter (`PI_SWEEP_INTERVAL_HOURS`, issue
1086
+ * #292). A SIBLING of makeLogReaper rather than a widening of it: that one's `.log`/`.json` filter and
1087
+ * logsDir scope are a documented contract, and these files have a different retention policy and a
1088
+ * different PII class. Same never-throws shape, same `0 = keep forever` sentinel, and safe to re-run --
1089
+ * it carries no state between calls.
513
1090
  *
514
- * Age at boot is the smaller half. The gate that matters is the one in readCanonical, which runs at
515
- * OPEN -- a worker that never restarts would otherwise keep resuming a transcript indefinitely, and a
516
- * stale transcript is a live input to a future job rather than debris (OQ-007).
1091
+ * Age on disk is the smaller half. The gate that matters is the one in readCanonical, which runs at
1092
+ * OPEN, because a stale transcript is a live INPUT to a future job rather than debris. Until #292 that
1093
+ * gate was also carrying the disk half alone, since a worker that never restarts never re-swept
1094
+ * (OQ-007, RESOLVED).
517
1095
  */
518
1096
  function reapSessions() {
519
1097
  if (!sessionsDir || ttlDays === 0) return;
@@ -522,19 +1100,66 @@ export function makeSessionStore({
522
1100
  try {
523
1101
  names = fs.readdirSync(sessionsDir);
524
1102
  } catch (err) {
525
- log("session_reaper_skipped", { reason: err?.message });
1103
+ log("session_reaper_skipped", { reason: scrubCredentials(err?.message) });
526
1104
  return;
527
1105
  }
528
1106
  for (const name of names) {
529
1107
  try {
530
1108
  const dir = join(sessionsDir, name);
531
- const st = fs.lstatSync(join(dir, SESSION_FILE_NAME));
532
- if (st.mtimeMs < cutoff) {
1109
+ // THE ENTRY ITSELF FIRST (issue #375), so nothing here is ever judged THROUGH a link. Before this,
1110
+ // a link whose target held a transcript was judged on that target's mtime, and the link was
1111
+ // removed once the target aged: the sweep's decision came from a file outside the store. Now only
1112
+ // a real directory is a key on this path, on both edges, which is the same rule `resolveSession`
1113
+ // and `promoteSession` now apply.
1114
+ //
1115
+ // `session_not_reaped`, NOT `session_reaper_skipped`. OQ-007's #337 amendment reserves the
1116
+ // `*_reaper_skipped` names for what a pass could not establish, and says that giving an everyday
1117
+ // verdict the fault's name puts an ordinary outcome into the grep an operator reads as trouble.
1118
+ // This pass established exactly what this entry is and decided to leave it, which is a verdict.
1119
+ const entry = fs.lstatSync(dir);
1120
+ if (!entry.isDirectory()) {
1121
+ log("session_not_reaped", { key: name, reason: "key-not-a-directory" });
1122
+ continue;
1123
+ }
1124
+ let mtimeMs;
1125
+ try {
1126
+ mtimeMs = fs.lstatSync(join(dir, SESSION_FILE_NAME)).mtimeMs;
1127
+ } catch (err) {
1128
+ // ENOENT ALONE falls back to the DIRECTORY's own mtime (issue #336). A key whose first promotion
1129
+ // died before any transcript landed has none to key on, so this loop used to log-and-skip it on
1130
+ // every pass forever, and anything leaked inside it -- a lock, an in-flight copy -- outlived the
1131
+ // store. A directory's mtime moves on every entry created or removed inside it, so a live key is
1132
+ // refreshed by its own promotions and a dead one is stamped at whatever last touched it.
1133
+ //
1134
+ // ENOENT-ONLY rather than unconditional, and that is the whole safety of it: an EIO or an EACCES
1135
+ // on the transcript is a disk fault, and a disk fault is not evidence that a key is old. Those
1136
+ // keep today's log-and-skip.
1137
+ if (err?.code !== "ENOENT") throw err;
1138
+ const dst = entry; // the entry's own stat, already taken and already proven a directory
1139
+ // The entry was already proven a real directory above (issue #375), so this arm is now only about
1140
+ // a key that has no transcript yet, and `dst` is that same stat rather than a second one.
1141
+ //
1142
+ // What the removal below is and is not, measured rather than assumed, because the obvious reading
1143
+ // is wrong: `rmSync(p, { recursive: true, force: true })` does NOT follow a symlink. On a link to
1144
+ // a real directory it removes the LINK and leaves the target untouched, and on a dangling link it
1145
+ // does nothing at all. So a link's TARGET was never at risk here even before this change; what
1146
+ // has changed is that a link is no longer JUDGED, which it was whenever its target held a
1147
+ // readable transcript -- the sweep's decision then came from a file outside the store entirely.
1148
+ mtimeMs = dst.mtimeMs;
1149
+ }
1150
+ if (mtimeMs < cutoff) {
1151
+ // ONE recursive remove, on the transcript's age where there is one and the directory's own where
1152
+ // there is not (above). Removing the
1153
+ // transcript first (so an absent stamp, which reads as `local`, can never sit beside a readable
1154
+ // transcript mid-sweep) was tried under #277 and withdrawn: a remove that then failed transiently
1155
+ // left the directory with no transcript, which this loop never looks at again, so a leaked lock
1156
+ // in it wedged the key for good. The race it closed needs a promotion landing on an EXPIRED key
1157
+ // during its own sweep on a deployment with a second venue; INT-SESSION-STORE-CONTRACT names it.
533
1158
  fs.rmSync(dir, { recursive: true, force: true });
534
1159
  log("reaped_session", { key: name });
535
1160
  }
536
1161
  } catch (err) {
537
- log("session_reaper_skipped", { key: name, reason: err?.message });
1162
+ log("session_reaper_skipped", { key: name, reason: scrubCredentials(err?.message) });
538
1163
  }
539
1164
  }
540
1165
  }