@edgehero/pi-dispatch 1.10.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +300 -148
- package/README.md +50 -0
- package/deploy/com.pi-dispatch.worker.plist +9 -3
- package/deploy/docker-compose.yml +49 -16
- package/deploy/egress-proxy.conf +32 -2
- package/deploy/nssm-install.cmd +12 -6
- package/deploy/pi-dispatch-egress-out.network +10 -0
- package/deploy/pi-dispatch-egress-proxy.container +50 -0
- package/deploy/pi-dispatch-netns-keeper.container +80 -0
- package/deploy/pi-dispatch-netns-keeper.network +18 -0
- package/deploy/pi-dispatch-valkey.container +51 -0
- package/deploy/pi-dispatch-valkey.network +16 -0
- package/deploy/receiver.service +6 -0
- package/deploy/worker-env-wrapper.cmd +11 -0
- package/deploy/worker-env-wrapper.sh +60 -34
- package/deploy/worker.service +18 -8
- package/package.json +14 -4
- package/src/azure-host.mjs +19 -0
- package/src/azure-identity.mjs +18 -2
- package/src/backend-conformance.mjs +71 -18
- package/src/backend-local.mjs +637 -21
- package/src/backend-podman.mjs +1168 -0
- package/src/backend-registry.mjs +86 -3
- package/src/backends.mjs +489 -37
- package/src/branch.mjs +7 -2
- package/src/cancel-cli.mjs +174 -0
- package/src/cancel-state.mjs +125 -0
- package/src/cli.mjs +188 -90
- package/src/config.mjs +503 -43
- package/src/connection.mjs +374 -8
- package/src/container-spec.mjs +102 -7
- package/src/daemon-facts.mjs +167 -0
- package/src/deployment-venue.mjs +158 -0
- package/src/docker-run.mjs +146 -15
- package/src/doctor.mjs +4701 -414
- package/src/egress-conf-copy.mjs +166 -0
- package/src/egress-proxy-state.mjs +151 -0
- package/src/egress.mjs +455 -25
- package/src/entry.mjs +27 -0
- package/src/env-allowlist.mjs +222 -40
- package/src/env-file.mjs +1869 -33
- package/src/exit-code.mjs +15 -0
- package/src/flow-gate.mjs +5 -3
- package/src/forgejo-host.mjs +19 -0
- package/src/forgejo-identity.mjs +21 -2
- package/src/get-token.mjs +67 -18
- package/src/git-dirty.mjs +9 -1
- package/src/git-hardening.mjs +33 -0
- package/src/github-app-setup.mjs +29 -12
- package/src/github-prompt.mjs +4 -1
- package/src/gitlab-host.mjs +19 -0
- package/src/gitlab-identity.mjs +19 -2
- package/src/host-registry.mjs +29 -2
- package/src/identity.mjs +29 -4
- package/src/image-preflight.mjs +46 -11
- package/src/image-ref.mjs +21 -0
- package/src/index.mjs +363 -13
- package/src/init.mjs +197 -38
- package/src/job-user.mjs +252 -0
- package/src/json-duplicates.mjs +204 -0
- package/src/live-probes.mjs +1020 -0
- package/src/materialize.mjs +4 -11
- package/src/netns-keeper.mjs +264 -0
- package/src/on-failure.mjs +119 -0
- package/src/outbox.mjs +7 -0
- package/src/podman-stack.mjs +1304 -0
- package/src/prepare-github.mjs +6 -6
- package/src/prepare-local.mjs +51 -17
- package/src/prepare.mjs +27 -6
- package/src/processor.mjs +505 -26
- package/src/provider-key.mjs +41 -0
- package/src/provider-steering.mjs +144 -0
- package/src/queue.mjs +35 -8
- package/src/redact.mjs +84 -0
- package/src/reserved-env.mjs +7 -3
- package/src/retention-sweep.mjs +178 -0
- package/src/run-container.mjs +181 -14
- package/src/run-history.mjs +105 -16
- package/src/runtime-observations.mjs +1152 -0
- package/src/runtime-settings.mjs +13 -8
- package/src/sandbox-cli.mjs +100 -95
- package/src/sandbox-store.mjs +612 -45
- package/src/sandbox.mjs +1459 -37
- package/src/schedules.mjs +16 -3
- package/src/secret-profiles.mjs +2 -1
- package/src/secrets.mjs +23 -6
- package/src/service-env.mjs +247 -0
- package/src/service.mjs +618 -28
- package/src/session-store.mjs +678 -53
- package/src/start.mjs +1348 -326
- package/src/transient.mjs +240 -0
- package/src/triggers-file.mjs +71 -15
- package/src/triggers.mjs +176 -19
- package/src/up.mjs +1399 -85
- package/src/valkey-auth.mjs +529 -0
- package/src/valkey-endpoint.mjs +367 -0
- package/src/watch-closer.mjs +158 -0
package/src/session-store.mjs
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
copyFileSync,
|
|
3
|
+
fstatSync,
|
|
3
4
|
lstatSync,
|
|
4
5
|
mkdirSync,
|
|
5
6
|
openSync,
|
|
6
7
|
closeSync,
|
|
8
|
+
readSync,
|
|
7
9
|
readFileSync,
|
|
8
10
|
readdirSync,
|
|
9
11
|
renameSync,
|
|
@@ -11,8 +13,12 @@ import {
|
|
|
11
13
|
unlinkSync,
|
|
12
14
|
writeFileSync,
|
|
13
15
|
} from "node:fs";
|
|
16
|
+
import { randomBytes } from "node:crypto";
|
|
14
17
|
import { join } from "node:path";
|
|
18
|
+
import { scrubCredentials } from "./redact.mjs";
|
|
15
19
|
import { sessionKeyFor } from "./session-key.mjs";
|
|
20
|
+
import { resolveBackendName } from "./backend-registry.mjs";
|
|
21
|
+
import { UNATTRIBUTED_BACKEND } from "./backends.mjs";
|
|
16
22
|
|
|
17
23
|
/**
|
|
18
24
|
* session-store.mjs -- the host side of a resumable session (INT-SESSION-STORE-CONTRACT).
|
|
@@ -72,7 +78,75 @@ const RESUME_CHAIN_FILE = "resume-chain";
|
|
|
72
78
|
* agent-influenced, since the transcript itself is agent-written.
|
|
73
79
|
*/
|
|
74
80
|
const CONTEXT_FILE = "context";
|
|
75
|
-
/**
|
|
81
|
+
/**
|
|
82
|
+
* The venue whose container produced the transcript beside it (issue #277): a backend name, one line.
|
|
83
|
+
*
|
|
84
|
+
* A SIDECAR, NOT KEY MATERIAL. `DES-SESSION-KEY-IS-DERIVED-NOT-INDEXED` makes the key a pure function of
|
|
85
|
+
* (kind, repo, ref), and a venue in the key would fork a trigger moved between venues into a second lineage
|
|
86
|
+
* that nothing ever sweeps. As a sidecar it GATES the one lineage instead: the move costs one cold start.
|
|
87
|
+
*
|
|
88
|
+
* ABSENT READS AS `UNATTRIBUTED_BACKEND`, never as the deployment default. Every transcript written before this
|
|
89
|
+
* stamp existed ran on `local`, and the default is a setting that can move under it.
|
|
90
|
+
*/
|
|
91
|
+
const VENUE_FILE = "venue";
|
|
92
|
+
/**
|
|
93
|
+
* What the venue stamp reads while a promotion is swapping the transcript under it. It matches no venue:
|
|
94
|
+
* parentheses are outside the charset every backend name is validated against (a test pins every name this
|
|
95
|
+
* build knows to it), so a key left holding this cold-starts on every venue until a promotion completes.
|
|
96
|
+
*/
|
|
97
|
+
const VENUE_PENDING = "(pending)";
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Per-promotion suffix for the transcript's in-flight copy, paired with the pid.
|
|
101
|
+
*
|
|
102
|
+
* The tmp name USED to be a fixed `<canonical>.incoming`, which was self-cleaning and harmless while the
|
|
103
|
+
* per-key lock guaranteed one writer. The stale-lock takeover concedes a window with two, and a SHARED tmp
|
|
104
|
+
* path voids exactly the atomicity the swap exists for: B's unlink removes A's in-flight copy, B's copy
|
|
105
|
+
* replaces it, and whichever renames second can put the other's half-written file at the canonical path --
|
|
106
|
+
* an agent handed a truncated conversation with no gate able to see it. `triggers-file.mjs` learned this
|
|
107
|
+
* on `triggers.json` and its `tmpPathFor` says the same thing.
|
|
108
|
+
*
|
|
109
|
+
* RANDOM BYTES, not just a pid and a counter, and the pid is why: two workers can share one. Two
|
|
110
|
+
* containers where node is pid 1, or two hosts on one shared `PI_SESSIONS_DIR` -- which is the very
|
|
111
|
+
* `OQ-031` shape the takeover's own skew analysis invokes -- produce byte-identical `<canonical>.1.0.incoming`
|
|
112
|
+
* names, measured, and a shared destination does tear (two processes copying 4 MiB to one path mixed both
|
|
113
|
+
* writers' bytes in 1 sample of 110). A pid-and-counter name separates two promotions inside ONE process,
|
|
114
|
+
* which is the one case that cannot happen, and leaves the case that can. The random half is what actually
|
|
115
|
+
* separates writers; the pid and counter stay because they make a straggler attributable.
|
|
116
|
+
*
|
|
117
|
+
* The cost is that a crash between the copy and the rename leaves a uniquely named straggler instead of one
|
|
118
|
+
* the next promotion overwrites; the reaper's recursive sweep of the key takes it with everything else --
|
|
119
|
+
* except under `PI_SESSIONS_TTL_DAYS=0`, where that sweep does not run at all, so stragglers accumulate
|
|
120
|
+
* there where the old shared name was self-cleaning. The same TTL-0 caveat the lock's takeover carries.
|
|
121
|
+
*
|
|
122
|
+
* The SIDECAR temps keep the shared name deliberately, and the asymmetry is the point rather than an
|
|
123
|
+
* oversight: a sidecar's worst case under a concurrent writer is a missing or half-written sidecar, and
|
|
124
|
+
* every one of those reads back as `null` and produces a cold start, which this store already treats as
|
|
125
|
+
* the safe outcome. The transcript's worst case is a corrupt transcript, which it does not.
|
|
126
|
+
*/
|
|
127
|
+
let tmpSeq = 0;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* A promotion lock older than this is a crashed writer's, not a live one's.
|
|
131
|
+
*
|
|
132
|
+
* `triggers-file.mjs` holds the same idiom at ten seconds; this is 360 times that, because the work under
|
|
133
|
+
* the two locks is not comparable. A trigger write is a read, one mutate and two syscalls. A
|
|
134
|
+
* promotion is a COPY of the container's transcript -- up to `PI_SESSION_MAX_BYTES`, 8 MiB by default --
|
|
135
|
+
* plus a rename and four small sidecar writes.
|
|
136
|
+
*
|
|
137
|
+
* AN ASSERTION ABOUT STORAGE, NOT A DERIVATION, and said plainly because `PI_SESSION_MAX_BYTES=0` is a
|
|
138
|
+
* supported setting and there is then no configured bound to derive from. An hour covers roughly three
|
|
139
|
+
* gigabytes at a megabyte a second, which is a slower store than anything this project will meet.
|
|
140
|
+
*
|
|
141
|
+
* WHAT MAKES A GENEROUS NUMBER THE RIGHT TRADE IS THE ASYMMETRY. Too short steals a LIVE writer's lock,
|
|
142
|
+
* and with the release-by-path residual below that degrades into the lock being functionally absent. Too
|
|
143
|
+
* long only delays recovery on a key nobody is watching, at one extra cold start per job until it passes.
|
|
144
|
+
*
|
|
145
|
+
* A CONSTANT RATHER THAN A KNOB, on triggers-file's precedent: the one value an operator would reach for
|
|
146
|
+
* is zero, and zero here means no lock at all.
|
|
147
|
+
*/
|
|
148
|
+
const LOCK_STALE_MS = 3_600_000;
|
|
149
|
+
/** Every sidecar format is a handful of bytes. Generous, and still nowhere near a job's wall clock. */
|
|
76
150
|
const SIDECAR_MAX_BYTES = 4096;
|
|
77
151
|
/**
|
|
78
152
|
* The host-effective provider and model as one token, or null when the job names neither.
|
|
@@ -93,6 +167,15 @@ function modelIdentity(job) {
|
|
|
93
167
|
return /^[a-z0-9][a-z0-9._:/-]{0,127}$/.test(id) ? id : null;
|
|
94
168
|
}
|
|
95
169
|
|
|
170
|
+
/**
|
|
171
|
+
* A venue name as the store handles it: a non-empty string, else `null`. A promotion stamps `String(venue)`,
|
|
172
|
+
* and `String(undefined)` is the word `undefined`, which is a valid-looking name; normalising first means a
|
|
173
|
+
* session with no resolvable venue is stamped with nothing rather than with that.
|
|
174
|
+
*/
|
|
175
|
+
function normaliseVenue(venue) {
|
|
176
|
+
return typeof venue === "string" && venue !== "" ? venue : null;
|
|
177
|
+
}
|
|
178
|
+
|
|
96
179
|
/**
|
|
97
180
|
* Read-path outcomes. Every one is a named cold start rather than a bare `false`: a feature that fails
|
|
98
181
|
* open is otherwise indistinguishable from a feature nobody switched on, which is precisely how "we
|
|
@@ -107,9 +190,12 @@ export function makeSessionStore({
|
|
|
107
190
|
maxAgeDays = 0,
|
|
108
191
|
maxResumeChain = 0,
|
|
109
192
|
maxContextPct = null,
|
|
193
|
+
// The deployment's default venue, `PI_BACKENDS[0]`, for a job that names none (#277). `null` is a
|
|
194
|
+
// dependency-injection seam, and a job whose venue cannot be resolved never resumes.
|
|
195
|
+
defaultBackend = null,
|
|
110
196
|
log = () => {},
|
|
111
197
|
now = () => Date.now(),
|
|
112
|
-
fs = { copyFileSync, lstatSync, mkdirSync, openSync, closeSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync },
|
|
198
|
+
fs = { copyFileSync, fstatSync, lstatSync, mkdirSync, openSync, closeSync, readSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync },
|
|
113
199
|
}) {
|
|
114
200
|
/**
|
|
115
201
|
* Stage this job's /session directory and decide whether it resumes.
|
|
@@ -121,6 +207,8 @@ export function makeSessionStore({
|
|
|
121
207
|
* /session mount at all (unarmed, or no key) -- which is byte-identical to a pre-feature job.
|
|
122
208
|
*/
|
|
123
209
|
function resolveSession(job, { jobDir, resolved = {}, piVersion = null } = {}) {
|
|
210
|
+
// Declared outside the try so a fault can take back what this call staged (below).
|
|
211
|
+
let hostDir = null;
|
|
124
212
|
try {
|
|
125
213
|
// Unreachable in a wired worker, and deliberately kept: resolveSession is only ever called for a
|
|
126
214
|
// job that armed run.resume (prepare-github.mjs), and processor.mjs refuses exactly that job
|
|
@@ -139,13 +227,91 @@ export function makeSessionStore({
|
|
|
139
227
|
// count is 78% of a 32k window and 2.5% of a 1M one. Carried on the session object rather than
|
|
140
228
|
// read again at promote time, so the reading is stamped with the model that produced it.
|
|
141
229
|
const modelId = modelIdentity(job);
|
|
142
|
-
|
|
230
|
+
// The venue this job will run in, resolved exactly as the registry dispatches it (#277). Carried on
|
|
231
|
+
// the session like `modelId`, so the promotion stamps the venue that produced the transcript rather
|
|
232
|
+
// than re-deriving one later.
|
|
233
|
+
const venue = normaliseVenue(resolveBackendName(job, defaultBackend));
|
|
234
|
+
hostDir = join(jobDir, "session");
|
|
143
235
|
const staged = join(hostDir, SESSION_FILE_NAME);
|
|
144
236
|
fs.mkdirSync(hostDir, { recursive: true, mode: 0o700 });
|
|
145
237
|
|
|
146
|
-
|
|
238
|
+
// The identity is split off the verdict rather than carried on it: this return is spread onto the
|
|
239
|
+
// session object the processor holds for the WHOLE run, and an inode number is bookkeeping for the
|
|
240
|
+
// next few lines, not state a promotion an hour later should be able to read.
|
|
241
|
+
const { ident: judged = null, dirIdent: judgedDir = null, ...judgedVerdict } = readCanonical(key, piVersion, modelId, venue);
|
|
242
|
+
let verdict = judgedVerdict;
|
|
147
243
|
if (verdict.resume) {
|
|
148
|
-
|
|
244
|
+
// THE BYTES COME OFF THE DESCRIPTOR THAT WAS JUDGED, never off the path a second time.
|
|
245
|
+
//
|
|
246
|
+
// The gates above ran by PATH, and so did this copy until issue #375's gate round: it was
|
|
247
|
+
// `copyFileSync(canonicalFile(key), staged)` followed by re-checks that also re-resolved the path.
|
|
248
|
+
// Every one of those resolutions is a separate walk of the same name, so an attacker who is a
|
|
249
|
+
// symlink DURING the copy and the original directory again before the re-check matched all three
|
|
250
|
+
// arms while the bytes came from somewhere else entirely. Measured on the pre-fix code by a plain
|
|
251
|
+
// second process doing rename/symlink/rename in a loop: 195 of 757 successful resumes in the
|
|
252
|
+
// review round's own harness, and 23 of 654 in a 90-second run of this file's, staged an
|
|
253
|
+
// attacker's transcript and reported `resumed`. The same shape one level down -- the
|
|
254
|
+
// A, B, A round trip on `current.jsonl` itself -- is what this contract used to state as a
|
|
255
|
+
// residual the identity re-check could only shrink.
|
|
256
|
+
//
|
|
257
|
+
// One `open` answers both. The descriptor is bound to the inode at the instant it is taken, so
|
|
258
|
+
// `fstat` on it cannot be lied to by a later swap, and bytes read from it cannot come from a file
|
|
259
|
+
// that replaced the name afterwards. What a swap can still do is make this refuse: the identity
|
|
260
|
+
// will not match what the gates judged, and the job cold-starts, which is the safe direction.
|
|
261
|
+
//
|
|
262
|
+
// Streamed in bounded chunks rather than read whole: `PI_SESSION_MAX_BYTES` defaults to 8 MiB but
|
|
263
|
+
// `0` means NO CAP, and buffering an uncapped transcript into memory to copy it would be a new
|
|
264
|
+
// way to hurt a host that the old `copyFileSync` never had.
|
|
265
|
+
let fd = null;
|
|
266
|
+
try {
|
|
267
|
+
fd = fs.openSync(canonicalFile(key), "r");
|
|
268
|
+
const opened = identityOf(fs.fstatSync(fd));
|
|
269
|
+
// TWO CHECKS, and neither subsumes the other. The gate round proved that the hard way, twice.
|
|
270
|
+
//
|
|
271
|
+
// The DESCRIPTOR check says the bytes about to be staged come off the inode this open took,
|
|
272
|
+
// and nothing can re-point it afterwards. It is what closes the A, B, A across the copy.
|
|
273
|
+
//
|
|
274
|
+
// The BY-PATH check after the copy says the name still resolves to the file the GATES judged.
|
|
275
|
+
// It is what the descriptor cannot say: every gate above ran by path, so a link planted after
|
|
276
|
+
// the first of those lstats has the whole ladder judge the attacker's files, and then the
|
|
277
|
+
// descriptor agrees with a `judged` that is already the attacker's. Dropping this check for
|
|
278
|
+
// one round reintroduced exactly that (measured: 2 attacker transcripts staged in 16,594
|
|
279
|
+
// resumes, where the by-path version staged none), and it is also what keeps the venue arm
|
|
280
|
+
// reachable: a promotion that wrote its `(pending)` sentinel and has not yet swapped moves no
|
|
281
|
+
// identity at all, so an identity-gated ladder never asks about it.
|
|
282
|
+
const swapped = () => {
|
|
283
|
+
const dirNow = inspectKeyDir(key);
|
|
284
|
+
return dirNow.reason === "key-not-a-directory"
|
|
285
|
+
? "key-not-a-directory"
|
|
286
|
+
: dirNow.ident !== judgedDir
|
|
287
|
+
? "transcript-replaced" // another REAL directory at the name: the gates judged its files
|
|
288
|
+
: readVenue(key) !== venue
|
|
289
|
+
? "venue-changed"
|
|
290
|
+
: readIdentity(canonicalFile(key)) !== judged
|
|
291
|
+
? "transcript-replaced"
|
|
292
|
+
: null;
|
|
293
|
+
};
|
|
294
|
+
// VENUE FIRST inside `swapped()`, as it has been since #277: a cross-venue promotion trips
|
|
295
|
+
// both and `venue-changed` names WHY where `transcript-replaced` only says something moved.
|
|
296
|
+
// The directory arm is ahead of both (issue #375), because a name that is not a directory is
|
|
297
|
+
// neither a venue move nor a swap.
|
|
298
|
+
const raced = opened !== judged ? (swapped() ?? "transcript-replaced") : null;
|
|
299
|
+
if (raced !== null) {
|
|
300
|
+
// A 0-byte staged file, the shape every cold start gets. If emptying fails, the catch
|
|
301
|
+
// below removes the staged directory rather than leave a copy behind.
|
|
302
|
+
fs.writeFileSync(staged, "");
|
|
303
|
+
verdict = COLD(raced);
|
|
304
|
+
} else {
|
|
305
|
+
copyFromDescriptor(fd, staged, verdict.bytes);
|
|
306
|
+
const after = swapped();
|
|
307
|
+
if (after !== null) {
|
|
308
|
+
fs.writeFileSync(staged, "");
|
|
309
|
+
verdict = COLD(after);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
} finally {
|
|
313
|
+
if (fd !== null) fs.closeSync(fd);
|
|
314
|
+
}
|
|
149
315
|
} else {
|
|
150
316
|
// A 0-BYTE FILE, not an absent one. pi's setSessionFile then takes its empty-file branch and
|
|
151
317
|
// writes its own header at this exact path, which marks the manager flushed -- so _persist
|
|
@@ -154,10 +320,22 @@ export function makeSessionStore({
|
|
|
154
320
|
fs.writeFileSync(staged, "");
|
|
155
321
|
}
|
|
156
322
|
log("session_resolved", { key, resume: verdict.resume, reason: verdict.reason });
|
|
157
|
-
return { hostDir, key, modelId, ...verdict };
|
|
323
|
+
return { hostDir, key, modelId, venue, ...verdict };
|
|
158
324
|
} catch (err) {
|
|
159
325
|
// A history fault must never fail the prepare that asked.
|
|
160
326
|
log("session_store_failed", { phase: "resolve", reason: err?.message });
|
|
327
|
+
// `null` means NO MOUNT AND NOTHING WRITTEN, so make the second half true, best effort. A fault after
|
|
328
|
+
// the copy -- a partial copy, or the venue re-check failing to empty a transcript another venue just
|
|
329
|
+
// promoted (#277) -- would otherwise leave that file under the job dir, which is mounted `/job:ro`:
|
|
330
|
+
// no `/session`, but the transcript readable at `/job/session/current.jsonl` all the same.
|
|
331
|
+
if (hostDir !== null) {
|
|
332
|
+
try {
|
|
333
|
+
fs.rmSync(hostDir, { recursive: true, force: true });
|
|
334
|
+
} catch {
|
|
335
|
+
// If this fails too, the copy stays under the job dir, which the container mounts before
|
|
336
|
+
// teardown removes it. Two consecutive disk faults on one path; nothing further to try.
|
|
337
|
+
}
|
|
338
|
+
}
|
|
161
339
|
return null;
|
|
162
340
|
}
|
|
163
341
|
}
|
|
@@ -170,6 +348,88 @@ export function makeSessionStore({
|
|
|
170
348
|
* one key is a real shape (REQ-QUEUE-BURST-NO-DROP), and last-write-wins there would interleave two
|
|
171
349
|
* agents' turns into one transcript.
|
|
172
350
|
*/
|
|
351
|
+
/**
|
|
352
|
+
* Take the per-key promotion lock, with ONE stale takeover. Returns `{ fd }`, or `{ locked: true }` when
|
|
353
|
+
* a LIVE writer holds it. Throws only for non-EEXIST failures, which is the doctrine this call site
|
|
354
|
+
* already carried: a read-only directory, a full disk or a vanished store all fail to create the lock
|
|
355
|
+
* too, and reporting those as `locked` sends an operator hunting a stuck file that does not exist.
|
|
356
|
+
*
|
|
357
|
+
* WITHOUT THIS, A PROCESS KILLED INSIDE THE LOCK WEDGED THE KEY FOREVER (issue #336). Later promotions
|
|
358
|
+
* reported `locked` until the reaper swept the key, and the reaper keys on the TRANSCRIPT's mtime: a key
|
|
359
|
+
* whose first promotion died before any transcript landed had none to key on, and with
|
|
360
|
+
* `PI_SESSIONS_TTL_DAYS=0` the reaper does not run at all. A takeover works in both of those, which is
|
|
361
|
+
* why it is the primary fix and the reaper's own repair is the secondary one.
|
|
362
|
+
*
|
|
363
|
+
* `lstatSync`, never `statSync`. The injected `fs` deliberately carries no `statSync`, this module's
|
|
364
|
+
* whole doctrine is lstat-in-a-key-directory, and a DANGLING link planted at this name throws ENOENT
|
|
365
|
+
* under `stat` on every attempt, which is the same wedge in a different coat.
|
|
366
|
+
*
|
|
367
|
+
* TWO DIFFERENT CLOCKS, and the comparison is between them: `now()` is this process's, `mtimeMs` is the
|
|
368
|
+
* FILESYSTEM's, which on a network mount is a server's. `OQ-031` already records two hosts sharing one
|
|
369
|
+
* working tree as a live hazard, and a shared `PI_SESSIONS_DIR` is exactly that shape. Both signs of the
|
|
370
|
+
* skew are bad, and differently. A server more than `LOCK_STALE_MS` BEHIND makes every live lock read as
|
|
371
|
+
* stale, so the takeover fires on every attempt and the conceded double-take window stops being rare;
|
|
372
|
+
* worse, `releaseLock` unlinks by PATH rather than by fd, so once A's lock is stolen A's release deletes
|
|
373
|
+
* B's. A server AHEAD makes the difference negative, so a genuinely crashed writer's lock is NEVER
|
|
374
|
+
* swept -- which is precisely today's behaviour, so under that skew this degrades to the status quo
|
|
375
|
+
* rather than to something worse. Nothing here closes either; `triggers-file.mjs` states the same pair.
|
|
376
|
+
*/
|
|
377
|
+
function takeLock(dir, key) {
|
|
378
|
+
const lock = join(dir, LOCK_FILE);
|
|
379
|
+
let sweptAgeMs = null;
|
|
380
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
381
|
+
try {
|
|
382
|
+
const fd = fs.openSync(lock, "wx"); // exclusive create IS the lock; no daemon, no lease
|
|
383
|
+
// Logged only AFTER the retake create SUCCEEDED. The unlink alone proves nothing, since a rival
|
|
384
|
+
// sweeper can win the recreate race, and a takeover line for a lock we did not get would send an
|
|
385
|
+
// operator reading a history that never happened.
|
|
386
|
+
if (sweptAgeMs !== null) log("session_lock_stale_taken", { key, ageMs: sweptAgeMs });
|
|
387
|
+
return { fd };
|
|
388
|
+
} catch (err) {
|
|
389
|
+
if (err?.code !== "EEXIST") throw err;
|
|
390
|
+
let mtimeMs;
|
|
391
|
+
try {
|
|
392
|
+
mtimeMs = fs.lstatSync(lock).mtimeMs;
|
|
393
|
+
} catch {
|
|
394
|
+
// Released between our open and our stat: the next create answers.
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
if (now() - mtimeMs <= LOCK_STALE_MS) return { locked: true };
|
|
398
|
+
try {
|
|
399
|
+
fs.unlinkSync(lock);
|
|
400
|
+
} catch {
|
|
401
|
+
// Someone else swept it first; the retry create answers who won.
|
|
402
|
+
}
|
|
403
|
+
sweptAgeMs = Math.round(now() - mtimeMs);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
return { locked: true };
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* Make the key directory, or answer that the name is not one (issue #375). The write-side twin of
|
|
411
|
+
* `inspectKeyDir`, separate because it CREATES: the read path must never make a directory for a key it is
|
|
412
|
+
* only asking about.
|
|
413
|
+
*
|
|
414
|
+
* An lstat that fails with anything but ENOENT is re-thrown, which the outer catch reports as
|
|
415
|
+
* `promote-failed`: a disk fault is not evidence about the shape of the name, and reporting it as one
|
|
416
|
+
* would send an operator hunting a symlink that does not exist. That is `takeLock`'s doctrine one step
|
|
417
|
+
* earlier.
|
|
418
|
+
*/
|
|
419
|
+
function ensureKeyDir(dir) {
|
|
420
|
+
let st = null;
|
|
421
|
+
try {
|
|
422
|
+
st = fs.lstatSync(dir);
|
|
423
|
+
} catch (err) {
|
|
424
|
+
if (err?.code !== "ENOENT") throw err;
|
|
425
|
+
}
|
|
426
|
+
if (st === null) {
|
|
427
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
428
|
+
st = fs.lstatSync(dir); // the second lstat: a link planted between the two is caught here
|
|
429
|
+
}
|
|
430
|
+
return st.isDirectory() ? `${st.dev}:${st.ino}` : null;
|
|
431
|
+
}
|
|
432
|
+
|
|
173
433
|
function promoteSession(session, { piVersion = null, context = null } = {}) {
|
|
174
434
|
// The second DI-seam backstop, and unreachable for the same reason as the `!sessionsDir` return
|
|
175
435
|
// above: sessionKeyFor is total and binary (null, or 32 hex chars), so resolveSession returns null
|
|
@@ -187,38 +447,150 @@ export function makeSessionStore({
|
|
|
187
447
|
}
|
|
188
448
|
|
|
189
449
|
const dir = keyDir(session.key);
|
|
190
|
-
|
|
450
|
+
// LSTAT BEFORE CREATE, and again after (issue #375). Recursive `mkdir` succeeds silently on an
|
|
451
|
+
// existing link to a directory, so asking it first and checking afterwards would accept exactly the
|
|
452
|
+
// shape this refuses; asking first means nothing already at the name is ever handed to `mkdir` at all,
|
|
453
|
+
// and the outcome does not depend on how `mkdir` treats a link, a file or a dangling link. Placed
|
|
454
|
+
// BEFORE `takeLock`, so a refused promotion never creates a lock file through a link either.
|
|
455
|
+
//
|
|
456
|
+
// The entry is NOT removed. It is not ours: this store creates key directories and nothing else, so
|
|
457
|
+
// whatever is sitting there was put there by an operator or by an attacker, and a sweep that deletes
|
|
458
|
+
// either is a worse answer than a refusal an operator can read. The key stays cold until they clear
|
|
459
|
+
// it, which is the loud direction.
|
|
460
|
+
const madeDir = ensureKeyDir(dir);
|
|
461
|
+
if (madeDir === null) {
|
|
462
|
+
log("session_promote_skipped", { key: session.key, reason: "key-not-a-directory" });
|
|
463
|
+
return { promoted: false, reason: "key-not-a-directory" };
|
|
464
|
+
}
|
|
465
|
+
// Is the key directory still the one `ensureKeyDir` made or found? SHAPE ALONE IS NOT ENOUGH, and
|
|
466
|
+
// the gate round proved it: a swap to another REAL directory passed a shape-only re-check and the
|
|
467
|
+
// promotion wrote the transcript and all four sidecars into it while reporting `promoted: true`.
|
|
468
|
+
const sameDir = () => inspectKeyDir(session.key).ident === madeDir;
|
|
191
469
|
|
|
192
470
|
const lock = join(dir, LOCK_FILE);
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
// vanished store all failed to create the lock too, and reporting those as `locked` sends an
|
|
199
|
-
// operator looking for a stuck lock file that does not exist. Anything else falls through to
|
|
200
|
-
// the outer catch and reports `promote-failed`, which is what actually happened.
|
|
201
|
-
if (err?.code !== "EEXIST") throw err;
|
|
471
|
+
// `locked` still means a LIVE writer and nothing else. A lock older than any plausible promotion is
|
|
472
|
+
// taken over rather than believed (see `takeLock`); anything that is not EEXIST still falls through
|
|
473
|
+
// to the outer catch and reports `promote-failed`, which is what actually happened.
|
|
474
|
+
const taken = takeLock(dir, session.key);
|
|
475
|
+
if (taken.locked) {
|
|
202
476
|
log("session_promote_skipped", { key: session.key, reason: "locked" });
|
|
203
477
|
return { promoted: false, reason: "locked" };
|
|
204
478
|
}
|
|
479
|
+
const fd = taken.fd;
|
|
205
480
|
try {
|
|
206
|
-
//
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
481
|
+
// RE-CHECKED UNDER THE LOCK (issue #375's gate round). `ensureKeyDir` ran before the lock, so a
|
|
482
|
+
// link swapped in between the two would take every write below it outside the store: measured on
|
|
483
|
+
// the first draft, the transcript and all four sidecars landed in the attacker's directory and
|
|
484
|
+
// the promotion still reported `promoted: true`. Asking again here closes that segment, and
|
|
485
|
+
// costs one lstat on a path already in the page cache.
|
|
486
|
+
//
|
|
487
|
+
// It does NOT close the whole window, and the residual is stated rather than implied: a swap
|
|
488
|
+
// after THIS check still redirects the writes, because Node exposes no `openat`/`renameat` and
|
|
489
|
+
// every write below therefore resolves the name again. What the re-check after the swap does is
|
|
490
|
+
// make that case reportable rather than silent.
|
|
491
|
+
//
|
|
492
|
+
// What the refusal costs where the swap landed after the lock was taken: the lock file is in the
|
|
493
|
+
// REAL key and `releaseLock` unlinks by path, which now resolves elsewhere, so the real key keeps
|
|
494
|
+
// a lock nobody holds and the next promotions report `locked` until the staleness takeover sweeps
|
|
495
|
+
// it an hour later. That is the same unlink-by-path residual `takeLock` already states for a
|
|
496
|
+
// takeover, reached by a different route, and it is the honest cost of refusing here rather than
|
|
497
|
+
// writing into whatever the name now points at.
|
|
498
|
+
if (!sameDir()) {
|
|
499
|
+
log("session_promote_skipped", { key: session.key, reason: "key-not-a-directory" });
|
|
500
|
+
return { promoted: false, reason: "key-not-a-directory" };
|
|
216
501
|
}
|
|
502
|
+
// Atomic swap: a reader either sees the old file or the new one, never a half-written one.
|
|
503
|
+
// PER WRITER (see `tmpSeq`), not the fixed `.incoming` this shared before the lock could be taken
|
|
504
|
+
// over. The random half is what separates two writers; the pid and counter are there to make a
|
|
505
|
+
// straggler attributable, and neither is a writer identity on its own. The removal below stays
|
|
506
|
+
// regardless: a name that is hard to guess is not a name that cannot be guessed.
|
|
507
|
+
const tmp = `${canonicalFile(session.key)}.${process.pid}.${tmpSeq++}.${randomBytes(6).toString("hex")}.incoming`;
|
|
508
|
+
// `copyFileSync` follows a link at the DESTINATION, so anything left at this name would receive
|
|
509
|
+
// the whole transcript. `removeTemp` takes every shape; see its own comment for why one call
|
|
510
|
+
// could not.
|
|
511
|
+
removeTemp(tmp);
|
|
217
512
|
fs.copyFileSync(staged, tmp);
|
|
513
|
+
// THE VENUE STAMP IS INVALIDATED BEFORE THE SWAP, and fatally (#277). Removing a stamp does not
|
|
514
|
+
// invalidate it, because an absent stamp reads as `local`; stamping only after the rename would
|
|
515
|
+
// leave a window, and a failed post-swap write a permanent state, in which one venue's transcript
|
|
516
|
+
// sits under another venue's stamp and the next job there resumes it. So every promotion first
|
|
517
|
+
// writes a stamp no venue matches. If that write fails, nothing has been swapped and the outer
|
|
518
|
+
// catch reports `promote-failed`; if the process dies after it, the key cold-starts everywhere
|
|
519
|
+
// until a promotion completes. A process KILLED inside this lock also leaks the lock itself, as it
|
|
520
|
+
// always has -- but it no longer wedges the key in the shapes that used to be permanent: the next
|
|
521
|
+
// promotion takes a lock older than `LOCK_STALE_MS` over, and the reaper reaches a key with no
|
|
522
|
+
// transcript through the directory's own mtime (issue #336). A lock whose mtime is in the FUTURE is
|
|
523
|
+
// still never stale, so under `PI_SESSIONS_TTL_DAYS=0` such a key stays wedged. Unconditional rather than only
|
|
524
|
+
// on a venue change, so the decision needs no read of the stamp under the lock and there is no
|
|
525
|
+
// branch to get wrong.
|
|
526
|
+
replaceSidecar(dir, VENUE_FILE, VENUE_PENDING);
|
|
527
|
+
// The temp's identity, read BEFORE the rename, is what makes the A, B, A reportable (issue #390).
|
|
528
|
+
// `rename` preserves the inode, so after a rename that landed where it was meant to, the canonical
|
|
529
|
+
// name holds THIS file. If the directory was swapped for the rename and swapped back afterwards,
|
|
530
|
+
// the canonical name resolves into the real key again and holds its OLD transcript, or nothing --
|
|
531
|
+
// a different inode either way. That is a fact the shape re-check below cannot see, because by
|
|
532
|
+
// then the shape is right again.
|
|
533
|
+
const tmpIdent = readIdentity(tmp);
|
|
218
534
|
fs.renameSync(tmp, canonicalFile(session.key));
|
|
219
|
-
|
|
220
|
-
//
|
|
221
|
-
//
|
|
535
|
+
// DETECTION for the segment prevention cannot reach (issue #375's gate round). A link swapped in
|
|
536
|
+
// after the check above takes this rename and every sidecar write with it, and the old code then
|
|
537
|
+
// reported `promoted: true` for a transcript that landed outside the store: the operator's record
|
|
538
|
+
// said the next run would resume work that is not there. This cannot undo the write, the bytes
|
|
539
|
+
// being already wherever the name pointed, and it catches such a swap only while it is STILL
|
|
540
|
+
// STANDING here.
|
|
541
|
+
if (!sameDir()) {
|
|
542
|
+
log("session_promote_skipped", { key: session.key, reason: "key-not-a-directory" });
|
|
543
|
+
return { promoted: false, reason: "key-not-a-directory" };
|
|
544
|
+
}
|
|
545
|
+
// AND THE A, B, A, which the check above cannot see because by then the shape is right again
|
|
546
|
+
// (issue #390). Measured with a deterministic probe before this existed: `{"promoted":true,
|
|
547
|
+
// "reason":"promoted"}` with the transcript in the attacker's directory and the real key holding
|
|
548
|
+
// only `pi-version`, `resume-chain` and `venue` -- the next job on that key cold-starts as
|
|
549
|
+
// `absent` while the record says the work was promoted, which is the silent no-op `CLAUDE.md`
|
|
550
|
+
// calls the worst outcome available. A 45 second unsynchronised live race did not hit the exact
|
|
551
|
+
// ordering (16,591 promotions, 3 refusals, 0 lies), so this is a deterministic-window finding
|
|
552
|
+
// rather than a frequent one, and its precondition is the one the whole store concedes: write
|
|
553
|
+
// access to `PI_SESSIONS_DIR`.
|
|
554
|
+
//
|
|
555
|
+
// DETECTION, NOT PREVENTION, like its neighbour: the bytes are already wherever the name pointed
|
|
556
|
+
// and nothing here can recall them. What it buys is that the RECORD stops claiming otherwise.
|
|
557
|
+
// A shape-only comparison would not do it -- the round that added the checks above proved a swap
|
|
558
|
+
// to another REAL directory passes one -- so this compares the inode the rename preserved.
|
|
559
|
+
if (tmpIdent === null || readIdentity(canonicalFile(session.key)) !== tmpIdent) {
|
|
560
|
+
log("session_promote_skipped", { key: session.key, reason: "transcript-diverted" });
|
|
561
|
+
return { promoted: false, reason: "transcript-diverted" };
|
|
562
|
+
}
|
|
563
|
+
// The real stamp FIRST of the post-swap writes, and the ordering is the contract. It is the only one
|
|
564
|
+
// whose absence MISATTRIBUTES rather than cold-starts: between the rename and this write the key reads
|
|
565
|
+
// `(pending)` and cold-starts on every venue, so every write placed ahead of it lengthens that window,
|
|
566
|
+
// and `pi-version`, the chain counter and the context reading all degrade to a cold start instead.
|
|
567
|
+
// (It used to be ordered ahead of the pi-version write because THAT write could throw. It no longer
|
|
568
|
+
// can, and the ordering that survives is this one.) A session with no venue (a DI seam) leaves the
|
|
569
|
+
// sentinel on purpose: no stamp is better than a guessed one.
|
|
570
|
+
const venue = normaliseVenue(session.venue);
|
|
571
|
+
if (venue !== null) writeSidecar(dir, VENUE_FILE, session.key, venue);
|
|
572
|
+
// THROUGH `writeSidecar` like every other sidecar (issue #336). It was the one plain write left, and
|
|
573
|
+
// it carried both halves of that exception: `writeFileSync` FOLLOWS a link, so a symlink planted at
|
|
574
|
+
// this name -- at a path anyone who knows the repository and the branch can compute -- turned a
|
|
575
|
+
// promotion into a truncating write of the link's target with the pi version as its payload; and it
|
|
576
|
+
// sat inside the try, so a failure AFTER the swap reported `promote-failed` for a promotion that had
|
|
577
|
+
// landed.
|
|
578
|
+
//
|
|
579
|
+
// A FAILED STAMP IS SAFE IN BOTH DIRECTIONS, which is why this one needs no sentinel of its own and
|
|
580
|
+
// the venue stamp does. With no prior stamp the next read gets `null` and cold-starts. With one, it
|
|
581
|
+
// survives beside the new transcript -- and it is still TRUE, because this job resumed only if that
|
|
582
|
+
// stamp already matched its own image's pi, and a job runs one image. The remaining case, a job that
|
|
583
|
+
// cold-started under a new pi and then failed this write, leaves the OLD version beside the new
|
|
584
|
+
// transcript, and the next job on that image reads a mismatch and cold-starts. Stale-but-true, or a
|
|
585
|
+
// cold start. Never a transcript resumed under a version that did not write it.
|
|
586
|
+
//
|
|
587
|
+
// `String(piVersion ?? "")` is load-bearing and must not be simplified to skipping the write: an empty
|
|
588
|
+
// file is refused by `readSidecar`'s own size check and reads as `null`, so a promotion that knows no
|
|
589
|
+
// version INVALIDATES the stamp. Skipping instead would leave a PREVIOUS version beside a transcript
|
|
590
|
+
// written by an unknown pi, and the next matching job would resume it.
|
|
591
|
+
writeSidecar(dir, PI_VERSION_FILE, session.key, String(piVersion ?? ""));
|
|
592
|
+
// The chain and context sidecars, immediately after the swap and under the same lock. NOT part of
|
|
593
|
+
// the swap itself, which is one rename and cannot be widened: what the lock buys them is that no
|
|
222
594
|
// other job can interleave, and what the ordering buys them is that they never describe a
|
|
223
595
|
// transcript older than the one now in place.
|
|
224
596
|
//
|
|
@@ -234,7 +606,8 @@ export function makeSessionStore({
|
|
|
234
606
|
try {
|
|
235
607
|
fs.unlinkSync(lock);
|
|
236
608
|
} catch {
|
|
237
|
-
// A
|
|
609
|
+
// A lock left behind here is taken over by the next promotion once it is stale (`takeLock`).
|
|
610
|
+
// Logged, never thrown: the promotion itself succeeded.
|
|
238
611
|
log("session_lock_stuck", { key: session.key });
|
|
239
612
|
}
|
|
240
613
|
}
|
|
@@ -273,21 +646,59 @@ export function makeSessionStore({
|
|
|
273
646
|
* cold start when it will in fact resume. The bookkeeping loss is logged and the truth is kept.
|
|
274
647
|
*/
|
|
275
648
|
function writeSidecar(dir, name, key, value) {
|
|
276
|
-
const file = join(dir, name);
|
|
277
|
-
const tmp = `${file}.incoming`;
|
|
278
649
|
try {
|
|
279
|
-
|
|
280
|
-
fs.unlinkSync(tmp);
|
|
281
|
-
} catch {
|
|
282
|
-
// Absent is the desired state.
|
|
283
|
-
}
|
|
284
|
-
fs.writeFileSync(tmp, value);
|
|
285
|
-
fs.renameSync(tmp, file);
|
|
650
|
+
replaceSidecar(dir, name, value);
|
|
286
651
|
} catch (err) {
|
|
287
652
|
log("session_sidecar_failed", { key, file: name, reason: err?.message });
|
|
288
653
|
}
|
|
289
654
|
}
|
|
290
655
|
|
|
656
|
+
/**
|
|
657
|
+
* Remove whatever is at a temp path, whatever SHAPE it has, before anything writes there.
|
|
658
|
+
*
|
|
659
|
+
* ONE RULE BECAUSE CHOOSING BETWEEN THE TWO CALLS WAS GOT WRONG TWICE, in opposite directions.
|
|
660
|
+
* `unlinkSync` alone cannot remove a DIRECTORY planted at the name, and at the venue sentinel -- the one
|
|
661
|
+
* sidecar write that is fatal -- that wedged every promotion on the key forever. `rmSync` alone does not
|
|
662
|
+
* remove a DANGLING SYMLINK: it resolves the path, finds nothing, and with `force` reports success while
|
|
663
|
+
* leaving the link (measured, and it is the same measurement the reaper's removal note records further down
|
|
664
|
+
* this file -- measured there for the reaper, and not carried here until it had cost a round). The next write then follows the surviving link and creates a file at its target, which
|
|
665
|
+
* is the write-through-a-link hole this whole series exists to close, and if that target is unreachable
|
|
666
|
+
* the write throws and the key is wedged again.
|
|
667
|
+
*
|
|
668
|
+
* So: `unlinkSync` first, which takes a file or a link INCLUDING a dangling one; then `rmSync` for the
|
|
669
|
+
* one shape it cannot take. A directory is removed with its subtree, which is bounded to a temp name
|
|
670
|
+
* inside the key directory and is the point rather than a side effect: nothing may be left at that name.
|
|
671
|
+
*/
|
|
672
|
+
function removeTemp(path) {
|
|
673
|
+
try {
|
|
674
|
+
fs.unlinkSync(path);
|
|
675
|
+
return;
|
|
676
|
+
} catch (err) {
|
|
677
|
+
if (err?.code === "ENOENT") return; // absent is the desired state
|
|
678
|
+
}
|
|
679
|
+
try {
|
|
680
|
+
fs.rmSync(path, { recursive: true, force: true });
|
|
681
|
+
} catch {
|
|
682
|
+
// Nothing further to try HERE, and the honest bound is worth stating: if both calls fail, anything
|
|
683
|
+
// left at the name is still there and the write that follows may go THROUGH it. The only way both
|
|
684
|
+
// fail is a key directory this process cannot write, and `takeLock`'s own exclusive create fails
|
|
685
|
+
// first in that state, so a promotion never reaches here with it -- which is why this is a bound
|
|
686
|
+
// rather than a hole.
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* `writeSidecar`'s link-safe temp-and-rename, and it THROWS. For the one write whose failure must stop a
|
|
692
|
+
* promotion rather than be logged past it: the venue sentinel, which runs before the swap (#277).
|
|
693
|
+
*/
|
|
694
|
+
function replaceSidecar(dir, name, value) {
|
|
695
|
+
const file = join(dir, name);
|
|
696
|
+
const tmp = `${file}.incoming`;
|
|
697
|
+
removeTemp(tmp);
|
|
698
|
+
fs.writeFileSync(tmp, value);
|
|
699
|
+
fs.renameSync(tmp, file);
|
|
700
|
+
}
|
|
701
|
+
|
|
291
702
|
/**
|
|
292
703
|
* The context sidecar, whose three cases are all different.
|
|
293
704
|
*
|
|
@@ -326,20 +737,138 @@ export function makeSessionStore({
|
|
|
326
737
|
return join(keyDir(key), SESSION_FILE_NAME);
|
|
327
738
|
}
|
|
328
739
|
|
|
740
|
+
/**
|
|
741
|
+
* Copy an already-open transcript to `dest`, in bounded chunks, and never more than `bytes`.
|
|
742
|
+
*
|
|
743
|
+
* The source is a DESCRIPTOR rather than a path, which is the whole point (issue #375): the bytes a job
|
|
744
|
+
* resumes must come off the same inode the gates judged, and any second resolution of the name is a second
|
|
745
|
+
* chance for something else to be standing there. `copyFileSync` cannot take one, so this is the copy.
|
|
746
|
+
*
|
|
747
|
+
* BOUNDED BY THE JUDGED SIZE, not by what the file turns out to hold. `inspectFile` capped the size
|
|
748
|
+
* against `PI_SESSION_MAX_BYTES` and the record reports that number, and a transcript still being written
|
|
749
|
+
* grows between the two: measured on the first draft of this copy, a file growing under it staged 2,460,306
|
|
750
|
+
* bytes against a 1,000,000 cap while the record said 306. Reading exactly what was judged makes the cap
|
|
751
|
+
* and the record true again. A SHORT read is refused rather than truncated silently, because a transcript
|
|
752
|
+
* cut off mid-line is a transcript pi will reject anyway. What REFUSES either case is the by-path identity
|
|
753
|
+
* re-check in `resolveSession`, since a transcript that grew or shrank has an identity that moved; this
|
|
754
|
+
* bound is what stops an unbounded append from being read at all, and it is deliberately not the thing
|
|
755
|
+
* that decides the verdict. A short read therefore needs no flag of its own: the re-check has it.
|
|
756
|
+
*
|
|
757
|
+
* 64 KiB because it is one buffer per resume and a transcript is usually a few hundred KiB; the loop is
|
|
758
|
+
* what keeps an uncapped `PI_SESSION_MAX_BYTES` from becoming a memory bound.
|
|
759
|
+
*
|
|
760
|
+
* THREE EQUIVALENT MUTANTS LIVE IN THIS FUNCTION, recorded so the next reader does not re-derive them
|
|
761
|
+
* (issue #391), because the lines around them ARE pinned and the difference is not visible by reading:
|
|
762
|
+
*
|
|
763
|
+
* - the buffer SIZING, `Math.min(64 * 1024, Math.max(1, bytes))` to a bare `64 * 1024`. The per-read
|
|
764
|
+
* clamp in the loop (`Math.min(buf.length, bytes - copied)`) is what bounds the read, so the sizing
|
|
765
|
+
* only decides how much memory a small file allocates.
|
|
766
|
+
* - the LOOP CONDITION, `copied < bytes` to `copied <= bytes`, for the same reason.
|
|
767
|
+
* - `Math.max(1, bytes)` to `bytes`, which is dead defensive code: `inspectFile` refuses a 0-byte
|
|
768
|
+
* transcript as `absent`, so `bytes === 0` never reaches this function at all. That unreachability
|
|
769
|
+
* is the thing a next reader is most likely to re-derive, which is why it is written down.
|
|
770
|
+
*
|
|
771
|
+
* All three were run against the whole suite and produce byte-identical staged output across transcript
|
|
772
|
+
* sizes either side of 64 KiB. What is NOT equivalent, and IS pinned by a descriptor count: the close
|
|
773
|
+
* below, the one in `resolveSession` that opened the descriptor this is handed, and the LOCK's close in
|
|
774
|
+
* `promoteSession` -- each of which leaks one descriptor per resume or per promotion in a process
|
|
775
|
+
* designed to run for weeks.
|
|
776
|
+
*/
|
|
777
|
+
function copyFromDescriptor(fd, dest, bytes) {
|
|
778
|
+
const buf = Buffer.allocUnsafe(Math.min(64 * 1024, Math.max(1, bytes)));
|
|
779
|
+
let out = null;
|
|
780
|
+
let copied = 0;
|
|
781
|
+
try {
|
|
782
|
+
out = fs.openSync(dest, "w");
|
|
783
|
+
while (copied < bytes) {
|
|
784
|
+
const read = fs.readSync(fd, buf, 0, Math.min(buf.length, bytes - copied), null);
|
|
785
|
+
if (read === 0) break;
|
|
786
|
+
fs.writeFileSync(out, buf.subarray(0, read));
|
|
787
|
+
copied += read;
|
|
788
|
+
}
|
|
789
|
+
} finally {
|
|
790
|
+
if (out !== null) fs.closeSync(out);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
/**
|
|
795
|
+
* Is the key's own name a real directory? `inspectFile`'s rule, one level up (issue #375).
|
|
796
|
+
*
|
|
797
|
+
* Every other read and write in here is an `lstat` on a name INSIDE the key directory, which is exactly
|
|
798
|
+
* what a key directory that is ITSELF a symlink defeats: the link is the directory, so each of those
|
|
799
|
+
* lstats resolves through it and every guarantee in this file lands wherever it points. Measured before
|
|
800
|
+
* the fix: a link planted at the derived path made `resolveSession` return `resumed` from a transcript
|
|
801
|
+
* outside the store, and `promoteSession` write `current.jsonl`, `pi-version`, `venue` and `resume-chain`
|
|
802
|
+
* through it. `mkdirSync(dir, { recursive: true })` succeeds on an existing link to a directory, so
|
|
803
|
+
* nothing on the write path noticed either.
|
|
804
|
+
*
|
|
805
|
+
* THE FINAL COMPONENT ONLY, deliberately. `PI_SESSIONS_DIR` and its ancestors may legitimately be links:
|
|
806
|
+
* macOS's own temp root is `/var -> private/var`, and moving the whole store behind a link is a supported
|
|
807
|
+
* layout. A `realpath` or an ancestor walk would refuse those, and would refuse every test fixture on
|
|
808
|
+
* this platform. The name this checks is the DERIVED one, which is what an attacker who knows the
|
|
809
|
+
* repository and the branch can precompute, and which nothing but this module should be creating.
|
|
810
|
+
*
|
|
811
|
+
* It answers the SHAPE only. It carried a `dev:ino` identity for the post-copy re-check in the first draft
|
|
812
|
+
* of this change, and the gate round removed the need for it: the staged bytes now come off one descriptor
|
|
813
|
+
* (see `resolveSession`), so a directory swapped under a job cannot change what that job reads, and there
|
|
814
|
+
* is nothing left for a directory identity to protect.
|
|
815
|
+
*/
|
|
816
|
+
function inspectKeyDir(key) {
|
|
817
|
+
let st;
|
|
818
|
+
try {
|
|
819
|
+
st = fs.lstatSync(keyDir(key));
|
|
820
|
+
} catch {
|
|
821
|
+
// EVERY failure reads as `absent`, not only ENOENT, and that is this edge's posture rather than an
|
|
822
|
+
// oversight: `inspectFile` one level down has always done the same, and a read that cannot ask costs a
|
|
823
|
+
// job its history and nothing else. Its write-side twin `ensureKeyDir` takes the opposite trade
|
|
824
|
+
// deliberately, because a promotion that cannot ask must not report the key's SHAPE as the reason.
|
|
825
|
+
// The cost, stated: an unreadable store cold-starts silently, indistinguishable from a fresh key.
|
|
826
|
+
return { ok: false, reason: "absent" };
|
|
827
|
+
}
|
|
828
|
+
if (!st.isDirectory()) return { ok: false, reason: "key-not-a-directory" };
|
|
829
|
+
// `dev:ino` and NOTHING else. The file rule's `dev:ino:size:mtime` is wrong for a directory: both move
|
|
830
|
+
// on every entry a promotion creates inside the key (64 -> 96 -> 1376 bytes on APFS, measured), so a
|
|
831
|
+
// re-check built on it reports a race on every quiet run. BOTH edges compare this identity: the write
|
|
832
|
+
// edge against what `ensureKeyDir` made or found, the read edge against what the gates were computed
|
|
833
|
+
// from. Comparing the SHAPE alone on either edge lets a swap for another REAL directory through, which
|
|
834
|
+
// is how it was found on the write edge and, one round later, on this one.
|
|
835
|
+
return { ok: true, ident: `${st.dev}:${st.ino}` };
|
|
836
|
+
}
|
|
837
|
+
|
|
329
838
|
/** The read path, gate by gate. The FIRST miss wins and names itself. */
|
|
330
|
-
function readCanonical(key, piVersion, modelId) {
|
|
839
|
+
function readCanonical(key, piVersion, modelId, venue) {
|
|
840
|
+
// FIRST, because every gate below it is an lstat inside this directory and none of them can see that
|
|
841
|
+
// the directory is not one. A regular file or a link to one read as `absent` before this arm existed,
|
|
842
|
+
// through the ENOTDIR their inner lstat threw; they are now named for what they are.
|
|
843
|
+
const dirCheck = inspectKeyDir(key);
|
|
844
|
+
if (!dirCheck.ok) return COLD(dirCheck.reason);
|
|
845
|
+
|
|
846
|
+
|
|
331
847
|
const file = canonicalFile(key);
|
|
332
848
|
const check = inspectFile(file);
|
|
333
849
|
if (!check.ok) return COLD(check.reason);
|
|
334
850
|
|
|
335
851
|
if (ttlDays > 0 && now() - check.mtimeMs > ttlDays * 86400000) return COLD("expired");
|
|
336
852
|
|
|
853
|
+
// The VENUE the transcript was written in (#277). Placed HERE, after the arms a venue move cannot cause
|
|
854
|
+
// -- a transcript that is absent, not a regular file, too large or past its TTL is all of those on every
|
|
855
|
+
// venue -- and AHEAD of both pi-version arms, because a venue move CAN cause those: image preflight is
|
|
856
|
+
// dispatched per venue, so another venue can report another pi or none. The first miss names itself,
|
|
857
|
+
// and before this arm a venue move named itself as a version change. Without it, a trigger moved
|
|
858
|
+
// between venues staged the old venue's transcript into the new venue's container with nothing refusing.
|
|
859
|
+
//
|
|
860
|
+
// FAILS CLOSED: a job whose venue cannot be resolved (a DI seam) never resumes, the pi-version gate's
|
|
861
|
+
// polarity. The token also covers a stamp left pending by an interrupted promotion and a stamp that is
|
|
862
|
+
// present but unreadable -- a misnaming stated rather than hidden, as `pi-version-changed` already does
|
|
863
|
+
// for an unreadable version stamp.
|
|
864
|
+
if (venue === null || readVenue(key) !== venue) return COLD("venue-changed");
|
|
865
|
+
|
|
337
866
|
// A transcript outlives the pi that wrote it, and pi's own docs record what then breaks: an older
|
|
338
867
|
// session's stored tool-call arguments may no longer match the current tool schema. We cannot
|
|
339
868
|
// repair that mid-run, so a version change is a cold start rather than a mid-run failure. An
|
|
340
869
|
// image that declares no version never resumes -- the safe direction, never "assume it matches".
|
|
341
870
|
if (piVersion === null) return COLD("pi-version-changed");
|
|
342
|
-
// Through the same guarded read as the
|
|
871
|
+
// Through the same guarded read as the other sidecars. This one predates them and was the
|
|
343
872
|
// one unguarded read left in the key directory; a symlink here would have decided a gate on the
|
|
344
873
|
// contents of some other file entirely.
|
|
345
874
|
const stamped = readSidecar(key, PI_VERSION_FILE);
|
|
@@ -425,7 +954,7 @@ export function makeSessionStore({
|
|
|
425
954
|
if (!Number.isFinite(started)) return COLD("conversation-too-old");
|
|
426
955
|
if (now() - started > maxAgeDays * 86400000) return COLD("conversation-too-old");
|
|
427
956
|
}
|
|
428
|
-
return { resume: true, reason: "resumed", bytes: check.bytes };
|
|
957
|
+
return { resume: true, reason: "resumed", bytes: check.bytes, ident: check.ident, dirIdent: dirCheck.ident };
|
|
429
958
|
}
|
|
430
959
|
|
|
431
960
|
/**
|
|
@@ -453,6 +982,25 @@ export function makeSessionStore({
|
|
|
453
982
|
}
|
|
454
983
|
}
|
|
455
984
|
|
|
985
|
+
/**
|
|
986
|
+
* The venue stamped beside a key's transcript (#277).
|
|
987
|
+
*
|
|
988
|
+
* ABSENT and UNREADABLE are different answers, and conflating them either way is a defect. An absent stamp
|
|
989
|
+
* is a transcript from before venues were recorded, which ran on `local`. A stamp that exists but cannot
|
|
990
|
+
* be read -- not a regular file, empty, oversized, unreadable -- matches nothing (`null`), so the key
|
|
991
|
+
* cold-starts. Absence is decided by `lstat` and ENOENT ALONE: `stat` or `existsSync` would follow a
|
|
992
|
+
* DANGLING symlink planted at this name, report it absent, and resume a transcript as `local` on the
|
|
993
|
+
* strength of a link that points nowhere. Any other `lstat` failure is not absence either.
|
|
994
|
+
*/
|
|
995
|
+
function readVenue(key) {
|
|
996
|
+
try {
|
|
997
|
+
fs.lstatSync(join(keyDir(key), VENUE_FILE));
|
|
998
|
+
} catch (err) {
|
|
999
|
+
return err?.code === "ENOENT" ? UNATTRIBUTED_BACKEND : null;
|
|
1000
|
+
}
|
|
1001
|
+
return readSidecar(key, VENUE_FILE);
|
|
1002
|
+
}
|
|
1003
|
+
|
|
456
1004
|
/**
|
|
457
1005
|
* The consecutive-delivery counter for a key, or 0 when there is not a readable one. Never throws and
|
|
458
1006
|
* never guesses: a missing, empty, corrupt or negative counter is 0, so the only way to be refused by
|
|
@@ -493,6 +1041,33 @@ export function makeSessionStore({
|
|
|
493
1041
|
* (`fs.readFile` off the clone following a symlink into a worker-host file). The repo's own habit is
|
|
494
1042
|
* the wrong one here: makeLogReaper uses statSync, which follows.
|
|
495
1043
|
*/
|
|
1044
|
+
/**
|
|
1045
|
+
* The canonical file's IDENTITY as one comparable token: device, inode, size, mtime.
|
|
1046
|
+
*
|
|
1047
|
+
* ONE STRING RATHER THAN FOUR FIELDS, so the comparison is one `!==` and there is no partial compare to
|
|
1048
|
+
* get wrong. `ino` is the field that does the work: a promotion renames a freshly created `.incoming`
|
|
1049
|
+
* file into place, so the inode behind the canonical path is a DIFFERENT one after every completed swap
|
|
1050
|
+
* -- including the A, B, A round trip whose venue stamp matches again, which is why this catches what the
|
|
1051
|
+
* venue re-check cannot. `dev` is what makes `ino` meaningful, since an inode number is unique per
|
|
1052
|
+
* device. `size` and `mtimeMs` narrow the inode-REUSE residual and nothing else: the swapped-away inode
|
|
1053
|
+
* is freed by its own rename and its number may be handed straight back to the next `.incoming` file.
|
|
1054
|
+
*/
|
|
1055
|
+
function identityOf(st) {
|
|
1056
|
+
return `${st.dev}:${st.ino}:${st.size}:${st.mtimeMs}`;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
/**
|
|
1060
|
+
* The identity now at a path, or `null` when there is nothing readable there -- which compares unequal to
|
|
1061
|
+
* every real identity, so a transcript that VANISHED between the gate and the re-check cold-starts too.
|
|
1062
|
+
*/
|
|
1063
|
+
function readIdentity(file) {
|
|
1064
|
+
try {
|
|
1065
|
+
return identityOf(fs.lstatSync(file));
|
|
1066
|
+
} catch {
|
|
1067
|
+
return null;
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
496
1071
|
function inspectFile(file) {
|
|
497
1072
|
let st;
|
|
498
1073
|
try {
|
|
@@ -503,17 +1078,20 @@ export function makeSessionStore({
|
|
|
503
1078
|
if (!st.isFile()) return { ok: false, reason: "not-a-regular-file" };
|
|
504
1079
|
if (st.size === 0) return { ok: false, reason: "absent" }; // a staged-but-unwritten transcript
|
|
505
1080
|
if (maxBytes > 0 && st.size > maxBytes) return { ok: false, reason: "too-large" };
|
|
506
|
-
return { ok: true, bytes: st.size, mtimeMs: st.mtimeMs };
|
|
1081
|
+
return { ok: true, bytes: st.size, mtimeMs: st.mtimeMs, ident: identityOf(st) };
|
|
507
1082
|
}
|
|
508
1083
|
|
|
509
1084
|
/**
|
|
510
|
-
*
|
|
511
|
-
*
|
|
512
|
-
*
|
|
1085
|
+
* The DISK sweep: at boot, and on the retention timer thereafter (`PI_SWEEP_INTERVAL_HOURS`, issue
|
|
1086
|
+
* #292). A SIBLING of makeLogReaper rather than a widening of it: that one's `.log`/`.json` filter and
|
|
1087
|
+
* logsDir scope are a documented contract, and these files have a different retention policy and a
|
|
1088
|
+
* different PII class. Same never-throws shape, same `0 = keep forever` sentinel, and safe to re-run --
|
|
1089
|
+
* it carries no state between calls.
|
|
513
1090
|
*
|
|
514
|
-
* Age
|
|
515
|
-
* OPEN
|
|
516
|
-
*
|
|
1091
|
+
* Age on disk is the smaller half. The gate that matters is the one in readCanonical, which runs at
|
|
1092
|
+
* OPEN, because a stale transcript is a live INPUT to a future job rather than debris. Until #292 that
|
|
1093
|
+
* gate was also carrying the disk half alone, since a worker that never restarts never re-swept
|
|
1094
|
+
* (OQ-007, RESOLVED).
|
|
517
1095
|
*/
|
|
518
1096
|
function reapSessions() {
|
|
519
1097
|
if (!sessionsDir || ttlDays === 0) return;
|
|
@@ -522,19 +1100,66 @@ export function makeSessionStore({
|
|
|
522
1100
|
try {
|
|
523
1101
|
names = fs.readdirSync(sessionsDir);
|
|
524
1102
|
} catch (err) {
|
|
525
|
-
log("session_reaper_skipped", { reason: err?.message });
|
|
1103
|
+
log("session_reaper_skipped", { reason: scrubCredentials(err?.message) });
|
|
526
1104
|
return;
|
|
527
1105
|
}
|
|
528
1106
|
for (const name of names) {
|
|
529
1107
|
try {
|
|
530
1108
|
const dir = join(sessionsDir, name);
|
|
531
|
-
|
|
532
|
-
|
|
1109
|
+
// THE ENTRY ITSELF FIRST (issue #375), so nothing here is ever judged THROUGH a link. Before this,
|
|
1110
|
+
// a link whose target held a transcript was judged on that target's mtime, and the link was
|
|
1111
|
+
// removed once the target aged: the sweep's decision came from a file outside the store. Now only
|
|
1112
|
+
// a real directory is a key on this path, on both edges, which is the same rule `resolveSession`
|
|
1113
|
+
// and `promoteSession` now apply.
|
|
1114
|
+
//
|
|
1115
|
+
// `session_not_reaped`, NOT `session_reaper_skipped`. OQ-007's #337 amendment reserves the
|
|
1116
|
+
// `*_reaper_skipped` names for what a pass could not establish, and says that giving an everyday
|
|
1117
|
+
// verdict the fault's name puts an ordinary outcome into the grep an operator reads as trouble.
|
|
1118
|
+
// This pass established exactly what this entry is and decided to leave it, which is a verdict.
|
|
1119
|
+
const entry = fs.lstatSync(dir);
|
|
1120
|
+
if (!entry.isDirectory()) {
|
|
1121
|
+
log("session_not_reaped", { key: name, reason: "key-not-a-directory" });
|
|
1122
|
+
continue;
|
|
1123
|
+
}
|
|
1124
|
+
let mtimeMs;
|
|
1125
|
+
try {
|
|
1126
|
+
mtimeMs = fs.lstatSync(join(dir, SESSION_FILE_NAME)).mtimeMs;
|
|
1127
|
+
} catch (err) {
|
|
1128
|
+
// ENOENT ALONE falls back to the DIRECTORY's own mtime (issue #336). A key whose first promotion
|
|
1129
|
+
// died before any transcript landed has none to key on, so this loop used to log-and-skip it on
|
|
1130
|
+
// every pass forever, and anything leaked inside it -- a lock, an in-flight copy -- outlived the
|
|
1131
|
+
// store. A directory's mtime moves on every entry created or removed inside it, so a live key is
|
|
1132
|
+
// refreshed by its own promotions and a dead one is stamped at whatever last touched it.
|
|
1133
|
+
//
|
|
1134
|
+
// ENOENT-ONLY rather than unconditional, and that is the whole safety of it: an EIO or an EACCES
|
|
1135
|
+
// on the transcript is a disk fault, and a disk fault is not evidence that a key is old. Those
|
|
1136
|
+
// keep today's log-and-skip.
|
|
1137
|
+
if (err?.code !== "ENOENT") throw err;
|
|
1138
|
+
const dst = entry; // the entry's own stat, already taken and already proven a directory
|
|
1139
|
+
// The entry was already proven a real directory above (issue #375), so this arm is now only about
|
|
1140
|
+
// a key that has no transcript yet, and `dst` is that same stat rather than a second one.
|
|
1141
|
+
//
|
|
1142
|
+
// What the removal below is and is not, measured rather than assumed, because the obvious reading
|
|
1143
|
+
// is wrong: `rmSync(p, { recursive: true, force: true })` does NOT follow a symlink. On a link to
|
|
1144
|
+
// a real directory it removes the LINK and leaves the target untouched, and on a dangling link it
|
|
1145
|
+
// does nothing at all. So a link's TARGET was never at risk here even before this change; what
|
|
1146
|
+
// has changed is that a link is no longer JUDGED, which it was whenever its target held a
|
|
1147
|
+
// readable transcript -- the sweep's decision then came from a file outside the store entirely.
|
|
1148
|
+
mtimeMs = dst.mtimeMs;
|
|
1149
|
+
}
|
|
1150
|
+
if (mtimeMs < cutoff) {
|
|
1151
|
+
// ONE recursive remove, on the transcript's age where there is one and the directory's own where
|
|
1152
|
+
// there is not (above). Removing the
|
|
1153
|
+
// transcript first (so an absent stamp, which reads as `local`, can never sit beside a readable
|
|
1154
|
+
// transcript mid-sweep) was tried under #277 and withdrawn: a remove that then failed transiently
|
|
1155
|
+
// left the directory with no transcript, which this loop never looks at again, so a leaked lock
|
|
1156
|
+
// in it wedged the key for good. The race it closed needs a promotion landing on an EXPIRED key
|
|
1157
|
+
// during its own sweep on a deployment with a second venue; INT-SESSION-STORE-CONTRACT names it.
|
|
533
1158
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
534
1159
|
log("reaped_session", { key: name });
|
|
535
1160
|
}
|
|
536
1161
|
} catch (err) {
|
|
537
|
-
log("session_reaper_skipped", { key: name, reason: err?.message });
|
|
1162
|
+
log("session_reaper_skipped", { key: name, reason: scrubCredentials(err?.message) });
|
|
538
1163
|
}
|
|
539
1164
|
}
|
|
540
1165
|
}
|