@dogfood-lab/findings 1.2.2 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/advise/advice-bundle.js +155 -155
- package/advise/index.js +5 -5
- package/advise/query.js +182 -182
- package/derive/dedupe.js +107 -107
- package/derive/derive-findings.js +187 -187
- package/derive/ids.js +48 -48
- package/derive/index.js +9 -9
- package/derive/load-records.js +153 -153
- package/derive/rules.js +415 -415
- package/derive/write-findings.js +63 -63
- package/index.js +11 -11
- package/lib/file-lock.js +359 -359
- package/package.json +2 -2
- package/reader.js +156 -156
- package/review/index.js +6 -6
- package/review/transitions.js +79 -79
- package/synthesis/doctrine-derivation.js +128 -128
- package/synthesis/index.js +8 -8
- package/synthesis/pattern-derivation.js +184 -184
- package/synthesis/recommendation-derivation.js +156 -156
- package/synthesis/validate-artifacts.js +46 -46
- package/synthesis/write-artifacts.js +75 -75
- package/validate.js +87 -87
package/lib/file-lock.js
CHANGED
|
@@ -1,359 +1,359 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Cross-process advisory file lock built on `openSync(path, 'wx')` exclusive-create.
|
|
3
|
-
*
|
|
4
|
-
* Why this exists: `appendEvent` in review/event-log.js does a read-modify-write
|
|
5
|
-
* of a YAML array (read existing events → push new → atomic-rename whole file).
|
|
6
|
-
* Two concurrent appends both read N events, both push, both rename — second
|
|
7
|
-
* rename wins, dropping the first event. The atomic rename eliminates partial-
|
|
8
|
-
* file corruption (worst case before wave 9), but cannot eliminate the lost-
|
|
9
|
-
* event window. F-PIPELINE-011 (W3-PIPE-001) closes that window at the choke
|
|
10
|
-
* point so the bug class is impossible to recur — Pattern #4 from the
|
|
11
|
-
* swarm-evidence catalog.
|
|
12
|
-
*
|
|
13
|
-
* Why an O_EXCL lock FILE (not a lock DIR): an early implementation used
|
|
14
|
-
* `mkdirSync` then a follow-up `writeFileSync(.../pid)` — but those are two
|
|
15
|
-
* separate fs ops, so a process that's mid-acquisition is in a state where
|
|
16
|
-
* the dir exists but the pid file does NOT. A racing acquirer sees the dir,
|
|
17
|
-
* tries to read pid, gets ENOENT, classifies as stale, removes the dir, and
|
|
18
|
-
* the original acquirer's pid write later trips on the dir being gone.
|
|
19
|
-
* Lock-FILE-with-O_EXCL avoids that: the `open(path, 'wx')` call returns a
|
|
20
|
-
* file descriptor in one syscall, and we write the pid through that fd —
|
|
21
|
-
* any racing acquirer that sees the file knows it's fully formed.
|
|
22
|
-
*
|
|
23
|
-
* Why not `proper-lockfile`: that package isn't a dep of this monorepo
|
|
24
|
-
* (verified at the start of wave 30). `open(path, 'wx')` is atomic on POSIX
|
|
25
|
-
* and Windows — the kernel guarantees exactly one creator wins, the loser
|
|
26
|
-
* sees `EEXIST`. The pattern has been in use for decades and works the same
|
|
27
|
-
* on every fs Node supports (NTFS, ext4, APFS, tmpfs, nfsv4).
|
|
28
|
-
*
|
|
29
|
-
* Why not `O_APPEND`: the format here is a YAML array, not JSONL. `O_APPEND`
|
|
30
|
-
* makes byte-append atomic for sub-`PIPE_BUF` writes on POSIX, but the
|
|
31
|
-
* read-modify-write of the whole array bypasses that — even with `O_APPEND`
|
|
32
|
-
* the events would not be a valid YAML array. Switching the format to JSONL
|
|
33
|
-
* is a bigger compat break than this contract should make. Lock instead.
|
|
34
|
-
*
|
|
35
|
-
* Stale lock recovery: a process that crashes while holding the lock leaves
|
|
36
|
-
* the file behind. We detect "stale" by reading the holder PID and using
|
|
37
|
-
* `process.kill(pid, 0)` to test liveness — `ESRCH` means the holder is
|
|
38
|
-
* gone and the lock is reclaimable. The reclaim itself is best-effort:
|
|
39
|
-
* another fresh acquirer might race the cleanup, but the worst case is an
|
|
40
|
-
* extra retry, never lost data. To prevent two reclaimers from both believing
|
|
41
|
-
* they won, the reclaim sequence is `unlink → open(wx)`; the second
|
|
42
|
-
* reclaimer's `open(wx)` will fail because the first reclaimer already
|
|
43
|
-
* created it.
|
|
44
|
-
*
|
|
45
|
-
* Concurrency caveat — out of scope: this is a *single-machine* lock. NFS or
|
|
46
|
-
* other distributed filesystems expose `open(wx)` semantics that may not be
|
|
47
|
-
* truly atomic across nodes. The dogfood pipeline runs on a single GitHub
|
|
48
|
-
* runner per dispatch, so this is sufficient. A multi-runner sharded ingest
|
|
49
|
-
* would need a real consensus lock — flagged in the JSDoc above
|
|
50
|
-
* `withFileLock` so the limitation is visible at the use site.
|
|
51
|
-
*/
|
|
52
|
-
|
|
53
|
-
import { mkdirSync, openSync, closeSync, writeSync, writeFileSync, readFileSync, existsSync, unlinkSync, linkSync, renameSync, statSync } from 'node:fs';
|
|
54
|
-
import { dirname } from 'node:path';
|
|
55
|
-
import { randomBytes } from 'node:crypto';
|
|
56
|
-
|
|
57
|
-
// 30s default timeout: appendEvent's critical section is small (~ms), but
|
|
58
|
-
// 50-way parallel ingests serializing through the lock can take a few seconds
|
|
59
|
-
// in aggregate. 30s leaves comfortable headroom for the dogfood pipeline's
|
|
60
|
-
// realistic concurrency (~10 parallel jobs per dispatch wave) without making
|
|
61
|
-
// CI hang on a runaway holder.
|
|
62
|
-
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
63
|
-
const DEFAULT_RETRY_INTERVAL_MS = 15;
|
|
64
|
-
const DEFAULT_STALE_AFTER_MS = 30_000;
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Compute the lock file path for a given target file.
|
|
68
|
-
* Co-locates with the target so a permission-restricted directory still works.
|
|
69
|
-
*
|
|
70
|
-
* Kept named `lockDirFor` for back-compat with the wave-30 design draft —
|
|
71
|
-
* the path now points at a regular file, not a directory, but the call sites
|
|
72
|
-
* don't care about the kind.
|
|
73
|
-
*
|
|
74
|
-
* @param {string} targetPath
|
|
75
|
-
* @returns {string}
|
|
76
|
-
*/
|
|
77
|
-
export function lockDirFor(targetPath) {
|
|
78
|
-
return `${targetPath}.lock`;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Test whether a process is alive. Returns false if `pid` is missing,
|
|
83
|
-
* non-numeric, or `process.kill(pid, 0)` rejects with `ESRCH`. Any other
|
|
84
|
-
* error (e.g. `EPERM` on a foreign-user pid) is treated as "alive" — better
|
|
85
|
-
* to wait out the lock than to steal a live one.
|
|
86
|
-
*
|
|
87
|
-
* @param {number|string|null|undefined} pid
|
|
88
|
-
* @returns {boolean}
|
|
89
|
-
*/
|
|
90
|
-
function isProcessAlive(pid) {
|
|
91
|
-
const n = typeof pid === 'number' ? pid : Number(pid);
|
|
92
|
-
if (!Number.isFinite(n) || n <= 0) return false;
|
|
93
|
-
try {
|
|
94
|
-
process.kill(n, 0);
|
|
95
|
-
return true;
|
|
96
|
-
} catch (err) {
|
|
97
|
-
if (err && err.code === 'ESRCH') return false;
|
|
98
|
-
return true;
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/**
|
|
103
|
-
* Atomically create the lock file with the holder's PID as content.
|
|
104
|
-
*
|
|
105
|
-
* Two-step pattern: write a temp file (containing this process's pid) in the
|
|
106
|
-
* same parent dir, then `linkSync(temp, lockPath)`. `link` is atomic on every
|
|
107
|
-
* fs Node supports — it either creates the link or fails with EEXIST; there
|
|
108
|
-
* is no observable intermediate state where the lock file exists but is
|
|
109
|
-
* empty. This closes the race that an earlier `open(wx)+write` design left
|
|
110
|
-
* open: a racing acquirer could read the lock file in the window between
|
|
111
|
-
* `open` and `write` and see it as empty (= stale) before the holder's pid
|
|
112
|
-
* was recorded.
|
|
113
|
-
*
|
|
114
|
-
* @param {string} lockPath
|
|
115
|
-
* @returns {boolean} true on success; false if the lock already exists.
|
|
116
|
-
*/
|
|
117
|
-
function atomicCreateLock(lockPath) {
|
|
118
|
-
const tmpPath = `${lockPath}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
|
|
119
|
-
// Write the temp file atomically (truncates if it somehow exists, which
|
|
120
|
-
// it never should given the pid+random suffix). Content is the holder pid.
|
|
121
|
-
writeFileSync(tmpPath, String(process.pid), 'utf-8');
|
|
122
|
-
try {
|
|
123
|
-
linkSync(tmpPath, lockPath);
|
|
124
|
-
return true;
|
|
125
|
-
} catch (err) {
|
|
126
|
-
if (err && err.code === 'EEXIST') return false;
|
|
127
|
-
throw err;
|
|
128
|
-
} finally {
|
|
129
|
-
// The temp file is no longer needed regardless of link outcome — the
|
|
130
|
-
// hardlink (on success) keeps the inode alive at lockPath; on failure
|
|
131
|
-
// the temp file is just garbage to clean up.
|
|
132
|
-
try { unlinkSync(tmpPath); } catch { /* best effort */ }
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
/**
|
|
137
|
-
* Attempt to acquire an exclusive file-lock on `lockPath`. Returns true if
|
|
138
|
-
* the lock was acquired; false if it is held by another live process OR
|
|
139
|
-
* by a process whose lock file is unreadable / pid-empty (treated as live
|
|
140
|
-
* with bounded mtime patience — see below).
|
|
141
|
-
*
|
|
142
|
-
* Stale recovery: a holder's PID file remains until its release `unlink`
|
|
143
|
-
* runs. If the holder PROCESS is gone (process.kill ESRCH), the lock is
|
|
144
|
-
* reclaimable. We do NOT treat empty/missing pid content as stale on its
|
|
145
|
-
* own — that was the wave-30 first-pass bug. Even with `linkSync` for
|
|
146
|
-
* atomic creation, on Windows a racing reader can land on a brief window
|
|
147
|
-
* where the pid file's content has not yet been committed to the dirent
|
|
148
|
-
* cache from the other process. Empty content + bounded-stale-mtime is the
|
|
149
|
-
* safer guard: we only reclaim if the lock file is ALSO older than
|
|
150
|
-
* `staleAfterMs`, the same boundary `proper-lockfile` uses.
|
|
151
|
-
*
|
|
152
|
-
* @param {string} lockPath
|
|
153
|
-
* @param {{ staleAfterMs?: number }} opts
|
|
154
|
-
* @returns {boolean}
|
|
155
|
-
*/
|
|
156
|
-
function tryAcquire(lockPath, opts = {}) {
|
|
157
|
-
const { staleAfterMs = DEFAULT_STALE_AFTER_MS } = opts;
|
|
158
|
-
|
|
159
|
-
if (atomicCreateLock(lockPath)) return true;
|
|
160
|
-
|
|
161
|
-
// Lock exists — test for staleness.
|
|
162
|
-
let pidRaw = null;
|
|
163
|
-
let mtimeMs = NaN;
|
|
164
|
-
try {
|
|
165
|
-
pidRaw = readFileSync(lockPath, 'utf-8').trim();
|
|
166
|
-
} catch {
|
|
167
|
-
// Read failed — fall through to the mtime check below; if we can't
|
|
168
|
-
// even stat it, treat as live.
|
|
169
|
-
}
|
|
170
|
-
try {
|
|
171
|
-
const st = statSync(lockPath);
|
|
172
|
-
mtimeMs = st.mtimeMs;
|
|
173
|
-
} catch { /* will be NaN, treated as live */ }
|
|
174
|
-
|
|
175
|
-
// PID-known case: trust process.kill to decide alive-vs-dead. This is the
|
|
176
|
-
// common case and gives fast crash recovery (sub-100ms typical).
|
|
177
|
-
if (pidRaw && pidRaw !== '') {
|
|
178
|
-
if (isProcessAlive(pidRaw)) return false;
|
|
179
|
-
|
|
180
|
-
// PID-dead reclaim via "rename to graveyard" — atomic claim that exactly
|
|
181
|
-
// one reclaimer wins. Without this, a sequence like:
|
|
182
|
-
// 1. A holds lock; A releases (unlinks); A's pid becomes dead
|
|
183
|
-
// 2. C atomicCreateLock succeeds — C owns lock with pid=C
|
|
184
|
-
// 3. B (had read pidRaw=A above) confirms A dead, calls unlinkSync — but
|
|
185
|
-
// the file is now C's lock! Now B and C both think they own it.
|
|
186
|
-
// produces a double-owner. The graveyard rename is the atomic CAS step:
|
|
187
|
-
// exactly one process can rename a given file at a given moment. We
|
|
188
|
-
// verify the rename succeeded AND the file we renamed still has the
|
|
189
|
-
// dead PID we expected; if the content shifted (someone else just
|
|
190
|
-
// re-acquired), put the file back.
|
|
191
|
-
return reclaimViaGraveyard(lockPath, pidRaw);
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
// PID-unknown case (empty/unreadable content). DO NOT treat as stale on
|
|
195
|
-
// content alone — Windows can present a brief window where the holder's
|
|
196
|
-
// file content is still flushing. Only reclaim if the lock is also older
|
|
197
|
-
// than `staleAfterMs`. With the default 30s staleAfterMs, fast appenders
|
|
198
|
-
// never trip this; only a true crash window does.
|
|
199
|
-
if (Number.isFinite(mtimeMs) && Date.now() - mtimeMs > staleAfterMs) {
|
|
200
|
-
if (process.env.FILE_LOCK_DEBUG) {
|
|
201
|
-
process.stderr.write(`[file-lock][pid=${process.pid}] STALE-RECLAIM (mtime-old) lockPath=${lockPath} ageMs=${Date.now() - mtimeMs}\n`);
|
|
202
|
-
}
|
|
203
|
-
// Use the same graveyard-rename CAS for the mtime-stale path. The PID
|
|
204
|
-
// is unknown so we use empty string as the "expected" content — the
|
|
205
|
-
// graveyard verification will accept whatever it reads (the put-back
|
|
206
|
-
// branch only triggers when actualPid is non-empty AND differs).
|
|
207
|
-
return reclaimViaGraveyard(lockPath, pidRaw || '');
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
// Treat as live; the holder owns the lock and will release it.
|
|
211
|
-
return false;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
/**
|
|
215
|
-
* Release a lock acquired via `tryAcquire`. Best-effort — a missing lock file
|
|
216
|
-
* is not an error (it means stale-lock recovery already cleaned it).
|
|
217
|
-
*
|
|
218
|
-
* @param {string} lockPath
|
|
219
|
-
*/
|
|
220
|
-
function release(lockPath) {
|
|
221
|
-
try { unlinkSync(lockPath); } catch { /* may already be gone */ }
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
/**
|
|
225
|
-
* Reclaim a stale lock via "rename to graveyard." Atomic CAS step: exactly
|
|
226
|
-
* one reclaimer can rename a file at a time. The winner verifies the renamed
|
|
227
|
-
* file still has the dead PID it expected (defending against the case where
|
|
228
|
-
* the lock content shifted between the read-pid step and the rename); if so,
|
|
229
|
-
* the winner unlinks the graveyard file and creates a fresh lock. If the
|
|
230
|
-
* content shifted, the winner puts the file back so the new owner is
|
|
231
|
-
* unaffected.
|
|
232
|
-
*
|
|
233
|
-
* @param {string} lockPath
|
|
234
|
-
* @param {string} expectedDeadPid - The PID we observed and confirmed dead.
|
|
235
|
-
* @returns {boolean}
|
|
236
|
-
*/
|
|
237
|
-
function reclaimViaGraveyard(lockPath, expectedDeadPid) {
|
|
238
|
-
const graveyardPath = `${lockPath}.gy.${process.pid}.${randomBytes(4).toString('hex')}`;
|
|
239
|
-
try {
|
|
240
|
-
renameSync(lockPath, graveyardPath);
|
|
241
|
-
} catch (err) {
|
|
242
|
-
// Another reclaimer beat us — let the retry loop sort it out.
|
|
243
|
-
return false;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
// We won the rename. Verify content.
|
|
247
|
-
let actualPid = null;
|
|
248
|
-
try {
|
|
249
|
-
actualPid = readFileSync(graveyardPath, 'utf-8').trim();
|
|
250
|
-
} catch { /* unreadable — treat as confirmation */ }
|
|
251
|
-
|
|
252
|
-
if (actualPid && actualPid !== expectedDeadPid) {
|
|
253
|
-
// Content shifted between our read and our rename. The "stale" lock
|
|
254
|
-
// we grabbed is actually a fresh acquisition by someone else.
|
|
255
|
-
// Put it back. If put-back fails (e.g., another reclaimer raced and
|
|
256
|
-
// already created a new lock at lockPath), discard the graveyard copy.
|
|
257
|
-
try {
|
|
258
|
-
renameSync(graveyardPath, lockPath);
|
|
259
|
-
if (process.env.FILE_LOCK_DEBUG) {
|
|
260
|
-
process.stderr.write(`[file-lock][pid=${process.pid}] CAS-FAIL-PUTBACK lockPath=${lockPath} expected=${expectedDeadPid} found=${actualPid}\n`);
|
|
261
|
-
}
|
|
262
|
-
} catch {
|
|
263
|
-
try { unlinkSync(graveyardPath); } catch { /* drop */ }
|
|
264
|
-
}
|
|
265
|
-
return false;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
if (process.env.FILE_LOCK_DEBUG) {
|
|
269
|
-
process.stderr.write(`[file-lock][pid=${process.pid}] STALE-RECLAIM (dead-pid) lockPath=${lockPath} pid=${expectedDeadPid}\n`);
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
try { unlinkSync(graveyardPath); } catch { /* drop */ }
|
|
273
|
-
return atomicCreateLock(lockPath);
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
/**
|
|
277
|
-
* Sleep synchronously for `ms` milliseconds via `Atomics.wait` on a tiny
|
|
278
|
-
* SharedArrayBuffer. We can't `await` here because the callers
|
|
279
|
-
* (`appendEvent`, `rebuildIndexes`) are sync APIs — the whole pipeline
|
|
280
|
-
* is sync from `ingest()` down to fs I/O, so introducing async here would
|
|
281
|
-
* cascade through six callers and break the back-compat contract on
|
|
282
|
-
* `appendEvent` that review-engine relies on.
|
|
283
|
-
*
|
|
284
|
-
* @param {number} ms
|
|
285
|
-
*/
|
|
286
|
-
function sleepSync(ms) {
|
|
287
|
-
const sab = new SharedArrayBuffer(4);
|
|
288
|
-
const view = new Int32Array(sab);
|
|
289
|
-
Atomics.wait(view, 0, 0, ms);
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
/**
|
|
293
|
-
* Run `fn` while holding an exclusive lock on `targetPath`. The lock is a
|
|
294
|
-
* sibling directory at `<targetPath>.lock` — see file header for the
|
|
295
|
-
* full design rationale.
|
|
296
|
-
*
|
|
297
|
-
* The lock is per-target, so two unrelated `appendEvent` calls writing to
|
|
298
|
-
* different daily log files do NOT serialize against each other.
|
|
299
|
-
*
|
|
300
|
-
* @template T
|
|
301
|
-
* @param {string} targetPath - The file being mutated; the lock dir is `<targetPath>.lock`.
|
|
302
|
-
* @param {() => T} fn - The critical section. Runs synchronously.
|
|
303
|
-
* @param {{
|
|
304
|
-
* timeoutMs?: number,
|
|
305
|
-
* retryIntervalMs?: number,
|
|
306
|
-
* staleAfterMs?: number
|
|
307
|
-
* }} [options]
|
|
308
|
-
* @returns {T}
|
|
309
|
-
*/
|
|
310
|
-
export function withFileLock(targetPath, fn, options = {}) {
|
|
311
|
-
const {
|
|
312
|
-
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
313
|
-
retryIntervalMs = DEFAULT_RETRY_INTERVAL_MS,
|
|
314
|
-
staleAfterMs = DEFAULT_STALE_AFTER_MS,
|
|
315
|
-
} = options;
|
|
316
|
-
|
|
317
|
-
const lockPath = lockDirFor(targetPath);
|
|
318
|
-
|
|
319
|
-
// Ensure the parent dir exists — otherwise mkdirSync(lockPath) fails with
|
|
320
|
-
// ENOENT and we mis-classify the lock as held.
|
|
321
|
-
try { mkdirSync(dirname(lockPath), { recursive: true }); } catch { /* may already exist */ }
|
|
322
|
-
|
|
323
|
-
const deadline = Date.now() + timeoutMs;
|
|
324
|
-
let acquired = false;
|
|
325
|
-
let lastErrCtx = null;
|
|
326
|
-
|
|
327
|
-
while (Date.now() < deadline) {
|
|
328
|
-
if (tryAcquire(lockPath, { staleAfterMs })) {
|
|
329
|
-
acquired = true;
|
|
330
|
-
break;
|
|
331
|
-
}
|
|
332
|
-
sleepSync(retryIntervalMs);
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
if (!acquired) {
|
|
336
|
-
const err = new Error(
|
|
337
|
-
`withFileLock: timed out after ${timeoutMs}ms waiting for ${lockPath}` +
|
|
338
|
-
(lastErrCtx ? ` (last error: ${lastErrCtx})` : '')
|
|
339
|
-
);
|
|
340
|
-
err.code = 'ELOCKTIMEOUT';
|
|
341
|
-
err.lockPath = lockPath;
|
|
342
|
-
throw err;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
try {
|
|
346
|
-
return fn();
|
|
347
|
-
} finally {
|
|
348
|
-
release(lockPath);
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
/**
|
|
353
|
-
* Test-only: probe whether a lock is currently held without acquiring it.
|
|
354
|
-
* @param {string} targetPath
|
|
355
|
-
* @returns {boolean}
|
|
356
|
-
*/
|
|
357
|
-
export function isLocked(targetPath) {
|
|
358
|
-
return existsSync(lockDirFor(targetPath));
|
|
359
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Cross-process advisory file lock built on `openSync(path, 'wx')` exclusive-create.
|
|
3
|
+
*
|
|
4
|
+
* Why this exists: `appendEvent` in review/event-log.js does a read-modify-write
|
|
5
|
+
* of a YAML array (read existing events → push new → atomic-rename whole file).
|
|
6
|
+
* Two concurrent appends both read N events, both push, both rename — second
|
|
7
|
+
* rename wins, dropping the first event. The atomic rename eliminates partial-
|
|
8
|
+
* file corruption (worst case before wave 9), but cannot eliminate the lost-
|
|
9
|
+
* event window. F-PIPELINE-011 (W3-PIPE-001) closes that window at the choke
|
|
10
|
+
* point so the bug class is impossible to recur — Pattern #4 from the
|
|
11
|
+
* swarm-evidence catalog.
|
|
12
|
+
*
|
|
13
|
+
* Why an O_EXCL lock FILE (not a lock DIR): an early implementation used
|
|
14
|
+
* `mkdirSync` then a follow-up `writeFileSync(.../pid)` — but those are two
|
|
15
|
+
* separate fs ops, so a process that's mid-acquisition is in a state where
|
|
16
|
+
* the dir exists but the pid file does NOT. A racing acquirer sees the dir,
|
|
17
|
+
* tries to read pid, gets ENOENT, classifies as stale, removes the dir, and
|
|
18
|
+
* the original acquirer's pid write later trips on the dir being gone.
|
|
19
|
+
* Lock-FILE-with-O_EXCL avoids that: the `open(path, 'wx')` call returns a
|
|
20
|
+
* file descriptor in one syscall, and we write the pid through that fd —
|
|
21
|
+
* any racing acquirer that sees the file knows it's fully formed.
|
|
22
|
+
*
|
|
23
|
+
* Why not `proper-lockfile`: that package isn't a dep of this monorepo
|
|
24
|
+
* (verified at the start of wave 30). `open(path, 'wx')` is atomic on POSIX
|
|
25
|
+
* and Windows — the kernel guarantees exactly one creator wins, the loser
|
|
26
|
+
* sees `EEXIST`. The pattern has been in use for decades and works the same
|
|
27
|
+
* on every fs Node supports (NTFS, ext4, APFS, tmpfs, nfsv4).
|
|
28
|
+
*
|
|
29
|
+
* Why not `O_APPEND`: the format here is a YAML array, not JSONL. `O_APPEND`
|
|
30
|
+
* makes byte-append atomic for sub-`PIPE_BUF` writes on POSIX, but the
|
|
31
|
+
* read-modify-write of the whole array bypasses that — even with `O_APPEND`
|
|
32
|
+
* the events would not be a valid YAML array. Switching the format to JSONL
|
|
33
|
+
* is a bigger compat break than this contract should make. Lock instead.
|
|
34
|
+
*
|
|
35
|
+
* Stale lock recovery: a process that crashes while holding the lock leaves
|
|
36
|
+
* the file behind. We detect "stale" by reading the holder PID and using
|
|
37
|
+
* `process.kill(pid, 0)` to test liveness — `ESRCH` means the holder is
|
|
38
|
+
* gone and the lock is reclaimable. The reclaim itself is best-effort:
|
|
39
|
+
* another fresh acquirer might race the cleanup, but the worst case is an
|
|
40
|
+
* extra retry, never lost data. To prevent two reclaimers from both believing
|
|
41
|
+
* they won, the reclaim sequence is `unlink → open(wx)`; the second
|
|
42
|
+
* reclaimer's `open(wx)` will fail because the first reclaimer already
|
|
43
|
+
* created it.
|
|
44
|
+
*
|
|
45
|
+
* Concurrency caveat — out of scope: this is a *single-machine* lock. NFS or
|
|
46
|
+
* other distributed filesystems expose `open(wx)` semantics that may not be
|
|
47
|
+
* truly atomic across nodes. The dogfood pipeline runs on a single GitHub
|
|
48
|
+
* runner per dispatch, so this is sufficient. A multi-runner sharded ingest
|
|
49
|
+
* would need a real consensus lock — flagged in the JSDoc above
|
|
50
|
+
* `withFileLock` so the limitation is visible at the use site.
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
import { mkdirSync, openSync, closeSync, writeSync, writeFileSync, readFileSync, existsSync, unlinkSync, linkSync, renameSync, statSync } from 'node:fs';
|
|
54
|
+
import { dirname } from 'node:path';
|
|
55
|
+
import { randomBytes } from 'node:crypto';
|
|
56
|
+
|
|
57
|
+
// 30s default timeout: appendEvent's critical section is small (~ms), but
|
|
58
|
+
// 50-way parallel ingests serializing through the lock can take a few seconds
|
|
59
|
+
// in aggregate. 30s leaves comfortable headroom for the dogfood pipeline's
|
|
60
|
+
// realistic concurrency (~10 parallel jobs per dispatch wave) without making
|
|
61
|
+
// CI hang on a runaway holder.
|
|
62
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
63
|
+
const DEFAULT_RETRY_INTERVAL_MS = 15;
|
|
64
|
+
const DEFAULT_STALE_AFTER_MS = 30_000;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Compute the lock file path for a given target file.
|
|
68
|
+
* Co-locates with the target so a permission-restricted directory still works.
|
|
69
|
+
*
|
|
70
|
+
* Kept named `lockDirFor` for back-compat with the wave-30 design draft —
|
|
71
|
+
* the path now points at a regular file, not a directory, but the call sites
|
|
72
|
+
* don't care about the kind.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} targetPath
|
|
75
|
+
* @returns {string}
|
|
76
|
+
*/
|
|
77
|
+
export function lockDirFor(targetPath) {
|
|
78
|
+
return `${targetPath}.lock`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Test whether a process is alive. Returns false if `pid` is missing,
|
|
83
|
+
* non-numeric, or `process.kill(pid, 0)` rejects with `ESRCH`. Any other
|
|
84
|
+
* error (e.g. `EPERM` on a foreign-user pid) is treated as "alive" — better
|
|
85
|
+
* to wait out the lock than to steal a live one.
|
|
86
|
+
*
|
|
87
|
+
* @param {number|string|null|undefined} pid
|
|
88
|
+
* @returns {boolean}
|
|
89
|
+
*/
|
|
90
|
+
function isProcessAlive(pid) {
|
|
91
|
+
const n = typeof pid === 'number' ? pid : Number(pid);
|
|
92
|
+
if (!Number.isFinite(n) || n <= 0) return false;
|
|
93
|
+
try {
|
|
94
|
+
process.kill(n, 0);
|
|
95
|
+
return true;
|
|
96
|
+
} catch (err) {
|
|
97
|
+
if (err && err.code === 'ESRCH') return false;
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Atomically create the lock file with the holder's PID as content.
|
|
104
|
+
*
|
|
105
|
+
* Two-step pattern: write a temp file (containing this process's pid) in the
|
|
106
|
+
* same parent dir, then `linkSync(temp, lockPath)`. `link` is atomic on every
|
|
107
|
+
* fs Node supports — it either creates the link or fails with EEXIST; there
|
|
108
|
+
* is no observable intermediate state where the lock file exists but is
|
|
109
|
+
* empty. This closes the race that an earlier `open(wx)+write` design left
|
|
110
|
+
* open: a racing acquirer could read the lock file in the window between
|
|
111
|
+
* `open` and `write` and see it as empty (= stale) before the holder's pid
|
|
112
|
+
* was recorded.
|
|
113
|
+
*
|
|
114
|
+
* @param {string} lockPath
|
|
115
|
+
* @returns {boolean} true on success; false if the lock already exists.
|
|
116
|
+
*/
|
|
117
|
+
function atomicCreateLock(lockPath) {
|
|
118
|
+
const tmpPath = `${lockPath}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
|
|
119
|
+
// Write the temp file atomically (truncates if it somehow exists, which
|
|
120
|
+
// it never should given the pid+random suffix). Content is the holder pid.
|
|
121
|
+
writeFileSync(tmpPath, String(process.pid), 'utf-8');
|
|
122
|
+
try {
|
|
123
|
+
linkSync(tmpPath, lockPath);
|
|
124
|
+
return true;
|
|
125
|
+
} catch (err) {
|
|
126
|
+
if (err && err.code === 'EEXIST') return false;
|
|
127
|
+
throw err;
|
|
128
|
+
} finally {
|
|
129
|
+
// The temp file is no longer needed regardless of link outcome — the
|
|
130
|
+
// hardlink (on success) keeps the inode alive at lockPath; on failure
|
|
131
|
+
// the temp file is just garbage to clean up.
|
|
132
|
+
try { unlinkSync(tmpPath); } catch { /* best effort */ }
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Attempt to acquire an exclusive file-lock on `lockPath`. Returns true if
|
|
138
|
+
* the lock was acquired; false if it is held by another live process OR
|
|
139
|
+
* by a process whose lock file is unreadable / pid-empty (treated as live
|
|
140
|
+
* with bounded mtime patience — see below).
|
|
141
|
+
*
|
|
142
|
+
* Stale recovery: a holder's PID file remains until its release `unlink`
|
|
143
|
+
* runs. If the holder PROCESS is gone (process.kill ESRCH), the lock is
|
|
144
|
+
* reclaimable. We do NOT treat empty/missing pid content as stale on its
|
|
145
|
+
* own — that was the wave-30 first-pass bug. Even with `linkSync` for
|
|
146
|
+
* atomic creation, on Windows a racing reader can land on a brief window
|
|
147
|
+
* where the pid file's content has not yet been committed to the dirent
|
|
148
|
+
* cache from the other process. Empty content + bounded-stale-mtime is the
|
|
149
|
+
* safer guard: we only reclaim if the lock file is ALSO older than
|
|
150
|
+
* `staleAfterMs`, the same boundary `proper-lockfile` uses.
|
|
151
|
+
*
|
|
152
|
+
* @param {string} lockPath
|
|
153
|
+
* @param {{ staleAfterMs?: number }} opts
|
|
154
|
+
* @returns {boolean}
|
|
155
|
+
*/
|
|
156
|
+
function tryAcquire(lockPath, opts = {}) {
|
|
157
|
+
const { staleAfterMs = DEFAULT_STALE_AFTER_MS } = opts;
|
|
158
|
+
|
|
159
|
+
if (atomicCreateLock(lockPath)) return true;
|
|
160
|
+
|
|
161
|
+
// Lock exists — test for staleness.
|
|
162
|
+
let pidRaw = null;
|
|
163
|
+
let mtimeMs = NaN;
|
|
164
|
+
try {
|
|
165
|
+
pidRaw = readFileSync(lockPath, 'utf-8').trim();
|
|
166
|
+
} catch {
|
|
167
|
+
// Read failed — fall through to the mtime check below; if we can't
|
|
168
|
+
// even stat it, treat as live.
|
|
169
|
+
}
|
|
170
|
+
try {
|
|
171
|
+
const st = statSync(lockPath);
|
|
172
|
+
mtimeMs = st.mtimeMs;
|
|
173
|
+
} catch { /* will be NaN, treated as live */ }
|
|
174
|
+
|
|
175
|
+
// PID-known case: trust process.kill to decide alive-vs-dead. This is the
|
|
176
|
+
// common case and gives fast crash recovery (sub-100ms typical).
|
|
177
|
+
if (pidRaw && pidRaw !== '') {
|
|
178
|
+
if (isProcessAlive(pidRaw)) return false;
|
|
179
|
+
|
|
180
|
+
// PID-dead reclaim via "rename to graveyard" — atomic claim that exactly
|
|
181
|
+
// one reclaimer wins. Without this, a sequence like:
|
|
182
|
+
// 1. A holds lock; A releases (unlinks); A's pid becomes dead
|
|
183
|
+
// 2. C atomicCreateLock succeeds — C owns lock with pid=C
|
|
184
|
+
// 3. B (had read pidRaw=A above) confirms A dead, calls unlinkSync — but
|
|
185
|
+
// the file is now C's lock! Now B and C both think they own it.
|
|
186
|
+
// produces a double-owner. The graveyard rename is the atomic CAS step:
|
|
187
|
+
// exactly one process can rename a given file at a given moment. We
|
|
188
|
+
// verify the rename succeeded AND the file we renamed still has the
|
|
189
|
+
// dead PID we expected; if the content shifted (someone else just
|
|
190
|
+
// re-acquired), put the file back.
|
|
191
|
+
return reclaimViaGraveyard(lockPath, pidRaw);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// PID-unknown case (empty/unreadable content). DO NOT treat as stale on
|
|
195
|
+
// content alone — Windows can present a brief window where the holder's
|
|
196
|
+
// file content is still flushing. Only reclaim if the lock is also older
|
|
197
|
+
// than `staleAfterMs`. With the default 30s staleAfterMs, fast appenders
|
|
198
|
+
// never trip this; only a true crash window does.
|
|
199
|
+
if (Number.isFinite(mtimeMs) && Date.now() - mtimeMs > staleAfterMs) {
|
|
200
|
+
if (process.env.FILE_LOCK_DEBUG) {
|
|
201
|
+
process.stderr.write(`[file-lock][pid=${process.pid}] STALE-RECLAIM (mtime-old) lockPath=${lockPath} ageMs=${Date.now() - mtimeMs}\n`);
|
|
202
|
+
}
|
|
203
|
+
// Use the same graveyard-rename CAS for the mtime-stale path. The PID
|
|
204
|
+
// is unknown so we use empty string as the "expected" content — the
|
|
205
|
+
// graveyard verification will accept whatever it reads (the put-back
|
|
206
|
+
// branch only triggers when actualPid is non-empty AND differs).
|
|
207
|
+
return reclaimViaGraveyard(lockPath, pidRaw || '');
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Treat as live; the holder owns the lock and will release it.
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Release a lock acquired via `tryAcquire`. Best-effort — a missing lock file
|
|
216
|
+
* is not an error (it means stale-lock recovery already cleaned it).
|
|
217
|
+
*
|
|
218
|
+
* @param {string} lockPath
|
|
219
|
+
*/
|
|
220
|
+
function release(lockPath) {
|
|
221
|
+
try { unlinkSync(lockPath); } catch { /* may already be gone */ }
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Reclaim a stale lock via "rename to graveyard." Atomic CAS step: exactly
|
|
226
|
+
* one reclaimer can rename a file at a time. The winner verifies the renamed
|
|
227
|
+
* file still has the dead PID it expected (defending against the case where
|
|
228
|
+
* the lock content shifted between the read-pid step and the rename); if so,
|
|
229
|
+
* the winner unlinks the graveyard file and creates a fresh lock. If the
|
|
230
|
+
* content shifted, the winner puts the file back so the new owner is
|
|
231
|
+
* unaffected.
|
|
232
|
+
*
|
|
233
|
+
* @param {string} lockPath
|
|
234
|
+
* @param {string} expectedDeadPid - The PID we observed and confirmed dead.
|
|
235
|
+
* @returns {boolean}
|
|
236
|
+
*/
|
|
237
|
+
function reclaimViaGraveyard(lockPath, expectedDeadPid) {
|
|
238
|
+
const graveyardPath = `${lockPath}.gy.${process.pid}.${randomBytes(4).toString('hex')}`;
|
|
239
|
+
try {
|
|
240
|
+
renameSync(lockPath, graveyardPath);
|
|
241
|
+
} catch (err) {
|
|
242
|
+
// Another reclaimer beat us — let the retry loop sort it out.
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// We won the rename. Verify content.
|
|
247
|
+
let actualPid = null;
|
|
248
|
+
try {
|
|
249
|
+
actualPid = readFileSync(graveyardPath, 'utf-8').trim();
|
|
250
|
+
} catch { /* unreadable — treat as confirmation */ }
|
|
251
|
+
|
|
252
|
+
if (actualPid && actualPid !== expectedDeadPid) {
|
|
253
|
+
// Content shifted between our read and our rename. The "stale" lock
|
|
254
|
+
// we grabbed is actually a fresh acquisition by someone else.
|
|
255
|
+
// Put it back. If put-back fails (e.g., another reclaimer raced and
|
|
256
|
+
// already created a new lock at lockPath), discard the graveyard copy.
|
|
257
|
+
try {
|
|
258
|
+
renameSync(graveyardPath, lockPath);
|
|
259
|
+
if (process.env.FILE_LOCK_DEBUG) {
|
|
260
|
+
process.stderr.write(`[file-lock][pid=${process.pid}] CAS-FAIL-PUTBACK lockPath=${lockPath} expected=${expectedDeadPid} found=${actualPid}\n`);
|
|
261
|
+
}
|
|
262
|
+
} catch {
|
|
263
|
+
try { unlinkSync(graveyardPath); } catch { /* drop */ }
|
|
264
|
+
}
|
|
265
|
+
return false;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (process.env.FILE_LOCK_DEBUG) {
|
|
269
|
+
process.stderr.write(`[file-lock][pid=${process.pid}] STALE-RECLAIM (dead-pid) lockPath=${lockPath} pid=${expectedDeadPid}\n`);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
try { unlinkSync(graveyardPath); } catch { /* drop */ }
|
|
273
|
+
return atomicCreateLock(lockPath);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Sleep synchronously for `ms` milliseconds via `Atomics.wait` on a tiny
|
|
278
|
+
* SharedArrayBuffer. We can't `await` here because the callers
|
|
279
|
+
* (`appendEvent`, `rebuildIndexes`) are sync APIs — the whole pipeline
|
|
280
|
+
* is sync from `ingest()` down to fs I/O, so introducing async here would
|
|
281
|
+
* cascade through six callers and break the back-compat contract on
|
|
282
|
+
* `appendEvent` that review-engine relies on.
|
|
283
|
+
*
|
|
284
|
+
* @param {number} ms
|
|
285
|
+
*/
|
|
286
|
+
function sleepSync(ms) {
|
|
287
|
+
const sab = new SharedArrayBuffer(4);
|
|
288
|
+
const view = new Int32Array(sab);
|
|
289
|
+
Atomics.wait(view, 0, 0, ms);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Run `fn` while holding an exclusive lock on `targetPath`. The lock is a
|
|
294
|
+
* sibling directory at `<targetPath>.lock` — see file header for the
|
|
295
|
+
* full design rationale.
|
|
296
|
+
*
|
|
297
|
+
* The lock is per-target, so two unrelated `appendEvent` calls writing to
|
|
298
|
+
* different daily log files do NOT serialize against each other.
|
|
299
|
+
*
|
|
300
|
+
* @template T
|
|
301
|
+
* @param {string} targetPath - The file being mutated; the lock dir is `<targetPath>.lock`.
|
|
302
|
+
* @param {() => T} fn - The critical section. Runs synchronously.
|
|
303
|
+
* @param {{
|
|
304
|
+
* timeoutMs?: number,
|
|
305
|
+
* retryIntervalMs?: number,
|
|
306
|
+
* staleAfterMs?: number
|
|
307
|
+
* }} [options]
|
|
308
|
+
* @returns {T}
|
|
309
|
+
*/
|
|
310
|
+
export function withFileLock(targetPath, fn, options = {}) {
|
|
311
|
+
const {
|
|
312
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
313
|
+
retryIntervalMs = DEFAULT_RETRY_INTERVAL_MS,
|
|
314
|
+
staleAfterMs = DEFAULT_STALE_AFTER_MS,
|
|
315
|
+
} = options;
|
|
316
|
+
|
|
317
|
+
const lockPath = lockDirFor(targetPath);
|
|
318
|
+
|
|
319
|
+
// Ensure the parent dir exists — otherwise mkdirSync(lockPath) fails with
|
|
320
|
+
// ENOENT and we mis-classify the lock as held.
|
|
321
|
+
try { mkdirSync(dirname(lockPath), { recursive: true }); } catch { /* may already exist */ }
|
|
322
|
+
|
|
323
|
+
const deadline = Date.now() + timeoutMs;
|
|
324
|
+
let acquired = false;
|
|
325
|
+
let lastErrCtx = null;
|
|
326
|
+
|
|
327
|
+
while (Date.now() < deadline) {
|
|
328
|
+
if (tryAcquire(lockPath, { staleAfterMs })) {
|
|
329
|
+
acquired = true;
|
|
330
|
+
break;
|
|
331
|
+
}
|
|
332
|
+
sleepSync(retryIntervalMs);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (!acquired) {
|
|
336
|
+
const err = new Error(
|
|
337
|
+
`withFileLock: timed out after ${timeoutMs}ms waiting for ${lockPath}` +
|
|
338
|
+
(lastErrCtx ? ` (last error: ${lastErrCtx})` : '')
|
|
339
|
+
);
|
|
340
|
+
err.code = 'ELOCKTIMEOUT';
|
|
341
|
+
err.lockPath = lockPath;
|
|
342
|
+
throw err;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
try {
|
|
346
|
+
return fn();
|
|
347
|
+
} finally {
|
|
348
|
+
release(lockPath);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Test-only: probe whether a lock is currently held without acquiring it.
|
|
354
|
+
* @param {string} targetPath
|
|
355
|
+
* @returns {boolean}
|
|
356
|
+
*/
|
|
357
|
+
export function isLocked(targetPath) {
|
|
358
|
+
return existsSync(lockDirFor(targetPath));
|
|
359
|
+
}
|