@dogfood-lab/findings 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +96 -0
- package/advise/advice-bundle.js +155 -0
- package/advise/index.js +5 -0
- package/advise/query.js +182 -0
- package/cli.js +936 -0
- package/derive/dedupe.js +107 -0
- package/derive/derive-findings.js +187 -0
- package/derive/ids.js +48 -0
- package/derive/index.js +9 -0
- package/derive/load-records.js +153 -0
- package/derive/rules.js +415 -0
- package/derive/write-findings.js +63 -0
- package/index.js +11 -0
- package/lib/atomic-write.js +47 -0
- package/lib/file-lock.js +359 -0
- package/lib/rename-with-retry.js +43 -0
- package/package.json +70 -0
- package/reader.js +156 -0
- package/review/event-log.js +177 -0
- package/review/index.js +6 -0
- package/review/review-engine.js +288 -0
- package/review/transitions.js +79 -0
- package/synthesis/doctrine-derivation.js +128 -0
- package/synthesis/index.js +8 -0
- package/synthesis/pattern-derivation.js +184 -0
- package/synthesis/recommendation-derivation.js +156 -0
- package/synthesis/validate-artifacts.js +46 -0
- package/synthesis/write-artifacts.js +75 -0
- package/validate.js +87 -0
package/lib/file-lock.js
ADDED
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-process advisory file lock built on `openSync(path, 'wx')` exclusive-create.
|
|
3
|
+
*
|
|
4
|
+
* Why this exists: `appendEvent` in review/event-log.js does a read-modify-write
|
|
5
|
+
* of a YAML array (read existing events → push new → atomic-rename whole file).
|
|
6
|
+
* Two concurrent appends both read N events, both push, both rename — second
|
|
7
|
+
* rename wins, dropping the first event. The atomic rename eliminates partial-
|
|
8
|
+
* file corruption (worst case before wave 9), but cannot eliminate the lost-
|
|
9
|
+
* event window. F-PIPELINE-011 (W3-PIPE-001) closes that window at the choke
|
|
10
|
+
* point so the bug class is impossible to recur — Pattern #4 from the
|
|
11
|
+
* swarm-evidence catalog.
|
|
12
|
+
*
|
|
13
|
+
* Why an O_EXCL lock FILE (not a lock DIR): an early implementation used
|
|
14
|
+
* `mkdirSync` then a follow-up `writeFileSync(.../pid)` — but those are two
|
|
15
|
+
* separate fs ops, so a process that's mid-acquisition is in a state where
|
|
16
|
+
* the dir exists but the pid file does NOT. A racing acquirer sees the dir,
|
|
17
|
+
* tries to read pid, gets ENOENT, classifies as stale, removes the dir, and
|
|
18
|
+
* the original acquirer's pid write later trips on the dir being gone.
|
|
19
|
+
* Lock-FILE-with-O_EXCL avoids that: the `open(path, 'wx')` call returns a
|
|
20
|
+
* file descriptor in one syscall, and we write the pid through that fd —
|
|
21
|
+
* any racing acquirer that sees the file knows it's fully formed.
|
|
22
|
+
*
|
|
23
|
+
* Why not `proper-lockfile`: that package isn't a dep of this monorepo
|
|
24
|
+
* (verified at the start of wave 30). `open(path, 'wx')` is atomic on POSIX
|
|
25
|
+
* and Windows — the kernel guarantees exactly one creator wins, the loser
|
|
26
|
+
* sees `EEXIST`. The pattern has been in use for decades and works the same
|
|
27
|
+
* on every fs Node supports (NTFS, ext4, APFS, tmpfs, nfsv4).
|
|
28
|
+
*
|
|
29
|
+
* Why not `O_APPEND`: the format here is a YAML array, not JSONL. `O_APPEND`
|
|
30
|
+
* makes byte-append atomic for sub-`PIPE_BUF` writes on POSIX, but the
|
|
31
|
+
* read-modify-write of the whole array bypasses that — even with `O_APPEND`
|
|
32
|
+
* the events would not be a valid YAML array. Switching the format to JSONL
|
|
33
|
+
* is a bigger compat break than this contract should make. Lock instead.
|
|
34
|
+
*
|
|
35
|
+
* Stale lock recovery: a process that crashes while holding the lock leaves
|
|
36
|
+
* the file behind. We detect "stale" by reading the holder PID and using
|
|
37
|
+
* `process.kill(pid, 0)` to test liveness — `ESRCH` means the holder is
|
|
38
|
+
* gone and the lock is reclaimable. The reclaim itself is best-effort:
|
|
39
|
+
* another fresh acquirer might race the cleanup, but the worst case is an
|
|
40
|
+
* extra retry, never lost data. To prevent two reclaimers from both believing
|
|
41
|
+
* they won, the reclaim sequence is `unlink → open(wx)`; the second
|
|
42
|
+
* reclaimer's `open(wx)` will fail because the first reclaimer already
|
|
43
|
+
* created it.
|
|
44
|
+
*
|
|
45
|
+
* Concurrency caveat — out of scope: this is a *single-machine* lock. NFS or
|
|
46
|
+
* other distributed filesystems expose `open(wx)` semantics that may not be
|
|
47
|
+
* truly atomic across nodes. The dogfood pipeline runs on a single GitHub
|
|
48
|
+
* runner per dispatch, so this is sufficient. A multi-runner sharded ingest
|
|
49
|
+
* would need a real consensus lock — flagged in the JSDoc above
|
|
50
|
+
* `withFileLock` so the limitation is visible at the use site.
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
import { mkdirSync, openSync, closeSync, writeSync, writeFileSync, readFileSync, existsSync, unlinkSync, linkSync, renameSync, statSync } from 'node:fs';
|
|
54
|
+
import { dirname } from 'node:path';
|
|
55
|
+
import { randomBytes } from 'node:crypto';
|
|
56
|
+
|
|
57
|
+
// 30s default timeout: appendEvent's critical section is small (~ms), but
|
|
58
|
+
// 50-way parallel ingests serializing through the lock can take a few seconds
|
|
59
|
+
// in aggregate. 30s leaves comfortable headroom for the dogfood pipeline's
|
|
60
|
+
// realistic concurrency (~10 parallel jobs per dispatch wave) without making
|
|
61
|
+
// CI hang on a runaway holder.
|
|
62
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
63
|
+
const DEFAULT_RETRY_INTERVAL_MS = 15;
|
|
64
|
+
const DEFAULT_STALE_AFTER_MS = 30_000;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Compute the lock file path for a given target file.
|
|
68
|
+
* Co-locates with the target so a permission-restricted directory still works.
|
|
69
|
+
*
|
|
70
|
+
* Kept named `lockDirFor` for back-compat with the wave-30 design draft —
|
|
71
|
+
* the path now points at a regular file, not a directory, but the call sites
|
|
72
|
+
* don't care about the kind.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} targetPath
|
|
75
|
+
* @returns {string}
|
|
76
|
+
*/
|
|
77
|
+
export function lockDirFor(targetPath) {
|
|
78
|
+
return `${targetPath}.lock`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Test whether a process is alive. Returns false if `pid` is missing,
|
|
83
|
+
* non-numeric, or `process.kill(pid, 0)` rejects with `ESRCH`. Any other
|
|
84
|
+
* error (e.g. `EPERM` on a foreign-user pid) is treated as "alive" — better
|
|
85
|
+
* to wait out the lock than to steal a live one.
|
|
86
|
+
*
|
|
87
|
+
* @param {number|string|null|undefined} pid
|
|
88
|
+
* @returns {boolean}
|
|
89
|
+
*/
|
|
90
|
+
function isProcessAlive(pid) {
|
|
91
|
+
const n = typeof pid === 'number' ? pid : Number(pid);
|
|
92
|
+
if (!Number.isFinite(n) || n <= 0) return false;
|
|
93
|
+
try {
|
|
94
|
+
process.kill(n, 0);
|
|
95
|
+
return true;
|
|
96
|
+
} catch (err) {
|
|
97
|
+
if (err && err.code === 'ESRCH') return false;
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Atomically create the lock file with the holder's PID as content.
|
|
104
|
+
*
|
|
105
|
+
* Two-step pattern: write a temp file (containing this process's pid) in the
|
|
106
|
+
* same parent dir, then `linkSync(temp, lockPath)`. `link` is atomic on every
|
|
107
|
+
* fs Node supports — it either creates the link or fails with EEXIST; there
|
|
108
|
+
* is no observable intermediate state where the lock file exists but is
|
|
109
|
+
* empty. This closes the race that an earlier `open(wx)+write` design left
|
|
110
|
+
* open: a racing acquirer could read the lock file in the window between
|
|
111
|
+
* `open` and `write` and see it as empty (= stale) before the holder's pid
|
|
112
|
+
* was recorded.
|
|
113
|
+
*
|
|
114
|
+
* @param {string} lockPath
|
|
115
|
+
* @returns {boolean} true on success; false if the lock already exists.
|
|
116
|
+
*/
|
|
117
|
+
function atomicCreateLock(lockPath) {
|
|
118
|
+
const tmpPath = `${lockPath}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
|
|
119
|
+
// Write the temp file atomically (truncates if it somehow exists, which
|
|
120
|
+
// it never should given the pid+random suffix). Content is the holder pid.
|
|
121
|
+
writeFileSync(tmpPath, String(process.pid), 'utf-8');
|
|
122
|
+
try {
|
|
123
|
+
linkSync(tmpPath, lockPath);
|
|
124
|
+
return true;
|
|
125
|
+
} catch (err) {
|
|
126
|
+
if (err && err.code === 'EEXIST') return false;
|
|
127
|
+
throw err;
|
|
128
|
+
} finally {
|
|
129
|
+
// The temp file is no longer needed regardless of link outcome — the
|
|
130
|
+
// hardlink (on success) keeps the inode alive at lockPath; on failure
|
|
131
|
+
// the temp file is just garbage to clean up.
|
|
132
|
+
try { unlinkSync(tmpPath); } catch { /* best effort */ }
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Attempt to acquire an exclusive file-lock on `lockPath`. Returns true if
|
|
138
|
+
* the lock was acquired; false if it is held by another live process OR
|
|
139
|
+
* by a process whose lock file is unreadable / pid-empty (treated as live
|
|
140
|
+
* with bounded mtime patience — see below).
|
|
141
|
+
*
|
|
142
|
+
* Stale recovery: a holder's PID file remains until its release `unlink`
|
|
143
|
+
* runs. If the holder PROCESS is gone (process.kill ESRCH), the lock is
|
|
144
|
+
* reclaimable. We do NOT treat empty/missing pid content as stale on its
|
|
145
|
+
* own — that was the wave-30 first-pass bug. Even with `linkSync` for
|
|
146
|
+
* atomic creation, on Windows a racing reader can land on a brief window
|
|
147
|
+
* where the pid file's content has not yet been committed to the dirent
|
|
148
|
+
* cache from the other process. Empty content + bounded-stale-mtime is the
|
|
149
|
+
* safer guard: we only reclaim if the lock file is ALSO older than
|
|
150
|
+
* `staleAfterMs`, the same boundary `proper-lockfile` uses.
|
|
151
|
+
*
|
|
152
|
+
* @param {string} lockPath
|
|
153
|
+
* @param {{ staleAfterMs?: number }} opts
|
|
154
|
+
* @returns {boolean}
|
|
155
|
+
*/
|
|
156
|
+
function tryAcquire(lockPath, opts = {}) {
|
|
157
|
+
const { staleAfterMs = DEFAULT_STALE_AFTER_MS } = opts;
|
|
158
|
+
|
|
159
|
+
if (atomicCreateLock(lockPath)) return true;
|
|
160
|
+
|
|
161
|
+
// Lock exists — test for staleness.
|
|
162
|
+
let pidRaw = null;
|
|
163
|
+
let mtimeMs = NaN;
|
|
164
|
+
try {
|
|
165
|
+
pidRaw = readFileSync(lockPath, 'utf-8').trim();
|
|
166
|
+
} catch {
|
|
167
|
+
// Read failed — fall through to the mtime check below; if we can't
|
|
168
|
+
// even stat it, treat as live.
|
|
169
|
+
}
|
|
170
|
+
try {
|
|
171
|
+
const st = statSync(lockPath);
|
|
172
|
+
mtimeMs = st.mtimeMs;
|
|
173
|
+
} catch { /* will be NaN, treated as live */ }
|
|
174
|
+
|
|
175
|
+
// PID-known case: trust process.kill to decide alive-vs-dead. This is the
|
|
176
|
+
// common case and gives fast crash recovery (sub-100ms typical).
|
|
177
|
+
if (pidRaw && pidRaw !== '') {
|
|
178
|
+
if (isProcessAlive(pidRaw)) return false;
|
|
179
|
+
|
|
180
|
+
// PID-dead reclaim via "rename to graveyard" — atomic claim that exactly
|
|
181
|
+
// one reclaimer wins. Without this, a sequence like:
|
|
182
|
+
// 1. A holds lock; A releases (unlinks); A's pid becomes dead
|
|
183
|
+
// 2. C atomicCreateLock succeeds — C owns lock with pid=C
|
|
184
|
+
// 3. B (had read pidRaw=A above) confirms A dead, calls unlinkSync — but
|
|
185
|
+
// the file is now C's lock! Now B and C both think they own it.
|
|
186
|
+
// produces a double-owner. The graveyard rename is the atomic CAS step:
|
|
187
|
+
// exactly one process can rename a given file at a given moment. We
|
|
188
|
+
// verify the rename succeeded AND the file we renamed still has the
|
|
189
|
+
// dead PID we expected; if the content shifted (someone else just
|
|
190
|
+
// re-acquired), put the file back.
|
|
191
|
+
return reclaimViaGraveyard(lockPath, pidRaw);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// PID-unknown case (empty/unreadable content). DO NOT treat as stale on
|
|
195
|
+
// content alone — Windows can present a brief window where the holder's
|
|
196
|
+
// file content is still flushing. Only reclaim if the lock is also older
|
|
197
|
+
// than `staleAfterMs`. With the default 30s staleAfterMs, fast appenders
|
|
198
|
+
// never trip this; only a true crash window does.
|
|
199
|
+
if (Number.isFinite(mtimeMs) && Date.now() - mtimeMs > staleAfterMs) {
|
|
200
|
+
if (process.env.FILE_LOCK_DEBUG) {
|
|
201
|
+
process.stderr.write(`[file-lock][pid=${process.pid}] STALE-RECLAIM (mtime-old) lockPath=${lockPath} ageMs=${Date.now() - mtimeMs}\n`);
|
|
202
|
+
}
|
|
203
|
+
// Use the same graveyard-rename CAS for the mtime-stale path. The PID
|
|
204
|
+
// is unknown so we use empty string as the "expected" content — the
|
|
205
|
+
// graveyard verification will accept whatever it reads (the put-back
|
|
206
|
+
// branch only triggers when actualPid is non-empty AND differs).
|
|
207
|
+
return reclaimViaGraveyard(lockPath, pidRaw || '');
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Treat as live; the holder owns the lock and will release it.
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Release a lock acquired via `tryAcquire`. Best-effort — a missing lock file
|
|
216
|
+
* is not an error (it means stale-lock recovery already cleaned it).
|
|
217
|
+
*
|
|
218
|
+
* @param {string} lockPath
|
|
219
|
+
*/
|
|
220
|
+
function release(lockPath) {
|
|
221
|
+
try { unlinkSync(lockPath); } catch { /* may already be gone */ }
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Reclaim a stale lock via "rename to graveyard." Atomic CAS step: exactly
|
|
226
|
+
* one reclaimer can rename a file at a time. The winner verifies the renamed
|
|
227
|
+
* file still has the dead PID it expected (defending against the case where
|
|
228
|
+
* the lock content shifted between the read-pid step and the rename); if so,
|
|
229
|
+
* the winner unlinks the graveyard file and creates a fresh lock. If the
|
|
230
|
+
* content shifted, the winner puts the file back so the new owner is
|
|
231
|
+
* unaffected.
|
|
232
|
+
*
|
|
233
|
+
* @param {string} lockPath
|
|
234
|
+
* @param {string} expectedDeadPid - The PID we observed and confirmed dead.
|
|
235
|
+
* @returns {boolean}
|
|
236
|
+
*/
|
|
237
|
+
function reclaimViaGraveyard(lockPath, expectedDeadPid) {
|
|
238
|
+
const graveyardPath = `${lockPath}.gy.${process.pid}.${randomBytes(4).toString('hex')}`;
|
|
239
|
+
try {
|
|
240
|
+
renameSync(lockPath, graveyardPath);
|
|
241
|
+
} catch (err) {
|
|
242
|
+
// Another reclaimer beat us — let the retry loop sort it out.
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// We won the rename. Verify content.
|
|
247
|
+
let actualPid = null;
|
|
248
|
+
try {
|
|
249
|
+
actualPid = readFileSync(graveyardPath, 'utf-8').trim();
|
|
250
|
+
} catch { /* unreadable — treat as confirmation */ }
|
|
251
|
+
|
|
252
|
+
if (actualPid && actualPid !== expectedDeadPid) {
|
|
253
|
+
// Content shifted between our read and our rename. The "stale" lock
|
|
254
|
+
// we grabbed is actually a fresh acquisition by someone else.
|
|
255
|
+
// Put it back. If put-back fails (e.g., another reclaimer raced and
|
|
256
|
+
// already created a new lock at lockPath), discard the graveyard copy.
|
|
257
|
+
try {
|
|
258
|
+
renameSync(graveyardPath, lockPath);
|
|
259
|
+
if (process.env.FILE_LOCK_DEBUG) {
|
|
260
|
+
process.stderr.write(`[file-lock][pid=${process.pid}] CAS-FAIL-PUTBACK lockPath=${lockPath} expected=${expectedDeadPid} found=${actualPid}\n`);
|
|
261
|
+
}
|
|
262
|
+
} catch {
|
|
263
|
+
try { unlinkSync(graveyardPath); } catch { /* drop */ }
|
|
264
|
+
}
|
|
265
|
+
return false;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (process.env.FILE_LOCK_DEBUG) {
|
|
269
|
+
process.stderr.write(`[file-lock][pid=${process.pid}] STALE-RECLAIM (dead-pid) lockPath=${lockPath} pid=${expectedDeadPid}\n`);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
try { unlinkSync(graveyardPath); } catch { /* drop */ }
|
|
273
|
+
return atomicCreateLock(lockPath);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Sleep synchronously for `ms` milliseconds via `Atomics.wait` on a tiny
|
|
278
|
+
* SharedArrayBuffer. We can't `await` here because the callers
|
|
279
|
+
* (`appendEvent`, `rebuildIndexes`) are sync APIs — the whole pipeline
|
|
280
|
+
* is sync from `ingest()` down to fs I/O, so introducing async here would
|
|
281
|
+
* cascade through six callers and break the back-compat contract on
|
|
282
|
+
* `appendEvent` that review-engine relies on.
|
|
283
|
+
*
|
|
284
|
+
* @param {number} ms
|
|
285
|
+
*/
|
|
286
|
+
function sleepSync(ms) {
|
|
287
|
+
const sab = new SharedArrayBuffer(4);
|
|
288
|
+
const view = new Int32Array(sab);
|
|
289
|
+
Atomics.wait(view, 0, 0, ms);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Run `fn` while holding an exclusive lock on `targetPath`. The lock is a
|
|
294
|
+
* sibling directory at `<targetPath>.lock` — see file header for the
|
|
295
|
+
* full design rationale.
|
|
296
|
+
*
|
|
297
|
+
* The lock is per-target, so two unrelated `appendEvent` calls writing to
|
|
298
|
+
* different daily log files do NOT serialize against each other.
|
|
299
|
+
*
|
|
300
|
+
* @template T
|
|
301
|
+
* @param {string} targetPath - The file being mutated; the lock dir is `<targetPath>.lock`.
|
|
302
|
+
* @param {() => T} fn - The critical section. Runs synchronously.
|
|
303
|
+
* @param {{
|
|
304
|
+
* timeoutMs?: number,
|
|
305
|
+
* retryIntervalMs?: number,
|
|
306
|
+
* staleAfterMs?: number
|
|
307
|
+
* }} [options]
|
|
308
|
+
* @returns {T}
|
|
309
|
+
*/
|
|
310
|
+
export function withFileLock(targetPath, fn, options = {}) {
|
|
311
|
+
const {
|
|
312
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
313
|
+
retryIntervalMs = DEFAULT_RETRY_INTERVAL_MS,
|
|
314
|
+
staleAfterMs = DEFAULT_STALE_AFTER_MS,
|
|
315
|
+
} = options;
|
|
316
|
+
|
|
317
|
+
const lockPath = lockDirFor(targetPath);
|
|
318
|
+
|
|
319
|
+
// Ensure the parent dir exists — otherwise mkdirSync(lockPath) fails with
|
|
320
|
+
// ENOENT and we mis-classify the lock as held.
|
|
321
|
+
try { mkdirSync(dirname(lockPath), { recursive: true }); } catch { /* may already exist */ }
|
|
322
|
+
|
|
323
|
+
const deadline = Date.now() + timeoutMs;
|
|
324
|
+
let acquired = false;
|
|
325
|
+
let lastErrCtx = null;
|
|
326
|
+
|
|
327
|
+
while (Date.now() < deadline) {
|
|
328
|
+
if (tryAcquire(lockPath, { staleAfterMs })) {
|
|
329
|
+
acquired = true;
|
|
330
|
+
break;
|
|
331
|
+
}
|
|
332
|
+
sleepSync(retryIntervalMs);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (!acquired) {
|
|
336
|
+
const err = new Error(
|
|
337
|
+
`withFileLock: timed out after ${timeoutMs}ms waiting for ${lockPath}` +
|
|
338
|
+
(lastErrCtx ? ` (last error: ${lastErrCtx})` : '')
|
|
339
|
+
);
|
|
340
|
+
err.code = 'ELOCKTIMEOUT';
|
|
341
|
+
err.lockPath = lockPath;
|
|
342
|
+
throw err;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
try {
|
|
346
|
+
return fn();
|
|
347
|
+
} finally {
|
|
348
|
+
release(lockPath);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Test-only: probe whether a lock is currently held without acquiring it.
|
|
354
|
+
* @param {string} targetPath
|
|
355
|
+
* @returns {boolean}
|
|
356
|
+
*/
|
|
357
|
+
export function isLocked(targetPath) {
|
|
358
|
+
return existsSync(lockDirFor(targetPath));
|
|
359
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* renameWithRetry — Windows-tolerant atomic rename.
|
|
3
|
+
*
|
|
4
|
+
* On Windows NTFS, `renameSync(tmp, target)` can throw EPERM (or EBUSY) even
|
|
5
|
+
* when the calling process holds the file lock. The most common cause is a
|
|
6
|
+
* transient handle held by a sibling — antivirus scanning the freshly-written
|
|
7
|
+
* temp, Search Indexer, Defender, a backup agent — across the rename window.
|
|
8
|
+
* The handle releases on its own within milliseconds, but the bare renameSync
|
|
9
|
+
* is unforgiving and surfaces the error to the operator.
|
|
10
|
+
*
|
|
11
|
+
* The fix is a bounded exponential-backoff retry. The W3-PIPE-001 wave-30
|
|
12
|
+
* receipt documents the 50/50 multi-process race-test outcome with this
|
|
13
|
+
* mitigation in place; without it, that test fails reliably on Windows CI.
|
|
14
|
+
*
|
|
15
|
+
* Synchronous-on-purpose: the call sites here (atomic-write helpers, event-log
|
|
16
|
+
* appender) are themselves synchronous, and Promise-based retries would
|
|
17
|
+
* leak the async boundary into otherwise-deterministic flush paths.
|
|
18
|
+
*
|
|
19
|
+
* @param {string} tmp - Source path (the just-written temp file).
|
|
20
|
+
* @param {string} dest - Destination path (the canonical artifact).
|
|
21
|
+
* @param {object} [opts]
|
|
22
|
+
* @param {number} [opts.retries=10] - Max retry attempts after the first failure.
|
|
23
|
+
* @param {number} [opts.baseMs=15] - Initial backoff in ms (doubles each step).
|
|
24
|
+
* @param {number} [opts.maxMs=200] - Cap on per-step backoff.
|
|
25
|
+
*/
|
|
26
|
+
import { renameSync } from 'node:fs';
|
|
27
|
+
|
|
28
|
+
export function renameWithRetry(tmp, dest, { retries = 10, baseMs = 15, maxMs = 200 } = {}) {
|
|
29
|
+
for (let i = 0; i <= retries; i++) {
|
|
30
|
+
try {
|
|
31
|
+
renameSync(tmp, dest);
|
|
32
|
+
return;
|
|
33
|
+
} catch (err) {
|
|
34
|
+
if ((err.code !== 'EPERM' && err.code !== 'EBUSY') || i === retries) throw err;
|
|
35
|
+
const delay = Math.min(baseMs * (1 << i), maxMs);
|
|
36
|
+
// Synchronous sleep — the public API is sync (matches renameSync), so
|
|
37
|
+
// setTimeout would change the contract. Spin-wait is acceptable here
|
|
38
|
+
// because the delays are bounded (≤200ms) and the failure mode is rare.
|
|
39
|
+
const until = Date.now() + delay;
|
|
40
|
+
while (Date.now() < until) { /* spin */ }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@dogfood-lab/findings",
|
|
3
|
+
"version": "1.2.1",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Finding contract spine for testing-os. Validates, reads, lists, and queries evidence-bound findings — the fourth contract alongside record, scenario, and policy.",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./index.js",
|
|
9
|
+
"./reader.js": "./reader.js",
|
|
10
|
+
"./validate.js": "./validate.js",
|
|
11
|
+
"./derive/*": "./derive/*",
|
|
12
|
+
"./review/*": "./review/*",
|
|
13
|
+
"./synthesis/*": "./synthesis/*",
|
|
14
|
+
"./advise/*": "./advise/*",
|
|
15
|
+
"./lib/*": "./lib/*"
|
|
16
|
+
},
|
|
17
|
+
"bin": {
|
|
18
|
+
"findings": "./cli.js"
|
|
19
|
+
},
|
|
20
|
+
"scripts": {
|
|
21
|
+
"test": "node --test findings.test.js derive/derive.test.js review/review.test.js synthesis/synthesis.test.js advise/advise.test.js lib/atomic-write.test.js"
|
|
22
|
+
},
|
|
23
|
+
"files": [
|
|
24
|
+
"index.js",
|
|
25
|
+
"cli.js",
|
|
26
|
+
"reader.js",
|
|
27
|
+
"validate.js",
|
|
28
|
+
"advise/",
|
|
29
|
+
"derive/",
|
|
30
|
+
"lib/",
|
|
31
|
+
"review/",
|
|
32
|
+
"synthesis/",
|
|
33
|
+
"README.md",
|
|
34
|
+
"LICENSE",
|
|
35
|
+
"!**/*.test.js",
|
|
36
|
+
"!**/*.test.mjs"
|
|
37
|
+
],
|
|
38
|
+
"publishConfig": {
|
|
39
|
+
"access": "public"
|
|
40
|
+
},
|
|
41
|
+
"dependencies": {
|
|
42
|
+
"@dogfood-lab/ingest": "^1.2.0",
|
|
43
|
+
"@dogfood-lab/schemas": "^1.2.0",
|
|
44
|
+
"ajv": "^8.18.0",
|
|
45
|
+
"ajv-formats": "^3.0.1",
|
|
46
|
+
"js-yaml": "^4.1.0"
|
|
47
|
+
},
|
|
48
|
+
"engines": {
|
|
49
|
+
"node": ">=20"
|
|
50
|
+
},
|
|
51
|
+
"author": "mcp-tool-shop",
|
|
52
|
+
"license": "MIT",
|
|
53
|
+
"repository": {
|
|
54
|
+
"type": "git",
|
|
55
|
+
"url": "https://github.com/dogfood-lab/testing-os.git",
|
|
56
|
+
"directory": "packages/findings"
|
|
57
|
+
},
|
|
58
|
+
"homepage": "https://github.com/dogfood-lab/testing-os",
|
|
59
|
+
"bugs": {
|
|
60
|
+
"url": "https://github.com/dogfood-lab/testing-os/issues"
|
|
61
|
+
},
|
|
62
|
+
"keywords": [
|
|
63
|
+
"testing-os",
|
|
64
|
+
"dogfood-lab",
|
|
65
|
+
"findings",
|
|
66
|
+
"evidence",
|
|
67
|
+
"audit",
|
|
68
|
+
"review"
|
|
69
|
+
]
|
|
70
|
+
}
|
package/reader.js
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finding reader/lister.
|
|
3
|
+
* Discovers findings from the filesystem, supports filtering and lookup.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { readdirSync, existsSync, statSync } from 'node:fs';
|
|
7
|
+
import { resolve, join, basename, extname } from 'node:path';
|
|
8
|
+
import { parseFinding, validateFinding } from './validate.js';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Discover all .yaml finding files under a root directory.
|
|
12
|
+
* Walks findings/<org>/<repo>/*.yaml
|
|
13
|
+
*
|
|
14
|
+
* @param {string} rootDir - The dogfood-labs repo root.
|
|
15
|
+
* @returns {string[]} Array of absolute paths to finding files.
|
|
16
|
+
*/
|
|
17
|
+
export function discoverFindings(rootDir) {
|
|
18
|
+
const findingsDir = resolve(rootDir, 'findings');
|
|
19
|
+
if (!existsSync(findingsDir)) return [];
|
|
20
|
+
|
|
21
|
+
const paths = [];
|
|
22
|
+
|
|
23
|
+
// Walk: findings/<org>/<repo>/*.yaml
|
|
24
|
+
for (const org of listDirs(findingsDir)) {
|
|
25
|
+
const orgDir = join(findingsDir, org);
|
|
26
|
+
for (const repo of listDirs(orgDir)) {
|
|
27
|
+
const repoDir = join(orgDir, repo);
|
|
28
|
+
for (const file of readdirSync(repoDir)) {
|
|
29
|
+
if (extname(file) === '.yaml') {
|
|
30
|
+
paths.push(resolve(repoDir, file));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
return paths.sort();
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Discover finding files from fixtures directory.
|
|
41
|
+
* @param {string} rootDir - The dogfood-labs repo root.
|
|
42
|
+
* @param {'valid' | 'invalid'} kind - Which fixture set.
|
|
43
|
+
* @returns {string[]} Array of absolute paths.
|
|
44
|
+
*/
|
|
45
|
+
export function discoverFixtures(rootDir, kind) {
|
|
46
|
+
const dir = resolve(rootDir, 'fixtures', 'findings', kind);
|
|
47
|
+
if (!existsSync(dir)) return [];
|
|
48
|
+
|
|
49
|
+
return readdirSync(dir)
|
|
50
|
+
.filter(f => extname(f) === '.yaml')
|
|
51
|
+
.map(f => resolve(dir, f))
|
|
52
|
+
.sort();
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Load all findings from disk (real or fixtures).
|
|
57
|
+
* Returns parsed + validated findings.
|
|
58
|
+
*
|
|
59
|
+
* @param {string} rootDir - The dogfood-labs repo root.
|
|
60
|
+
* @param {{ fixtures?: boolean, fixtureKind?: 'valid' | 'invalid' }} opts
|
|
61
|
+
* @returns {Array<{ path: string, data: object | null, valid: boolean, errors: Array }>}
|
|
62
|
+
*/
|
|
63
|
+
export function loadFindings(rootDir, opts = {}) {
|
|
64
|
+
const paths = opts.fixtures
|
|
65
|
+
? discoverFixtures(rootDir, opts.fixtureKind || 'valid')
|
|
66
|
+
: discoverFindings(rootDir);
|
|
67
|
+
|
|
68
|
+
return paths.map(filePath => {
|
|
69
|
+
const { data, error } = parseFinding(filePath);
|
|
70
|
+
if (error) {
|
|
71
|
+
return { path: filePath, data: null, valid: false, errors: [{ path: '/', message: error }] };
|
|
72
|
+
}
|
|
73
|
+
const result = validateFinding(data);
|
|
74
|
+
return { path: filePath, data, ...result };
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Find a single finding by its finding_id.
|
|
80
|
+
* Searches real findings first, then fixtures.
|
|
81
|
+
*
|
|
82
|
+
* @param {string} rootDir - The dogfood-labs repo root.
|
|
83
|
+
* @param {string} findingId - The finding_id to look up.
|
|
84
|
+
* @returns {{ path: string, data: object, valid: boolean, errors: Array } | null}
|
|
85
|
+
*/
|
|
86
|
+
export function findById(rootDir, findingId) {
|
|
87
|
+
// Search real findings
|
|
88
|
+
for (const filePath of discoverFindings(rootDir)) {
|
|
89
|
+
const { data } = parseFinding(filePath);
|
|
90
|
+
if (data && data.finding_id === findingId) {
|
|
91
|
+
const result = validateFinding(data);
|
|
92
|
+
return { path: filePath, data, ...result };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Search valid fixtures
|
|
97
|
+
for (const filePath of discoverFixtures(rootDir, 'valid')) {
|
|
98
|
+
const { data } = parseFinding(filePath);
|
|
99
|
+
if (data && data.finding_id === findingId) {
|
|
100
|
+
const result = validateFinding(data);
|
|
101
|
+
return { path: filePath, data, ...result };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Filter a list of loaded findings.
|
|
110
|
+
*
|
|
111
|
+
* @param {Array<{ data: object }>} findings - Loaded findings.
|
|
112
|
+
* @param {{ repo?: string, status?: string, surface?: string, issueKind?: string, transferScope?: string }} filters
|
|
113
|
+
* @returns {Array}
|
|
114
|
+
*/
|
|
115
|
+
export function filterFindings(findings, filters = {}) {
|
|
116
|
+
return findings.filter(f => {
|
|
117
|
+
if (!f.data) return false;
|
|
118
|
+
if (filters.repo && f.data.repo !== filters.repo) return false;
|
|
119
|
+
if (filters.status && f.data.status !== filters.status) return false;
|
|
120
|
+
if (filters.surface && f.data.product_surface !== filters.surface) return false;
|
|
121
|
+
if (filters.issueKind && f.data.issue_kind !== filters.issueKind) return false;
|
|
122
|
+
if (filters.transferScope && f.data.transfer_scope !== filters.transferScope) return false;
|
|
123
|
+
return true;
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Check for duplicate finding_ids across all findings.
|
|
129
|
+
* @param {Array<{ data: object, path: string }>} findings
|
|
130
|
+
* @returns {Array<{ findingId: string, paths: string[] }>}
|
|
131
|
+
*/
|
|
132
|
+
export function findDuplicates(findings) {
|
|
133
|
+
const seen = new Map();
|
|
134
|
+
for (const f of findings) {
|
|
135
|
+
if (!f.data || !f.data.finding_id) continue;
|
|
136
|
+
const id = f.data.finding_id;
|
|
137
|
+
if (!seen.has(id)) seen.set(id, []);
|
|
138
|
+
seen.get(id).push(f.path);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return Array.from(seen.entries())
|
|
142
|
+
.filter(([, paths]) => paths.length > 1)
|
|
143
|
+
.map(([findingId, paths]) => ({ findingId, paths }));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** List subdirectories of a directory. */
|
|
147
|
+
function listDirs(dir) {
|
|
148
|
+
if (!existsSync(dir)) return [];
|
|
149
|
+
return readdirSync(dir).filter(name => {
|
|
150
|
+
try {
|
|
151
|
+
return statSync(join(dir, name)).isDirectory();
|
|
152
|
+
} catch {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
}
|