@objectstack/metadata-fs 17.0.0-rc.6 → 17.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +304 -0
- package/dist/index.cjs +289 -29
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +174 -2
- package/dist/index.d.ts +174 -2
- package/dist/index.js +289 -29
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.d.ts
CHANGED
|
@@ -25,10 +25,18 @@ declare class FileSystemRepository implements MetadataRepository {
|
|
|
25
25
|
private readonly heads;
|
|
26
26
|
/** Next seq counter, hydrated from the log on `start()`. */
|
|
27
27
|
private nextSeq;
|
|
28
|
-
/** Paths we wrote ourselves; suppress the resulting chokidar event. */
|
|
29
|
-
private readonly selfWrites;
|
|
30
28
|
private watcher;
|
|
31
29
|
private started;
|
|
30
|
+
/** Pending reconciliation sweep (#9339). Chained, never overlapping. */
|
|
31
|
+
private resyncTimer;
|
|
32
|
+
/** False before the watcher is armed and from `close()` onwards. */
|
|
33
|
+
private resyncEnabled;
|
|
34
|
+
/**
|
|
35
|
+
* Sweep read faults already reported, keyed `CODE @ path`, so a standing
|
|
36
|
+
* fault is announced once rather than every 2s (AGENTS.md: say it once, at
|
|
37
|
+
* the first degradation). An entry is cleared when that path reads again.
|
|
38
|
+
*/
|
|
39
|
+
private readonly resyncFaults;
|
|
32
40
|
constructor(opts: FileSystemRepositoryOptions);
|
|
33
41
|
/**
|
|
34
42
|
* Attach the repository. **Creates nothing on disk** (#7000).
|
|
@@ -110,6 +118,170 @@ declare class FileSystemRepository implements MetadataRepository {
|
|
|
110
118
|
*/
|
|
111
119
|
private trackWrittenPath;
|
|
112
120
|
private startWatcher;
|
|
121
|
+
/**
|
|
122
|
+
* Publish the `delete` face of an externally-observed removal.
|
|
123
|
+
*
|
|
124
|
+
* Extracted from `handleFsChange` unchanged so the reconciliation sweep
|
|
125
|
+
* (#9339) can reuse it **verbatim** rather than growing a second copy of the
|
|
126
|
+
* event shape. The one-line invariant: the caller already holds the per-key
|
|
127
|
+
* mutex, and `!currentHead` is the content-keyed suppression that makes our
|
|
128
|
+
* own `delete()` a no-op here.
|
|
129
|
+
*/
|
|
130
|
+
private publishExternalDelete;
|
|
131
|
+
private startResync;
|
|
132
|
+
private stopResync;
|
|
133
|
+
/**
|
|
134
|
+
* Schedule the next sweep — chained, never `setInterval` (#9339).
|
|
135
|
+
*
|
|
136
|
+
* A chained timeout cannot stack: the next sweep is armed only once the
|
|
137
|
+
* previous one has finished, so a saturated runner degrades to *fewer*
|
|
138
|
+
* sweeps instead of a growing backlog of overlapping tree walks. The timer
|
|
139
|
+
* is `unref`ed because a backstop must never be the reason a process stays
|
|
140
|
+
* alive.
|
|
141
|
+
*/
|
|
142
|
+
private scheduleResync;
|
|
143
|
+
/**
|
|
144
|
+
* Announce a sweep read that could not run — the non-silence half of #8895's
|
|
145
|
+
* "discriminate or propagate".
|
|
146
|
+
*
|
|
147
|
+
* ## Why `error` and not `warn`
|
|
148
|
+
*
|
|
149
|
+
* AGENTS.md decides the level with one question: *after the degradation, does
|
|
150
|
+
* the system still look "normal" from the outside while something it claims
|
|
151
|
+
* is persisted has not actually landed?* Here it does. Nothing throws, the
|
|
152
|
+
* watcher stays armed, `getWatched()` stays populated, `start()` succeeded —
|
|
153
|
+
* and the repository's index quietly stops tracking what is on disk. That is
|
|
154
|
+
* the rule's second limb verbatim ("persisted state and runtime state
|
|
155
|
+
* disagree"), not the functional-degradation limb: no capability is visibly
|
|
156
|
+
* smaller, so nobody finds out by using the missing thing.
|
|
157
|
+
*
|
|
158
|
+
* The counter-argument — *this is only a backstop, the watcher is still the
|
|
159
|
+
* fast path* — is why the level is arguable, and it does not survive the
|
|
160
|
+
* failing errno. The sharp case is fd exhaustion: EMFILE/ENFILE break this
|
|
161
|
+
* `readdir` and chokidar's `fs.watchFile` polling **at the same time and for
|
|
162
|
+
* the same reason**, so the fast path is not an independent fallback under
|
|
163
|
+
* precisely the load that produces this fault. A backstop that is silently
|
|
164
|
+
* absent whenever it is most needed is a durability-shaped degradation.
|
|
165
|
+
*
|
|
166
|
+
* ⚠️ AGENTS.md also warns against over-applying `error`, and the discipline
|
|
167
|
+
* that answers it is the ledger, not a quieter level: an `error` owes the
|
|
168
|
+
* consequence and the fix, said **once** at the first degradation rather than
|
|
169
|
+
* once per failed read. A sweep runs every 2s forever, so an unlatched
|
|
170
|
+
* `console.error` here would be the mirror-image failure the same rule names.
|
|
171
|
+
*
|
|
172
|
+
* ⛔ It deliberately does NOT throw. This runs on a background timer; taking
|
|
173
|
+
* a process down on a transient EACCES would be worse than the bug. The bar
|
|
174
|
+
* met here is non-silence, not propagation.
|
|
175
|
+
*
|
|
176
|
+
* The channel is `console.error` because this class has no logger: nothing is
|
|
177
|
+
* injected through `FileSystemRepositoryOptions`, and widening that public
|
|
178
|
+
* surface to carry one is out of scope for this fix.
|
|
179
|
+
*/
|
|
180
|
+
private reportResyncFault;
|
|
181
|
+
/** Re-arm reporting for a path that reads again, so a recurrence is heard. */
|
|
182
|
+
private clearResyncFault;
|
|
183
|
+
/**
|
|
184
|
+
* Content-keyed reconciliation sweep — the backstop that makes external-edit
|
|
185
|
+
* detection a guarantee rather than a single chance (#9339, #7282).
|
|
186
|
+
*
|
|
187
|
+
* ## Why the watcher alone cannot be the guarantee
|
|
188
|
+
*
|
|
189
|
+
* An external write to `<root>/<type>/<name>.json` reaches a subscriber only
|
|
190
|
+
* if chokidar notices it, and under `usePolling` it gets **exactly one**
|
|
191
|
+
* opportunity to do so: the write advances the type directory's mtime once,
|
|
192
|
+
* and chokidar re-reads a directory only when its stat *strictly advances*,
|
|
193
|
+
* so every later poll compares an unchanged stat and can never rediscover
|
|
194
|
+
* the file. Measured on #9339 with a fault-injection harness: with the one
|
|
195
|
+
* read suppressed, fifteen further poll ticks never find the new file, and a
|
|
196
|
+
* 20s deadline and a 200s deadline buy the same single attempt. That is the
|
|
197
|
+
* structural reason behind #7282's empirical finding that the event is
|
|
198
|
+
* "never delivered, not slow", and why widening the deadline (#7208) and
|
|
199
|
+
* lowering `interval` were both spent before they were tried.
|
|
200
|
+
*
|
|
201
|
+
* At least six independent one-shot gates sit on that single attempt,
|
|
202
|
+
* spanning three layers — the kernel timestamp (the directory mtime does not
|
|
203
|
+
* strictly advance), chokidar's readdir throttle and readdir snapshot, and
|
|
204
|
+
* chokidar's emit gates (`_throttle('add')`, a stale `_pendingWrites` entry,
|
|
205
|
+
* the `awaitWriteFinish` ENOENT early return). Each one produces a
|
|
206
|
+
* byte-identical observable: no event, ever, for that path.
|
|
207
|
+
*
|
|
208
|
+
* ## Why this shape, and not a narrower one
|
|
209
|
+
*
|
|
210
|
+
* ⚠️ The six are indistinguishable at the point of failure, so **any fix
|
|
211
|
+
* that has to name which gate fired is a fix for one member of a family** —
|
|
212
|
+
* which is exactly how #7282 was closed and exactly why it reopened. This
|
|
213
|
+
* sweep never asks. It compares what is on disk against `heads`, the index
|
|
214
|
+
* that already defines what this repository believes it holds, and publishes
|
|
215
|
+
* the divergence through the same `handleFsChange` the watcher feeds. It is
|
|
216
|
+
* therefore robust across all six *by construction*, and equally across a
|
|
217
|
+
* seventh nobody has found: the only property it relies on is that the bytes
|
|
218
|
+
* on disk stopped matching the index.
|
|
219
|
+
*
|
|
220
|
+
* `put()` is unaffected and keeps its direct registration (`trackWrittenPath`
|
|
221
|
+
* calls `watcher.add` and bypasses the whole chain, which is why the `put()`
|
|
222
|
+
* half of this family was already closed by #7336 and the external-write half
|
|
223
|
+
* was not).
|
|
224
|
+
*
|
|
225
|
+
* ## Cost, and why it is bounded
|
|
226
|
+
*
|
|
227
|
+
* One pass over `<root>/<type>/*.json` per sweep — the same walk `start()`
|
|
228
|
+
* already performs once — with no retry loop inside it and no work at all
|
|
229
|
+
* when nothing diverged. Sweeps are chained, so they cannot overlap; the
|
|
230
|
+
* timer is `unref`ed and dies with `close()`; and it is armed only alongside
|
|
231
|
+
* the watcher, so a `disableWatch` repository pays nothing.
|
|
232
|
+
*
|
|
233
|
+
* Discovery is by content, never by stat: a stat pre-filter would reintroduce
|
|
234
|
+
* a time key of exactly the kind this replaces.
|
|
235
|
+
*/
|
|
236
|
+
private resync;
|
|
237
|
+
/**
|
|
238
|
+
* Translate a watcher event into a `MetadataEvent`, or drop it.
|
|
239
|
+
*
|
|
240
|
+
* ## Self-writes are suppressed by content identity, never by a clock (#7335)
|
|
241
|
+
*
|
|
242
|
+
* This used to open with `if (this.selfWrites.has(absPath)) return;` — a
|
|
243
|
+
* `Set` that `put()`/`delete()` added the path to and a `setTimeout(…, 200)`
|
|
244
|
+
* cleared. That check discarded **every** event for a recently-written path
|
|
245
|
+
* without ever looking at what the watcher had actually observed, which is
|
|
246
|
+
* the whole defect: with `usePolling`, chokidar compares state once per
|
|
247
|
+
* `interval`, so our write and an external edit landing between two ticks
|
|
248
|
+
* are delivered as **one** event carrying the *external* content. Dropping
|
|
249
|
+
* it on a wall clock destroyed the only notification that edit would ever
|
|
250
|
+
* produce.
|
|
251
|
+
*
|
|
252
|
+
* Measured on `origin/main` @ `69fde55`, 40 iterations, poll phase
|
|
253
|
+
* randomised so the delivery lag samples `[0, interval)` uniformly:
|
|
254
|
+
*
|
|
255
|
+
* delivery lag < 200ms → 7 runs → external edit SWALLOWED, every time
|
|
256
|
+
* delivery lag > 200ms → 33 runs → external edit delivered, every time
|
|
257
|
+
*
|
|
258
|
+
* A perfect split on the wall-clock boundary, and the reason earlier
|
|
259
|
+
* instrumentation saw 0/360: a *fixed* pre-edit sleep phase-locks the poll,
|
|
260
|
+
* pinning the lag (measured: 519–585ms across 25 runs) safely outside the
|
|
261
|
+
* window. Nothing about the window was rare — it was unsampled.
|
|
262
|
+
*
|
|
263
|
+
* What remains is the check that was already doing the real work one step
|
|
264
|
+
* down, and it needs no timer because it compares the content the watcher
|
|
265
|
+
* **read** against the index:
|
|
266
|
+
*
|
|
267
|
+
* - `add`/`change` — `currentHead === hash` drops the event when the bytes
|
|
268
|
+
* on disk are the bytes we last published. `put()` sets that head in the
|
|
269
|
+
* same continuation as its `rename`, and `awaitWriteFinish` holds the
|
|
270
|
+
* event for a further `stabilityThreshold`, so it is never late.
|
|
271
|
+
* - `unlink` — `!currentHead` drops the event when the index already
|
|
272
|
+
* agrees the item is gone. `delete()` retires the head *before* it
|
|
273
|
+
* unlinks, precisely because this face gets no `awaitWriteFinish` delay.
|
|
274
|
+
*
|
|
275
|
+
* Both faces are pinned together in `test/self-write-suppression.test.ts`.
|
|
276
|
+
*
|
|
277
|
+
* Note the deliberate limit: identity is judged on what round-trips through
|
|
278
|
+
* the file, so a spec whose in-memory form does not (a `Date`, which
|
|
279
|
+
* canonicalises to `{}` in memory but to an ISO string once written and
|
|
280
|
+
* re-read) is republished as an external `update`. That predates this change
|
|
281
|
+
* and is independent of it — such a spec already fails `put().version ===
|
|
282
|
+
* get().hash`, and the 200ms window never covered it either, expiring some
|
|
283
|
+
* 360ms before the event it would have had to catch.
|
|
284
|
+
*/
|
|
113
285
|
private handleFsChange;
|
|
114
286
|
}
|
|
115
287
|
|
package/dist/index.js
CHANGED
|
@@ -202,6 +202,8 @@ var matchRefFilter = (ref, filter) => {
|
|
|
202
202
|
return true;
|
|
203
203
|
};
|
|
204
204
|
var matchEvent = (evt, filter) => matchRefFilter(evt.ref, filter);
|
|
205
|
+
var RESYNC_INTERVAL_MS = 2e3;
|
|
206
|
+
var isEnoent = (err) => err?.code === "ENOENT";
|
|
205
207
|
var FileSystemRepository = class {
|
|
206
208
|
constructor(opts) {
|
|
207
209
|
this.mutex = new KeyedMutex();
|
|
@@ -210,10 +212,18 @@ var FileSystemRepository = class {
|
|
|
210
212
|
this.heads = /* @__PURE__ */ new Map();
|
|
211
213
|
/** Next seq counter, hydrated from the log on `start()`. */
|
|
212
214
|
this.nextSeq = 1;
|
|
213
|
-
/** Paths we wrote ourselves; suppress the resulting chokidar event. */
|
|
214
|
-
this.selfWrites = /* @__PURE__ */ new Set();
|
|
215
215
|
this.watcher = null;
|
|
216
216
|
this.started = false;
|
|
217
|
+
/** Pending reconciliation sweep (#9339). Chained, never overlapping. */
|
|
218
|
+
this.resyncTimer = null;
|
|
219
|
+
/** False before the watcher is armed and from `close()` onwards. */
|
|
220
|
+
this.resyncEnabled = false;
|
|
221
|
+
/**
|
|
222
|
+
* Sweep read faults already reported, keyed `CODE @ path`, so a standing
|
|
223
|
+
* fault is announced once rather than every 2s (AGENTS.md: say it once, at
|
|
224
|
+
* the first degradation). An entry is cleared when that path reads again.
|
|
225
|
+
*/
|
|
226
|
+
this.resyncFaults = /* @__PURE__ */ new Set();
|
|
217
227
|
this.org = opts.org;
|
|
218
228
|
this.fsActor = opts.fsActor ?? "fs";
|
|
219
229
|
this.disableWatch = opts.disableWatch ?? false;
|
|
@@ -260,6 +270,7 @@ var FileSystemRepository = class {
|
|
|
260
270
|
if (this.started && !this.disableWatch && !this.watcher) this.startWatcher();
|
|
261
271
|
}
|
|
262
272
|
async close() {
|
|
273
|
+
this.stopResync();
|
|
263
274
|
if (this.watcher) {
|
|
264
275
|
await this.watcher.close();
|
|
265
276
|
this.watcher = null;
|
|
@@ -377,12 +388,7 @@ var FileSystemRepository = class {
|
|
|
377
388
|
const file = itemPath(this.layout, ref.type, ref.name);
|
|
378
389
|
await this.ensureRoot();
|
|
379
390
|
await fs2.mkdir(typeDir(this.layout, ref.type), { recursive: true });
|
|
380
|
-
|
|
381
|
-
try {
|
|
382
|
-
await writeJsonAtomic(file, spec);
|
|
383
|
-
} finally {
|
|
384
|
-
setTimeout(() => this.selfWrites.delete(file), 200);
|
|
385
|
-
}
|
|
391
|
+
await writeJsonAtomic(file, spec);
|
|
386
392
|
this.trackWrittenPath(file);
|
|
387
393
|
this.heads.set(key, hash);
|
|
388
394
|
const evt = {
|
|
@@ -424,13 +430,13 @@ var FileSystemRepository = class {
|
|
|
424
430
|
}
|
|
425
431
|
const file = itemPath(this.layout, ref.type, ref.name);
|
|
426
432
|
await this.ensureRoot();
|
|
427
|
-
this.
|
|
433
|
+
this.heads.delete(key);
|
|
428
434
|
try {
|
|
429
435
|
if (existsSync2(file)) await fs2.unlink(file);
|
|
430
|
-
}
|
|
431
|
-
|
|
436
|
+
} catch (err) {
|
|
437
|
+
if (currentHead !== null) this.heads.set(key, currentHead);
|
|
438
|
+
throw err;
|
|
432
439
|
}
|
|
433
|
-
this.heads.delete(key);
|
|
434
440
|
const seq = this.nextSeq++;
|
|
435
441
|
const ts = this.now().toISOString();
|
|
436
442
|
const evt = {
|
|
@@ -565,9 +571,278 @@ var FileSystemRepository = class {
|
|
|
565
571
|
w.on("change", (p) => void this.handleFsChange(p, "change"));
|
|
566
572
|
w.on("unlink", (p) => void this.handleFsChange(p, "unlink"));
|
|
567
573
|
this.watcher = w;
|
|
574
|
+
this.startResync();
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* Publish the `delete` face of an externally-observed removal.
|
|
578
|
+
*
|
|
579
|
+
* Extracted from `handleFsChange` unchanged so the reconciliation sweep
|
|
580
|
+
* (#9339) can reuse it **verbatim** rather than growing a second copy of the
|
|
581
|
+
* event shape. The one-line invariant: the caller already holds the per-key
|
|
582
|
+
* mutex, and `!currentHead` is the content-keyed suppression that makes our
|
|
583
|
+
* own `delete()` a no-op here.
|
|
584
|
+
*/
|
|
585
|
+
async publishExternalDelete(ref, key) {
|
|
586
|
+
const currentHead = this.heads.get(key) ?? null;
|
|
587
|
+
if (!currentHead) return;
|
|
588
|
+
this.heads.delete(key);
|
|
589
|
+
const seq = this.nextSeq++;
|
|
590
|
+
const evt = {
|
|
591
|
+
seq,
|
|
592
|
+
op: "delete",
|
|
593
|
+
ref: { ...ref, version: void 0 },
|
|
594
|
+
hash: null,
|
|
595
|
+
parentHash: currentHead,
|
|
596
|
+
actor: this.fsActor,
|
|
597
|
+
ts: this.now().toISOString(),
|
|
598
|
+
source: "fs"
|
|
599
|
+
};
|
|
600
|
+
await this.log.append(evt);
|
|
601
|
+
this.broker.publish(evt);
|
|
602
|
+
}
|
|
603
|
+
startResync() {
|
|
604
|
+
this.resyncEnabled = true;
|
|
605
|
+
this.scheduleResync();
|
|
606
|
+
}
|
|
607
|
+
stopResync() {
|
|
608
|
+
this.resyncEnabled = false;
|
|
609
|
+
if (this.resyncTimer) {
|
|
610
|
+
clearTimeout(this.resyncTimer);
|
|
611
|
+
this.resyncTimer = null;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* Schedule the next sweep — chained, never `setInterval` (#9339).
|
|
616
|
+
*
|
|
617
|
+
* A chained timeout cannot stack: the next sweep is armed only once the
|
|
618
|
+
* previous one has finished, so a saturated runner degrades to *fewer*
|
|
619
|
+
* sweeps instead of a growing backlog of overlapping tree walks. The timer
|
|
620
|
+
* is `unref`ed because a backstop must never be the reason a process stays
|
|
621
|
+
* alive.
|
|
622
|
+
*/
|
|
623
|
+
scheduleResync() {
|
|
624
|
+
if (!this.resyncEnabled || this.resyncTimer) return;
|
|
625
|
+
const timer = setTimeout(() => {
|
|
626
|
+
this.resyncTimer = null;
|
|
627
|
+
void this.resync().finally(() => this.scheduleResync());
|
|
628
|
+
}, RESYNC_INTERVAL_MS);
|
|
629
|
+
timer.unref?.();
|
|
630
|
+
this.resyncTimer = timer;
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Announce a sweep read that could not run — the non-silence half of #8895's
|
|
634
|
+
* "discriminate or propagate".
|
|
635
|
+
*
|
|
636
|
+
* ## Why `error` and not `warn`
|
|
637
|
+
*
|
|
638
|
+
* AGENTS.md decides the level with one question: *after the degradation, does
|
|
639
|
+
* the system still look "normal" from the outside while something it claims
|
|
640
|
+
* is persisted has not actually landed?* Here it does. Nothing throws, the
|
|
641
|
+
* watcher stays armed, `getWatched()` stays populated, `start()` succeeded —
|
|
642
|
+
* and the repository's index quietly stops tracking what is on disk. That is
|
|
643
|
+
* the rule's second limb verbatim ("persisted state and runtime state
|
|
644
|
+
* disagree"), not the functional-degradation limb: no capability is visibly
|
|
645
|
+
* smaller, so nobody finds out by using the missing thing.
|
|
646
|
+
*
|
|
647
|
+
* The counter-argument — *this is only a backstop, the watcher is still the
|
|
648
|
+
* fast path* — is why the level is arguable, and it does not survive the
|
|
649
|
+
* failing errno. The sharp case is fd exhaustion: EMFILE/ENFILE break this
|
|
650
|
+
* `readdir` and chokidar's `fs.watchFile` polling **at the same time and for
|
|
651
|
+
* the same reason**, so the fast path is not an independent fallback under
|
|
652
|
+
* precisely the load that produces this fault. A backstop that is silently
|
|
653
|
+
* absent whenever it is most needed is a durability-shaped degradation.
|
|
654
|
+
*
|
|
655
|
+
* ⚠️ AGENTS.md also warns against over-applying `error`, and the discipline
|
|
656
|
+
* that answers it is the ledger, not a quieter level: an `error` owes the
|
|
657
|
+
* consequence and the fix, said **once** at the first degradation rather than
|
|
658
|
+
* once per failed read. A sweep runs every 2s forever, so an unlatched
|
|
659
|
+
* `console.error` here would be the mirror-image failure the same rule names.
|
|
660
|
+
*
|
|
661
|
+
* ⛔ It deliberately does NOT throw. This runs on a background timer; taking
|
|
662
|
+
* a process down on a transient EACCES would be worse than the bug. The bar
|
|
663
|
+
* met here is non-silence, not propagation.
|
|
664
|
+
*
|
|
665
|
+
* The channel is `console.error` because this class has no logger: nothing is
|
|
666
|
+
* injected through `FileSystemRepositoryOptions`, and widening that public
|
|
667
|
+
* surface to carry one is out of scope for this fix.
|
|
668
|
+
*/
|
|
669
|
+
reportResyncFault(target, err) {
|
|
670
|
+
const code = err?.code ?? "UNKNOWN";
|
|
671
|
+
const key = `${code} @ ${target}`;
|
|
672
|
+
if (this.resyncFaults.has(key)) return;
|
|
673
|
+
this.resyncFaults.add(key);
|
|
674
|
+
console.error(
|
|
675
|
+
`[FileSystemRepository] metadata reconciliation sweep could not read ${target} (${code}). CONSEQUENCE: external edits under this path are no longer reconciled, so this repository's index and its watch() subscribers can drift from what is on disk while everything keeps reporting healthy. The chokidar watcher is not an independent fallback here \u2014 fd exhaustion degrades both. FIX: restore read access to the path; the sweep recovers by itself on the first successful read. Reported once per path and error code.`
|
|
676
|
+
);
|
|
677
|
+
}
|
|
678
|
+
/** Re-arm reporting for a path that reads again, so a recurrence is heard. */
|
|
679
|
+
clearResyncFault(target) {
|
|
680
|
+
if (this.resyncFaults.size === 0) return;
|
|
681
|
+
const suffix = ` @ ${target}`;
|
|
682
|
+
for (const key of this.resyncFaults) {
|
|
683
|
+
if (key.endsWith(suffix)) this.resyncFaults.delete(key);
|
|
684
|
+
}
|
|
568
685
|
}
|
|
686
|
+
/**
|
|
687
|
+
* Content-keyed reconciliation sweep — the backstop that makes external-edit
|
|
688
|
+
* detection a guarantee rather than a single chance (#9339, #7282).
|
|
689
|
+
*
|
|
690
|
+
* ## Why the watcher alone cannot be the guarantee
|
|
691
|
+
*
|
|
692
|
+
* An external write to `<root>/<type>/<name>.json` reaches a subscriber only
|
|
693
|
+
* if chokidar notices it, and under `usePolling` it gets **exactly one**
|
|
694
|
+
* opportunity to do so: the write advances the type directory's mtime once,
|
|
695
|
+
* and chokidar re-reads a directory only when its stat *strictly advances*,
|
|
696
|
+
* so every later poll compares an unchanged stat and can never rediscover
|
|
697
|
+
* the file. Measured on #9339 with a fault-injection harness: with the one
|
|
698
|
+
* read suppressed, fifteen further poll ticks never find the new file, and a
|
|
699
|
+
* 20s deadline and a 200s deadline buy the same single attempt. That is the
|
|
700
|
+
* structural reason behind #7282's empirical finding that the event is
|
|
701
|
+
* "never delivered, not slow", and why widening the deadline (#7208) and
|
|
702
|
+
* lowering `interval` were both spent before they were tried.
|
|
703
|
+
*
|
|
704
|
+
* At least six independent one-shot gates sit on that single attempt,
|
|
705
|
+
* spanning three layers — the kernel timestamp (the directory mtime does not
|
|
706
|
+
* strictly advance), chokidar's readdir throttle and readdir snapshot, and
|
|
707
|
+
* chokidar's emit gates (`_throttle('add')`, a stale `_pendingWrites` entry,
|
|
708
|
+
* the `awaitWriteFinish` ENOENT early return). Each one produces a
|
|
709
|
+
* byte-identical observable: no event, ever, for that path.
|
|
710
|
+
*
|
|
711
|
+
* ## Why this shape, and not a narrower one
|
|
712
|
+
*
|
|
713
|
+
* ⚠️ The six are indistinguishable at the point of failure, so **any fix
|
|
714
|
+
* that has to name which gate fired is a fix for one member of a family** —
|
|
715
|
+
* which is exactly how #7282 was closed and exactly why it reopened. This
|
|
716
|
+
* sweep never asks. It compares what is on disk against `heads`, the index
|
|
717
|
+
* that already defines what this repository believes it holds, and publishes
|
|
718
|
+
* the divergence through the same `handleFsChange` the watcher feeds. It is
|
|
719
|
+
* therefore robust across all six *by construction*, and equally across a
|
|
720
|
+
* seventh nobody has found: the only property it relies on is that the bytes
|
|
721
|
+
* on disk stopped matching the index.
|
|
722
|
+
*
|
|
723
|
+
* `put()` is unaffected and keeps its direct registration (`trackWrittenPath`
|
|
724
|
+
* calls `watcher.add` and bypasses the whole chain, which is why the `put()`
|
|
725
|
+
* half of this family was already closed by #7336 and the external-write half
|
|
726
|
+
* was not).
|
|
727
|
+
*
|
|
728
|
+
* ## Cost, and why it is bounded
|
|
729
|
+
*
|
|
730
|
+
* One pass over `<root>/<type>/*.json` per sweep — the same walk `start()`
|
|
731
|
+
* already performs once — with no retry loop inside it and no work at all
|
|
732
|
+
* when nothing diverged. Sweeps are chained, so they cannot overlap; the
|
|
733
|
+
* timer is `unref`ed and dies with `close()`; and it is armed only alongside
|
|
734
|
+
* the watcher, so a `disableWatch` repository pays nothing.
|
|
735
|
+
*
|
|
736
|
+
* Discovery is by content, never by stat: a stat pre-filter would reintroduce
|
|
737
|
+
* a time key of exactly the kind this replaces.
|
|
738
|
+
*/
|
|
739
|
+
async resync() {
|
|
740
|
+
const root = this.layout.root;
|
|
741
|
+
let entries = [];
|
|
742
|
+
try {
|
|
743
|
+
entries = await fs2.readdir(root, { withFileTypes: true });
|
|
744
|
+
this.clearResyncFault(root);
|
|
745
|
+
} catch (err) {
|
|
746
|
+
if (!isEnoent(err)) this.reportResyncFault(root, err);
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
const onDisk = /* @__PURE__ */ new Set();
|
|
750
|
+
const unreadableTypes = /* @__PURE__ */ new Set();
|
|
751
|
+
for (const entry of entries) {
|
|
752
|
+
if (!entry.isDirectory()) continue;
|
|
753
|
+
if (entry.name.startsWith(".")) continue;
|
|
754
|
+
const dir = path3.join(root, entry.name);
|
|
755
|
+
let files = [];
|
|
756
|
+
try {
|
|
757
|
+
files = await fs2.readdir(dir);
|
|
758
|
+
this.clearResyncFault(dir);
|
|
759
|
+
} catch (err) {
|
|
760
|
+
if (!isEnoent(err)) {
|
|
761
|
+
this.reportResyncFault(dir, err);
|
|
762
|
+
unreadableTypes.add(entry.name);
|
|
763
|
+
}
|
|
764
|
+
continue;
|
|
765
|
+
}
|
|
766
|
+
for (const file of files) {
|
|
767
|
+
if (!file.endsWith(".json") || file.startsWith(".")) continue;
|
|
768
|
+
const abs = path3.join(dir, file);
|
|
769
|
+
const parsed = parseItemPath(this.layout, abs);
|
|
770
|
+
if (!parsed) continue;
|
|
771
|
+
const ref = {
|
|
772
|
+
org: this.org,
|
|
773
|
+
type: parsed.type,
|
|
774
|
+
name: parsed.name
|
|
775
|
+
};
|
|
776
|
+
const key = refKey(ref);
|
|
777
|
+
onDisk.add(key);
|
|
778
|
+
const before = this.heads.get(key);
|
|
779
|
+
await this.handleFsChange(abs, "add");
|
|
780
|
+
if (this.heads.get(key) !== before) {
|
|
781
|
+
this.trackWrittenPath(abs);
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
for (const key of [...this.heads.keys()]) {
|
|
786
|
+
if (onDisk.has(key)) continue;
|
|
787
|
+
const ref = parseRefKey(key);
|
|
788
|
+
if (!ref) continue;
|
|
789
|
+
if (unreadableTypes.has(ref.type)) continue;
|
|
790
|
+
const file = itemPath(this.layout, ref.type, ref.name);
|
|
791
|
+
await this.mutex.run(key, async () => {
|
|
792
|
+
if (existsSync2(file)) return;
|
|
793
|
+
await this.publishExternalDelete(ref, key);
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
/**
|
|
798
|
+
* Translate a watcher event into a `MetadataEvent`, or drop it.
|
|
799
|
+
*
|
|
800
|
+
* ## Self-writes are suppressed by content identity, never by a clock (#7335)
|
|
801
|
+
*
|
|
802
|
+
* This used to open with `if (this.selfWrites.has(absPath)) return;` — a
|
|
803
|
+
* `Set` that `put()`/`delete()` added the path to and a `setTimeout(…, 200)`
|
|
804
|
+
* cleared. That check discarded **every** event for a recently-written path
|
|
805
|
+
* without ever looking at what the watcher had actually observed, which is
|
|
806
|
+
* the whole defect: with `usePolling`, chokidar compares state once per
|
|
807
|
+
* `interval`, so our write and an external edit landing between two ticks
|
|
808
|
+
* are delivered as **one** event carrying the *external* content. Dropping
|
|
809
|
+
* it on a wall clock destroyed the only notification that edit would ever
|
|
810
|
+
* produce.
|
|
811
|
+
*
|
|
812
|
+
* Measured on `origin/main` @ `69fde55`, 40 iterations, poll phase
|
|
813
|
+
* randomised so the delivery lag samples `[0, interval)` uniformly:
|
|
814
|
+
*
|
|
815
|
+
* delivery lag < 200ms → 7 runs → external edit SWALLOWED, every time
|
|
816
|
+
* delivery lag > 200ms → 33 runs → external edit delivered, every time
|
|
817
|
+
*
|
|
818
|
+
* A perfect split on the wall-clock boundary, and the reason earlier
|
|
819
|
+
* instrumentation saw 0/360: a *fixed* pre-edit sleep phase-locks the poll,
|
|
820
|
+
* pinning the lag (measured: 519–585ms across 25 runs) safely outside the
|
|
821
|
+
* window. Nothing about the window was rare — it was unsampled.
|
|
822
|
+
*
|
|
823
|
+
* What remains is the check that was already doing the real work one step
|
|
824
|
+
* down, and it needs no timer because it compares the content the watcher
|
|
825
|
+
* **read** against the index:
|
|
826
|
+
*
|
|
827
|
+
* - `add`/`change` — `currentHead === hash` drops the event when the bytes
|
|
828
|
+
* on disk are the bytes we last published. `put()` sets that head in the
|
|
829
|
+
* same continuation as its `rename`, and `awaitWriteFinish` holds the
|
|
830
|
+
* event for a further `stabilityThreshold`, so it is never late.
|
|
831
|
+
* - `unlink` — `!currentHead` drops the event when the index already
|
|
832
|
+
* agrees the item is gone. `delete()` retires the head *before* it
|
|
833
|
+
* unlinks, precisely because this face gets no `awaitWriteFinish` delay.
|
|
834
|
+
*
|
|
835
|
+
* Both faces are pinned together in `test/self-write-suppression.test.ts`.
|
|
836
|
+
*
|
|
837
|
+
* Note the deliberate limit: identity is judged on what round-trips through
|
|
838
|
+
* the file, so a spec whose in-memory form does not (a `Date`, which
|
|
839
|
+
* canonicalises to `{}` in memory but to an ISO string once written and
|
|
840
|
+
* re-read) is republished as an external `update`. That predates this change
|
|
841
|
+
* and is independent of it — such a spec already fails `put().version ===
|
|
842
|
+
* get().hash`, and the 200ms window never covered it either, expiring some
|
|
843
|
+
* 360ms before the event it would have had to catch.
|
|
844
|
+
*/
|
|
569
845
|
async handleFsChange(absPath, kind) {
|
|
570
|
-
if (this.selfWrites.has(absPath)) return;
|
|
571
846
|
const parsed = parseItemPath(this.layout, absPath);
|
|
572
847
|
if (!parsed) return;
|
|
573
848
|
const ref = {
|
|
@@ -578,22 +853,7 @@ var FileSystemRepository = class {
|
|
|
578
853
|
const key = refKey(ref);
|
|
579
854
|
await this.mutex.run(key, async () => {
|
|
580
855
|
if (kind === "unlink") {
|
|
581
|
-
|
|
582
|
-
if (!currentHead2) return;
|
|
583
|
-
this.heads.delete(key);
|
|
584
|
-
const seq2 = this.nextSeq++;
|
|
585
|
-
const evt2 = {
|
|
586
|
-
seq: seq2,
|
|
587
|
-
op: "delete",
|
|
588
|
-
ref: { ...ref, version: void 0 },
|
|
589
|
-
hash: null,
|
|
590
|
-
parentHash: currentHead2,
|
|
591
|
-
actor: this.fsActor,
|
|
592
|
-
ts: this.now().toISOString(),
|
|
593
|
-
source: "fs"
|
|
594
|
-
};
|
|
595
|
-
await this.log.append(evt2);
|
|
596
|
-
this.broker.publish(evt2);
|
|
856
|
+
await this.publishExternalDelete(ref, key);
|
|
597
857
|
return;
|
|
598
858
|
}
|
|
599
859
|
const body = await readJson(absPath);
|