@objectstack/metadata-fs 17.0.0 → 17.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/dist/index.cjs +291 -17
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +160 -0
- package/dist/index.d.ts +160 -0
- package/dist/index.js +291 -17
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -27,6 +27,25 @@ declare class FileSystemRepository implements MetadataRepository {
|
|
|
27
27
|
private nextSeq;
|
|
28
28
|
private watcher;
|
|
29
29
|
private started;
|
|
30
|
+
/** Pending reconciliation sweep (#9339). Chained, never overlapping. */
|
|
31
|
+
private resyncTimer;
|
|
32
|
+
/** False before the watcher is armed and from `close()` onwards. */
|
|
33
|
+
private resyncEnabled;
|
|
34
|
+
/**
|
|
35
|
+
* Sweep read faults already reported, keyed `CODE @ path`, so a standing
|
|
36
|
+
* fault is announced once rather than every 2s (AGENTS.md: say it once, at
|
|
37
|
+
* the first degradation). An entry is cleared when that path reads again.
|
|
38
|
+
*/
|
|
39
|
+
private readonly resyncFaults;
|
|
40
|
+
/**
|
|
41
|
+
* Bumped by every `close()`. `watch()` reads it before its deferred log
|
|
42
|
+
* replay starts and hands the comparison to `createWatchIterable`, so a
|
|
43
|
+
* subscription that registers AFTER the shutdown sweep terminates on
|
|
44
|
+
* arrival instead of parking forever (#11127). A counter rather than a
|
|
45
|
+
* boolean because `start()` may follow `close()`: a repository restart must
|
|
46
|
+
* not poison the watchers opened after it.
|
|
47
|
+
*/
|
|
48
|
+
private closeGeneration;
|
|
30
49
|
constructor(opts: FileSystemRepositoryOptions);
|
|
31
50
|
/**
|
|
32
51
|
* Attach the repository. **Creates nothing on disk** (#7000).
|
|
@@ -55,6 +74,31 @@ declare class FileSystemRepository implements MetadataRepository {
|
|
|
55
74
|
* gets armed, so "external edits are detected" survives the change.
|
|
56
75
|
*/
|
|
57
76
|
private ensureRoot;
|
|
77
|
+
/**
|
|
78
|
+
* Shut the repository down, ending every live `watch()` iterator.
|
|
79
|
+
*
|
|
80
|
+
* **Shutdown terminates; it does not emit** — invariant 8 in
|
|
81
|
+
* `@objectstack/metadata-core`'s `repository.ts`, and the reason this method
|
|
82
|
+
* reaches the broker at all. It used to retire the chokidar watcher and the
|
|
83
|
+
* resync sweep and stop there. The broker has no teardown of its own
|
|
84
|
+
* (`subscribe`/`unsubscribe` add to and delete from a plain `Set`), and each
|
|
85
|
+
* iterator parks its pending `next()` on a `waiter` that only a broker
|
|
86
|
+
* `push` or the iterator's own terminator can settle. After `close()` the
|
|
87
|
+
* chokidar source was gone, so no `push` could arrive; the subscriber was
|
|
88
|
+
* still registered, and nothing ran its terminator. A consumer holding a
|
|
89
|
+
* `for await` at shutdown — `MetadataManager.startRepositoryWatch()` is
|
|
90
|
+
* exactly that shape — therefore never saw its loop end, for EVERY
|
|
91
|
+
* subscription shape including `watch({})`.
|
|
92
|
+
*
|
|
93
|
+
* Termination is expressed as termination: each subscription's
|
|
94
|
+
* `terminate()`, which is the same routine the consumer's own
|
|
95
|
+
* `iterator.return()` runs, so no consumer has to tell "the repository shut
|
|
96
|
+
* down under me" apart from "I broke my own loop". A synthetic drain event
|
|
97
|
+
* would be the wrong shape and was measured to be so (#11021): the
|
|
98
|
+
* subscriptions most in need of draining are exactly the ones whose filter
|
|
99
|
+
* or numeric `since` drops it, and delivering an event has never ended an
|
|
100
|
+
* iterator.
|
|
101
|
+
*/
|
|
58
102
|
close(): Promise<void>;
|
|
59
103
|
get(ref: MetaRef): Promise<MetadataItem | null>;
|
|
60
104
|
getByHash(ref: MetaRef, hash: string): Promise<MetadataItem | null>;
|
|
@@ -108,6 +152,122 @@ declare class FileSystemRepository implements MetadataRepository {
|
|
|
108
152
|
*/
|
|
109
153
|
private trackWrittenPath;
|
|
110
154
|
private startWatcher;
|
|
155
|
+
/**
|
|
156
|
+
* Publish the `delete` face of an externally-observed removal.
|
|
157
|
+
*
|
|
158
|
+
* Extracted from `handleFsChange` unchanged so the reconciliation sweep
|
|
159
|
+
* (#9339) can reuse it **verbatim** rather than growing a second copy of the
|
|
160
|
+
* event shape. The one-line invariant: the caller already holds the per-key
|
|
161
|
+
* mutex, and `!currentHead` is the content-keyed suppression that makes our
|
|
162
|
+
* own `delete()` a no-op here.
|
|
163
|
+
*/
|
|
164
|
+
private publishExternalDelete;
|
|
165
|
+
private startResync;
|
|
166
|
+
private stopResync;
|
|
167
|
+
/**
|
|
168
|
+
* Schedule the next sweep — chained, never `setInterval` (#9339).
|
|
169
|
+
*
|
|
170
|
+
* A chained timeout cannot stack: the next sweep is armed only once the
|
|
171
|
+
* previous one has finished, so a saturated runner degrades to *fewer*
|
|
172
|
+
* sweeps instead of a growing backlog of overlapping tree walks. The timer
|
|
173
|
+
* is `unref`ed because a backstop must never be the reason a process stays
|
|
174
|
+
* alive.
|
|
175
|
+
*/
|
|
176
|
+
private scheduleResync;
|
|
177
|
+
/**
|
|
178
|
+
* Announce a sweep read that could not run — the non-silence half of #8895's
|
|
179
|
+
* "discriminate or propagate".
|
|
180
|
+
*
|
|
181
|
+
* ## Why `error` and not `warn`
|
|
182
|
+
*
|
|
183
|
+
* AGENTS.md decides the level with one question: *after the degradation, does
|
|
184
|
+
* the system still look "normal" from the outside while something it claims
|
|
185
|
+
* is persisted has not actually landed?* Here it does. Nothing throws, the
|
|
186
|
+
* watcher stays armed, `getWatched()` stays populated, `start()` succeeded —
|
|
187
|
+
* and the repository's index quietly stops tracking what is on disk. That is
|
|
188
|
+
* the rule's second limb verbatim ("persisted state and runtime state
|
|
189
|
+
* disagree"), not the functional-degradation limb: no capability is visibly
|
|
190
|
+
* smaller, so nobody finds out by using the missing thing.
|
|
191
|
+
*
|
|
192
|
+
* The counter-argument — *this is only a backstop, the watcher is still the
|
|
193
|
+
* fast path* — is why the level is arguable, and it does not survive the
|
|
194
|
+
* failing errno. The sharp case is fd exhaustion: EMFILE/ENFILE break this
|
|
195
|
+
* `readdir` and chokidar's `fs.watchFile` polling **at the same time and for
|
|
196
|
+
* the same reason**, so the fast path is not an independent fallback under
|
|
197
|
+
* precisely the load that produces this fault. A backstop that is silently
|
|
198
|
+
* absent whenever it is most needed is a durability-shaped degradation.
|
|
199
|
+
*
|
|
200
|
+
* ⚠️ AGENTS.md also warns against over-applying `error`, and the discipline
|
|
201
|
+
* that answers it is the ledger, not a quieter level: an `error` owes the
|
|
202
|
+
* consequence and the fix, said **once** at the first degradation rather than
|
|
203
|
+
* once per failed read. A sweep runs every 2s forever, so an unlatched
|
|
204
|
+
* `console.error` here would be the mirror-image failure the same rule names.
|
|
205
|
+
*
|
|
206
|
+
* ⛔ It deliberately does NOT throw. This runs on a background timer; taking
|
|
207
|
+
* a process down on a transient EACCES would be worse than the bug. The bar
|
|
208
|
+
* met here is non-silence, not propagation.
|
|
209
|
+
*
|
|
210
|
+
* The channel is `console.error` because this class has no logger: nothing is
|
|
211
|
+
* injected through `FileSystemRepositoryOptions`, and widening that public
|
|
212
|
+
* surface to carry one is out of scope for this fix.
|
|
213
|
+
*/
|
|
214
|
+
private reportResyncFault;
|
|
215
|
+
/** Re-arm reporting for a path that reads again, so a recurrence is heard. */
|
|
216
|
+
private clearResyncFault;
|
|
217
|
+
/**
|
|
218
|
+
* Content-keyed reconciliation sweep — the backstop that makes external-edit
|
|
219
|
+
* detection a guarantee rather than a single chance (#9339, #7282).
|
|
220
|
+
*
|
|
221
|
+
* ## Why the watcher alone cannot be the guarantee
|
|
222
|
+
*
|
|
223
|
+
* An external write to `<root>/<type>/<name>.json` reaches a subscriber only
|
|
224
|
+
* if chokidar notices it, and under `usePolling` it gets **exactly one**
|
|
225
|
+
* opportunity to do so: the write advances the type directory's mtime once,
|
|
226
|
+
* and chokidar re-reads a directory only when its stat *strictly advances*,
|
|
227
|
+
* so every later poll compares an unchanged stat and can never rediscover
|
|
228
|
+
* the file. Measured on #9339 with a fault-injection harness: with the one
|
|
229
|
+
* read suppressed, fifteen further poll ticks never find the new file, and a
|
|
230
|
+
* 20s deadline and a 200s deadline buy the same single attempt. That is the
|
|
231
|
+
* structural reason behind #7282's empirical finding that the event is
|
|
232
|
+
* "never delivered, not slow", and why widening the deadline (#7208) and
|
|
233
|
+
* lowering `interval` were both spent before they were tried.
|
|
234
|
+
*
|
|
235
|
+
* At least six independent one-shot gates sit on that single attempt,
|
|
236
|
+
* spanning three layers — the kernel timestamp (the directory mtime does not
|
|
237
|
+
* strictly advance), chokidar's readdir throttle and readdir snapshot, and
|
|
238
|
+
* chokidar's emit gates (`_throttle('add')`, a stale `_pendingWrites` entry,
|
|
239
|
+
* the `awaitWriteFinish` ENOENT early return). Each one produces a
|
|
240
|
+
* byte-identical observable: no event, ever, for that path.
|
|
241
|
+
*
|
|
242
|
+
* ## Why this shape, and not a narrower one
|
|
243
|
+
*
|
|
244
|
+
* ⚠️ The six are indistinguishable at the point of failure, so **any fix
|
|
245
|
+
* that has to name which gate fired is a fix for one member of a family** —
|
|
246
|
+
* which is exactly how #7282 was closed and exactly why it reopened. This
|
|
247
|
+
* sweep never asks. It compares what is on disk against `heads`, the index
|
|
248
|
+
* that already defines what this repository believes it holds, and publishes
|
|
249
|
+
* the divergence through the same `handleFsChange` the watcher feeds. It is
|
|
250
|
+
* therefore robust across all six *by construction*, and equally across a
|
|
251
|
+
* seventh nobody has found: the only property it relies on is that the bytes
|
|
252
|
+
* on disk stopped matching the index.
|
|
253
|
+
*
|
|
254
|
+
* `put()` is unaffected and keeps its direct registration (`trackWrittenPath`
|
|
255
|
+
* calls `watcher.add` and bypasses the whole chain, which is why the `put()`
|
|
256
|
+
* half of this family was already closed by #7336 and the external-write half
|
|
257
|
+
* was not).
|
|
258
|
+
*
|
|
259
|
+
* ## Cost, and why it is bounded
|
|
260
|
+
*
|
|
261
|
+
* One pass over `<root>/<type>/*.json` per sweep — the same walk `start()`
|
|
262
|
+
* already performs once — with no retry loop inside it and no work at all
|
|
263
|
+
* when nothing diverged. Sweeps are chained, so they cannot overlap; the
|
|
264
|
+
* timer is `unref`ed and dies with `close()`; and it is armed only alongside
|
|
265
|
+
* the watcher, so a `disableWatch` repository pays nothing.
|
|
266
|
+
*
|
|
267
|
+
* Discovery is by content, never by stat: a stat pre-filter would reintroduce
|
|
268
|
+
* a time key of exactly the kind this replaces.
|
|
269
|
+
*/
|
|
270
|
+
private resync;
|
|
111
271
|
/**
|
|
112
272
|
* Translate a watcher event into a `MetadataEvent`, or drop it.
|
|
113
273
|
*
|
package/dist/index.js
CHANGED
|
@@ -111,6 +111,16 @@ function createBroker(matches) {
|
|
|
111
111
|
if (!matches(evt, s.filter)) continue;
|
|
112
112
|
s.push(evt);
|
|
113
113
|
}
|
|
114
|
+
},
|
|
115
|
+
terminateAll: () => {
|
|
116
|
+
const snapshot = Array.from(subs);
|
|
117
|
+
subs.clear();
|
|
118
|
+
for (const s of snapshot) {
|
|
119
|
+
try {
|
|
120
|
+
s.terminate();
|
|
121
|
+
} catch {
|
|
122
|
+
}
|
|
123
|
+
}
|
|
114
124
|
}
|
|
115
125
|
};
|
|
116
126
|
}
|
|
@@ -125,6 +135,10 @@ function createWatchIterable(args) {
|
|
|
125
135
|
const subscriber = {
|
|
126
136
|
filter: args.filter,
|
|
127
137
|
closed: false,
|
|
138
|
+
// Assigned below, once `close` exists. Termination and the consumer's own
|
|
139
|
+
// `return()` are ONE routine, deliberately: invariant 8 requires shutdown
|
|
140
|
+
// to be indistinguishable from `iterator.return()`.
|
|
141
|
+
terminate: () => void 0,
|
|
128
142
|
push: (evt) => {
|
|
129
143
|
if (subscriber.closed) return;
|
|
130
144
|
const k = evtKey(evt);
|
|
@@ -173,6 +187,8 @@ function createWatchIterable(args) {
|
|
|
173
187
|
}
|
|
174
188
|
return { value: void 0, done: true };
|
|
175
189
|
};
|
|
190
|
+
subscriber.terminate = close;
|
|
191
|
+
if (args.arrivesClosed?.()) close();
|
|
176
192
|
const iterator = {
|
|
177
193
|
next: () => {
|
|
178
194
|
if (closed) return Promise.resolve({ value: void 0, done: true });
|
|
@@ -202,6 +218,8 @@ var matchRefFilter = (ref, filter) => {
|
|
|
202
218
|
return true;
|
|
203
219
|
};
|
|
204
220
|
var matchEvent = (evt, filter) => matchRefFilter(evt.ref, filter);
|
|
221
|
+
var RESYNC_INTERVAL_MS = 2e3;
|
|
222
|
+
var isEnoent = (err) => err?.code === "ENOENT";
|
|
205
223
|
var FileSystemRepository = class {
|
|
206
224
|
constructor(opts) {
|
|
207
225
|
this.mutex = new KeyedMutex();
|
|
@@ -212,6 +230,25 @@ var FileSystemRepository = class {
|
|
|
212
230
|
this.nextSeq = 1;
|
|
213
231
|
this.watcher = null;
|
|
214
232
|
this.started = false;
|
|
233
|
+
/** Pending reconciliation sweep (#9339). Chained, never overlapping. */
|
|
234
|
+
this.resyncTimer = null;
|
|
235
|
+
/** False before the watcher is armed and from `close()` onwards. */
|
|
236
|
+
this.resyncEnabled = false;
|
|
237
|
+
/**
|
|
238
|
+
* Sweep read faults already reported, keyed `CODE @ path`, so a standing
|
|
239
|
+
* fault is announced once rather than every 2s (AGENTS.md: say it once, at
|
|
240
|
+
* the first degradation). An entry is cleared when that path reads again.
|
|
241
|
+
*/
|
|
242
|
+
this.resyncFaults = /* @__PURE__ */ new Set();
|
|
243
|
+
/**
|
|
244
|
+
* Bumped by every `close()`. `watch()` reads it before its deferred log
|
|
245
|
+
* replay starts and hands the comparison to `createWatchIterable`, so a
|
|
246
|
+
* subscription that registers AFTER the shutdown sweep terminates on
|
|
247
|
+
* arrival instead of parking forever (#11127). A counter rather than a
|
|
248
|
+
* boolean because `start()` may follow `close()`: a repository restart must
|
|
249
|
+
* not poison the watchers opened after it.
|
|
250
|
+
*/
|
|
251
|
+
this.closeGeneration = 0;
|
|
215
252
|
this.org = opts.org;
|
|
216
253
|
this.fsActor = opts.fsActor ?? "fs";
|
|
217
254
|
this.disableWatch = opts.disableWatch ?? false;
|
|
@@ -257,7 +294,35 @@ var FileSystemRepository = class {
|
|
|
257
294
|
await fs2.mkdir(this.layout.root, { recursive: true });
|
|
258
295
|
if (this.started && !this.disableWatch && !this.watcher) this.startWatcher();
|
|
259
296
|
}
|
|
297
|
+
/**
|
|
298
|
+
* Shut the repository down, ending every live `watch()` iterator.
|
|
299
|
+
*
|
|
300
|
+
* **Shutdown terminates; it does not emit** — invariant 8 in
|
|
301
|
+
* `@objectstack/metadata-core`'s `repository.ts`, and the reason this method
|
|
302
|
+
* reaches the broker at all. It used to retire the chokidar watcher and the
|
|
303
|
+
* resync sweep and stop there. The broker has no teardown of its own
|
|
304
|
+
* (`subscribe`/`unsubscribe` add to and delete from a plain `Set`), and each
|
|
305
|
+
* iterator parks its pending `next()` on a `waiter` that only a broker
|
|
306
|
+
* `push` or the iterator's own terminator can settle. After `close()` the
|
|
307
|
+
* chokidar source was gone, so no `push` could arrive; the subscriber was
|
|
308
|
+
* still registered, and nothing ran its terminator. A consumer holding a
|
|
309
|
+
* `for await` at shutdown — `MetadataManager.startRepositoryWatch()` is
|
|
310
|
+
* exactly that shape — therefore never saw its loop end, for EVERY
|
|
311
|
+
* subscription shape including `watch({})`.
|
|
312
|
+
*
|
|
313
|
+
* Termination is expressed as termination: each subscription's
|
|
314
|
+
* `terminate()`, which is the same routine the consumer's own
|
|
315
|
+
* `iterator.return()` runs, so no consumer has to tell "the repository shut
|
|
316
|
+
* down under me" apart from "I broke my own loop". A synthetic drain event
|
|
317
|
+
* would be the wrong shape and was measured to be so (#11021): the
|
|
318
|
+
* subscriptions most in need of draining are exactly the ones whose filter
|
|
319
|
+
* or numeric `since` drops it, and delivering an event has never ended an
|
|
320
|
+
* iterator.
|
|
321
|
+
*/
|
|
260
322
|
async close() {
|
|
323
|
+
this.stopResync();
|
|
324
|
+
this.closeGeneration++;
|
|
325
|
+
this.broker.terminateAll();
|
|
261
326
|
if (this.watcher) {
|
|
262
327
|
await this.watcher.close();
|
|
263
328
|
this.watcher = null;
|
|
@@ -332,6 +397,7 @@ var FileSystemRepository = class {
|
|
|
332
397
|
if (matchEvent(evt, filter)) replay.push(evt);
|
|
333
398
|
}
|
|
334
399
|
})();
|
|
400
|
+
const generation = this.closeGeneration;
|
|
335
401
|
return deferredIterable(promise.then(
|
|
336
402
|
() => createWatchIterable({
|
|
337
403
|
filter,
|
|
@@ -339,7 +405,8 @@ var FileSystemRepository = class {
|
|
|
339
405
|
replay,
|
|
340
406
|
broker: this.broker,
|
|
341
407
|
matches: matchEvent,
|
|
342
|
-
branchKeyOf: (e) => e.ref.org
|
|
408
|
+
branchKeyOf: (e) => e.ref.org,
|
|
409
|
+
arrivesClosed: () => this.closeGeneration !== generation
|
|
343
410
|
})
|
|
344
411
|
));
|
|
345
412
|
}
|
|
@@ -558,6 +625,228 @@ var FileSystemRepository = class {
|
|
|
558
625
|
w.on("change", (p) => void this.handleFsChange(p, "change"));
|
|
559
626
|
w.on("unlink", (p) => void this.handleFsChange(p, "unlink"));
|
|
560
627
|
this.watcher = w;
|
|
628
|
+
this.startResync();
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Publish the `delete` face of an externally-observed removal.
|
|
632
|
+
*
|
|
633
|
+
* Extracted from `handleFsChange` unchanged so the reconciliation sweep
|
|
634
|
+
* (#9339) can reuse it **verbatim** rather than growing a second copy of the
|
|
635
|
+
* event shape. The one-line invariant: the caller already holds the per-key
|
|
636
|
+
* mutex, and `!currentHead` is the content-keyed suppression that makes our
|
|
637
|
+
* own `delete()` a no-op here.
|
|
638
|
+
*/
|
|
639
|
+
async publishExternalDelete(ref, key) {
|
|
640
|
+
const currentHead = this.heads.get(key) ?? null;
|
|
641
|
+
if (!currentHead) return;
|
|
642
|
+
this.heads.delete(key);
|
|
643
|
+
const seq = this.nextSeq++;
|
|
644
|
+
const evt = {
|
|
645
|
+
seq,
|
|
646
|
+
op: "delete",
|
|
647
|
+
ref: { ...ref, version: void 0 },
|
|
648
|
+
hash: null,
|
|
649
|
+
parentHash: currentHead,
|
|
650
|
+
actor: this.fsActor,
|
|
651
|
+
ts: this.now().toISOString(),
|
|
652
|
+
source: "fs"
|
|
653
|
+
};
|
|
654
|
+
await this.log.append(evt);
|
|
655
|
+
this.broker.publish(evt);
|
|
656
|
+
}
|
|
657
|
+
startResync() {
|
|
658
|
+
this.resyncEnabled = true;
|
|
659
|
+
this.scheduleResync();
|
|
660
|
+
}
|
|
661
|
+
stopResync() {
|
|
662
|
+
this.resyncEnabled = false;
|
|
663
|
+
if (this.resyncTimer) {
|
|
664
|
+
clearTimeout(this.resyncTimer);
|
|
665
|
+
this.resyncTimer = null;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* Schedule the next sweep — chained, never `setInterval` (#9339).
|
|
670
|
+
*
|
|
671
|
+
* A chained timeout cannot stack: the next sweep is armed only once the
|
|
672
|
+
* previous one has finished, so a saturated runner degrades to *fewer*
|
|
673
|
+
* sweeps instead of a growing backlog of overlapping tree walks. The timer
|
|
674
|
+
* is `unref`ed because a backstop must never be the reason a process stays
|
|
675
|
+
* alive.
|
|
676
|
+
*/
|
|
677
|
+
scheduleResync() {
|
|
678
|
+
if (!this.resyncEnabled || this.resyncTimer) return;
|
|
679
|
+
const timer = setTimeout(() => {
|
|
680
|
+
this.resyncTimer = null;
|
|
681
|
+
void this.resync().finally(() => this.scheduleResync());
|
|
682
|
+
}, RESYNC_INTERVAL_MS);
|
|
683
|
+
timer.unref?.();
|
|
684
|
+
this.resyncTimer = timer;
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Announce a sweep read that could not run — the non-silence half of #8895's
|
|
688
|
+
* "discriminate or propagate".
|
|
689
|
+
*
|
|
690
|
+
* ## Why `error` and not `warn`
|
|
691
|
+
*
|
|
692
|
+
* AGENTS.md decides the level with one question: *after the degradation, does
|
|
693
|
+
* the system still look "normal" from the outside while something it claims
|
|
694
|
+
* is persisted has not actually landed?* Here it does. Nothing throws, the
|
|
695
|
+
* watcher stays armed, `getWatched()` stays populated, `start()` succeeded —
|
|
696
|
+
* and the repository's index quietly stops tracking what is on disk. That is
|
|
697
|
+
* the rule's second limb verbatim ("persisted state and runtime state
|
|
698
|
+
* disagree"), not the functional-degradation limb: no capability is visibly
|
|
699
|
+
* smaller, so nobody finds out by using the missing thing.
|
|
700
|
+
*
|
|
701
|
+
* The counter-argument — *this is only a backstop, the watcher is still the
|
|
702
|
+
* fast path* — is why the level is arguable, and it does not survive the
|
|
703
|
+
* failing errno. The sharp case is fd exhaustion: EMFILE/ENFILE break this
|
|
704
|
+
* `readdir` and chokidar's `fs.watchFile` polling **at the same time and for
|
|
705
|
+
* the same reason**, so the fast path is not an independent fallback under
|
|
706
|
+
* precisely the load that produces this fault. A backstop that is silently
|
|
707
|
+
* absent whenever it is most needed is a durability-shaped degradation.
|
|
708
|
+
*
|
|
709
|
+
* ⚠️ AGENTS.md also warns against over-applying `error`, and the discipline
|
|
710
|
+
* that answers it is the ledger, not a quieter level: an `error` owes the
|
|
711
|
+
* consequence and the fix, said **once** at the first degradation rather than
|
|
712
|
+
* once per failed read. A sweep runs every 2s forever, so an unlatched
|
|
713
|
+
* `console.error` here would be the mirror-image failure the same rule names.
|
|
714
|
+
*
|
|
715
|
+
* ⛔ It deliberately does NOT throw. This runs on a background timer; taking
|
|
716
|
+
* a process down on a transient EACCES would be worse than the bug. The bar
|
|
717
|
+
* met here is non-silence, not propagation.
|
|
718
|
+
*
|
|
719
|
+
* The channel is `console.error` because this class has no logger: nothing is
|
|
720
|
+
* injected through `FileSystemRepositoryOptions`, and widening that public
|
|
721
|
+
* surface to carry one is out of scope for this fix.
|
|
722
|
+
*/
|
|
723
|
+
reportResyncFault(target, err) {
|
|
724
|
+
const code = err?.code ?? "UNKNOWN";
|
|
725
|
+
const key = `${code} @ ${target}`;
|
|
726
|
+
if (this.resyncFaults.has(key)) return;
|
|
727
|
+
this.resyncFaults.add(key);
|
|
728
|
+
console.error(
|
|
729
|
+
`[FileSystemRepository] metadata reconciliation sweep could not read ${target} (${code}). CONSEQUENCE: external edits under this path are no longer reconciled, so this repository's index and its watch() subscribers can drift from what is on disk while everything keeps reporting healthy. The chokidar watcher is not an independent fallback here \u2014 fd exhaustion degrades both. FIX: restore read access to the path; the sweep recovers by itself on the first successful read. Reported once per path and error code.`
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
/** Re-arm reporting for a path that reads again, so a recurrence is heard. */
|
|
733
|
+
clearResyncFault(target) {
|
|
734
|
+
if (this.resyncFaults.size === 0) return;
|
|
735
|
+
const suffix = ` @ ${target}`;
|
|
736
|
+
for (const key of this.resyncFaults) {
|
|
737
|
+
if (key.endsWith(suffix)) this.resyncFaults.delete(key);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
/**
|
|
741
|
+
* Content-keyed reconciliation sweep — the backstop that makes external-edit
|
|
742
|
+
* detection a guarantee rather than a single chance (#9339, #7282).
|
|
743
|
+
*
|
|
744
|
+
* ## Why the watcher alone cannot be the guarantee
|
|
745
|
+
*
|
|
746
|
+
* An external write to `<root>/<type>/<name>.json` reaches a subscriber only
|
|
747
|
+
* if chokidar notices it, and under `usePolling` it gets **exactly one**
|
|
748
|
+
* opportunity to do so: the write advances the type directory's mtime once,
|
|
749
|
+
* and chokidar re-reads a directory only when its stat *strictly advances*,
|
|
750
|
+
* so every later poll compares an unchanged stat and can never rediscover
|
|
751
|
+
* the file. Measured on #9339 with a fault-injection harness: with the one
|
|
752
|
+
* read suppressed, fifteen further poll ticks never find the new file, and a
|
|
753
|
+
* 20s deadline and a 200s deadline buy the same single attempt. That is the
|
|
754
|
+
* structural reason behind #7282's empirical finding that the event is
|
|
755
|
+
* "never delivered, not slow", and why widening the deadline (#7208) and
|
|
756
|
+
* lowering `interval` were both spent before they were tried.
|
|
757
|
+
*
|
|
758
|
+
* At least six independent one-shot gates sit on that single attempt,
|
|
759
|
+
* spanning three layers — the kernel timestamp (the directory mtime does not
|
|
760
|
+
* strictly advance), chokidar's readdir throttle and readdir snapshot, and
|
|
761
|
+
* chokidar's emit gates (`_throttle('add')`, a stale `_pendingWrites` entry,
|
|
762
|
+
* the `awaitWriteFinish` ENOENT early return). Each one produces a
|
|
763
|
+
* byte-identical observable: no event, ever, for that path.
|
|
764
|
+
*
|
|
765
|
+
* ## Why this shape, and not a narrower one
|
|
766
|
+
*
|
|
767
|
+
* ⚠️ The six are indistinguishable at the point of failure, so **any fix
|
|
768
|
+
* that has to name which gate fired is a fix for one member of a family** —
|
|
769
|
+
* which is exactly how #7282 was closed and exactly why it reopened. This
|
|
770
|
+
* sweep never asks. It compares what is on disk against `heads`, the index
|
|
771
|
+
* that already defines what this repository believes it holds, and publishes
|
|
772
|
+
* the divergence through the same `handleFsChange` the watcher feeds. It is
|
|
773
|
+
* therefore robust across all six *by construction*, and equally across a
|
|
774
|
+
* seventh nobody has found: the only property it relies on is that the bytes
|
|
775
|
+
* on disk stopped matching the index.
|
|
776
|
+
*
|
|
777
|
+
* `put()` is unaffected and keeps its direct registration (`trackWrittenPath`
|
|
778
|
+
* calls `watcher.add` and bypasses the whole chain, which is why the `put()`
|
|
779
|
+
* half of this family was already closed by #7336 and the external-write half
|
|
780
|
+
* was not).
|
|
781
|
+
*
|
|
782
|
+
* ## Cost, and why it is bounded
|
|
783
|
+
*
|
|
784
|
+
* One pass over `<root>/<type>/*.json` per sweep — the same walk `start()`
|
|
785
|
+
* already performs once — with no retry loop inside it and no work at all
|
|
786
|
+
* when nothing diverged. Sweeps are chained, so they cannot overlap; the
|
|
787
|
+
* timer is `unref`ed and dies with `close()`; and it is armed only alongside
|
|
788
|
+
* the watcher, so a `disableWatch` repository pays nothing.
|
|
789
|
+
*
|
|
790
|
+
* Discovery is by content, never by stat: a stat pre-filter would reintroduce
|
|
791
|
+
* a time key of exactly the kind this replaces.
|
|
792
|
+
*/
|
|
793
|
+
async resync() {
|
|
794
|
+
const root = this.layout.root;
|
|
795
|
+
let entries = [];
|
|
796
|
+
try {
|
|
797
|
+
entries = await fs2.readdir(root, { withFileTypes: true });
|
|
798
|
+
this.clearResyncFault(root);
|
|
799
|
+
} catch (err) {
|
|
800
|
+
if (!isEnoent(err)) this.reportResyncFault(root, err);
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
const onDisk = /* @__PURE__ */ new Set();
|
|
804
|
+
const unreadableTypes = /* @__PURE__ */ new Set();
|
|
805
|
+
for (const entry of entries) {
|
|
806
|
+
if (!entry.isDirectory()) continue;
|
|
807
|
+
if (entry.name.startsWith(".")) continue;
|
|
808
|
+
const dir = path3.join(root, entry.name);
|
|
809
|
+
let files = [];
|
|
810
|
+
try {
|
|
811
|
+
files = await fs2.readdir(dir);
|
|
812
|
+
this.clearResyncFault(dir);
|
|
813
|
+
} catch (err) {
|
|
814
|
+
if (!isEnoent(err)) {
|
|
815
|
+
this.reportResyncFault(dir, err);
|
|
816
|
+
unreadableTypes.add(entry.name);
|
|
817
|
+
}
|
|
818
|
+
continue;
|
|
819
|
+
}
|
|
820
|
+
for (const file of files) {
|
|
821
|
+
if (!file.endsWith(".json") || file.startsWith(".")) continue;
|
|
822
|
+
const abs = path3.join(dir, file);
|
|
823
|
+
const parsed = parseItemPath(this.layout, abs);
|
|
824
|
+
if (!parsed) continue;
|
|
825
|
+
const ref = {
|
|
826
|
+
org: this.org,
|
|
827
|
+
type: parsed.type,
|
|
828
|
+
name: parsed.name
|
|
829
|
+
};
|
|
830
|
+
const key = refKey(ref);
|
|
831
|
+
onDisk.add(key);
|
|
832
|
+
const before = this.heads.get(key);
|
|
833
|
+
await this.handleFsChange(abs, "add");
|
|
834
|
+
if (this.heads.get(key) !== before) {
|
|
835
|
+
this.trackWrittenPath(abs);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
for (const key of [...this.heads.keys()]) {
|
|
840
|
+
if (onDisk.has(key)) continue;
|
|
841
|
+
const ref = parseRefKey(key);
|
|
842
|
+
if (!ref) continue;
|
|
843
|
+
if (unreadableTypes.has(ref.type)) continue;
|
|
844
|
+
const file = itemPath(this.layout, ref.type, ref.name);
|
|
845
|
+
await this.mutex.run(key, async () => {
|
|
846
|
+
if (existsSync2(file)) return;
|
|
847
|
+
await this.publishExternalDelete(ref, key);
|
|
848
|
+
});
|
|
849
|
+
}
|
|
561
850
|
}
|
|
562
851
|
/**
|
|
563
852
|
* Translate a watcher event into a `MetadataEvent`, or drop it.
|
|
@@ -618,22 +907,7 @@ var FileSystemRepository = class {
|
|
|
618
907
|
const key = refKey(ref);
|
|
619
908
|
await this.mutex.run(key, async () => {
|
|
620
909
|
if (kind === "unlink") {
|
|
621
|
-
|
|
622
|
-
if (!currentHead2) return;
|
|
623
|
-
this.heads.delete(key);
|
|
624
|
-
const seq2 = this.nextSeq++;
|
|
625
|
-
const evt2 = {
|
|
626
|
-
seq: seq2,
|
|
627
|
-
op: "delete",
|
|
628
|
-
ref: { ...ref, version: void 0 },
|
|
629
|
-
hash: null,
|
|
630
|
-
parentHash: currentHead2,
|
|
631
|
-
actor: this.fsActor,
|
|
632
|
-
ts: this.now().toISOString(),
|
|
633
|
-
source: "fs"
|
|
634
|
-
};
|
|
635
|
-
await this.log.append(evt2);
|
|
636
|
-
this.broker.publish(evt2);
|
|
910
|
+
await this.publishExternalDelete(ref, key);
|
|
637
911
|
return;
|
|
638
912
|
}
|
|
639
913
|
const body = await readJson(absPath);
|