@augmentd-labs/canvas-edge 0.0.0-stage → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/LICENSE +661 -0
  2. package/NOTICE +114 -0
  3. package/README.md +76 -2
  4. package/bin/canvas-edge +2 -0
  5. package/bin/healthcheck.mjs +9 -0
  6. package/node_modules/canvas-stored/LICENSE +661 -0
  7. package/node_modules/canvas-stored/NOTICE +85 -0
  8. package/node_modules/canvas-stored/README.md +472 -0
  9. package/node_modules/canvas-stored/package.json +26 -0
  10. package/node_modules/canvas-stored/src/backends/BackendManager.js +60 -0
  11. package/node_modules/canvas-stored/src/backends/StorageBackend.js +72 -0
  12. package/node_modules/canvas-stored/src/backends/cacache/index.js +130 -0
  13. package/node_modules/canvas-stored/src/backends/canvas/index.js +722 -0
  14. package/node_modules/canvas-stored/src/backends/file/index.js +658 -0
  15. package/node_modules/canvas-stored/src/backends/gdrive/index.js +924 -0
  16. package/node_modules/canvas-stored/src/backends/http/index.js +47 -0
  17. package/node_modules/canvas-stored/src/backends/s3/index.js +25 -0
  18. package/node_modules/canvas-stored/src/cache/index.js +65 -0
  19. package/node_modules/canvas-stored/src/extractors/audio.js +53 -0
  20. package/node_modules/canvas-stored/src/extractors/image.js +90 -0
  21. package/node_modules/canvas-stored/src/extractors/index.js +41 -0
  22. package/node_modules/canvas-stored/src/extractors/text.js +65 -0
  23. package/node_modules/canvas-stored/src/index/index.js +360 -0
  24. package/node_modules/canvas-stored/src/index.js +2058 -0
  25. package/node_modules/canvas-stored/src/sync/JobQueue.js +0 -0
  26. package/node_modules/canvas-stored/src/sync/Ledger.js +138 -0
  27. package/node_modules/canvas-stored/src/sync/Mirror.js +995 -0
  28. package/node_modules/canvas-stored/src/sync/SyncQueue.js +111 -0
  29. package/node_modules/canvas-stored/src/sync/keys.js +140 -0
  30. package/node_modules/canvas-stored/src/sync/worker.js +52 -0
  31. package/node_modules/canvas-stored/src/utils/checksum.js +32 -0
  32. package/node_modules/canvas-stored/src/utils/common.js +37 -0
  33. package/node_modules/canvas-stored/src/utils/mime.js +107 -0
  34. package/node_modules/canvas-stored/src/utils/mount.js +99 -0
  35. package/package.json +49 -4
  36. package/src/EdgeClient.js +129 -0
  37. package/src/control.js +48 -0
  38. package/src/daemon.js +88 -0
  39. package/src/env.js +182 -0
  40. package/src/fuse-runtime.js +173 -0
  41. package/src/index.js +26 -0
  42. package/src/mirror-runtime.js +232 -0
  43. package/src/remote-config.js +69 -0
  44. package/src/runtime.js +61 -0
@@ -0,0 +1,2058 @@
1
+ import EventEmitter2 from 'eventemitter2';
2
+ import path from 'path';
3
+ import crypto from 'crypto';
4
+ import { createReadStream, createWriteStream, promises as fsp } from 'fs';
5
+ import { once } from 'events';
6
+ import { Readable } from 'stream';
7
+ import { pipeline } from 'stream/promises';
8
+ import Debug from 'debug';
9
+ import Cache from './cache/index.js';
10
+ import BackendManager from './backends/BackendManager.js';
11
+ import Index from './index/index.js';
12
+ import SyncQueue from './sync/SyncQueue.js';
13
+ import JobQueue from './sync/JobQueue.js';
14
+ import Ledger from './sync/Ledger.js';
15
+ import Mirror from './sync/Mirror.js';
16
+ import { isBuffer, isFile, isStream, resolveStoredPaths } from './utils/common.js';
17
+ import { checksumBuffer, formatId } from './utils/checksum.js';
18
+ import { detectMimeType, detectMimeFromHead } from './utils/mime.js';
19
+
20
+ const debug = Debug('stored');
21
+
22
+ const HEAD_BYTES = 4096;
23
+
24
+ // A filesystem move reaches the watcher as unlink(old) + add(new). Unlink
25
+ // processing is held back this long so an add carrying the same inode can
26
+ // claim it as a rename (in-place URL rewrite, no orphan churn). Expired
27
+ // holds process as genuine deletions.
28
+ const RENAME_WINDOW_MS = 1000;
29
+
30
+ // A copy/move writes and deletes through the same watched roots the file
31
+ // backends observe, so our own writes come back as add/unlink events. Affected
32
+ // keys are suppressed for the duration of the operation plus this grace period
33
+ // (chokidar's awaitWriteFinish alone is 200ms, and an unlink is held for
34
+ // RENAME_WINDOW_MS on top) — without it a move is seen a second time as a
35
+ // delete of the source, which downstream drops the location for.
36
+ const MOVE_SUPPRESS_MS = 3000;
37
+
38
+ export default class Stored extends EventEmitter2 {
39
+ #cache;
40
+ #backends;
41
+ #index;
42
+ #config;
43
+ #paths;
44
+ #syncQueue;
45
+ #extract; // optional injected metadata extractor: (source,{mimeType,key})→Promise<obj>
46
+ // `${backend}|${dev}:${ino}` → { data, timer } — unlink events held for
47
+ // RENAME_WINDOW_MS awaiting a same-inode add (see #handleFileEvent).
48
+ #pendingUnlinks = new Map();
49
+ // `${backend}:${key}` → refcount. Watcher events for these keys are ours
50
+ // (an in-flight copy/move) and must not be re-processed as external change.
51
+ #suppressedKeys = new Map();
52
+ // id → [{ sourceLocation, target, destKey }] — moves onto a `type:'remote'`
53
+ // backend, held until the SyncQueue confirms the destination write. The
54
+ // source is only removed once the copy is durable somewhere else.
55
+ #pendingMoves = new Map();
56
+ #retention = null;
57
+ #retainedDb = null;
58
+ #sweepTimer = null;
59
+
60
+ constructor(config = {}) {
61
+ // Wildcards (':' delimiter) let consumers bind `object:*` across backends.
62
+ super({ wildcard: true, delimiter: ':', maxListeners: 100, verboseMemoryLeak: false });
63
+ this.#paths = resolveStoredPaths(config);
64
+ this.#config = {
65
+ checksums: config.checksums || ['sha256'],
66
+ primaryChecksum: config.primaryChecksum || 'sha256',
67
+ ...config,
68
+ };
69
+ // Retention window for displaced bytes (overwrite/delete on a local
70
+ // file backend): `{ days }`; absent/false = off. See #retain().
71
+ const days = config.retention ? Number(config.retention.days ?? 30) : 0;
72
+ this.#retention = days > 0 ? { days, sweepEveryMs: Number(config.retention.sweepEveryMs ?? 60 * 60_000) } : null;
73
+
74
+ this.#extract = typeof config.extract === 'function' ? config.extract : null;
75
+
76
+ this.#cache = new Cache({ path: this.#paths.cache, algorithms: this.#config.checksums });
77
+ this.#backends = new BackendManager();
78
+ // The index's change log is re-emitted as `change` events (one per
79
+ // entry, after the transaction that produced them committed).
80
+ this.#index = new Index(this.#paths.index, {
81
+ changes: config.changes,
82
+ onChange: (entries) => { for (const entry of entries) this.emit('change', entry); },
83
+ });
84
+
85
+ // Background sync queue for remote backends: file targets are copied by
86
+ // a worker thread, network drivers (gdrive, …) are committed in-process
87
+ // through their live backend (async I/O — nothing to offload, and the
88
+ // driver's credentials/path cache stay on the main thread).
89
+ this.#syncQueue = new SyncQueue({ resolveBackend: (name) => this.#backends.get(name) });
90
+ this.#syncQueue.on('synced', ({ id, results }) => this.#handleSyncResult(id, results));
91
+ this.#syncQueue.on('error', (err) => this.emit('error', err));
92
+
93
+ debug('Stored initialized');
94
+ }
95
+
96
+ // ─────────────────────────────────────────────────────────────────────────
97
+ // Getters
98
+ // ─────────────────────────────────────────────────────────────────────────
99
+
100
+ get root() { return this.#paths.root; }
101
+ get paths() { return { ...this.#paths }; }
102
+ get dataPath() { return this.#paths.data; }
103
+ get cache() { return this.#cache; }
104
+ get index() { return this.#index; }
105
+ get backends() { return this.#backends; }
106
+
107
+ // ─────────────────────────────────────────────────────────────────────────
108
+ // Backend Management
109
+ // ─────────────────────────────────────────────────────────────────────────
110
+
111
+ addBackend(name, config) {
112
+ const backend = this.#backends.add(name, config);
113
+
114
+ backend.on('file:add', e => this.#handleFileEvent('file:add', e));
115
+ backend.on('file:change', e => this.#handleFileEvent('file:change', e));
116
+ backend.on('file:unlink', e => this.#handleFileEvent('file:unlink', e));
117
+ // Generic change events from non-file backends — forwarded as-is for
118
+ // consumers; they carry {backend, kind, key, ...}.
119
+ backend.on('object:add', e => this.emit('object:add', e));
120
+ backend.on('object:change', e => this.emit('object:change', e));
121
+ backend.on('object:unlink', e => this.emit('object:unlink', e));
122
+ backend.on('backend:state', e => this.emit('backend:state', e));
123
+ backend.on('scan:start', e => this.emit('scan:start', e));
124
+ backend.on('scan:complete', e => this.emit('scan:complete', e));
125
+ backend.on('error', e => this.emit('error', e));
126
+
127
+ if (config.watch) backend.watch();
128
+ return backend;
129
+ }
130
+
131
+ // Unregister a backend and drop its now-dangling locations from the index.
132
+ // Entries left with no locations are removed (the index is derived state —
133
+ // synapsd documents are durable and rebuildable via scan()).
134
+ async removeBackend(name) {
135
+ const removed = await this.#backends.remove(name);
136
+ if (removed) this.#removeMissingLocations(name, new Set());
137
+ return removed;
138
+ }
139
+
140
+ listBackends() { return this.#backends.list(); }
141
+ getBackend(name) { return this.#backends.get(name); }
142
+
143
+ /**
144
+ * Parse a canonical `stored://<backend>/<key>` URL. The backend name may
145
+ * itself contain colons (e.g. `workspace:data`), so we split on the first
146
+ * `/` after the scheme. Returns { backend, key } or null if malformed.
147
+ */
148
+ #parseStoredUrl(url) {
149
+ const prefix = 'stored://';
150
+ if (typeof url !== 'string' || !url.startsWith(prefix)) return null;
151
+ const rest = url.slice(prefix.length);
152
+ const slash = rest.indexOf('/');
153
+ if (slash < 0) return null;
154
+ return { backend: rest.slice(0, slash), key: rest.slice(slash + 1) };
155
+ }
156
+
157
+ // Fetch bytes directly by `stored://` URL, bypassing the content index.
158
+ // Returns Buffer | null (never throws on bad input).
159
+ async getByUrl(url) {
160
+ const p = this.#parseStoredUrl(url);
161
+ const backend = p && this.#backends.get(p.backend);
162
+ return backend ? backend.get(p.key) : null;
163
+ }
164
+
165
+ // Same as getByUrl but returns a Readable | null.
166
+ async getStreamByUrl(url) {
167
+ const p = this.#parseStoredUrl(url);
168
+ const backend = p && this.#backends.get(p.backend);
169
+ return backend ? backend.get(p.key, { stream: true }) : null;
170
+ }
171
+
172
+ /**
173
+ * Ranged stream for HTTP Range / media streaming. Returns
174
+ * `{ stream, ranged }` where `ranged` is true only when the backend actually
175
+ * served the requested byte window (`getRange`). Backends without ranged
176
+ * reads (e.g. remote proxies) fall back to a full stream with `ranged:false`,
177
+ * so the caller can respond 200 instead of a length-mismatched 206.
178
+ * `range` = { start, end } with an inclusive `end`. Null on a miss.
179
+ */
180
+ async getRangeStreamByUrl(url, range) {
181
+ const p = this.#parseStoredUrl(url);
182
+ const backend = p && this.#backends.get(p.backend);
183
+ if (!backend) return null;
184
+ if (typeof backend.getRange === 'function') {
185
+ const stream = await backend.getRange(p.key, range);
186
+ if (stream) return { stream, ranged: true };
187
+ }
188
+ const stream = await backend.get(p.key, { stream: true });
189
+ return stream ? { stream, ranged: false } : null;
190
+ }
191
+
192
+ /**
193
+ * Delete the bytes behind a `stored://<backend>/<key>` URL. Does not touch
194
+ * the consumer's document index (synapsd owns that) — callers trim
195
+ * `locations[]` — but DOES drop the location from stored's own index: an
196
+ * index that still claims bytes we just deleted makes a later `copy()` skip
197
+ * the target as "already there" and report success for a no-op.
198
+ * Returns { ok:boolean, reason?:'malformed-url'|'unknown-backend'|'read-only-backend' }.
199
+ */
200
+ async deleteByUrl(url) {
201
+ const p = this.#parseStoredUrl(url);
202
+ if (!p) return { ok: false, reason: 'malformed-url' };
203
+ const backend = this.#backends.get(p.backend);
204
+ if (!backend) return { ok: false, reason: 'unknown-backend' };
205
+ if (!backend.canDelete) return { ok: false, reason: 'read-only-backend' };
206
+ await this.#retain(p.backend, p.key, this.#index.get(`${p.backend}:${p.key}`));
207
+ const deleted = !!(await backend.delete(p.key));
208
+ if (deleted) this.#dropLocation(p.backend, p.key);
209
+ return { ok: deleted };
210
+ }
211
+
212
+ /**
213
+ * Forget one location we just deleted the bytes for. Emits
214
+ * `object:location:remove` when the content survives elsewhere; drops the
215
+ * whole entry when that was its last home.
216
+ */
217
+ #dropLocation(backendName, key) {
218
+ const meta = this.#index.get(`${backendName}:${key}`);
219
+ if (!meta) return;
220
+ const removed = (meta.locations || []).find(l => l.backend === backendName && l.key === key);
221
+ const remaining = (meta.locations || []).filter(l => !(l.backend === backendName && l.key === key));
222
+ if (remaining.length === 0) {
223
+ this.#index.delete(meta.id);
224
+ return;
225
+ }
226
+ const saved = this.#index.put(meta.id, { ...meta, locations: remaining });
227
+ if (removed) this.#emitLocationEvent('location:remove', saved, removed, { reason: 'deleted' });
228
+ }
229
+
230
+ // ─────────────────────────────────────────────────────────────────────────
231
+ // Core API — cache-first writes, cache-first reads
232
+ // ─────────────────────────────────────────────────────────────────────────
233
+
234
+ /**
235
+ * Store a blob. Accepts Buffer | string (content) | filesystem path |
236
+ * Readable. Paths and streams are ingested without ever materializing the
237
+ * whole blob in memory. `backends` is required (no implicit fan-out).
238
+ * Returns { ok:true, id, key, size, mimeType, checksums, locations, ... }
239
+ * or { ok:false, reason:'no-targets'|'unknown-backend' }.
240
+ */
241
+ async put(blob, options = {}) {
242
+ const { key, backends, metadata = {} } = options;
243
+ if (!Array.isArray(backends) || backends.length === 0) return { ok: false, reason: 'no-targets' };
244
+
245
+ const targets = backends
246
+ .map(name => ({ name, backend: this.#backends.get(name) }))
247
+ .filter(t => t.backend);
248
+ if (!targets.length) return { ok: false, reason: 'unknown-backend' };
249
+
250
+ const ingestOptions = { mtime: options.mtime ?? null };
251
+ const ingest = (isBuffer(blob) || (typeof blob === 'string' && !isFile(blob)))
252
+ ? await this.#ingestMemory(blob, targets, key, options.mimeType, ingestOptions)
253
+ : await this.#ingestStream(blob, targets, key, options.mimeType, ingestOptions);
254
+
255
+ // Caller-supplied metadata + inline-extracted (EXIF/GPS/dimensions/media)
256
+ // → persisted under `custom`, returned to caller + on the 'put' event.
257
+ const extracted = ingest.extracted && Object.keys(ingest.extracted).length ? ingest.extracted : null;
258
+ const meta = this.#index.put(ingest.id, {
259
+ checksums: ingest.checksums,
260
+ size: ingest.size,
261
+ mimeType: ingest.mimeType,
262
+ locations: ingest.locations,
263
+ custom: extracted ? { ...metadata, ...extracted } : metadata,
264
+ }, { origin: options.origin });
265
+
266
+ if (ingest.remoteTargets.length) {
267
+ this.#syncQueue.enqueue({ id: ingest.id, cacheRoot: this.#cache.root, cacheKey: ingest.id, targets: ingest.remoteTargets });
268
+ }
269
+
270
+ this.emit('put', { id: ingest.id, key: ingest.finalKey, metadata: meta });
271
+ debug(`PUT ${ingest.id.slice(0, 19)}... → ${targets.map(t => t.name).join(', ')}`);
272
+ return { ok: true, key: ingest.finalKey, ...meta };
273
+ }
274
+
275
+ async get(idOrKey) { return this.#read(idOrKey, false); }
276
+ async getStream(idOrKey) { return this.#read(idOrKey, true); }
277
+
278
+ async delete(idOrKey, options = {}) {
279
+ const meta = this.#index.get(idOrKey);
280
+ if (!meta) return { ok: false, reason: 'not-found' };
281
+
282
+ this.#cache.delete(meta.id).catch(() => {});
283
+
284
+ let targets = meta.locations || [];
285
+ if (Array.isArray(options.urls)) {
286
+ const set = new Set(options.urls);
287
+ targets = targets.filter(l => set.has(`stored://${l.backend}/${l.key}`));
288
+ } else if (Array.isArray(options.backends)) {
289
+ targets = targets.filter(l => options.backends.includes(l.backend));
290
+ }
291
+
292
+ const deleted = [];
293
+ const removed = new Set();
294
+ for (const loc of targets) {
295
+ const backend = this.#backends.get(loc.backend);
296
+ if (backend) await this.#retain(loc.backend, loc.key, meta);
297
+ if (backend && await backend.delete(loc.key)) {
298
+ deleted.push(loc.backend);
299
+ removed.add(loc);
300
+ }
301
+ }
302
+
303
+ const remaining = (meta.locations || []).filter(l => !removed.has(l));
304
+ if (remaining.length === 0) {
305
+ this.#index.delete(meta.id);
306
+ } else {
307
+ meta.locations = remaining;
308
+ this.#index.put(meta.id, meta);
309
+ // Content survives elsewhere — consumers patch locations[] in place
310
+ // rather than treating this as the object going away.
311
+ for (const loc of removed) this.#emitLocationEvent('location:remove', meta, loc, { reason: 'deleted' });
312
+ }
313
+
314
+ this.emit('delete', { id: meta.id, backends: deleted });
315
+ return { ok: true, deleted, kept: remaining.map(l => l.backend) };
316
+ }
317
+
318
+ async stat(idOrKey) { return this.#index.get(idOrKey); }
319
+ async has(idOrKey) { return this.#index.has(idOrKey); }
320
+
321
+ // ─────────────────────────────────────────────────────────────────────────
322
+ // Change log + listing — the feed remote mirrors tail
323
+ // ─────────────────────────────────────────────────────────────────────────
324
+
325
+ /** See Index.changes(): `{ changes, head, oldest, cursor, cursorTooOld }`. */
326
+ changes(options = {}) { return this.#index.changes(options); }
327
+ head() { return this.#index.head(); }
328
+ oldest() { return this.#index.oldest(); }
329
+ trimChanges(options = {}) { return this.#index.trimChanges(options); }
330
+
331
+ /** Page one backend's indexed keys in key order: `{ objects, cursor }`. */
332
+ listObjects(backendName, options = {}) {
333
+ if (!this.#backends.get(backendName)) return { objects: [], cursor: null, reason: 'unknown-backend' };
334
+ return this.#index.locationsByBackend(backendName, options);
335
+ }
336
+
337
+ /**
338
+ * Suppress watcher echoes for keys an external caller is about to write
339
+ * or delete through the backend directly. Returns the release function.
340
+ */
341
+ holdKeys(keys) { return this.#holdKeys(Array.isArray(keys) ? keys : [keys]); }
342
+
343
+ // ─────────────────────────────────────────────────────────────────────────
344
+ // Keyed object writes — precondition-checked, succession-preserving
345
+ // ─────────────────────────────────────────────────────────────────────────
346
+
347
+ /**
348
+ * Write `source` (Buffer | string content | file path | Readable) to
349
+ * `backend:key`, replacing whatever is there, with HTTP-style preconditions
350
+ * evaluated against the index right before the swap:
351
+ * - `ifMatch`: the sha256 (or content id) the caller believes is there
352
+ * - `ifNoneMatch: '*'`: the key must be free
353
+ * - `sha256`: what the bytes must hash to (a corrupted upload is refused)
354
+ * The bytes are staged on the backend's own filesystem and renamed into
355
+ * place, then the SAME watcher path a local edit takes is fed with the
356
+ * resulting stat — so consumers see one `object:add` (with `previous` on
357
+ * an edit) exactly as if the file had been dropped into the folder, and
358
+ * the backend's own watcher echo is suppressed.
359
+ *
360
+ * Returns `{ ok:true, id, sha256, size, mtime, seq, previous, unchanged? }`
361
+ * or `{ ok:false, reason, ... }` — `reason:'precondition-failed'` carries
362
+ * `current` (`{ id, sha256, size, mtime }` | null) so callers can report
363
+ * what is actually there.
364
+ */
365
+ async writeObject(backendName, key, source, options = {}) {
366
+ const backend = this.#backends.get(backendName);
367
+ if (!backend) return { ok: false, reason: 'unknown-backend', backend: backendName };
368
+ if (!backend.canWrite) return { ok: false, reason: 'read-only-target', backend: backendName };
369
+ if (backend.type !== 'local' || typeof backend.commit !== 'function') {
370
+ return { ok: false, reason: 'unsupported-backend', backend: backendName };
371
+ }
372
+ const destKey = this.#normalizeKey(key, { nfc: true });
373
+ if (!destKey || !this.#isSafeKey(destKey)) return { ok: false, reason: 'invalid-key', key };
374
+ const live = await this.#verifyBackendRoot(backend);
375
+ if (!live.ok) return { ok: false, reason: 'target-offline', backend: backendName, detail: live.reason };
376
+
377
+ const pathKey = `${backendName}:${destKey}`;
378
+ const tempDir = backend.tempDir || path.join(this.#paths.cache, '.tmp');
379
+ const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
380
+ await fsp.mkdir(tempDir, { recursive: true });
381
+
382
+ try {
383
+ const stream = isBuffer(source) ? Readable.from([source])
384
+ : (typeof source === 'string' && !isFile(source)) ? Readable.from([Buffer.from(source)])
385
+ : (isStream(source) ? source : createReadStream(source));
386
+ const { checksums, size, head } = await this.#hashToFile(stream, tempPath);
387
+ const id = formatId(checksums, this.#config.primaryChecksum);
388
+ const sha256 = checksums.sha256 ?? null;
389
+ if (options.sha256 && sha256 && String(options.sha256).toLowerCase() !== sha256) {
390
+ return { ok: false, reason: 'checksum-mismatch', expected: String(options.sha256).toLowerCase(), actual: sha256 };
391
+ }
392
+
393
+ const current = this.#index.get(pathKey);
394
+ const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
395
+ const failed = this.#checkPrecondition(current, currentLoc, options);
396
+ if (failed) return failed;
397
+
398
+ if (current && current.id === id) {
399
+ // Same bytes already there — nothing to write, nothing to log.
400
+ let mtime = currentLoc?.mtime ?? null;
401
+ if (options.mtime != null && typeof backend.utimes === 'function') {
402
+ await backend.utimes(destKey, options.mtime).catch(() => {});
403
+ const st = await backend.stat(destKey).catch(() => null);
404
+ if (st?.modified != null && currentLoc) {
405
+ mtime = st.modified;
406
+ this.#index.put(current.id, {
407
+ ...current,
408
+ locations: current.locations.map(l => (l === currentLoc ? { ...l, mtime: st.modified, size: st.size } : l)),
409
+ });
410
+ }
411
+ }
412
+ return { ok: true, unchanged: true, id, sha256, checksums, size: current.size, mtime, seq: this.#index.head(), previous: null };
413
+ }
414
+
415
+ const mimeType = options.mimeType || await detectMimeFromHead(head, destKey);
416
+ const release = this.#holdKeys([pathKey]);
417
+ try {
418
+ if (current) await this.#retain(backendName, destKey, current);
419
+ let placed = null;
420
+ if (typeof backend.renameFrom === 'function') {
421
+ try { placed = await backend.renameFrom(destKey, tempPath); }
422
+ catch (err) { if (err.code !== 'EXDEV') throw err; }
423
+ }
424
+ if (!placed) {
425
+ await backend.commit(destKey, tempPath);
426
+ placed = await backend.stat(destKey).catch(() => null);
427
+ }
428
+ if (options.mtime != null && typeof backend.utimes === 'function') {
429
+ await backend.utimes(destKey, options.mtime).catch(() => {});
430
+ placed = (await backend.stat(destKey).catch(() => null)) || placed;
431
+ }
432
+ const absPath = typeof backend.resolveKeyPath === 'function' ? backend.resolveKeyPath(destKey) : null;
433
+ this.#handleFileEvent(current ? 'file:change' : 'file:add', {
434
+ backend: backendName,
435
+ key: destKey,
436
+ path: absPath,
437
+ checksums,
438
+ mimeType,
439
+ size,
440
+ modified: placed?.modified,
441
+ dev: placed?.dev,
442
+ ino: placed?.ino,
443
+ ...(options.origin ? { origin: options.origin } : {}),
444
+ }, { force: true });
445
+
446
+ const saved = this.#index.get(id);
447
+ const loc = saved?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
448
+ debug(`WRITE ${pathKey} ← ${id.slice(0, 19)}...${current ? ` (replaces ${current.id.slice(0, 19)}...)` : ''}`);
449
+ return {
450
+ ok: true,
451
+ id,
452
+ sha256,
453
+ checksums,
454
+ size,
455
+ mimeType,
456
+ mtime: loc?.mtime ?? placed?.modified ?? null,
457
+ seq: this.#index.head(),
458
+ previous: current ? { id: current.id, checksums: current.checksums } : null,
459
+ };
460
+ } finally {
461
+ release();
462
+ }
463
+ } finally {
464
+ await fsp.rm(tempPath, { force: true }).catch(() => {});
465
+ }
466
+ }
467
+
468
+ // ─────────────────────────────────────────────────────────────────────────
469
+ // Retention — displaced bytes stay recoverable for a window
470
+ // ─────────────────────────────────────────────────────────────────────────
471
+ //
472
+ // Every path that overwrites or deletes bytes on a local file backend
473
+ // (writeObject over an existing key, removeObject, remove()/deleteByUrl,
474
+ // an `onConflict:'overwrite'` transfer) first asks the backend to retain
475
+ // the current bytes under their sha256. The `retained` sub-db records
476
+ // when and from where; `sweepRetained` drops entries older than the
477
+ // window. Digest-addressed, so the same bytes displaced from ten keys are
478
+ // kept once. Off unless Stored was given `retention: { days }`.
479
+
480
+ get retention() { return this.#retention ? { ...this.#retention } : null; }
481
+
482
+ get #retained() {
483
+ if (!this.#retainedDb) this.#retainedDb = this.#index.openDB('retained');
484
+ return this.#retainedDb;
485
+ }
486
+
487
+ async #retain(backendName, key, meta) {
488
+ if (!this.#retention || !meta) return null;
489
+ const backend = this.#backends.get(backendName);
490
+ if (!backend || backend.type !== 'local' || typeof backend.retain !== 'function') return null;
491
+ const sha256 = meta.checksums?.sha256 ? String(meta.checksums.sha256).toLowerCase() : null;
492
+ if (!sha256) return null;
493
+ try {
494
+ const kept = await backend.retain(key, sha256);
495
+ if (!kept) return null;
496
+ const now = Date.now();
497
+ const prev = this.#retained.get(sha256) || null;
498
+ const keys = [...new Set([...(prev?.keys || []), `${backendName}:${key}`])].slice(-20);
499
+ const entry = {
500
+ sha256,
501
+ backend: prev?.backend || backendName,
502
+ size: kept.size ?? prev?.size ?? meta.size ?? null,
503
+ mimeType: meta.mimeType ?? prev?.mimeType ?? null,
504
+ id: meta.id ?? prev?.id ?? null,
505
+ keys,
506
+ firstAt: prev?.firstAt ?? now,
507
+ lastAt: now,
508
+ };
509
+ this.#retained.putSync(sha256, entry);
510
+ this.#scheduleSweep();
511
+ return entry;
512
+ } catch (err) {
513
+ debug(`retain ${backendName}:${key} failed: ${err.message}`);
514
+ this.emit('error', Object.assign(new Error(`retention failed for ${backendName}:${key}: ${err.message}`), { code: 'RETAIN_FAILED' }));
515
+ return null;
516
+ }
517
+ }
518
+
519
+ #scheduleSweep() {
520
+ if (this.#sweepTimer || !this.#retention) return;
521
+ this.#sweepTimer = setInterval(() => { this.sweepRetained().catch(() => {}); }, this.#retention.sweepEveryMs);
522
+ this.#sweepTimer.unref?.();
523
+ }
524
+
525
+ /** Retained entries, newest displaced first: `[{ sha256, backend, size, mimeType, id, keys, firstAt, lastAt }]`. */
526
+ listRetained({ backend = null, key = null, limit = 1000 } = {}) {
527
+ if (!this.#retention) return [];
528
+ const out = [];
529
+ for (const { value } of this.#retained.getRange()) {
530
+ if (!value?.sha256) continue;
531
+ if (backend && value.backend !== backend && !value.keys?.some((k) => k.startsWith(`${backend}:`))) continue;
532
+ if (key && !value.keys?.some((k) => k === key || k.endsWith(`:${key}`))) continue;
533
+ out.push(value);
534
+ }
535
+ out.sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0));
536
+ return out.slice(0, limit);
537
+ }
538
+
539
+ getRetained(sha256) {
540
+ if (!this.#retention) return null;
541
+ return this.#retained.get(String(sha256).toLowerCase()) || null;
542
+ }
543
+
544
+ /** Readable stream of retained bytes, or null. */
545
+ async retainedStream(sha256) {
546
+ const entry = this.getRetained(sha256);
547
+ const backend = entry && this.#backends.get(entry.backend);
548
+ if (!backend || typeof backend.retainedPath !== 'function') return null;
549
+ const p = backend.retainedPath(entry.sha256);
550
+ if (!await fsp.stat(p).catch(() => null)) return null;
551
+ return createReadStream(p);
552
+ }
553
+
554
+ /**
555
+ * Put retained bytes back at `backend:key` through the ordinary keyed
556
+ * write (succession, change log, preconditions). `ifNoneMatch:'*'` refuses
557
+ * an occupied key; pass `ifMatch` (digest) to replace a known current.
558
+ */
559
+ async restoreRetained(sha256, { backend, key, ifMatch = null, ifNoneMatch = null, origin = null, mtime = null } = {}) {
560
+ const entry = this.getRetained(sha256);
561
+ if (!entry) return { ok: false, reason: 'not-found', sha256 };
562
+ const stream = await this.retainedStream(entry.sha256);
563
+ if (!stream) return { ok: false, reason: 'not-found', sha256, detail: 'retained bytes missing on disk' };
564
+ const target = backend || entry.backend;
565
+ const targetKey = key || entry.keys?.at(-1)?.slice(target.length + 1) || null;
566
+ if (!targetKey) return { ok: false, reason: 'invalid-key', key };
567
+ const res = await this.writeObject(target, targetKey, stream, {
568
+ sha256: entry.sha256, mimeType: entry.mimeType || undefined,
569
+ ...(ifMatch != null ? { ifMatch } : {}), ...(ifNoneMatch != null ? { ifNoneMatch } : {}),
570
+ ...(origin ? { origin } : {}), ...(mtime != null ? { mtime } : {}),
571
+ });
572
+ return res.ok ? { ...res, key: targetKey, backend: target, restored: entry.sha256 } : res;
573
+ }
574
+
575
+ /** Drop entries (and their bytes) displaced longer than the window ago. Returns `{ swept, kept }`. */
576
+ async sweepRetained({ olderThanMs = null, now = Date.now() } = {}) {
577
+ if (!this.#retention) return { swept: 0, kept: 0 };
578
+ const window = olderThanMs ?? this.#retention.days * 86_400_000;
579
+ let swept = 0;
580
+ let kept = 0;
581
+ for (const { key: sha, value } of [...this.#retained.getRange()]) {
582
+ if (!value?.sha256) continue;
583
+ if ((now - (value.lastAt ?? 0)) < window) { kept += 1; continue; }
584
+ const backend = this.#backends.get(value.backend);
585
+ if (backend && typeof backend.dropRetained === 'function') await backend.dropRetained(value.sha256).catch(() => {});
586
+ this.#retained.removeSync(sha);
587
+ swept += 1;
588
+ }
589
+ if (swept) debug(`retention sweep: ${swept} dropped, ${kept} kept`);
590
+ return { swept, kept };
591
+ }
592
+
593
+ async forgetRetained(sha256) {
594
+ const entry = this.getRetained(sha256);
595
+ if (!entry) return false;
596
+ const backend = this.#backends.get(entry.backend);
597
+ if (backend && typeof backend.dropRetained === 'function') await backend.dropRetained(entry.sha256).catch(() => {});
598
+ this.#retained.removeSync(entry.sha256);
599
+ return true;
600
+ }
601
+
602
+ /**
603
+ * Delete the bytes at `backend:key` (precondition-checked like
604
+ * writeObject) and process it as the genuine unlink it is: consumers get
605
+ * `object:unlink`, the change log gets a `delete`.
606
+ */
607
+ async removeObject(backendName, key, options = {}) {
608
+ const backend = this.#backends.get(backendName);
609
+ if (!backend) return { ok: false, reason: 'unknown-backend', backend: backendName };
610
+ if (!backend.canDelete) return { ok: false, reason: 'read-only-backend', backend: backendName };
611
+ const destKey = this.#normalizeKey(key, { nfc: true });
612
+ if (!destKey || !this.#isSafeKey(destKey)) return { ok: false, reason: 'invalid-key', key };
613
+
614
+ const pathKey = `${backendName}:${destKey}`;
615
+ const current = this.#index.get(pathKey);
616
+ const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
617
+ if (!current) return { ok: false, reason: 'not-found', key: destKey };
618
+ const failed = this.#checkPrecondition(current, currentLoc, options);
619
+ if (failed) return failed;
620
+
621
+ const release = this.#holdKeys([pathKey]);
622
+ try {
623
+ // false = bytes already gone; the location still goes. A remote
624
+ // driver re-evaluates the precondition at the other end (412 →
625
+ // typed error, nothing is unlinked here).
626
+ await this.#retain(backendName, destKey, current);
627
+ await backend.delete(destKey, { ifMatch: options.ifMatch, origin: options.origin });
628
+ this.#processUnlink({ backend: backendName, key: destKey, ...(options.origin ? { origin: options.origin } : {}) });
629
+ debug(`REMOVE ${pathKey}`);
630
+ return { ok: true, id: current.id, sha256: current.checksums?.sha256 ?? null, seq: this.#index.head() };
631
+ } finally {
632
+ release();
633
+ }
634
+ }
635
+
636
+ /**
637
+ * Rename `backend:from` to `backend:to` on the same backend: one
638
+ * `rename(2)` where the driver supports it, identity and inode intact,
639
+ * `object:move` for consumers, a single `rename` change-log entry.
640
+ * Refuses an occupied destination (`target-exists`).
641
+ */
642
+ async renameObject(backendName, from, to, options = {}) {
643
+ const backend = this.#backends.get(backendName);
644
+ if (!backend) return { ok: false, reason: 'unknown-backend', backend: backendName };
645
+ const fromKey = this.#normalizeKey(from, { nfc: true });
646
+ const toKey = this.#normalizeKey(to, { nfc: true });
647
+ if (!fromKey || !this.#isSafeKey(fromKey)) return { ok: false, reason: 'invalid-key', key: from };
648
+ if (!toKey || !this.#isSafeKey(toKey)) return { ok: false, reason: 'invalid-key', key: to };
649
+ if (fromKey === toKey) return { ok: false, reason: 'same-key', key: fromKey };
650
+
651
+ const current = this.#index.get(`${backendName}:${fromKey}`);
652
+ const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === fromKey) || null;
653
+ if (!current) return { ok: false, reason: 'not-found', key: fromKey };
654
+ const failed = this.#checkPrecondition(current, currentLoc, options);
655
+ if (failed) return failed;
656
+ if (this.#index.get(`${backendName}:${toKey}`) || await backend.stat(toKey).catch(() => null)) {
657
+ return { ok: false, reason: 'target-exists', key: toKey };
658
+ }
659
+
660
+ const result = await this.move(`${backendName}:${fromKey}`, {
661
+ to: backendName,
662
+ key: toKey,
663
+ from: `stored://${backendName}/${fromKey}`,
664
+ onConflict: 'error',
665
+ origin: options.origin,
666
+ ifMatch: options.ifMatch,
667
+ });
668
+ if (!result.ok) return result;
669
+ return {
670
+ ok: true,
671
+ id: current.id,
672
+ sha256: current.checksums?.sha256 ?? null,
673
+ from: fromKey,
674
+ to: toKey,
675
+ state: result.state,
676
+ seq: this.#index.head(),
677
+ };
678
+ }
679
+
680
+ // `If-Match` / `If-None-Match` semantics over the index. Returns the typed
681
+ // failure (with what is actually there) or null when the write may go on.
682
+ #checkPrecondition(current, currentLoc, { ifMatch = null, ifNoneMatch = null } = {}) {
683
+ const strip = (v) => String(v).trim().replace(/^W\//, '').replace(/^"|"$/g, '').toLowerCase();
684
+ const describe = () => ({
685
+ ok: false,
686
+ reason: 'precondition-failed',
687
+ code: 'PRECONDITION_FAILED',
688
+ current: current ? {
689
+ id: current.id,
690
+ sha256: current.checksums?.sha256 ?? null,
691
+ size: currentLoc?.size ?? current.size ?? null,
692
+ mtime: currentLoc?.mtime ?? null,
693
+ } : null,
694
+ });
695
+ if (ifNoneMatch != null && String(ifNoneMatch).trim() === '*' && current) return describe();
696
+ if (ifMatch != null) {
697
+ if (!current) return describe();
698
+ const want = strip(ifMatch);
699
+ const have = String(current.checksums?.sha256 || '').toLowerCase();
700
+ if (want !== '*' && want !== have && want !== String(current.id).toLowerCase()) return describe();
701
+ }
702
+ return null;
703
+ }
704
+
705
+ // Collapse duplicate slashes, trim edge slashes; optionally NFC-normalize
706
+ // (the canonical spelling for keys that arrive over the wire from other
707
+ // platforms — macOS reports NFD).
708
+ #normalizeKey(key, { nfc = false } = {}) {
709
+ let out = String(key ?? '').replace(/\/{2,}/g, '/').replace(/^\/+|\/+$/g, '');
710
+ if (nfc) out = out.normalize('NFC');
711
+ return out;
712
+ }
713
+
714
+ // A key must stay inside the backend root: no empty, '.' or '..' segments,
715
+ // no NUL, no absolute/backslash-absolute forms.
716
+ #isSafeKey(key) {
717
+ if (typeof key !== 'string' || !key.length || key.includes('\0')) return false;
718
+ if (path.isAbsolute(key) || /^[a-zA-Z]:[\\/]/.test(key)) return false;
719
+ return key.split('/').every(seg => seg.length > 0 && seg !== '.' && seg !== '..');
720
+ }
721
+
722
+ // ─────────────────────────────────────────────────────────────────────────
723
+ // Copy / move — location mutations that preserve content identity
724
+ // ─────────────────────────────────────────────────────────────────────────
725
+
726
+ /**
727
+ * Copy the bytes behind `idOrKey` onto one or more other backends. Content
728
+ * identity is unchanged: the same index entry simply gains locations.
729
+ *
730
+ * Bytes are streamed through a hashing pass and the result is checked
731
+ * against the indexed id before anything is committed — the stream is being
732
+ * paid for regardless, and it catches a source that rotted or was rewritten
733
+ * behind our back (returns `checksum-mismatch` rather than silently
734
+ * propagating the wrong bytes under the old id).
735
+ *
736
+ * @param {string} idOrKey Content id or `{backend}:{key}`
737
+ * @param {object} options
738
+ * @param {string|string[]} options.to Target backend name(s)
739
+ * @param {string} [options.key] Key on the targets (defaults to the source key)
740
+ * @param {string} [options.from] Source backend name or `stored://` URL
741
+ * @param {'error'|'rename'|'overwrite'} [options.onConflict='error'] What to do
742
+ * when other content already occupies the destination key: refuse (default),
743
+ * pick the next free `name-1.ext`, or replace it.
744
+ * @returns {Promise<{ ok:true, id, added:string[], locations } | { ok:false, reason:string }>}
745
+ */
746
+ async copy(idOrKey, options = {}) {
747
+ const plan = await this.#planTransfer(idOrKey, options);
748
+ if (!plan.ok) return plan;
749
+
750
+ const { meta, sourceLocation, targets, destKey } = plan;
751
+ if (!targets.length) {
752
+ return { ok: true, id: meta.id, added: [], unchanged: true, locations: await this.locations(meta.id) };
753
+ }
754
+
755
+ const release = this.#holdKeys(targets.map(t => `${t.name}:${t.key || destKey}`));
756
+ try {
757
+ const transferred = await this.#streamToTargets(meta, sourceLocation, targets, destKey, this.#transferOptions(options));
758
+ if (!transferred.ok) return transferred;
759
+ // A conflict-inbox upload placed nothing at the key: no location.
760
+ if (transferred.conflict && !transferred.locations.length) {
761
+ return { ok: true, id: meta.id, added: [], conflict: transferred.conflict, locations: await this.locations(meta.id) };
762
+ }
763
+
764
+ // Displace-then-record in one transaction: the change log must
765
+ // never show the new owner of a key before the old one let go.
766
+ const { saved, previous } = this.#index.transaction(() => {
767
+ const previous = this.#displaceTargets(targets, destKey, meta.id, options);
768
+ const current = this.#index.get(meta.id) || meta;
769
+ const saved = this.#index.put(meta.id, { ...current, locations: this.#mergeLocations(current, transferred.locations) }, { origin: options.origin });
770
+ return { saved, previous };
771
+ });
772
+ if (transferred.remoteTargets.length) {
773
+ this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
774
+ }
775
+
776
+ for (const loc of transferred.locations) {
777
+ this.#emitLocationEvent('location:add', saved, loc, previous ? { previous } : {});
778
+ }
779
+ debug(`COPY ${meta.id.slice(0, 19)}... → ${targets.map(t => t.name).join(', ')}/${destKey}`);
780
+ return {
781
+ ok: true,
782
+ id: meta.id,
783
+ added: transferred.locations.map(l => `stored://${l.backend}/${l.key}`),
784
+ locations: saved.locations.map(l => this.#describeLocation(l)),
785
+ ...(transferred.result ? { remote: transferred.result } : {}),
786
+ ...(transferred.conflict ? { conflict: transferred.conflict } : {}),
787
+ };
788
+ } finally {
789
+ release();
790
+ }
791
+ }
792
+
793
+ /**
794
+ * Move the bytes behind `idOrKey` to another backend: a copy followed by
795
+ * removal of the source location.
796
+ *
797
+ * Ordering is load-bearing — the source is never dropped until the
798
+ * destination is durable. For a `type:'remote'` target that means waiting
799
+ * for the SyncQueue, so the call returns `state:'pending'` and completes on
800
+ * the `synced` event; a failed sync leaves the object as a copy rather than
801
+ * losing it. Within one filesystem the whole transfer collapses to a single
802
+ * `rename(2)` (no bytes cross userspace, no re-hash — the inode is the
803
+ * proof of identity).
804
+ *
805
+ * Emits `object:move` on completion — consumers must patch `locations[]` in
806
+ * place, NOT unlink + re-add, or the document loses its identity.
807
+ *
808
+ * @returns {Promise<{ ok:true, id, state:'complete'|'pending', from, to, locations } | { ok:false, reason:string }>}
809
+ */
810
+ async move(idOrKey, options = {}) {
811
+ const plan = await this.#planTransfer(idOrKey, options);
812
+ if (!plan.ok) return plan;
813
+
814
+ const { meta, sourceLocation, sourceBackend, targets, destKey } = plan;
815
+ if (!targets.length) {
816
+ return { ok: true, id: meta.id, state: 'complete', unchanged: true, locations: await this.locations(meta.id) };
817
+ }
818
+ // Fan-out is `copy`'s job; a move has exactly one destination, otherwise
819
+ // "which one may the source be dropped for?" has no answer.
820
+ if (targets.length > 1) return { ok: false, reason: 'move-single-target' };
821
+ if (!sourceBackend.canDelete) return { ok: false, reason: 'source-not-removable', backend: sourceLocation.backend };
822
+
823
+ const target = targets[0];
824
+ // Conflict resolution may have renamed this target's key (see
825
+ // #planTransfer) — everything below addresses the resolved one.
826
+ const targetKey = target.key || destKey;
827
+ const release = this.#holdKeys([
828
+ `${sourceLocation.backend}:${sourceLocation.key}`,
829
+ `${target.name}:${targetKey}`,
830
+ ]);
831
+ try {
832
+ // Fast path — same filesystem, both file backends.
833
+ const renamed = await this.#tryRename(sourceBackend, sourceLocation.key, target.backend, targetKey);
834
+ if (renamed) {
835
+ // The source's folder may be empty now (a hub delete moves the
836
+ // last file of a folder to trash) — drop the shell like delete() does.
837
+ if (typeof sourceBackend.pruneEmptyParents === 'function') await sourceBackend.pruneEmptyParents(sourceLocation.key).catch(() => {});
838
+ const location = this.#buildLocation(target.name, targetKey, true, {
839
+ size: renamed.size, mtime: renamed.modified, dev: renamed.dev, ino: renamed.ino,
840
+ });
841
+ debug(`MOVE (rename) ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
842
+ return this.#index.transaction(() => {
843
+ const previous = this.#displaceTargets(targets, destKey, meta.id, options);
844
+ return this.#finalizeMove(meta.id, sourceLocation, location, { removeSourceBytes: false, origin: options.origin, previous });
845
+ });
846
+ }
847
+
848
+ // Same remote backend, new key: the driver renames server-side
849
+ // (one request, no bytes through here, identity intact).
850
+ if (sourceLocation.backend === target.name && sourceBackend.type === 'remote' && typeof sourceBackend.rename === 'function') {
851
+ const renamedRemote = await this.#tryRemoteRename(sourceBackend, sourceLocation, targetKey, options);
852
+ if (!renamedRemote.ok) return renamedRemote;
853
+ debug(`MOVE (remote rename) ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
854
+ return this.#index.transaction(() => {
855
+ const previous = this.#displaceTargets(targets, destKey, meta.id, options);
856
+ return this.#finalizeMove(meta.id, sourceLocation, renamedRemote.location, { removeSourceBytes: false, origin: options.origin, previous });
857
+ });
858
+ }
859
+
860
+ const transferred = await this.#streamToTargets(meta, sourceLocation, targets, destKey, this.#transferOptions(options));
861
+ if (!transferred.ok) return transferred;
862
+ const location = transferred.locations[0];
863
+
864
+ // Add-first: the destination location is recorded before the source
865
+ // is touched, so a crash here leaves a copy — never an orphan.
866
+ const { saved, previous } = this.#index.transaction(() => {
867
+ const previous = this.#displaceTargets(targets, destKey, meta.id, options);
868
+ const current = this.#index.get(meta.id) || meta;
869
+ const saved = this.#index.put(meta.id, { ...current, locations: this.#mergeLocations(current, transferred.locations) }, { origin: options.origin });
870
+ return { saved, previous };
871
+ });
872
+ this.#emitLocationEvent('location:add', saved, location, previous ? { previous } : {});
873
+
874
+ if (transferred.remoteTargets.length) {
875
+ const pending = this.#pendingMoves.get(meta.id) || [];
876
+ pending.push({ sourceLocation, target: target.name, destKey: targetKey });
877
+ this.#pendingMoves.set(meta.id, pending);
878
+ this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
879
+ debug(`MOVE (pending sync) ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
880
+ return {
881
+ ok: true,
882
+ id: meta.id,
883
+ state: 'pending',
884
+ from: this.#endpoint(sourceLocation),
885
+ to: this.#endpoint(location),
886
+ locations: saved.locations.map(l => this.#describeLocation(l)),
887
+ };
888
+ }
889
+
890
+ debug(`MOVE ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
891
+ return this.#finalizeMove(meta.id, sourceLocation, location, { removeSourceBytes: true, origin: options.origin });
892
+ } finally {
893
+ release();
894
+ }
895
+ }
896
+
897
+ /**
898
+ * Where does the content behind `idOrKey` actually live? Returns the
899
+ * canonical, resolvable `stored://<backend>/<key>` URLs plus per-location
900
+ * metadata. Single source of truth for the URL grammar — consumers map
901
+ * these straight into synapsd documents and read them back via getByUrl.
902
+ */
903
+ async locations(idOrKey) {
904
+ const meta = this.#index.get(idOrKey);
905
+ return (meta?.locations || []).map(l => this.#describeLocation(l));
906
+ }
907
+
908
+ #describeLocation(l) {
909
+ const backend = this.#backends.get(l.backend);
910
+ // Live backend first (a mount may have been re-detected since indexing),
911
+ // falling back to the snapshot on the location for offline backends.
912
+ const remote = backend ? backend.remote : (l.source?.remote === true);
913
+ return {
914
+ url: `stored://${l.backend}/${l.key}`,
915
+ nativeUrl: backend ? backend.nativeUrl(l.key) : null,
916
+ backend: l.backend,
917
+ key: l.key,
918
+ driver: l.driver,
919
+ synced: l.synced,
920
+ size: l.size,
921
+ remote,
922
+ transport: backend ? backend.transport : (l.source?.transport ?? null),
923
+ source: l.source,
924
+ };
925
+ }
926
+
927
+ // Iterate all indexed metadata entries.
928
+ async *list() {
929
+ for (const [, meta] of this.#index.entries()) yield meta;
930
+ }
931
+
932
+ // Iterate a single backend's native listing (raw keys, not index entries).
933
+ async *listBackend(name, options = {}) {
934
+ const backend = this.#backends.get(name);
935
+ if (backend) yield* backend.list(options);
936
+ }
937
+
938
+ // ─────────────────────────────────────────────────────────────────────────
939
+ // Scan / Index
940
+ // ─────────────────────────────────────────────────────────────────────────
941
+
942
+ /**
943
+ * Index existing objects from one backend (or all). `options` is forwarded
944
+ * verbatim to each backend's `scan()` so each driver defines its own knobs.
945
+ * File backends get a size+mtime skip predicate so unchanged files are not
946
+ * re-hashed. Returns { ok:true, backend, count, files } | { ok:false, reason }.
947
+ */
948
+ async scan(backendName, options = {}) {
949
+ const { onFile, ...backendOptions } = options;
950
+ const list = backendName
951
+ ? [this.#backends.get(backendName)].filter(Boolean)
952
+ : this.#backends.all();
953
+ if (backendName && !list.length) return { ok: false, reason: 'unknown-backend' };
954
+
955
+ const files = [];
956
+ let complete = true;
957
+ let errors = null;
958
+ for (const backend of list) {
959
+ // Per-file ingest into the stored index — idempotent, guarded so the
960
+ // streaming path (backend onFile) and the returned-rows path don't
961
+ // double-process a key.
962
+ const processed = new Set();
963
+ const ingest = (file) => {
964
+ if (processed.has(file.key)) return;
965
+ processed.add(file.key);
966
+ if (!file.checksums) return;
967
+ const id = formatId(file.checksums, this.#config.primaryChecksum);
968
+ const existing = this.#index.get(id);
969
+ const locations = existing?.locations || [];
970
+ const match = locations.find(l => l.backend === file.backend && l.key === file.key);
971
+ if (match) {
972
+ match.size = file.size;
973
+ match.mtime = file.modified;
974
+ if (file.ino != null) { match.ino = file.ino; match.dev = file.dev; }
975
+ } else {
976
+ locations.push(this.#buildLocation(file.backend, file.key, true, {
977
+ size: file.size,
978
+ mtime: file.modified,
979
+ // Inode identity — enables rename matching (same
980
+ // (dev,ino) at a new path is a move, not delete+add).
981
+ ...(file.ino != null ? { ino: file.ino, dev: file.dev } : {}),
982
+ }));
983
+ }
984
+ this.#index.put(id, { checksums: file.checksums, size: file.size, mimeType: file.mimeType, locations });
985
+ };
986
+
987
+ // Inode identity snapshot for rename matching: same (dev, ino) at a
988
+ // NEW path with unchanged size+mtime is a move — reuse the indexed
989
+ // checksums instead of rehashing, so a folder rename (thousands of
990
+ // "moves") collapses into cheap URL rewrites.
991
+ const inodeMap = this.#buildInodeMap(backend.name);
992
+
993
+ const result = await backend.scan({
994
+ algorithms: this.#config.checksums,
995
+ knownChecksums: (k, st) =>
996
+ this.#knownIfUnchanged(backend.name, k, st) || this.#knownByInode(inodeMap, k, st),
997
+ ...backendOptions,
998
+ // Stream-through: index the file, then hand it to the caller so
999
+ // consumers (workspace doc upserts) see results as the walk runs.
1000
+ onFile: async (file) => {
1001
+ ingest(file);
1002
+ if (typeof onFile === 'function') await onFile(file);
1003
+ },
1004
+ });
1005
+ // Snapshot shape { files, complete, errors } (file driver) or a bare
1006
+ // rows array (legacy/simple drivers — treated as a complete walk).
1007
+ // Backends whose scan() reports via events instead (non-array, no
1008
+ // files[]) are not content-addressable here.
1009
+ const snapshot = Array.isArray(result)
1010
+ ? { files: result, complete: true, errors: null }
1011
+ : (Array.isArray(result?.files) ? result : null);
1012
+ if (!snapshot) continue;
1013
+
1014
+ if (snapshot.complete === false) complete = false;
1015
+ if (snapshot.errors) errors = { ...(errors || {}), [backend.name]: snapshot.errors };
1016
+
1017
+ // Backends that ignore onFile still get their rows indexed here.
1018
+ for (const file of snapshot.files) ingest(file);
1019
+
1020
+ // Removal only ever runs against a usable snapshot: a dead/absent
1021
+ // root (errors.root) means the backend is offline — nothing is
1022
+ // removed. Hash-failed rows are still rows, so their keys count as
1023
+ // present; unreadable subtrees carry their prior entries forward.
1024
+ if (snapshot.errors?.root) continue;
1025
+ const presentKeys = new Set(snapshot.files.map(file => file.key));
1026
+ const erroredPrefixes = (snapshot.errors?.dirs || []).map(d => d.prefix).filter(Boolean);
1027
+ this.#removeMissingLocations(backend.name, presentKeys, erroredPrefixes);
1028
+ files.push(...snapshot.files);
1029
+ }
1030
+ return { ok: true, backend: backendName ?? null, count: files.length, files, complete, errors };
1031
+ }
1032
+
1033
+ /**
1034
+ * Cheap structural walk of one backend (dirs + file count, no hashing).
1035
+ * Returns { ok, dirs, files } or { ok:false } when the backend does not
1036
+ * support it (only file backends do).
1037
+ */
1038
+ async shape(backendName) {
1039
+ const backend = this.#backends.get(backendName);
1040
+ if (!backend) return { ok: false, reason: 'unknown-backend' };
1041
+ if (typeof backend.shape !== 'function') return { ok: false, reason: 'unsupported' };
1042
+ const { dirs, files } = await backend.shape();
1043
+ return { ok: true, dirs, files };
1044
+ }
1045
+
1046
+ // ─────────────────────────────────────────────────────────────────────────
1047
+ // Lifecycle
1048
+ // ─────────────────────────────────────────────────────────────────────────
1049
+
1050
+ async stop() {
1051
+ if (this.#sweepTimer) { clearInterval(this.#sweepTimer); this.#sweepTimer = null; }
1052
+ await this.#syncQueue.stop();
1053
+ await this.#backends.stopAll();
1054
+ // Flush held unlinks (watchers are stopped — no add can claim them now)
1055
+ // so the index reflects them before it closes.
1056
+ for (const [pendingKey, { data, timer }] of this.#pendingUnlinks) {
1057
+ clearTimeout(timer);
1058
+ this.#pendingUnlinks.delete(pendingKey);
1059
+ this.#processUnlink(data);
1060
+ }
1061
+ this.#index.close();
1062
+ debug('Stopped');
1063
+ }
1064
+
1065
+ // ─────────────────────────────────────────────────────────────────────────
1066
+ // Private — sync result handling
1067
+ // ─────────────────────────────────────────────────────────────────────────
1068
+
1069
+ #handleSyncResult(id, results) {
1070
+ const meta = this.#index.get(id);
1071
+ if (!meta) return;
1072
+
1073
+ for (const r of results) {
1074
+ if (!r.success) continue;
1075
+ const loc = meta.locations.find(l => l.backend === r.backend);
1076
+ if (loc) loc.synced = true;
1077
+ }
1078
+
1079
+ this.#index.put(meta.id, meta);
1080
+ this.emit('synced', { id, results });
1081
+ this.#resolvePendingMoves(id, results);
1082
+ }
1083
+
1084
+ // ─────────────────────────────────────────────────────────────────────────
1085
+ // Private — copy / move
1086
+ // ─────────────────────────────────────────────────────────────────────────
1087
+
1088
+ /**
1089
+ * Validate a copy/move request before a single byte is read: content must
1090
+ * exist, the source must be readable and online, every target must exist, be
1091
+ * writable and be online. Targets that already hold `destKey` are dropped
1092
+ * (a copy onto an existing location is a no-op, not an error).
1093
+ */
1094
+ async #planTransfer(idOrKey, options = {}) {
1095
+ const meta = this.#index.get(idOrKey);
1096
+ if (!meta) return { ok: false, reason: 'not-found' };
1097
+
1098
+ const onConflict = ['error', 'overwrite', 'rename'].includes(options.onConflict)
1099
+ ? options.onConflict
1100
+ : 'error';
1101
+
1102
+ const toNames = (Array.isArray(options.to) ? options.to : [options.to]).filter(Boolean).map(String);
1103
+ if (!toNames.length) return { ok: false, reason: 'no-targets' };
1104
+
1105
+ const sourceLocation = this.#resolveSourceLocation(meta, options.from);
1106
+ if (!sourceLocation) return { ok: false, reason: 'no-source' };
1107
+ const sourceBackend = this.#backends.get(sourceLocation.backend);
1108
+ const sourceLive = await this.#verifyBackendRoot(sourceBackend);
1109
+ if (!sourceLive.ok) {
1110
+ return { ok: false, reason: 'source-offline', backend: sourceLocation.backend, detail: sourceLive.reason };
1111
+ }
1112
+
1113
+ // Normalize the destination key: a template that leaves an empty
1114
+ // segment ('Fotky//2019') writes a file the OS reports back under the
1115
+ // collapsed path, and the watcher would then index it a SECOND time as
1116
+ // a new location. The index must spell keys the way the filesystem does.
1117
+ const destKey = this.#normalizeKey(options.key || sourceLocation.key);
1118
+ if (!destKey) return { ok: false, reason: 'invalid-key' };
1119
+ const conflictKey = options.conflictKey ? this.#normalizeKey(options.conflictKey) : null;
1120
+ const targets = [];
1121
+ for (const name of toNames) {
1122
+ const backend = this.#backends.get(name);
1123
+ if (!backend) return { ok: false, reason: 'unknown-backend', backend: name };
1124
+ if (!backend.canWrite) return { ok: false, reason: 'read-only-target', backend: name };
1125
+ if (name === sourceLocation.backend && destKey === sourceLocation.key) continue;
1126
+ // Already recorded there — but only skip if the bytes are REALLY
1127
+ // there. An index that outlived its file (deleted behind our back,
1128
+ // an unmounted share, a failed sync) must not turn a copy into a
1129
+ // silent no-op that reports success.
1130
+ const ours = (meta.locations || []).some(l => l.backend === name && l.key === destKey);
1131
+ const present = await backend.stat(destKey).catch(() => null);
1132
+ if (ours && present) continue;
1133
+ if (ours) debug(`Stale location ${name}:${destKey} (indexed, no bytes) — re-transferring`);
1134
+
1135
+ const live = await this.#verifyBackendRoot(backend);
1136
+ if (!live.ok) return { ok: false, reason: 'target-offline', backend: name, detail: live.reason };
1137
+
1138
+ // Somebody else's bytes are sitting on the destination key. Writing
1139
+ // over them is silent data loss, so it takes an explicit policy.
1140
+ let key = destKey;
1141
+ let overwrite = false;
1142
+ if (present && !ours && !this.#isSameContent(name, destKey, meta.id)) {
1143
+ if (onConflict === 'error') return { ok: false, reason: 'target-exists', backend: name, key: destKey };
1144
+ if (onConflict === 'rename') {
1145
+ // A caller-chosen conflict name (`… (conflict from laptop …).ext`)
1146
+ // wins when free; otherwise the generic `name-1.ext` ladder.
1147
+ const wanted = conflictKey && conflictKey !== destKey && !(await backend.stat(conflictKey).catch(() => null))
1148
+ ? conflictKey : null;
1149
+ key = wanted || await this.#freeKey(backend, destKey);
1150
+ debug(`Conflict on ${name}:${destKey} — using ${key}`);
1151
+ }
1152
+ // 'overwrite' keeps destKey and lets the commit replace it; the
1153
+ // displaced content is reconciled after the bytes landed.
1154
+ if (onConflict === 'overwrite') overwrite = true;
1155
+ }
1156
+ targets.push({ name, backend, key, overwrite });
1157
+ }
1158
+
1159
+ return { ok: true, meta, sourceLocation, sourceBackend, targets, destKey };
1160
+ }
1161
+
1162
+ /** Do the bytes already on `key` happen to BE this content? Then it is not a conflict. */
1163
+ #isSameContent(backendName, key, id) {
1164
+ return this.#index.get(`${backendName}:${key}`)?.id === id;
1165
+ }
1166
+
1167
+ /**
1168
+ * First free variant of `key`: `photo.jpg` → `photo-1.jpg` → `photo-2.jpg`.
1169
+ * Two photos taken in the same second produce the same templated name, and
1170
+ * silently overwriting one with the other is the worst possible outcome.
1171
+ */
1172
+ async #freeKey(backend, key) {
1173
+ const dot = key.lastIndexOf('.');
1174
+ const slash = key.lastIndexOf('/');
1175
+ const hasExt = dot > slash + 1;
1176
+ const stem = hasExt ? key.slice(0, dot) : key;
1177
+ const ext = hasExt ? key.slice(dot) : '';
1178
+
1179
+ for (let n = 1; n <= 1000; n += 1) {
1180
+ const candidate = `${stem}-${n}${ext}`;
1181
+ if (!(await backend.stat(candidate).catch(() => null))) return candidate;
1182
+ }
1183
+ throw new Error(`No free key for ${key} after 1000 attempts`);
1184
+ }
1185
+
1186
+ /**
1187
+ * Which location do we read from? An explicit `from` (backend name or
1188
+ * `stored://` URL) wins; otherwise the cheapest readable one — local disk
1189
+ * before a network mount, so a copy between two local backends never pulls
1190
+ * the bytes over the LAN just because the NAS location was indexed first.
1191
+ */
1192
+ #resolveSourceLocation(meta, from) {
1193
+ const locations = meta.locations || [];
1194
+ if (from) {
1195
+ const parsed = this.#parseStoredUrl(from);
1196
+ const match = parsed
1197
+ ? locations.find(l => l.backend === parsed.backend && l.key === parsed.key)
1198
+ : locations.find(l => l.backend === from);
1199
+ if (!match) return null;
1200
+ const backend = this.#backends.get(match.backend);
1201
+ return backend && backend.canRead && match.synced ? match : null;
1202
+ }
1203
+ return this.#readableLocations(meta)[0] || null;
1204
+ }
1205
+
1206
+ /**
1207
+ * Liveness gate. For file backends an absent mountpoint must fail the whole
1208
+ * transfer up front: writing into the empty directory where a NAS used to be
1209
+ * silently creates a second, divergent copy — and reading from one yields
1210
+ * "not found" for content that is merely offline.
1211
+ */
1212
+ async #verifyBackendRoot(backend) {
1213
+ if (!backend) return { ok: false, reason: 'unknown-backend' };
1214
+ if (typeof backend.verifyRoot !== 'function') return { ok: true };
1215
+ return backend.verifyRoot();
1216
+ }
1217
+
1218
+ /**
1219
+ * Stream source bytes onto the targets through one hashing pass, verifying
1220
+ * content identity before committing. The temp file is staged on the first
1221
+ * local target's filesystem so `#commit` can hardlink instead of copying.
1222
+ */
1223
+ async #streamToTargets(meta, sourceLocation, targets, destKey, options = {}) {
1224
+ const sourceBackend = this.#backends.get(sourceLocation.backend);
1225
+ // One remote target that speaks `putStream`: stream straight to it —
1226
+ // no temp file, no cache entry, no SyncQueue. The hub verifies the
1227
+ // digest (`X-Canvas-Sha256`) and answers the precondition itself.
1228
+ if (targets.length === 1 && targets[0].backend.type === 'remote' && typeof targets[0].backend.putStream === 'function') {
1229
+ return this.#streamToRemote(meta, sourceBackend, sourceLocation, targets[0], destKey, options);
1230
+ }
1231
+ const source = await sourceBackend.get(sourceLocation.key, { stream: true });
1232
+ if (!source) return { ok: false, reason: 'source-unreadable', backend: sourceLocation.backend, key: sourceLocation.key };
1233
+
1234
+ const firstLocal = targets.find(t => t.backend.type === 'local')?.backend;
1235
+ const tempDir = firstLocal ? firstLocal.tempDir : path.join(this.#paths.cache, '.tmp');
1236
+ const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
1237
+ await fsp.mkdir(tempDir, { recursive: true });
1238
+
1239
+ try {
1240
+ const { checksums, size } = await this.#hashToFile(source, tempPath);
1241
+ const id = formatId(checksums, this.#config.primaryChecksum);
1242
+ if (id !== meta.id) {
1243
+ debug(`Transfer aborted: ${sourceLocation.backend}:${sourceLocation.key} hashes to ${id}, indexed as ${meta.id}`);
1244
+ return { ok: false, reason: 'checksum-mismatch', expected: meta.id, actual: id };
1245
+ }
1246
+ const { locations, remoteTargets } = await this.#commit(
1247
+ targets, destKey, meta.id, { file: tempPath }, { checksums, size, mimeType: meta.mimeType }, options,
1248
+ );
1249
+ return { ok: true, locations, remoteTargets, size };
1250
+ } catch (err) {
1251
+ return { ok: false, reason: 'transfer-failed', error: err.message };
1252
+ } finally {
1253
+ await fsp.rm(tempPath, { force: true }).catch(() => {});
1254
+ }
1255
+ }
1256
+
1257
+ // Protocol options a copy/move forwards verbatim to a remote `putStream`.
1258
+ #transferOptions(options = {}) {
1259
+ const out = { mtime: options.mtime };
1260
+ for (const k of ['ifMatch', 'ifNoneMatch', 'origin', 'conflictOf', 'conflictMode', 'baseSha256', 'deviceName']) {
1261
+ if (options[k] != null) out[k] = options[k];
1262
+ }
1263
+ return out;
1264
+ }
1265
+
1266
+ // Typed driver failures → the `{ ok:false, reason }` vocabulary the
1267
+ // keyed-write API already speaks (412 → `precondition-failed` + `current`).
1268
+ #remoteFailure(err, backendName, key) {
1269
+ const code = err?.code || null;
1270
+ const base = { ok: false, code, backend: backendName, key, error: err?.message, status: err?.status ?? null };
1271
+ if (code === 'PRECONDITION_FAILED') return { ...base, reason: 'precondition-failed', current: err.current ?? null };
1272
+ if (code === 'TARGET_EXISTS') return { ...base, reason: 'target-exists' };
1273
+ if (code === 'NOT_FOUND') return { ...base, reason: 'not-found' };
1274
+ if (code === 'OFFLINE') return { ...base, reason: 'target-offline', detail: 'offline' };
1275
+ if (code === 'UNAUTHORIZED') return { ...base, reason: 'target-offline', detail: 'unauthorized' };
1276
+ return { ...base, reason: 'transfer-failed' };
1277
+ }
1278
+
1279
+ async #streamToRemote(meta, sourceBackend, sourceLocation, target, destKey, options = {}) {
1280
+ const targetKey = target.key || destKey;
1281
+ const source = await sourceBackend.get(sourceLocation.key, { stream: true });
1282
+ if (!source) return { ok: false, reason: 'source-unreadable', backend: sourceLocation.backend, key: sourceLocation.key };
1283
+ const sha256 = meta.checksums?.sha256 ?? null;
1284
+ try {
1285
+ const res = await target.backend.putStream(targetKey, source, {
1286
+ ...options,
1287
+ sha256,
1288
+ mtime: options.mtime ?? sourceLocation.mtime ?? null,
1289
+ mimeType: meta.mimeType || undefined,
1290
+ size: sourceLocation.size ?? meta.size ?? null,
1291
+ });
1292
+ if (res?.conflict) {
1293
+ // Inbox mode places nothing at the key; rename mode wrote an
1294
+ // ordinary object at the conflict key.
1295
+ const locations = options.conflictMode === 'rename'
1296
+ ? [this.#buildLocation(target.name, targetKey, true, {
1297
+ size: sourceLocation.size ?? meta.size, ...(sourceLocation.mtime != null ? { mtime: sourceLocation.mtime } : {}),
1298
+ ...(sha256 ? { ino: sha256, dev: target.backend.dev ?? null } : {}),
1299
+ })]
1300
+ : [];
1301
+ return { ok: true, locations, remoteTargets: [], size: meta.size, conflict: res };
1302
+ }
1303
+ if (sha256 && res?.sha256 && res.sha256 !== sha256) {
1304
+ return { ok: false, reason: 'checksum-mismatch', expected: meta.id, actual: `sha256:${res.sha256}` };
1305
+ }
1306
+ const location = this.#buildLocation(target.name, targetKey, true, {
1307
+ size: res?.size ?? sourceLocation.size ?? meta.size,
1308
+ ...(res?.mtime != null ? { mtime: res.mtime } : (sourceLocation.mtime != null ? { mtime: sourceLocation.mtime } : {})),
1309
+ ...(sha256 ? { ino: sha256, dev: target.backend.dev ?? null } : {}),
1310
+ });
1311
+ return { ok: true, locations: [location], remoteTargets: [], size: location.size, result: res };
1312
+ } catch (err) {
1313
+ source.destroy?.();
1314
+ debug(`Remote transfer to ${target.name}:${targetKey} failed: ${err.code || ''} ${err.message}`);
1315
+ return this.#remoteFailure(err, target.name, targetKey);
1316
+ }
1317
+ }
1318
+
1319
+ // Server-side rename on a remote driver; the location keeps its identity.
1320
+ async #tryRemoteRename(backend, sourceLocation, targetKey, options = {}) {
1321
+ try {
1322
+ const r = await backend.rename(sourceLocation.key, targetKey, { ifMatch: options.ifMatch, origin: options.origin });
1323
+ const location = this.#buildLocation(backend.name, targetKey, true, {
1324
+ size: r?.size ?? sourceLocation.size,
1325
+ ...((r?.modified ?? sourceLocation.mtime) != null ? { mtime: r?.modified ?? sourceLocation.mtime } : {}),
1326
+ ...(sourceLocation.ino != null ? { ino: sourceLocation.ino, dev: sourceLocation.dev } : {}),
1327
+ });
1328
+ return { ok: true, location, result: r };
1329
+ } catch (err) {
1330
+ return this.#remoteFailure(err, backend.name, sourceLocation.key);
1331
+ }
1332
+ }
1333
+
1334
+ /**
1335
+ * Same-filesystem shortcut: one `rename(2)` instead of reading and rewriting
1336
+ * the whole object. Returns the destination stat on success, or null when
1337
+ * the shortcut does not apply (different filesystems, non-file drivers) so
1338
+ * the caller falls back to a stream copy. No re-hash — the inode is carried
1339
+ * over intact, so the bytes are the same bytes by construction.
1340
+ */
1341
+ async #tryRename(sourceBackend, sourceKey, targetBackend, destKey) {
1342
+ if (typeof sourceBackend?.resolveKeyPath !== 'function') return null;
1343
+ if (typeof targetBackend?.renameFrom !== 'function' || !targetBackend.root) return null;
1344
+
1345
+ const sourcePath = sourceBackend.resolveKeyPath(sourceKey);
1346
+ const [sourceStat, targetStat] = await Promise.all([
1347
+ fsp.stat(sourcePath).catch(() => null),
1348
+ fsp.stat(targetBackend.root).catch(() => null),
1349
+ ]);
1350
+ if (!sourceStat || !targetStat || sourceStat.dev !== targetStat.dev) return null;
1351
+
1352
+ try {
1353
+ return await targetBackend.renameFrom(destKey, sourcePath);
1354
+ } catch (err) {
1355
+ if (err.code === 'EXDEV') return null; // mount table lied; copy instead
1356
+ throw err;
1357
+ }
1358
+ }
1359
+
1360
+ /**
1361
+ * Complete a move: drop the source bytes (unless a rename already relocated
1362
+ * them), swap the location in the index, emit `object:move`.
1363
+ *
1364
+ * A failed source delete is reported as `source-delete-failed` with
1365
+ * `degradedToCopy` — the content is safely at the destination, there is just
1366
+ * a stale second copy left behind. That is a cleanup problem, never data
1367
+ * loss, and the caller must be able to tell the two apart.
1368
+ */
1369
+ async #finalizeMove(id, sourceLocation, newLocation, { removeSourceBytes, origin = null, previous = null }) {
1370
+ const release = this.#holdKeys([`${sourceLocation.backend}:${sourceLocation.key}`]);
1371
+ try {
1372
+ if (removeSourceBytes) {
1373
+ const sourceBackend = this.#backends.get(sourceLocation.backend);
1374
+ const deleted = await sourceBackend.delete(sourceLocation.key).catch(() => false);
1375
+ if (!deleted) {
1376
+ return {
1377
+ ok: false,
1378
+ reason: 'source-delete-failed',
1379
+ degradedToCopy: true,
1380
+ id,
1381
+ from: this.#endpoint(sourceLocation),
1382
+ to: this.#endpoint(newLocation),
1383
+ };
1384
+ }
1385
+ }
1386
+
1387
+ // Re-read: a pending move completes on the sync callback, long after
1388
+ // the plan was made, and the entry may have gained locations since.
1389
+ const current = this.#index.get(id);
1390
+ if (!current) return { ok: false, reason: 'not-found' };
1391
+ const locations = this.#mergeLocations(current, [newLocation])
1392
+ .filter(l => !(l.backend === sourceLocation.backend && l.key === sourceLocation.key));
1393
+ // Same backend, new key = a rename for the change log (one entry,
1394
+ // `from` carried), not a delete + put pair.
1395
+ const rename = sourceLocation.backend === newLocation.backend
1396
+ ? { backend: newLocation.backend, from: sourceLocation.key, to: newLocation.key }
1397
+ : null;
1398
+ const saved = this.#index.put(id, { ...current, locations }, { origin, rename });
1399
+
1400
+ this.#emitObject('move', {
1401
+ id,
1402
+ checksums: saved.checksums,
1403
+ from: this.#endpoint(sourceLocation),
1404
+ to: this.#endpoint(newLocation),
1405
+ location: this.#describeLocation(newLocation),
1406
+ locations: saved.locations.map(l => this.#describeLocation(l)),
1407
+ ...(previous ? { previous } : {}),
1408
+ ...(origin ? { origin } : {}),
1409
+ });
1410
+
1411
+ return {
1412
+ ok: true,
1413
+ id,
1414
+ state: 'complete',
1415
+ from: this.#endpoint(sourceLocation),
1416
+ to: this.#endpoint(newLocation),
1417
+ locations: saved.locations.map(l => this.#describeLocation(l)),
1418
+ };
1419
+ } finally {
1420
+ release();
1421
+ }
1422
+ }
1423
+
1424
+ /** Finish moves that were waiting on a remote destination write. */
1425
+ #resolvePendingMoves(id, results = []) {
1426
+ const pending = this.#pendingMoves.get(id);
1427
+ if (!pending) return;
1428
+
1429
+ const succeeded = new Set(results.filter(r => r.success).map(r => r.backend));
1430
+ const failed = new Set(results.filter(r => !r.success).map(r => r.backend));
1431
+ const remaining = [];
1432
+
1433
+ for (const entry of pending) {
1434
+ if (succeeded.has(entry.target)) {
1435
+ const meta = this.#index.get(id);
1436
+ const location = (meta?.locations || []).find(l => l.backend === entry.target && l.key === entry.destKey);
1437
+ if (location) {
1438
+ this.#finalizeMove(id, entry.sourceLocation, location, { removeSourceBytes: true })
1439
+ .catch(err => this.emit('error', err));
1440
+ }
1441
+ } else if (failed.has(entry.target)) {
1442
+ // Destination write failed — the source stays exactly where it
1443
+ // is. The object is unharmed; the move simply did not happen.
1444
+ this.emit('error', Object.assign(
1445
+ new Error(`Move to ${entry.target} failed; source retained at ${entry.sourceLocation.backend}:${entry.sourceLocation.key}`),
1446
+ { id, code: 'MOVE_SYNC_FAILED' },
1447
+ ));
1448
+ } else {
1449
+ remaining.push(entry);
1450
+ }
1451
+ }
1452
+
1453
+ if (remaining.length) this.#pendingMoves.set(id, remaining);
1454
+ else this.#pendingMoves.delete(id);
1455
+ }
1456
+
1457
+ /**
1458
+ * After an `onConflict:'overwrite'` transfer landed, reconcile the content
1459
+ * that used to own each overwritten key: strip the stale location from
1460
+ * the displaced entry (drop the entry when that was its last home) and
1461
+ * tell consumers with the same succession vocabulary a watcher-observed
1462
+ * edit uses — `object:unlink` carrying `successor` — so the document
1463
+ * behind the old bytes is re-pointed or orphaned, never left claiming a
1464
+ * path it lost. Returns the first displaced `{ id, checksums }` (for the
1465
+ * `previous` hint on the follow-up add) or null.
1466
+ */
1467
+ #displaceTargets(targets, destKey, newId, options = {}) {
1468
+ let previous = null;
1469
+ for (const t of targets) {
1470
+ if (!t.overwrite) continue;
1471
+ const displaced = this.#displace(t.name, t.key || destKey, newId, options);
1472
+ if (displaced && !previous) previous = displaced;
1473
+ }
1474
+ return previous;
1475
+ }
1476
+
1477
+ #displace(backendName, key, newId, { origin = null } = {}) {
1478
+ const prev = this.#index.get(`${backendName}:${key}`);
1479
+ if (!prev || prev.id === newId) return null;
1480
+ const remaining = (prev.locations || []).filter(l => !(l.backend === backendName && l.key === key));
1481
+ if (remaining.length === 0) this.#index.delete(prev.id, { origin });
1482
+ else this.#index.put(prev.id, { ...prev, locations: remaining }, { origin });
1483
+ this.#emitObject('unlink', {
1484
+ backend: backendName,
1485
+ key,
1486
+ id: prev.id,
1487
+ checksums: prev.checksums,
1488
+ locations: remaining,
1489
+ successor: { id: newId },
1490
+ reason: 'overwritten',
1491
+ ...(origin ? { origin } : {}),
1492
+ });
1493
+ debug(`Displaced ${prev.id.slice(0, 19)}... from ${backendName}:${key}`);
1494
+ return { id: prev.id, checksums: prev.checksums };
1495
+ }
1496
+
1497
+ /** Upsert locations by (backend, key), preserving order. */
1498
+ #mergeLocations(meta, added = []) {
1499
+ const out = [...(meta.locations || [])];
1500
+ for (const loc of added) {
1501
+ const at = out.findIndex(l => l.backend === loc.backend && l.key === loc.key);
1502
+ if (at >= 0) out[at] = loc;
1503
+ else out.push(loc);
1504
+ }
1505
+ return out;
1506
+ }
1507
+
1508
+ #endpoint(location) {
1509
+ return {
1510
+ backend: location.backend,
1511
+ key: location.key,
1512
+ url: `stored://${location.backend}/${location.key}`,
1513
+ };
1514
+ }
1515
+
1516
+ #emitLocationEvent(suffix, meta, location, extra = {}) {
1517
+ this.#emitObject(suffix, {
1518
+ id: meta.id,
1519
+ checksums: meta.checksums,
1520
+ backend: location.backend,
1521
+ key: location.key,
1522
+ location: this.#describeLocation(location),
1523
+ locations: (meta.locations || []).map(l => this.#describeLocation(l)),
1524
+ ...extra,
1525
+ });
1526
+ }
1527
+
1528
+ /**
1529
+ * Suppress watcher events for keys we are about to write or delete
1530
+ * ourselves. Refcounted (concurrent transfers may overlap) and released on a
1531
+ * delay, because the events arrive well after the syscall returns.
1532
+ */
1533
+ #holdKeys(allKeys) {
1534
+ // Drivers that already filter their own echoes (a canvas hub feed
1535
+ // stamped with our origin) need no hold — and holding would swallow
1536
+ // a genuine remote change that lands right after our write.
1537
+ const keys = allKeys.filter((pathKey) => {
1538
+ const name = this.#backends.list().filter((n) => pathKey.startsWith(`${n}:`)).sort((a, b) => b.length - a.length)[0];
1539
+ const backend = name ? this.#backends.get(name) : null;
1540
+ return !(backend && backend.suppressEchoes === false);
1541
+ });
1542
+ for (const key of keys) this.#suppressedKeys.set(key, (this.#suppressedKeys.get(key) || 0) + 1);
1543
+ let released = false;
1544
+ return () => {
1545
+ if (released) return;
1546
+ released = true;
1547
+ const timer = setTimeout(() => {
1548
+ for (const key of keys) {
1549
+ const next = (this.#suppressedKeys.get(key) || 1) - 1;
1550
+ if (next <= 0) this.#suppressedKeys.delete(key);
1551
+ else this.#suppressedKeys.set(key, next);
1552
+ }
1553
+ }, MOVE_SUPPRESS_MS);
1554
+ timer.unref?.();
1555
+ };
1556
+ }
1557
+
1558
+ // ─────────────────────────────────────────────────────────────────────────
1559
+ // Private — reads
1560
+ // ─────────────────────────────────────────────────────────────────────────
1561
+
1562
+ /**
1563
+ * Synced locations whose backend is registered and readable, cheapest first:
1564
+ * local disk, then network mounts, then `type:'remote'` protocol backends
1565
+ * (S3/HTTP — a real request, not just a slower filesystem).
1566
+ */
1567
+ #readableLocations(meta) {
1568
+ const cost = (backend) => (backend.type === 'remote' ? 2 : (backend.remote ? 1 : 0));
1569
+ return (meta?.locations || [])
1570
+ .filter(l => l.synced)
1571
+ .map(l => ({ l, backend: this.#backends.get(l.backend) }))
1572
+ .filter(({ backend }) => backend && backend.canRead)
1573
+ .sort((a, b) => cost(a.backend) - cost(b.backend))
1574
+ .map(({ l }) => l);
1575
+ }
1576
+
1577
+ async #read(idOrKey, stream) {
1578
+ const meta = this.#index.get(idOrKey);
1579
+ if (!meta) return null;
1580
+
1581
+ // 1. Cache by content id.
1582
+ if (stream) {
1583
+ const info = await this.#cache.getInfo(meta.id).catch(() => null);
1584
+ if (info) return this.#cache.getStream(meta.id);
1585
+ } else {
1586
+ try { return (await this.#cache.get(meta.id)).data; } catch { /* miss */ }
1587
+ }
1588
+
1589
+ // 2. Best available backend location — local before remote (see
1590
+ // #readableLocations); a LAN round-trip must never win over a copy
1591
+ // sitting on local disk just because it was indexed first.
1592
+ const location = this.#readableLocations(meta)[0];
1593
+ const backend = location && this.#backends.get(location.backend);
1594
+ if (!backend) return null;
1595
+
1596
+ const data = await backend.get(location.key, { stream });
1597
+ if (!stream && Buffer.isBuffer(data)) this.#cache.put(meta.id, data).catch(() => {});
1598
+ return data;
1599
+ }
1600
+
1601
+ // ─────────────────────────────────────────────────────────────────────────
1602
+ // Private — ingest (streaming + in-memory)
1603
+ // ─────────────────────────────────────────────────────────────────────────
1604
+
1605
+ // Best-effort extraction hook — bytes are in hand at ingest (works even for
1606
+ // remote/S3-destined blobs, which still pass through the server here). Never
1607
+ // throws; returns {} when disabled/unsupported/failed.
1608
+ async #maybeExtract(source, mimeType, key) {
1609
+ if (!this.#extract) { return {}; }
1610
+ try { return (await this.#extract(source, { mimeType, key })) || {}; }
1611
+ catch (e) { debug(`extract hook failed (${mimeType}): ${e.message}`); return {}; }
1612
+ }
1613
+
1614
+ // Buffer / string: already resident, write directly (no temp file).
1615
+ async #ingestMemory(blob, targets, key, mimeHint, options = {}) {
1616
+ const data = isBuffer(blob) ? blob : Buffer.from(blob);
1617
+ const checksums = checksumBuffer(data, this.#config.checksums);
1618
+ const id = formatId(checksums, this.#config.primaryChecksum);
1619
+ const finalKey = key || this.#generateKey(checksums);
1620
+ const mimeType = mimeHint || await detectMimeType(data);
1621
+
1622
+ const extracted = await this.#maybeExtract({ data }, mimeType, finalKey);
1623
+ const { locations, remoteTargets } = await this.#commit(targets, finalKey, id, { data }, { checksums, size: data.length, mimeType }, options);
1624
+ return { id, finalKey, checksums, size: data.length, mimeType, locations, remoteTargets, extracted };
1625
+ }
1626
+
1627
+ // Path / stream: stream through a hash pass into a temp file on the primary
1628
+ // local backend's filesystem, then commit (hardlink/rename) to targets.
1629
+ async #ingestStream(blob, targets, key, mimeHint, options = {}) {
1630
+ const source = isStream(blob) ? blob : createReadStream(blob);
1631
+ const firstLocal = targets.find(t => t.backend.type === 'local')?.backend;
1632
+ const tempDir = firstLocal ? firstLocal.tempDir : path.join(this.#paths.cache, '.tmp');
1633
+ const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
1634
+ await fsp.mkdir(tempDir, { recursive: true });
1635
+
1636
+ let result;
1637
+ try {
1638
+ const { checksums, size, head } = await this.#hashToFile(source, tempPath);
1639
+ const id = formatId(checksums, this.#config.primaryChecksum);
1640
+ const finalKey = key || this.#generateKey(checksums);
1641
+ const mimeType = mimeHint || await detectMimeFromHead(head, finalKey);
1642
+
1643
+ // Extract from the whole temp file (head=4KB is too small for EXIF),
1644
+ // before the finally-block deletes it.
1645
+ const extracted = await this.#maybeExtract({ file: tempPath }, mimeType, finalKey);
1646
+ const { locations, remoteTargets } = await this.#commit(targets, finalKey, id, { file: tempPath }, { checksums, size, mimeType }, options);
1647
+ result = { id, finalKey, checksums, size, mimeType, locations, remoteTargets, extracted };
1648
+ } finally {
1649
+ await fsp.rm(tempPath, { force: true }).catch(() => {});
1650
+ }
1651
+ return result;
1652
+ }
1653
+
1654
+ // Single streaming pass: hash (all algorithms), measure size, peek the head
1655
+ // for mime sniffing, and write to `tempPath`. Never buffers the whole blob.
1656
+ async #hashToFile(source, tempPath) {
1657
+ const hashes = this.#config.checksums.map(algo => ({ algo, hash: crypto.createHash(algo) }));
1658
+ const ws = createWriteStream(tempPath);
1659
+ let size = 0;
1660
+ const headChunks = [];
1661
+ let headLen = 0;
1662
+
1663
+ try {
1664
+ for await (const chunk of source) {
1665
+ const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
1666
+ for (const { hash } of hashes) hash.update(buf);
1667
+ size += buf.length;
1668
+ if (headLen < HEAD_BYTES) {
1669
+ const slice = buf.subarray(0, HEAD_BYTES - headLen);
1670
+ headChunks.push(slice);
1671
+ headLen += slice.length;
1672
+ }
1673
+ if (!ws.write(buf)) await once(ws, 'drain');
1674
+ }
1675
+ ws.end();
1676
+ await once(ws, 'finish');
1677
+ } catch (err) {
1678
+ ws.destroy();
1679
+ throw err;
1680
+ }
1681
+
1682
+ const checksums = {};
1683
+ for (const { algo, hash } of hashes) checksums[algo] = hash.digest('hex');
1684
+ return { checksums, size, head: Buffer.concat(headChunks) };
1685
+ }
1686
+
1687
+ // Place ingested bytes on every target: local backends get the bytes now
1688
+ // (buffer write or hardlink/copy from temp); remote targets get a cache
1689
+ // entry + a SyncQueue placeholder. Returns { locations, remoteTargets }.
1690
+ async #commit(targets, finalKey, id, source, meta, options = {}) {
1691
+ const locations = [];
1692
+ const remoteTargets = [];
1693
+
1694
+ for (const { name, backend, key, overwrite } of targets) {
1695
+ // A target may carry its own key: conflict resolution can rename per
1696
+ // backend (`photo.jpg` is free on one, taken on another). `put()`
1697
+ // targets never do, so they all land on `finalKey`.
1698
+ const targetKey = key || finalKey;
1699
+ if (backend.type === 'local') {
1700
+ if (overwrite) await this.#retain(name, targetKey, this.#index.get(`${name}:${targetKey}`));
1701
+ if (source.data) await backend.put(targetKey, source.data);
1702
+ else await backend.commit(targetKey, source.file);
1703
+ if (options.mtime != null && typeof backend.utimes === 'function') {
1704
+ await backend.utimes(targetKey, options.mtime).catch(() => {});
1705
+ }
1706
+ // Record the on-disk identity (mtime + inode) the way the
1707
+ // watcher/scan paths do, so a later rename of this file pairs
1708
+ // as a move and a rescan can skip the rehash.
1709
+ const st = typeof backend.stat === 'function' ? await backend.stat(targetKey).catch(() => null) : null;
1710
+ locations.push(this.#buildLocation(name, targetKey, true, {
1711
+ size: st?.size ?? meta.size,
1712
+ ...(st?.modified != null ? { mtime: st.modified } : {}),
1713
+ ...(st?.ino != null ? { ino: st.ino, dev: st.dev } : {}),
1714
+ }));
1715
+ } else {
1716
+ locations.push(this.#buildLocation(name, targetKey, false, { size: meta.size }));
1717
+ remoteTargets.push({ name, driver: backend.config.driver, root: backend.config.root, key: targetKey });
1718
+ }
1719
+ }
1720
+
1721
+ if (remoteTargets.length) {
1722
+ const cacheMeta = { key: finalKey, checksums: meta.checksums, size: meta.size, mimeType: meta.mimeType };
1723
+ if (source.data) await this.#cache.put(id, source.data, cacheMeta);
1724
+ else await pipeline(createReadStream(source.file), this.#cache.putStream(id, cacheMeta));
1725
+ }
1726
+
1727
+ return { locations, remoteTargets };
1728
+ }
1729
+
1730
+ // Skip-hash predicate for scan: returns the cached descriptor when a file's
1731
+ // size+mtime match an existing indexed location, else null.
1732
+ #knownIfUnchanged(backendName, key, stat) {
1733
+ const meta = this.#index.get(`${backendName}:${key}`);
1734
+ const loc = meta?.locations?.find(l => l.backend === backendName && l.key === key);
1735
+ if (!loc || loc.size !== stat.size || loc.mtime !== stat.mtime) return null;
1736
+ return { checksums: meta.checksums, mimeType: meta.mimeType };
1737
+ }
1738
+
1739
+ // One-shot snapshot of a backend's indexed locations keyed by inode
1740
+ // identity, built per scan (the index is LMDB — a per-file reverse lookup
1741
+ // would be O(N²)).
1742
+ #buildInodeMap(backendName) {
1743
+ const map = new Map();
1744
+ for (const [, meta] of this.#index.entries()) {
1745
+ for (const loc of meta.locations || []) {
1746
+ if (loc.backend !== backendName || loc.ino == null) continue;
1747
+ map.set(`${loc.dev}:${loc.ino}`, {
1748
+ key: loc.key,
1749
+ size: loc.size,
1750
+ mtime: loc.mtime,
1751
+ checksums: meta.checksums,
1752
+ mimeType: meta.mimeType,
1753
+ });
1754
+ }
1755
+ }
1756
+ return map;
1757
+ }
1758
+
1759
+ // Rename-match skip-hash: the path is new, but the inode is one we already
1760
+ // indexed and size+mtime are untouched — trust the prior checksums. (Also
1761
+ // covers hardlink twins: same inode ⇒ same bytes by definition.)
1762
+ #knownByInode(inodeMap, key, stat) {
1763
+ if (stat?.ino == null) return null;
1764
+ const prior = inodeMap.get(`${stat.dev}:${stat.ino}`);
1765
+ if (!prior || prior.key === key) return null;
1766
+ if (prior.size !== stat.size || prior.mtime !== stat.mtime) return null;
1767
+ return { checksums: prior.checksums, mimeType: prior.mimeType };
1768
+ }
1769
+
1770
+ #generateKey(checksums) {
1771
+ const hash = checksums[this.#config.primaryChecksum];
1772
+ return `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash}`;
1773
+ }
1774
+
1775
+ #buildLocation(backendName, key, synced, extra = {}) {
1776
+ const backend = this.#backends.get(backendName);
1777
+ const config = backend?.config || {};
1778
+
1779
+ return {
1780
+ backend: backendName,
1781
+ driver: config.driver || null,
1782
+ key,
1783
+ synced,
1784
+ ...extra,
1785
+ source: this.#buildSourceDescriptor(backendName, key, config),
1786
+ };
1787
+ }
1788
+
1789
+ #buildSourceDescriptor(backendName, key, config = {}) {
1790
+ const backend = this.#backends.get(backendName);
1791
+ const [providerHint, ...accountHintParts] = String(backendName || '').split(':').filter(Boolean);
1792
+ const provider = config.provider || providerHint || config.driver || 'unknown';
1793
+ const account = config.account
1794
+ || (accountHintParts.length > 0 ? accountHintParts.join(':') : (providerHint || backendName || 'default'));
1795
+ const container = config.container
1796
+ || config.bucket
1797
+ || config.share
1798
+ || config.folder
1799
+ || (config.root ? path.basename(path.resolve(config.root)) : 'root');
1800
+
1801
+ // `remote`/`transport` are recorded on the location itself, not just read
1802
+ // off the live backend: a location must stay self-describing after its
1803
+ // backend is unmounted or dropped from config (the offline case is
1804
+ // exactly when a consumer needs to know "this lived on the NAS").
1805
+ // Sparse on purpose — local locations are the overwhelming majority and
1806
+ // carry no extra bytes.
1807
+ const remote = backend ? backend.remote : (config.remote === true);
1808
+ const transport = backend ? backend.transport : (config.transport ?? null);
1809
+
1810
+ return {
1811
+ provider,
1812
+ account,
1813
+ container,
1814
+ path: key,
1815
+ ...(remote ? { remote: true } : {}),
1816
+ ...(remote && transport ? { transport } : {}),
1817
+ };
1818
+ }
1819
+
1820
+ // `erroredPrefixes`: subtrees the scan could not read (EACCES etc.) — their
1821
+ // prior entries are carried forward, never treated as deleted. Key presence
1822
+ // is compared NFC-normalized: macOS reports NFD, so without this every
1823
+ // accented filename would diff as a perpetual delete+add pair.
1824
+ #removeMissingLocations(backendName, presentKeys, erroredPrefixes = []) {
1825
+ const nfc = (k) => String(k).normalize('NFC');
1826
+ const present = new Set([...presentKeys].map(nfc));
1827
+ const underErroredPrefix = (key) => erroredPrefixes.some((prefix) =>
1828
+ key === prefix || key.startsWith(`${prefix}/`) || key.startsWith(`${prefix}\\`));
1829
+ for (const [id, meta] of this.#index.entries()) {
1830
+ const nextLocations = (meta.locations || []).filter(location =>
1831
+ location.backend !== backendName
1832
+ || present.has(nfc(location.key))
1833
+ || underErroredPrefix(location.key)
1834
+ );
1835
+
1836
+ if (nextLocations.length === (meta.locations || []).length) continue;
1837
+
1838
+ if (nextLocations.length === 0) {
1839
+ this.#index.delete(id);
1840
+ continue;
1841
+ }
1842
+
1843
+ this.#index.put(id, { ...meta, locations: nextLocations });
1844
+ }
1845
+ }
1846
+
1847
+ // Emit a file event under both its typed name (`file:<suffix>`) and the
1848
+ // generic `object:<suffix>` (kind:'file') so consumers can bind either.
1849
+ #emitObject(suffix, payload) {
1850
+ this.emit(`file:${suffix}`, payload);
1851
+ this.emit(`object:${suffix}`, { kind: 'file', ...payload });
1852
+ }
1853
+
1854
+ // `options.force` bypasses the echo suppression — used when stored itself
1855
+ // wrote the bytes (writeObject) and feeds the resulting stat through the
1856
+ // very same path a watcher-observed change takes.
1857
+ #handleFileEvent(event, data, options = {}) {
1858
+ const pathKey = `${data.backend}:${data.key}`;
1859
+ // Our own copy/move touching a watched root — the index was already
1860
+ // updated by the operation itself. Re-processing the echo would emit a
1861
+ // spurious add (or, worse, an unlink that drops the location we just
1862
+ // moved the object to).
1863
+ if (!options.force && this.#suppressedKeys.has(pathKey)) {
1864
+ // Our own echo carries exactly the bytes we just indexed under the
1865
+ // key; anything else arriving inside the hold is a genuine change
1866
+ // made underneath the transfer and must not be lost. Unlink echoes
1867
+ // (the source of a move) are always ours.
1868
+ const indexed = data.checksums ? this.#index.get(pathKey) : null;
1869
+ const sameBytes = !!indexed && formatId(data.checksums, this.#config.primaryChecksum) === indexed.id;
1870
+ if (event === 'file:unlink') {
1871
+ // Ours (the source of a move, a remove-before-link) — or a
1872
+ // genuine delete landing inside the window. Decide once the
1873
+ // hold is over, from what is actually on disk.
1874
+ debug(`Deferred unlink under in-flight transfer ${pathKey}`);
1875
+ this.#recheckUnlink(data);
1876
+ return;
1877
+ }
1878
+ if (!data.checksums || sameBytes) {
1879
+ debug(`Suppressed ${event} for in-flight transfer ${pathKey}`);
1880
+ return;
1881
+ }
1882
+ debug(`Held key ${pathKey} changed underneath the transfer — processing ${event}`);
1883
+ }
1884
+ const origin = data.origin || null;
1885
+ const location = this.#buildLocation(data.backend, data.key, true, {
1886
+ ...(data.size != null ? { size: data.size } : {}),
1887
+ ...(data.modified != null ? { mtime: data.modified } : {}),
1888
+ ...(data.ino != null ? { ino: data.ino, dev: data.dev } : {}),
1889
+ });
1890
+ // Refresh an already-indexed location in place (size/mtime/inode may
1891
+ // have been unknown — a `put()`-created location learns its inode from
1892
+ // the first watcher event it gets) or append a new one.
1893
+ const upsertLocation = (locations) => {
1894
+ const at = locations.findIndex(l => l.backend === data.backend && l.key === data.key);
1895
+ if (at >= 0) locations[at] = { ...locations[at], ...location };
1896
+ else locations.push(location);
1897
+ return locations;
1898
+ };
1899
+
1900
+ if (event === 'file:add' && data.checksums) {
1901
+ const id = formatId(data.checksums, this.#config.primaryChecksum);
1902
+ // An `add` on a path the index already attributes to other bytes
1903
+ // (chokidar coalesces a write that follows a rename into one add)
1904
+ // is a succession: take the change path so the previous owner is
1905
+ // displaced and never re-claims the key.
1906
+ const occupant = this.#index.get(pathKey);
1907
+ if (occupant && occupant.id !== id) {
1908
+ this.#handleFileEvent('file:change', data, { ...options, force: true });
1909
+ return;
1910
+ }
1911
+
1912
+ // Rename pairing: an add whose inode matches a held unlink is the
1913
+ // second half of a move. Same content → silently drop the old-path
1914
+ // location (no unlink emission, no identity churn) and let the add
1915
+ // land the new path on the same id, as ONE index write so the
1916
+ // change log records a rename. Content changed mid-move → release
1917
+ // the held unlink first; the succession flow handles it.
1918
+ let renamedFrom = null;
1919
+ if (data.ino != null) {
1920
+ const pendingKey = `${data.backend}|${data.dev}:${data.ino}`;
1921
+ const pending = this.#pendingUnlinks.get(pendingKey);
1922
+ if (pending) {
1923
+ clearTimeout(pending.timer);
1924
+ this.#pendingUnlinks.delete(pendingKey);
1925
+ const oldMeta = this.#index.get(`${pending.data.backend}:${pending.data.key}`);
1926
+ if (oldMeta && oldMeta.id === id && pending.data.backend === data.backend) {
1927
+ renamedFrom = pending.data.key;
1928
+ } else {
1929
+ this.#processUnlink(pending.data);
1930
+ }
1931
+ }
1932
+ }
1933
+
1934
+ const existing = this.#index.get(id);
1935
+ const locations = upsertLocation((existing?.locations || []).filter(l =>
1936
+ !(renamedFrom && l.backend === data.backend && l.key === renamedFrom)
1937
+ ));
1938
+
1939
+ this.#index.put(id, {
1940
+ checksums: data.checksums,
1941
+ size: data.size,
1942
+ mimeType: data.mimeType,
1943
+ locations,
1944
+ }, {
1945
+ origin,
1946
+ rename: renamedFrom ? { backend: data.backend, from: renamedFrom, to: data.key } : null,
1947
+ });
1948
+ this.#emitObject('add', { ...data, id, locations, ...(renamedFrom ? { renamedFrom } : {}) });
1949
+
1950
+ } else if (event === 'file:change' && data.checksums) {
1951
+ const newId = formatId(data.checksums, this.#config.primaryChecksum);
1952
+ const oldMeta = this.#index.get(pathKey);
1953
+ // Same path, new bytes = a successor under content identity. Carry
1954
+ // the predecessor's identity on the add event so consumers can
1955
+ // migrate curated placements instead of orphaning them. The two
1956
+ // index writes share one transaction: the change log then holds a
1957
+ // single `put` for the key, never a transient delete.
1958
+ let previous = null;
1959
+ const locations = this.#index.transaction(() => {
1960
+ if (oldMeta && oldMeta.id !== newId) {
1961
+ previous = { id: oldMeta.id, checksums: oldMeta.checksums };
1962
+ oldMeta.locations = oldMeta.locations.filter(l =>
1963
+ !(l.backend === data.backend && l.key === data.key)
1964
+ );
1965
+ if (oldMeta.locations.length === 0) {
1966
+ this.#index.delete(oldMeta.id, { origin });
1967
+ } else {
1968
+ this.#index.put(oldMeta.id, oldMeta, { origin });
1969
+ }
1970
+ }
1971
+
1972
+ const existing = this.#index.get(newId);
1973
+ const next = upsertLocation(existing?.locations || []);
1974
+ this.#index.put(newId, {
1975
+ checksums: data.checksums,
1976
+ size: data.size,
1977
+ mimeType: data.mimeType,
1978
+ locations: next,
1979
+ }, { origin });
1980
+ return next;
1981
+ });
1982
+ if (previous) {
1983
+ this.#emitObject('unlink', { ...data, id: previous.id, checksums: previous.checksums, successor: { id: newId, checksums: data.checksums } });
1984
+ }
1985
+ this.#emitObject('add', { ...data, id: newId, locations, previous });
1986
+
1987
+ } else if (event === 'file:unlink') {
1988
+ // Hold the unlink briefly when we know the file's inode — if a
1989
+ // same-inode add arrives inside the window this was a move, not a
1990
+ // deletion, and consumers never see the transient absence.
1991
+ const meta = this.#index.get(pathKey);
1992
+ const loc = meta?.locations?.find(l => l.backend === data.backend && l.key === data.key);
1993
+ if (loc?.ino != null) {
1994
+ const pendingKey = `${data.backend}|${loc.dev}:${loc.ino}`;
1995
+ // A second unlink for the same inode (hardlink twin) processes
1996
+ // the held one immediately — one hold per inode.
1997
+ const prior = this.#pendingUnlinks.get(pendingKey);
1998
+ if (prior) {
1999
+ clearTimeout(prior.timer);
2000
+ this.#pendingUnlinks.delete(pendingKey);
2001
+ this.#processUnlink(prior.data);
2002
+ }
2003
+ const timer = setTimeout(() => {
2004
+ this.#pendingUnlinks.delete(pendingKey);
2005
+ this.#processUnlink(data);
2006
+ }, RENAME_WINDOW_MS);
2007
+ timer.unref?.();
2008
+ this.#pendingUnlinks.set(pendingKey, { data, timer });
2009
+ return;
2010
+ }
2011
+ this.#processUnlink(data);
2012
+ }
2013
+ }
2014
+
2015
+ // An unlink observed while its key was held: once the hold expires, a key
2016
+ // the index still claims but the backend no longer has was a real delete
2017
+ // (the user removed the file right after we wrote it) and is processed
2018
+ // through the normal path; the echo of our own remove/rename is a no-op
2019
+ // (the transfer already rewrote the index, or the file is back).
2020
+ #recheckUnlink(data) {
2021
+ const pathKey = `${data.backend}:${data.key}`;
2022
+ const timer = setTimeout(async () => {
2023
+ if (this.#suppressedKeys.has(pathKey)) { this.#recheckUnlink(data); return; }
2024
+ const meta = this.#index.get(pathKey);
2025
+ if (!meta) return;
2026
+ const backend = this.#backends.get(data.backend);
2027
+ if (!backend) return;
2028
+ const present = await backend.stat(data.key).catch(() => null);
2029
+ if (present) return;
2030
+ debug(`Deferred unlink confirmed for ${pathKey}`);
2031
+ this.#handleFileEvent('file:unlink', data);
2032
+ }, MOVE_SUPPRESS_MS + 100);
2033
+ timer.unref?.();
2034
+ }
2035
+
2036
+ // The genuine-deletion path for a file:unlink (immediate when the inode is
2037
+ // unknown, deferred through #pendingUnlinks otherwise).
2038
+ #processUnlink(data) {
2039
+ const pathKey = `${data.backend}:${data.key}`;
2040
+ const meta = this.#index.get(pathKey);
2041
+ const origin = data.origin || null;
2042
+ if (meta) {
2043
+ meta.locations = meta.locations.filter(l =>
2044
+ !(l.backend === data.backend && l.key === data.key)
2045
+ );
2046
+ if (meta.locations.length === 0) {
2047
+ this.#index.delete(meta.id, { origin });
2048
+ } else {
2049
+ this.#index.put(meta.id, meta, { origin });
2050
+ }
2051
+ this.#emitObject('unlink', { ...data, id: meta.id, checksums: meta.checksums, locations: meta.locations });
2052
+ } else {
2053
+ this.#emitObject('unlink', data);
2054
+ }
2055
+ }
2056
+ }
2057
+
2058
+ export { Mirror, JobQueue, Ledger, BackendManager, Index };