@augmentd-labs/canvas-edge 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +114 -0
- package/README.md +76 -2
- package/bin/canvas-edge +2 -0
- package/bin/healthcheck.mjs +9 -0
- package/node_modules/canvas-stored/LICENSE +661 -0
- package/node_modules/canvas-stored/NOTICE +85 -0
- package/node_modules/canvas-stored/README.md +472 -0
- package/node_modules/canvas-stored/package.json +26 -0
- package/node_modules/canvas-stored/src/backends/BackendManager.js +60 -0
- package/node_modules/canvas-stored/src/backends/StorageBackend.js +72 -0
- package/node_modules/canvas-stored/src/backends/cacache/index.js +130 -0
- package/node_modules/canvas-stored/src/backends/canvas/index.js +722 -0
- package/node_modules/canvas-stored/src/backends/file/index.js +658 -0
- package/node_modules/canvas-stored/src/backends/gdrive/index.js +924 -0
- package/node_modules/canvas-stored/src/backends/http/index.js +47 -0
- package/node_modules/canvas-stored/src/backends/s3/index.js +25 -0
- package/node_modules/canvas-stored/src/cache/index.js +65 -0
- package/node_modules/canvas-stored/src/extractors/audio.js +53 -0
- package/node_modules/canvas-stored/src/extractors/image.js +90 -0
- package/node_modules/canvas-stored/src/extractors/index.js +41 -0
- package/node_modules/canvas-stored/src/extractors/text.js +65 -0
- package/node_modules/canvas-stored/src/index/index.js +360 -0
- package/node_modules/canvas-stored/src/index.js +2058 -0
- package/node_modules/canvas-stored/src/sync/JobQueue.js +0 -0
- package/node_modules/canvas-stored/src/sync/Ledger.js +138 -0
- package/node_modules/canvas-stored/src/sync/Mirror.js +995 -0
- package/node_modules/canvas-stored/src/sync/SyncQueue.js +111 -0
- package/node_modules/canvas-stored/src/sync/keys.js +140 -0
- package/node_modules/canvas-stored/src/sync/worker.js +52 -0
- package/node_modules/canvas-stored/src/utils/checksum.js +32 -0
- package/node_modules/canvas-stored/src/utils/common.js +37 -0
- package/node_modules/canvas-stored/src/utils/mime.js +107 -0
- package/node_modules/canvas-stored/src/utils/mount.js +99 -0
- package/package.json +49 -4
- package/src/EdgeClient.js +129 -0
- package/src/control.js +48 -0
- package/src/daemon.js +88 -0
- package/src/env.js +182 -0
- package/src/fuse-runtime.js +173 -0
- package/src/index.js +26 -0
- package/src/mirror-runtime.js +232 -0
- package/src/remote-config.js +69 -0
- package/src/runtime.js +61 -0
|
@@ -0,0 +1,2058 @@
|
|
|
1
|
+
import EventEmitter2 from 'eventemitter2';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import crypto from 'crypto';
|
|
4
|
+
import { createReadStream, createWriteStream, promises as fsp } from 'fs';
|
|
5
|
+
import { once } from 'events';
|
|
6
|
+
import { Readable } from 'stream';
|
|
7
|
+
import { pipeline } from 'stream/promises';
|
|
8
|
+
import Debug from 'debug';
|
|
9
|
+
import Cache from './cache/index.js';
|
|
10
|
+
import BackendManager from './backends/BackendManager.js';
|
|
11
|
+
import Index from './index/index.js';
|
|
12
|
+
import SyncQueue from './sync/SyncQueue.js';
|
|
13
|
+
import JobQueue from './sync/JobQueue.js';
|
|
14
|
+
import Ledger from './sync/Ledger.js';
|
|
15
|
+
import Mirror from './sync/Mirror.js';
|
|
16
|
+
import { isBuffer, isFile, isStream, resolveStoredPaths } from './utils/common.js';
|
|
17
|
+
import { checksumBuffer, formatId } from './utils/checksum.js';
|
|
18
|
+
import { detectMimeType, detectMimeFromHead } from './utils/mime.js';
|
|
19
|
+
|
|
20
|
+
const debug = Debug('stored');
|
|
21
|
+
|
|
22
|
+
const HEAD_BYTES = 4096;
|
|
23
|
+
|
|
24
|
+
// A filesystem move reaches the watcher as unlink(old) + add(new). Unlink
|
|
25
|
+
// processing is held back this long so an add carrying the same inode can
|
|
26
|
+
// claim it as a rename (in-place URL rewrite, no orphan churn). Expired
|
|
27
|
+
// holds process as genuine deletions.
|
|
28
|
+
const RENAME_WINDOW_MS = 1000;
|
|
29
|
+
|
|
30
|
+
// A copy/move writes and deletes through the same watched roots the file
|
|
31
|
+
// backends observe, so our own writes come back as add/unlink events. Affected
|
|
32
|
+
// keys are suppressed for the duration of the operation plus this grace period
|
|
33
|
+
// (chokidar's awaitWriteFinish alone is 200ms, and an unlink is held for
|
|
34
|
+
// RENAME_WINDOW_MS on top) — without it a move is seen a second time as a
|
|
35
|
+
// delete of the source, which downstream drops the location for.
|
|
36
|
+
const MOVE_SUPPRESS_MS = 3000;
|
|
37
|
+
|
|
38
|
+
export default class Stored extends EventEmitter2 {
|
|
39
|
+
#cache;
|
|
40
|
+
#backends;
|
|
41
|
+
#index;
|
|
42
|
+
#config;
|
|
43
|
+
#paths;
|
|
44
|
+
#syncQueue;
|
|
45
|
+
#extract; // optional injected metadata extractor: (source,{mimeType,key})→Promise<obj>
|
|
46
|
+
// `${backend}|${dev}:${ino}` → { data, timer } — unlink events held for
|
|
47
|
+
// RENAME_WINDOW_MS awaiting a same-inode add (see #handleFileEvent).
|
|
48
|
+
#pendingUnlinks = new Map();
|
|
49
|
+
// `${backend}:${key}` → refcount. Watcher events for these keys are ours
|
|
50
|
+
// (an in-flight copy/move) and must not be re-processed as external change.
|
|
51
|
+
#suppressedKeys = new Map();
|
|
52
|
+
// id → [{ sourceLocation, target, destKey }] — moves onto a `type:'remote'`
|
|
53
|
+
// backend, held until the SyncQueue confirms the destination write. The
|
|
54
|
+
// source is only removed once the copy is durable somewhere else.
|
|
55
|
+
#pendingMoves = new Map();
|
|
56
|
+
#retention = null;
|
|
57
|
+
#retainedDb = null;
|
|
58
|
+
#sweepTimer = null;
|
|
59
|
+
|
|
60
|
+
constructor(config = {}) {
|
|
61
|
+
// Wildcards (':' delimiter) let consumers bind `object:*` across backends.
|
|
62
|
+
super({ wildcard: true, delimiter: ':', maxListeners: 100, verboseMemoryLeak: false });
|
|
63
|
+
this.#paths = resolveStoredPaths(config);
|
|
64
|
+
this.#config = {
|
|
65
|
+
checksums: config.checksums || ['sha256'],
|
|
66
|
+
primaryChecksum: config.primaryChecksum || 'sha256',
|
|
67
|
+
...config,
|
|
68
|
+
};
|
|
69
|
+
// Retention window for displaced bytes (overwrite/delete on a local
|
|
70
|
+
// file backend): `{ days }`; absent/false = off. See #retain().
|
|
71
|
+
const days = config.retention ? Number(config.retention.days ?? 30) : 0;
|
|
72
|
+
this.#retention = days > 0 ? { days, sweepEveryMs: Number(config.retention.sweepEveryMs ?? 60 * 60_000) } : null;
|
|
73
|
+
|
|
74
|
+
this.#extract = typeof config.extract === 'function' ? config.extract : null;
|
|
75
|
+
|
|
76
|
+
this.#cache = new Cache({ path: this.#paths.cache, algorithms: this.#config.checksums });
|
|
77
|
+
this.#backends = new BackendManager();
|
|
78
|
+
// The index's change log is re-emitted as `change` events (one per
|
|
79
|
+
// entry, after the transaction that produced them committed).
|
|
80
|
+
this.#index = new Index(this.#paths.index, {
|
|
81
|
+
changes: config.changes,
|
|
82
|
+
onChange: (entries) => { for (const entry of entries) this.emit('change', entry); },
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
// Background sync queue for remote backends: file targets are copied by
|
|
86
|
+
// a worker thread, network drivers (gdrive, …) are committed in-process
|
|
87
|
+
// through their live backend (async I/O — nothing to offload, and the
|
|
88
|
+
// driver's credentials/path cache stay on the main thread).
|
|
89
|
+
this.#syncQueue = new SyncQueue({ resolveBackend: (name) => this.#backends.get(name) });
|
|
90
|
+
this.#syncQueue.on('synced', ({ id, results }) => this.#handleSyncResult(id, results));
|
|
91
|
+
this.#syncQueue.on('error', (err) => this.emit('error', err));
|
|
92
|
+
|
|
93
|
+
debug('Stored initialized');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
97
|
+
// Getters
|
|
98
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
99
|
+
|
|
100
|
+
get root() { return this.#paths.root; }
|
|
101
|
+
get paths() { return { ...this.#paths }; }
|
|
102
|
+
get dataPath() { return this.#paths.data; }
|
|
103
|
+
get cache() { return this.#cache; }
|
|
104
|
+
get index() { return this.#index; }
|
|
105
|
+
get backends() { return this.#backends; }
|
|
106
|
+
|
|
107
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
108
|
+
// Backend Management
|
|
109
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
110
|
+
|
|
111
|
+
addBackend(name, config) {
|
|
112
|
+
const backend = this.#backends.add(name, config);
|
|
113
|
+
|
|
114
|
+
backend.on('file:add', e => this.#handleFileEvent('file:add', e));
|
|
115
|
+
backend.on('file:change', e => this.#handleFileEvent('file:change', e));
|
|
116
|
+
backend.on('file:unlink', e => this.#handleFileEvent('file:unlink', e));
|
|
117
|
+
// Generic change events from non-file backends — forwarded as-is for
|
|
118
|
+
// consumers; they carry {backend, kind, key, ...}.
|
|
119
|
+
backend.on('object:add', e => this.emit('object:add', e));
|
|
120
|
+
backend.on('object:change', e => this.emit('object:change', e));
|
|
121
|
+
backend.on('object:unlink', e => this.emit('object:unlink', e));
|
|
122
|
+
backend.on('backend:state', e => this.emit('backend:state', e));
|
|
123
|
+
backend.on('scan:start', e => this.emit('scan:start', e));
|
|
124
|
+
backend.on('scan:complete', e => this.emit('scan:complete', e));
|
|
125
|
+
backend.on('error', e => this.emit('error', e));
|
|
126
|
+
|
|
127
|
+
if (config.watch) backend.watch();
|
|
128
|
+
return backend;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Unregister a backend and drop its now-dangling locations from the index.
|
|
132
|
+
// Entries left with no locations are removed (the index is derived state —
|
|
133
|
+
// synapsd documents are durable and rebuildable via scan()).
|
|
134
|
+
async removeBackend(name) {
|
|
135
|
+
const removed = await this.#backends.remove(name);
|
|
136
|
+
if (removed) this.#removeMissingLocations(name, new Set());
|
|
137
|
+
return removed;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
listBackends() { return this.#backends.list(); }
|
|
141
|
+
getBackend(name) { return this.#backends.get(name); }
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Parse a canonical `stored://<backend>/<key>` URL. The backend name may
|
|
145
|
+
* itself contain colons (e.g. `workspace:data`), so we split on the first
|
|
146
|
+
* `/` after the scheme. Returns { backend, key } or null if malformed.
|
|
147
|
+
*/
|
|
148
|
+
#parseStoredUrl(url) {
|
|
149
|
+
const prefix = 'stored://';
|
|
150
|
+
if (typeof url !== 'string' || !url.startsWith(prefix)) return null;
|
|
151
|
+
const rest = url.slice(prefix.length);
|
|
152
|
+
const slash = rest.indexOf('/');
|
|
153
|
+
if (slash < 0) return null;
|
|
154
|
+
return { backend: rest.slice(0, slash), key: rest.slice(slash + 1) };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Fetch bytes directly by `stored://` URL, bypassing the content index.
|
|
158
|
+
// Returns Buffer | null (never throws on bad input).
|
|
159
|
+
async getByUrl(url) {
|
|
160
|
+
const p = this.#parseStoredUrl(url);
|
|
161
|
+
const backend = p && this.#backends.get(p.backend);
|
|
162
|
+
return backend ? backend.get(p.key) : null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Same as getByUrl but returns a Readable | null.
|
|
166
|
+
async getStreamByUrl(url) {
|
|
167
|
+
const p = this.#parseStoredUrl(url);
|
|
168
|
+
const backend = p && this.#backends.get(p.backend);
|
|
169
|
+
return backend ? backend.get(p.key, { stream: true }) : null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Ranged stream for HTTP Range / media streaming. Returns
|
|
174
|
+
* `{ stream, ranged }` where `ranged` is true only when the backend actually
|
|
175
|
+
* served the requested byte window (`getRange`). Backends without ranged
|
|
176
|
+
* reads (e.g. remote proxies) fall back to a full stream with `ranged:false`,
|
|
177
|
+
* so the caller can respond 200 instead of a length-mismatched 206.
|
|
178
|
+
* `range` = { start, end } with an inclusive `end`. Null on a miss.
|
|
179
|
+
*/
|
|
180
|
+
async getRangeStreamByUrl(url, range) {
|
|
181
|
+
const p = this.#parseStoredUrl(url);
|
|
182
|
+
const backend = p && this.#backends.get(p.backend);
|
|
183
|
+
if (!backend) return null;
|
|
184
|
+
if (typeof backend.getRange === 'function') {
|
|
185
|
+
const stream = await backend.getRange(p.key, range);
|
|
186
|
+
if (stream) return { stream, ranged: true };
|
|
187
|
+
}
|
|
188
|
+
const stream = await backend.get(p.key, { stream: true });
|
|
189
|
+
return stream ? { stream, ranged: false } : null;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Delete the bytes behind a `stored://<backend>/<key>` URL. Does not touch
|
|
194
|
+
* the consumer's document index (synapsd owns that) — callers trim
|
|
195
|
+
* `locations[]` — but DOES drop the location from stored's own index: an
|
|
196
|
+
* index that still claims bytes we just deleted makes a later `copy()` skip
|
|
197
|
+
* the target as "already there" and report success for a no-op.
|
|
198
|
+
* Returns { ok:boolean, reason?:'malformed-url'|'unknown-backend'|'read-only-backend' }.
|
|
199
|
+
*/
|
|
200
|
+
async deleteByUrl(url) {
|
|
201
|
+
const p = this.#parseStoredUrl(url);
|
|
202
|
+
if (!p) return { ok: false, reason: 'malformed-url' };
|
|
203
|
+
const backend = this.#backends.get(p.backend);
|
|
204
|
+
if (!backend) return { ok: false, reason: 'unknown-backend' };
|
|
205
|
+
if (!backend.canDelete) return { ok: false, reason: 'read-only-backend' };
|
|
206
|
+
await this.#retain(p.backend, p.key, this.#index.get(`${p.backend}:${p.key}`));
|
|
207
|
+
const deleted = !!(await backend.delete(p.key));
|
|
208
|
+
if (deleted) this.#dropLocation(p.backend, p.key);
|
|
209
|
+
return { ok: deleted };
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Forget one location we just deleted the bytes for. Emits
|
|
214
|
+
* `object:location:remove` when the content survives elsewhere; drops the
|
|
215
|
+
* whole entry when that was its last home.
|
|
216
|
+
*/
|
|
217
|
+
#dropLocation(backendName, key) {
|
|
218
|
+
const meta = this.#index.get(`${backendName}:${key}`);
|
|
219
|
+
if (!meta) return;
|
|
220
|
+
const removed = (meta.locations || []).find(l => l.backend === backendName && l.key === key);
|
|
221
|
+
const remaining = (meta.locations || []).filter(l => !(l.backend === backendName && l.key === key));
|
|
222
|
+
if (remaining.length === 0) {
|
|
223
|
+
this.#index.delete(meta.id);
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
const saved = this.#index.put(meta.id, { ...meta, locations: remaining });
|
|
227
|
+
if (removed) this.#emitLocationEvent('location:remove', saved, removed, { reason: 'deleted' });
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
231
|
+
// Core API — cache-first writes, cache-first reads
|
|
232
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Store a blob. Accepts Buffer | string (content) | filesystem path |
|
|
236
|
+
* Readable. Paths and streams are ingested without ever materializing the
|
|
237
|
+
* whole blob in memory. `backends` is required (no implicit fan-out).
|
|
238
|
+
* Returns { ok:true, id, key, size, mimeType, checksums, locations, ... }
|
|
239
|
+
* or { ok:false, reason:'no-targets'|'unknown-backend' }.
|
|
240
|
+
*/
|
|
241
|
+
async put(blob, options = {}) {
|
|
242
|
+
const { key, backends, metadata = {} } = options;
|
|
243
|
+
if (!Array.isArray(backends) || backends.length === 0) return { ok: false, reason: 'no-targets' };
|
|
244
|
+
|
|
245
|
+
const targets = backends
|
|
246
|
+
.map(name => ({ name, backend: this.#backends.get(name) }))
|
|
247
|
+
.filter(t => t.backend);
|
|
248
|
+
if (!targets.length) return { ok: false, reason: 'unknown-backend' };
|
|
249
|
+
|
|
250
|
+
const ingestOptions = { mtime: options.mtime ?? null };
|
|
251
|
+
const ingest = (isBuffer(blob) || (typeof blob === 'string' && !isFile(blob)))
|
|
252
|
+
? await this.#ingestMemory(blob, targets, key, options.mimeType, ingestOptions)
|
|
253
|
+
: await this.#ingestStream(blob, targets, key, options.mimeType, ingestOptions);
|
|
254
|
+
|
|
255
|
+
// Caller-supplied metadata + inline-extracted (EXIF/GPS/dimensions/media)
|
|
256
|
+
// → persisted under `custom`, returned to caller + on the 'put' event.
|
|
257
|
+
const extracted = ingest.extracted && Object.keys(ingest.extracted).length ? ingest.extracted : null;
|
|
258
|
+
const meta = this.#index.put(ingest.id, {
|
|
259
|
+
checksums: ingest.checksums,
|
|
260
|
+
size: ingest.size,
|
|
261
|
+
mimeType: ingest.mimeType,
|
|
262
|
+
locations: ingest.locations,
|
|
263
|
+
custom: extracted ? { ...metadata, ...extracted } : metadata,
|
|
264
|
+
}, { origin: options.origin });
|
|
265
|
+
|
|
266
|
+
if (ingest.remoteTargets.length) {
|
|
267
|
+
this.#syncQueue.enqueue({ id: ingest.id, cacheRoot: this.#cache.root, cacheKey: ingest.id, targets: ingest.remoteTargets });
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
this.emit('put', { id: ingest.id, key: ingest.finalKey, metadata: meta });
|
|
271
|
+
debug(`PUT ${ingest.id.slice(0, 19)}... → ${targets.map(t => t.name).join(', ')}`);
|
|
272
|
+
return { ok: true, key: ingest.finalKey, ...meta };
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async get(idOrKey) { return this.#read(idOrKey, false); }
|
|
276
|
+
async getStream(idOrKey) { return this.#read(idOrKey, true); }
|
|
277
|
+
|
|
278
|
+
async delete(idOrKey, options = {}) {
|
|
279
|
+
const meta = this.#index.get(idOrKey);
|
|
280
|
+
if (!meta) return { ok: false, reason: 'not-found' };
|
|
281
|
+
|
|
282
|
+
this.#cache.delete(meta.id).catch(() => {});
|
|
283
|
+
|
|
284
|
+
let targets = meta.locations || [];
|
|
285
|
+
if (Array.isArray(options.urls)) {
|
|
286
|
+
const set = new Set(options.urls);
|
|
287
|
+
targets = targets.filter(l => set.has(`stored://${l.backend}/${l.key}`));
|
|
288
|
+
} else if (Array.isArray(options.backends)) {
|
|
289
|
+
targets = targets.filter(l => options.backends.includes(l.backend));
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const deleted = [];
|
|
293
|
+
const removed = new Set();
|
|
294
|
+
for (const loc of targets) {
|
|
295
|
+
const backend = this.#backends.get(loc.backend);
|
|
296
|
+
if (backend) await this.#retain(loc.backend, loc.key, meta);
|
|
297
|
+
if (backend && await backend.delete(loc.key)) {
|
|
298
|
+
deleted.push(loc.backend);
|
|
299
|
+
removed.add(loc);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const remaining = (meta.locations || []).filter(l => !removed.has(l));
|
|
304
|
+
if (remaining.length === 0) {
|
|
305
|
+
this.#index.delete(meta.id);
|
|
306
|
+
} else {
|
|
307
|
+
meta.locations = remaining;
|
|
308
|
+
this.#index.put(meta.id, meta);
|
|
309
|
+
// Content survives elsewhere — consumers patch locations[] in place
|
|
310
|
+
// rather than treating this as the object going away.
|
|
311
|
+
for (const loc of removed) this.#emitLocationEvent('location:remove', meta, loc, { reason: 'deleted' });
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
this.emit('delete', { id: meta.id, backends: deleted });
|
|
315
|
+
return { ok: true, deleted, kept: remaining.map(l => l.backend) };
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
async stat(idOrKey) { return this.#index.get(idOrKey); }
|
|
319
|
+
async has(idOrKey) { return this.#index.has(idOrKey); }
|
|
320
|
+
|
|
321
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
322
|
+
// Change log + listing — the feed remote mirrors tail
|
|
323
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
324
|
+
|
|
325
|
+
/** See Index.changes(): `{ changes, head, oldest, cursor, cursorTooOld }`. */
|
|
326
|
+
changes(options = {}) { return this.#index.changes(options); }
|
|
327
|
+
head() { return this.#index.head(); }
|
|
328
|
+
oldest() { return this.#index.oldest(); }
|
|
329
|
+
trimChanges(options = {}) { return this.#index.trimChanges(options); }
|
|
330
|
+
|
|
331
|
+
/** Page one backend's indexed keys in key order: `{ objects, cursor }`. */
|
|
332
|
+
listObjects(backendName, options = {}) {
|
|
333
|
+
if (!this.#backends.get(backendName)) return { objects: [], cursor: null, reason: 'unknown-backend' };
|
|
334
|
+
return this.#index.locationsByBackend(backendName, options);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Suppress watcher echoes for keys an external caller is about to write
|
|
339
|
+
* or delete through the backend directly. Returns the release function.
|
|
340
|
+
*/
|
|
341
|
+
holdKeys(keys) { return this.#holdKeys(Array.isArray(keys) ? keys : [keys]); }
|
|
342
|
+
|
|
343
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
344
|
+
// Keyed object writes — precondition-checked, succession-preserving
|
|
345
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* Write `source` (Buffer | string content | file path | Readable) to
|
|
349
|
+
* `backend:key`, replacing whatever is there, with HTTP-style preconditions
|
|
350
|
+
* evaluated against the index right before the swap:
|
|
351
|
+
* - `ifMatch`: the sha256 (or content id) the caller believes is there
|
|
352
|
+
* - `ifNoneMatch: '*'`: the key must be free
|
|
353
|
+
* - `sha256`: what the bytes must hash to (a corrupted upload is refused)
|
|
354
|
+
* The bytes are staged on the backend's own filesystem and renamed into
|
|
355
|
+
* place, then the SAME watcher path a local edit takes is fed with the
|
|
356
|
+
* resulting stat — so consumers see one `object:add` (with `previous` on
|
|
357
|
+
* an edit) exactly as if the file had been dropped into the folder, and
|
|
358
|
+
* the backend's own watcher echo is suppressed.
|
|
359
|
+
*
|
|
360
|
+
* Returns `{ ok:true, id, sha256, size, mtime, seq, previous, unchanged? }`
|
|
361
|
+
* or `{ ok:false, reason, ... }` — `reason:'precondition-failed'` carries
|
|
362
|
+
* `current` (`{ id, sha256, size, mtime }` | null) so callers can report
|
|
363
|
+
* what is actually there.
|
|
364
|
+
*/
|
|
365
|
+
async writeObject(backendName, key, source, options = {}) {
|
|
366
|
+
const backend = this.#backends.get(backendName);
|
|
367
|
+
if (!backend) return { ok: false, reason: 'unknown-backend', backend: backendName };
|
|
368
|
+
if (!backend.canWrite) return { ok: false, reason: 'read-only-target', backend: backendName };
|
|
369
|
+
if (backend.type !== 'local' || typeof backend.commit !== 'function') {
|
|
370
|
+
return { ok: false, reason: 'unsupported-backend', backend: backendName };
|
|
371
|
+
}
|
|
372
|
+
const destKey = this.#normalizeKey(key, { nfc: true });
|
|
373
|
+
if (!destKey || !this.#isSafeKey(destKey)) return { ok: false, reason: 'invalid-key', key };
|
|
374
|
+
const live = await this.#verifyBackendRoot(backend);
|
|
375
|
+
if (!live.ok) return { ok: false, reason: 'target-offline', backend: backendName, detail: live.reason };
|
|
376
|
+
|
|
377
|
+
const pathKey = `${backendName}:${destKey}`;
|
|
378
|
+
const tempDir = backend.tempDir || path.join(this.#paths.cache, '.tmp');
|
|
379
|
+
const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
380
|
+
await fsp.mkdir(tempDir, { recursive: true });
|
|
381
|
+
|
|
382
|
+
try {
|
|
383
|
+
const stream = isBuffer(source) ? Readable.from([source])
|
|
384
|
+
: (typeof source === 'string' && !isFile(source)) ? Readable.from([Buffer.from(source)])
|
|
385
|
+
: (isStream(source) ? source : createReadStream(source));
|
|
386
|
+
const { checksums, size, head } = await this.#hashToFile(stream, tempPath);
|
|
387
|
+
const id = formatId(checksums, this.#config.primaryChecksum);
|
|
388
|
+
const sha256 = checksums.sha256 ?? null;
|
|
389
|
+
if (options.sha256 && sha256 && String(options.sha256).toLowerCase() !== sha256) {
|
|
390
|
+
return { ok: false, reason: 'checksum-mismatch', expected: String(options.sha256).toLowerCase(), actual: sha256 };
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
const current = this.#index.get(pathKey);
|
|
394
|
+
const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
|
|
395
|
+
const failed = this.#checkPrecondition(current, currentLoc, options);
|
|
396
|
+
if (failed) return failed;
|
|
397
|
+
|
|
398
|
+
if (current && current.id === id) {
|
|
399
|
+
// Same bytes already there — nothing to write, nothing to log.
|
|
400
|
+
let mtime = currentLoc?.mtime ?? null;
|
|
401
|
+
if (options.mtime != null && typeof backend.utimes === 'function') {
|
|
402
|
+
await backend.utimes(destKey, options.mtime).catch(() => {});
|
|
403
|
+
const st = await backend.stat(destKey).catch(() => null);
|
|
404
|
+
if (st?.modified != null && currentLoc) {
|
|
405
|
+
mtime = st.modified;
|
|
406
|
+
this.#index.put(current.id, {
|
|
407
|
+
...current,
|
|
408
|
+
locations: current.locations.map(l => (l === currentLoc ? { ...l, mtime: st.modified, size: st.size } : l)),
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
return { ok: true, unchanged: true, id, sha256, checksums, size: current.size, mtime, seq: this.#index.head(), previous: null };
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
const mimeType = options.mimeType || await detectMimeFromHead(head, destKey);
|
|
416
|
+
const release = this.#holdKeys([pathKey]);
|
|
417
|
+
try {
|
|
418
|
+
if (current) await this.#retain(backendName, destKey, current);
|
|
419
|
+
let placed = null;
|
|
420
|
+
if (typeof backend.renameFrom === 'function') {
|
|
421
|
+
try { placed = await backend.renameFrom(destKey, tempPath); }
|
|
422
|
+
catch (err) { if (err.code !== 'EXDEV') throw err; }
|
|
423
|
+
}
|
|
424
|
+
if (!placed) {
|
|
425
|
+
await backend.commit(destKey, tempPath);
|
|
426
|
+
placed = await backend.stat(destKey).catch(() => null);
|
|
427
|
+
}
|
|
428
|
+
if (options.mtime != null && typeof backend.utimes === 'function') {
|
|
429
|
+
await backend.utimes(destKey, options.mtime).catch(() => {});
|
|
430
|
+
placed = (await backend.stat(destKey).catch(() => null)) || placed;
|
|
431
|
+
}
|
|
432
|
+
const absPath = typeof backend.resolveKeyPath === 'function' ? backend.resolveKeyPath(destKey) : null;
|
|
433
|
+
this.#handleFileEvent(current ? 'file:change' : 'file:add', {
|
|
434
|
+
backend: backendName,
|
|
435
|
+
key: destKey,
|
|
436
|
+
path: absPath,
|
|
437
|
+
checksums,
|
|
438
|
+
mimeType,
|
|
439
|
+
size,
|
|
440
|
+
modified: placed?.modified,
|
|
441
|
+
dev: placed?.dev,
|
|
442
|
+
ino: placed?.ino,
|
|
443
|
+
...(options.origin ? { origin: options.origin } : {}),
|
|
444
|
+
}, { force: true });
|
|
445
|
+
|
|
446
|
+
const saved = this.#index.get(id);
|
|
447
|
+
const loc = saved?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
|
|
448
|
+
debug(`WRITE ${pathKey} ← ${id.slice(0, 19)}...${current ? ` (replaces ${current.id.slice(0, 19)}...)` : ''}`);
|
|
449
|
+
return {
|
|
450
|
+
ok: true,
|
|
451
|
+
id,
|
|
452
|
+
sha256,
|
|
453
|
+
checksums,
|
|
454
|
+
size,
|
|
455
|
+
mimeType,
|
|
456
|
+
mtime: loc?.mtime ?? placed?.modified ?? null,
|
|
457
|
+
seq: this.#index.head(),
|
|
458
|
+
previous: current ? { id: current.id, checksums: current.checksums } : null,
|
|
459
|
+
};
|
|
460
|
+
} finally {
|
|
461
|
+
release();
|
|
462
|
+
}
|
|
463
|
+
} finally {
|
|
464
|
+
await fsp.rm(tempPath, { force: true }).catch(() => {});
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
469
|
+
// Retention — displaced bytes stay recoverable for a window
|
|
470
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
471
|
+
//
|
|
472
|
+
// Every path that overwrites or deletes bytes on a local file backend
|
|
473
|
+
// (writeObject over an existing key, removeObject, remove()/deleteByUrl,
|
|
474
|
+
// an `onConflict:'overwrite'` transfer) first asks the backend to retain
|
|
475
|
+
// the current bytes under their sha256. The `retained` sub-db records
|
|
476
|
+
// when and from where; `sweepRetained` drops entries older than the
|
|
477
|
+
// window. Digest-addressed, so the same bytes displaced from ten keys are
|
|
478
|
+
// kept once. Off unless Stored was given `retention: { days }`.
|
|
479
|
+
|
|
480
|
+
get retention() { return this.#retention ? { ...this.#retention } : null; }
|
|
481
|
+
|
|
482
|
+
get #retained() {
|
|
483
|
+
if (!this.#retainedDb) this.#retainedDb = this.#index.openDB('retained');
|
|
484
|
+
return this.#retainedDb;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
async #retain(backendName, key, meta) {
|
|
488
|
+
if (!this.#retention || !meta) return null;
|
|
489
|
+
const backend = this.#backends.get(backendName);
|
|
490
|
+
if (!backend || backend.type !== 'local' || typeof backend.retain !== 'function') return null;
|
|
491
|
+
const sha256 = meta.checksums?.sha256 ? String(meta.checksums.sha256).toLowerCase() : null;
|
|
492
|
+
if (!sha256) return null;
|
|
493
|
+
try {
|
|
494
|
+
const kept = await backend.retain(key, sha256);
|
|
495
|
+
if (!kept) return null;
|
|
496
|
+
const now = Date.now();
|
|
497
|
+
const prev = this.#retained.get(sha256) || null;
|
|
498
|
+
const keys = [...new Set([...(prev?.keys || []), `${backendName}:${key}`])].slice(-20);
|
|
499
|
+
const entry = {
|
|
500
|
+
sha256,
|
|
501
|
+
backend: prev?.backend || backendName,
|
|
502
|
+
size: kept.size ?? prev?.size ?? meta.size ?? null,
|
|
503
|
+
mimeType: meta.mimeType ?? prev?.mimeType ?? null,
|
|
504
|
+
id: meta.id ?? prev?.id ?? null,
|
|
505
|
+
keys,
|
|
506
|
+
firstAt: prev?.firstAt ?? now,
|
|
507
|
+
lastAt: now,
|
|
508
|
+
};
|
|
509
|
+
this.#retained.putSync(sha256, entry);
|
|
510
|
+
this.#scheduleSweep();
|
|
511
|
+
return entry;
|
|
512
|
+
} catch (err) {
|
|
513
|
+
debug(`retain ${backendName}:${key} failed: ${err.message}`);
|
|
514
|
+
this.emit('error', Object.assign(new Error(`retention failed for ${backendName}:${key}: ${err.message}`), { code: 'RETAIN_FAILED' }));
|
|
515
|
+
return null;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
#scheduleSweep() {
|
|
520
|
+
if (this.#sweepTimer || !this.#retention) return;
|
|
521
|
+
this.#sweepTimer = setInterval(() => { this.sweepRetained().catch(() => {}); }, this.#retention.sweepEveryMs);
|
|
522
|
+
this.#sweepTimer.unref?.();
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/** Retained entries, newest displaced first: `[{ sha256, backend, size, mimeType, id, keys, firstAt, lastAt }]`. */
|
|
526
|
+
listRetained({ backend = null, key = null, limit = 1000 } = {}) {
|
|
527
|
+
if (!this.#retention) return [];
|
|
528
|
+
const out = [];
|
|
529
|
+
for (const { value } of this.#retained.getRange()) {
|
|
530
|
+
if (!value?.sha256) continue;
|
|
531
|
+
if (backend && value.backend !== backend && !value.keys?.some((k) => k.startsWith(`${backend}:`))) continue;
|
|
532
|
+
if (key && !value.keys?.some((k) => k === key || k.endsWith(`:${key}`))) continue;
|
|
533
|
+
out.push(value);
|
|
534
|
+
}
|
|
535
|
+
out.sort((a, b) => (b.lastAt ?? 0) - (a.lastAt ?? 0));
|
|
536
|
+
return out.slice(0, limit);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
getRetained(sha256) {
|
|
540
|
+
if (!this.#retention) return null;
|
|
541
|
+
return this.#retained.get(String(sha256).toLowerCase()) || null;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** Readable stream of retained bytes, or null. */
|
|
545
|
+
async retainedStream(sha256) {
|
|
546
|
+
const entry = this.getRetained(sha256);
|
|
547
|
+
const backend = entry && this.#backends.get(entry.backend);
|
|
548
|
+
if (!backend || typeof backend.retainedPath !== 'function') return null;
|
|
549
|
+
const p = backend.retainedPath(entry.sha256);
|
|
550
|
+
if (!await fsp.stat(p).catch(() => null)) return null;
|
|
551
|
+
return createReadStream(p);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Put retained bytes back at `backend:key` through the ordinary keyed
|
|
556
|
+
* write (succession, change log, preconditions). `ifNoneMatch:'*'` refuses
|
|
557
|
+
* an occupied key; pass `ifMatch` (digest) to replace a known current.
|
|
558
|
+
*/
|
|
559
|
+
async restoreRetained(sha256, { backend, key, ifMatch = null, ifNoneMatch = null, origin = null, mtime = null } = {}) {
|
|
560
|
+
const entry = this.getRetained(sha256);
|
|
561
|
+
if (!entry) return { ok: false, reason: 'not-found', sha256 };
|
|
562
|
+
const stream = await this.retainedStream(entry.sha256);
|
|
563
|
+
if (!stream) return { ok: false, reason: 'not-found', sha256, detail: 'retained bytes missing on disk' };
|
|
564
|
+
const target = backend || entry.backend;
|
|
565
|
+
const targetKey = key || entry.keys?.at(-1)?.slice(target.length + 1) || null;
|
|
566
|
+
if (!targetKey) return { ok: false, reason: 'invalid-key', key };
|
|
567
|
+
const res = await this.writeObject(target, targetKey, stream, {
|
|
568
|
+
sha256: entry.sha256, mimeType: entry.mimeType || undefined,
|
|
569
|
+
...(ifMatch != null ? { ifMatch } : {}), ...(ifNoneMatch != null ? { ifNoneMatch } : {}),
|
|
570
|
+
...(origin ? { origin } : {}), ...(mtime != null ? { mtime } : {}),
|
|
571
|
+
});
|
|
572
|
+
return res.ok ? { ...res, key: targetKey, backend: target, restored: entry.sha256 } : res;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/** Drop entries (and their bytes) displaced longer than the window ago. Returns `{ swept, kept }`. */
|
|
576
|
+
async sweepRetained({ olderThanMs = null, now = Date.now() } = {}) {
|
|
577
|
+
if (!this.#retention) return { swept: 0, kept: 0 };
|
|
578
|
+
const window = olderThanMs ?? this.#retention.days * 86_400_000;
|
|
579
|
+
let swept = 0;
|
|
580
|
+
let kept = 0;
|
|
581
|
+
for (const { key: sha, value } of [...this.#retained.getRange()]) {
|
|
582
|
+
if (!value?.sha256) continue;
|
|
583
|
+
if ((now - (value.lastAt ?? 0)) < window) { kept += 1; continue; }
|
|
584
|
+
const backend = this.#backends.get(value.backend);
|
|
585
|
+
if (backend && typeof backend.dropRetained === 'function') await backend.dropRetained(value.sha256).catch(() => {});
|
|
586
|
+
this.#retained.removeSync(sha);
|
|
587
|
+
swept += 1;
|
|
588
|
+
}
|
|
589
|
+
if (swept) debug(`retention sweep: ${swept} dropped, ${kept} kept`);
|
|
590
|
+
return { swept, kept };
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
async forgetRetained(sha256) {
|
|
594
|
+
const entry = this.getRetained(sha256);
|
|
595
|
+
if (!entry) return false;
|
|
596
|
+
const backend = this.#backends.get(entry.backend);
|
|
597
|
+
if (backend && typeof backend.dropRetained === 'function') await backend.dropRetained(entry.sha256).catch(() => {});
|
|
598
|
+
this.#retained.removeSync(entry.sha256);
|
|
599
|
+
return true;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Delete the bytes at `backend:key` (precondition-checked like
|
|
604
|
+
* writeObject) and process it as the genuine unlink it is: consumers get
|
|
605
|
+
* `object:unlink`, the change log gets a `delete`.
|
|
606
|
+
*/
|
|
607
|
+
async removeObject(backendName, key, options = {}) {
|
|
608
|
+
const backend = this.#backends.get(backendName);
|
|
609
|
+
if (!backend) return { ok: false, reason: 'unknown-backend', backend: backendName };
|
|
610
|
+
if (!backend.canDelete) return { ok: false, reason: 'read-only-backend', backend: backendName };
|
|
611
|
+
const destKey = this.#normalizeKey(key, { nfc: true });
|
|
612
|
+
if (!destKey || !this.#isSafeKey(destKey)) return { ok: false, reason: 'invalid-key', key };
|
|
613
|
+
|
|
614
|
+
const pathKey = `${backendName}:${destKey}`;
|
|
615
|
+
const current = this.#index.get(pathKey);
|
|
616
|
+
const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
|
|
617
|
+
if (!current) return { ok: false, reason: 'not-found', key: destKey };
|
|
618
|
+
const failed = this.#checkPrecondition(current, currentLoc, options);
|
|
619
|
+
if (failed) return failed;
|
|
620
|
+
|
|
621
|
+
const release = this.#holdKeys([pathKey]);
|
|
622
|
+
try {
|
|
623
|
+
// false = bytes already gone; the location still goes. A remote
|
|
624
|
+
// driver re-evaluates the precondition at the other end (412 →
|
|
625
|
+
// typed error, nothing is unlinked here).
|
|
626
|
+
await this.#retain(backendName, destKey, current);
|
|
627
|
+
await backend.delete(destKey, { ifMatch: options.ifMatch, origin: options.origin });
|
|
628
|
+
this.#processUnlink({ backend: backendName, key: destKey, ...(options.origin ? { origin: options.origin } : {}) });
|
|
629
|
+
debug(`REMOVE ${pathKey}`);
|
|
630
|
+
return { ok: true, id: current.id, sha256: current.checksums?.sha256 ?? null, seq: this.#index.head() };
|
|
631
|
+
} finally {
|
|
632
|
+
release();
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Rename `backend:from` to `backend:to` on the same backend: one
|
|
638
|
+
* `rename(2)` where the driver supports it, identity and inode intact,
|
|
639
|
+
* `object:move` for consumers, a single `rename` change-log entry.
|
|
640
|
+
* Refuses an occupied destination (`target-exists`).
|
|
641
|
+
*/
|
|
642
|
+
async renameObject(backendName, from, to, options = {}) {
|
|
643
|
+
const backend = this.#backends.get(backendName);
|
|
644
|
+
if (!backend) return { ok: false, reason: 'unknown-backend', backend: backendName };
|
|
645
|
+
const fromKey = this.#normalizeKey(from, { nfc: true });
|
|
646
|
+
const toKey = this.#normalizeKey(to, { nfc: true });
|
|
647
|
+
if (!fromKey || !this.#isSafeKey(fromKey)) return { ok: false, reason: 'invalid-key', key: from };
|
|
648
|
+
if (!toKey || !this.#isSafeKey(toKey)) return { ok: false, reason: 'invalid-key', key: to };
|
|
649
|
+
if (fromKey === toKey) return { ok: false, reason: 'same-key', key: fromKey };
|
|
650
|
+
|
|
651
|
+
const current = this.#index.get(`${backendName}:${fromKey}`);
|
|
652
|
+
const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === fromKey) || null;
|
|
653
|
+
if (!current) return { ok: false, reason: 'not-found', key: fromKey };
|
|
654
|
+
const failed = this.#checkPrecondition(current, currentLoc, options);
|
|
655
|
+
if (failed) return failed;
|
|
656
|
+
if (this.#index.get(`${backendName}:${toKey}`) || await backend.stat(toKey).catch(() => null)) {
|
|
657
|
+
return { ok: false, reason: 'target-exists', key: toKey };
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
const result = await this.move(`${backendName}:${fromKey}`, {
|
|
661
|
+
to: backendName,
|
|
662
|
+
key: toKey,
|
|
663
|
+
from: `stored://${backendName}/${fromKey}`,
|
|
664
|
+
onConflict: 'error',
|
|
665
|
+
origin: options.origin,
|
|
666
|
+
ifMatch: options.ifMatch,
|
|
667
|
+
});
|
|
668
|
+
if (!result.ok) return result;
|
|
669
|
+
return {
|
|
670
|
+
ok: true,
|
|
671
|
+
id: current.id,
|
|
672
|
+
sha256: current.checksums?.sha256 ?? null,
|
|
673
|
+
from: fromKey,
|
|
674
|
+
to: toKey,
|
|
675
|
+
state: result.state,
|
|
676
|
+
seq: this.#index.head(),
|
|
677
|
+
};
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// `If-Match` / `If-None-Match` semantics over the index. Returns the typed
|
|
681
|
+
// failure (with what is actually there) or null when the write may go on.
|
|
682
|
+
#checkPrecondition(current, currentLoc, { ifMatch = null, ifNoneMatch = null } = {}) {
|
|
683
|
+
const strip = (v) => String(v).trim().replace(/^W\//, '').replace(/^"|"$/g, '').toLowerCase();
|
|
684
|
+
const describe = () => ({
|
|
685
|
+
ok: false,
|
|
686
|
+
reason: 'precondition-failed',
|
|
687
|
+
code: 'PRECONDITION_FAILED',
|
|
688
|
+
current: current ? {
|
|
689
|
+
id: current.id,
|
|
690
|
+
sha256: current.checksums?.sha256 ?? null,
|
|
691
|
+
size: currentLoc?.size ?? current.size ?? null,
|
|
692
|
+
mtime: currentLoc?.mtime ?? null,
|
|
693
|
+
} : null,
|
|
694
|
+
});
|
|
695
|
+
if (ifNoneMatch != null && String(ifNoneMatch).trim() === '*' && current) return describe();
|
|
696
|
+
if (ifMatch != null) {
|
|
697
|
+
if (!current) return describe();
|
|
698
|
+
const want = strip(ifMatch);
|
|
699
|
+
const have = String(current.checksums?.sha256 || '').toLowerCase();
|
|
700
|
+
if (want !== '*' && want !== have && want !== String(current.id).toLowerCase()) return describe();
|
|
701
|
+
}
|
|
702
|
+
return null;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// Collapse duplicate slashes, trim edge slashes; optionally NFC-normalize
|
|
706
|
+
// (the canonical spelling for keys that arrive over the wire from other
|
|
707
|
+
// platforms — macOS reports NFD).
|
|
708
|
+
#normalizeKey(key, { nfc = false } = {}) {
|
|
709
|
+
let out = String(key ?? '').replace(/\/{2,}/g, '/').replace(/^\/+|\/+$/g, '');
|
|
710
|
+
if (nfc) out = out.normalize('NFC');
|
|
711
|
+
return out;
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
// A key must stay inside the backend root: no empty, '.' or '..' segments,
|
|
715
|
+
// no NUL, no absolute/backslash-absolute forms.
|
|
716
|
+
#isSafeKey(key) {
|
|
717
|
+
if (typeof key !== 'string' || !key.length || key.includes('\0')) return false;
|
|
718
|
+
if (path.isAbsolute(key) || /^[a-zA-Z]:[\\/]/.test(key)) return false;
|
|
719
|
+
return key.split('/').every(seg => seg.length > 0 && seg !== '.' && seg !== '..');
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
723
|
+
// Copy / move — location mutations that preserve content identity
|
|
724
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Copy the bytes behind `idOrKey` onto one or more other backends. Content
|
|
728
|
+
* identity is unchanged: the same index entry simply gains locations.
|
|
729
|
+
*
|
|
730
|
+
* Bytes are streamed through a hashing pass and the result is checked
|
|
731
|
+
* against the indexed id before anything is committed — the stream is being
|
|
732
|
+
* paid for regardless, and it catches a source that rotted or was rewritten
|
|
733
|
+
* behind our back (returns `checksum-mismatch` rather than silently
|
|
734
|
+
* propagating the wrong bytes under the old id).
|
|
735
|
+
*
|
|
736
|
+
* @param {string} idOrKey Content id or `{backend}:{key}`
|
|
737
|
+
* @param {object} options
|
|
738
|
+
* @param {string|string[]} options.to Target backend name(s)
|
|
739
|
+
* @param {string} [options.key] Key on the targets (defaults to the source key)
|
|
740
|
+
* @param {string} [options.from] Source backend name or `stored://` URL
|
|
741
|
+
* @param {'error'|'rename'|'overwrite'} [options.onConflict='error'] What to do
|
|
742
|
+
* when other content already occupies the destination key: refuse (default),
|
|
743
|
+
* pick the next free `name-1.ext`, or replace it.
|
|
744
|
+
* @returns {Promise<{ ok:true, id, added:string[], locations } | { ok:false, reason:string }>}
|
|
745
|
+
*/
|
|
746
|
+
async copy(idOrKey, options = {}) {
|
|
747
|
+
const plan = await this.#planTransfer(idOrKey, options);
|
|
748
|
+
if (!plan.ok) return plan;
|
|
749
|
+
|
|
750
|
+
const { meta, sourceLocation, targets, destKey } = plan;
|
|
751
|
+
if (!targets.length) {
|
|
752
|
+
return { ok: true, id: meta.id, added: [], unchanged: true, locations: await this.locations(meta.id) };
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
const release = this.#holdKeys(targets.map(t => `${t.name}:${t.key || destKey}`));
|
|
756
|
+
try {
|
|
757
|
+
const transferred = await this.#streamToTargets(meta, sourceLocation, targets, destKey, this.#transferOptions(options));
|
|
758
|
+
if (!transferred.ok) return transferred;
|
|
759
|
+
// A conflict-inbox upload placed nothing at the key: no location.
|
|
760
|
+
if (transferred.conflict && !transferred.locations.length) {
|
|
761
|
+
return { ok: true, id: meta.id, added: [], conflict: transferred.conflict, locations: await this.locations(meta.id) };
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
// Displace-then-record in one transaction: the change log must
|
|
765
|
+
// never show the new owner of a key before the old one let go.
|
|
766
|
+
const { saved, previous } = this.#index.transaction(() => {
|
|
767
|
+
const previous = this.#displaceTargets(targets, destKey, meta.id, options);
|
|
768
|
+
const current = this.#index.get(meta.id) || meta;
|
|
769
|
+
const saved = this.#index.put(meta.id, { ...current, locations: this.#mergeLocations(current, transferred.locations) }, { origin: options.origin });
|
|
770
|
+
return { saved, previous };
|
|
771
|
+
});
|
|
772
|
+
if (transferred.remoteTargets.length) {
|
|
773
|
+
this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
for (const loc of transferred.locations) {
|
|
777
|
+
this.#emitLocationEvent('location:add', saved, loc, previous ? { previous } : {});
|
|
778
|
+
}
|
|
779
|
+
debug(`COPY ${meta.id.slice(0, 19)}... → ${targets.map(t => t.name).join(', ')}/${destKey}`);
|
|
780
|
+
return {
|
|
781
|
+
ok: true,
|
|
782
|
+
id: meta.id,
|
|
783
|
+
added: transferred.locations.map(l => `stored://${l.backend}/${l.key}`),
|
|
784
|
+
locations: saved.locations.map(l => this.#describeLocation(l)),
|
|
785
|
+
...(transferred.result ? { remote: transferred.result } : {}),
|
|
786
|
+
...(transferred.conflict ? { conflict: transferred.conflict } : {}),
|
|
787
|
+
};
|
|
788
|
+
} finally {
|
|
789
|
+
release();
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
/**
|
|
794
|
+
* Move the bytes behind `idOrKey` to another backend: a copy followed by
|
|
795
|
+
* removal of the source location.
|
|
796
|
+
*
|
|
797
|
+
* Ordering is load-bearing — the source is never dropped until the
|
|
798
|
+
* destination is durable. For a `type:'remote'` target that means waiting
|
|
799
|
+
* for the SyncQueue, so the call returns `state:'pending'` and completes on
|
|
800
|
+
* the `synced` event; a failed sync leaves the object as a copy rather than
|
|
801
|
+
* losing it. Within one filesystem the whole transfer collapses to a single
|
|
802
|
+
* `rename(2)` (no bytes cross userspace, no re-hash — the inode is the
|
|
803
|
+
* proof of identity).
|
|
804
|
+
*
|
|
805
|
+
* Emits `object:move` on completion — consumers must patch `locations[]` in
|
|
806
|
+
* place, NOT unlink + re-add, or the document loses its identity.
|
|
807
|
+
*
|
|
808
|
+
* @returns {Promise<{ ok:true, id, state:'complete'|'pending', from, to, locations } | { ok:false, reason:string }>}
|
|
809
|
+
*/
|
|
810
|
+
async move(idOrKey, options = {}) {
|
|
811
|
+
const plan = await this.#planTransfer(idOrKey, options);
|
|
812
|
+
if (!plan.ok) return plan;
|
|
813
|
+
|
|
814
|
+
const { meta, sourceLocation, sourceBackend, targets, destKey } = plan;
|
|
815
|
+
if (!targets.length) {
|
|
816
|
+
return { ok: true, id: meta.id, state: 'complete', unchanged: true, locations: await this.locations(meta.id) };
|
|
817
|
+
}
|
|
818
|
+
// Fan-out is `copy`'s job; a move has exactly one destination, otherwise
|
|
819
|
+
// "which one may the source be dropped for?" has no answer.
|
|
820
|
+
if (targets.length > 1) return { ok: false, reason: 'move-single-target' };
|
|
821
|
+
if (!sourceBackend.canDelete) return { ok: false, reason: 'source-not-removable', backend: sourceLocation.backend };
|
|
822
|
+
|
|
823
|
+
const target = targets[0];
|
|
824
|
+
// Conflict resolution may have renamed this target's key (see
|
|
825
|
+
// #planTransfer) — everything below addresses the resolved one.
|
|
826
|
+
const targetKey = target.key || destKey;
|
|
827
|
+
const release = this.#holdKeys([
|
|
828
|
+
`${sourceLocation.backend}:${sourceLocation.key}`,
|
|
829
|
+
`${target.name}:${targetKey}`,
|
|
830
|
+
]);
|
|
831
|
+
try {
|
|
832
|
+
// Fast path — same filesystem, both file backends.
|
|
833
|
+
const renamed = await this.#tryRename(sourceBackend, sourceLocation.key, target.backend, targetKey);
|
|
834
|
+
if (renamed) {
|
|
835
|
+
// The source's folder may be empty now (a hub delete moves the
|
|
836
|
+
// last file of a folder to trash) — drop the shell like delete() does.
|
|
837
|
+
if (typeof sourceBackend.pruneEmptyParents === 'function') await sourceBackend.pruneEmptyParents(sourceLocation.key).catch(() => {});
|
|
838
|
+
const location = this.#buildLocation(target.name, targetKey, true, {
|
|
839
|
+
size: renamed.size, mtime: renamed.modified, dev: renamed.dev, ino: renamed.ino,
|
|
840
|
+
});
|
|
841
|
+
debug(`MOVE (rename) ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
|
|
842
|
+
return this.#index.transaction(() => {
|
|
843
|
+
const previous = this.#displaceTargets(targets, destKey, meta.id, options);
|
|
844
|
+
return this.#finalizeMove(meta.id, sourceLocation, location, { removeSourceBytes: false, origin: options.origin, previous });
|
|
845
|
+
});
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
// Same remote backend, new key: the driver renames server-side
|
|
849
|
+
// (one request, no bytes through here, identity intact).
|
|
850
|
+
if (sourceLocation.backend === target.name && sourceBackend.type === 'remote' && typeof sourceBackend.rename === 'function') {
|
|
851
|
+
const renamedRemote = await this.#tryRemoteRename(sourceBackend, sourceLocation, targetKey, options);
|
|
852
|
+
if (!renamedRemote.ok) return renamedRemote;
|
|
853
|
+
debug(`MOVE (remote rename) ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
|
|
854
|
+
return this.#index.transaction(() => {
|
|
855
|
+
const previous = this.#displaceTargets(targets, destKey, meta.id, options);
|
|
856
|
+
return this.#finalizeMove(meta.id, sourceLocation, renamedRemote.location, { removeSourceBytes: false, origin: options.origin, previous });
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
const transferred = await this.#streamToTargets(meta, sourceLocation, targets, destKey, this.#transferOptions(options));
|
|
861
|
+
if (!transferred.ok) return transferred;
|
|
862
|
+
const location = transferred.locations[0];
|
|
863
|
+
|
|
864
|
+
// Add-first: the destination location is recorded before the source
|
|
865
|
+
// is touched, so a crash here leaves a copy — never an orphan.
|
|
866
|
+
const { saved, previous } = this.#index.transaction(() => {
|
|
867
|
+
const previous = this.#displaceTargets(targets, destKey, meta.id, options);
|
|
868
|
+
const current = this.#index.get(meta.id) || meta;
|
|
869
|
+
const saved = this.#index.put(meta.id, { ...current, locations: this.#mergeLocations(current, transferred.locations) }, { origin: options.origin });
|
|
870
|
+
return { saved, previous };
|
|
871
|
+
});
|
|
872
|
+
this.#emitLocationEvent('location:add', saved, location, previous ? { previous } : {});
|
|
873
|
+
|
|
874
|
+
if (transferred.remoteTargets.length) {
|
|
875
|
+
const pending = this.#pendingMoves.get(meta.id) || [];
|
|
876
|
+
pending.push({ sourceLocation, target: target.name, destKey: targetKey });
|
|
877
|
+
this.#pendingMoves.set(meta.id, pending);
|
|
878
|
+
this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
|
|
879
|
+
debug(`MOVE (pending sync) ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
|
|
880
|
+
return {
|
|
881
|
+
ok: true,
|
|
882
|
+
id: meta.id,
|
|
883
|
+
state: 'pending',
|
|
884
|
+
from: this.#endpoint(sourceLocation),
|
|
885
|
+
to: this.#endpoint(location),
|
|
886
|
+
locations: saved.locations.map(l => this.#describeLocation(l)),
|
|
887
|
+
};
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
debug(`MOVE ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
|
|
891
|
+
return this.#finalizeMove(meta.id, sourceLocation, location, { removeSourceBytes: true, origin: options.origin });
|
|
892
|
+
} finally {
|
|
893
|
+
release();
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
/**
|
|
898
|
+
* Where does the content behind `idOrKey` actually live? Returns the
|
|
899
|
+
* canonical, resolvable `stored://<backend>/<key>` URLs plus per-location
|
|
900
|
+
* metadata. Single source of truth for the URL grammar — consumers map
|
|
901
|
+
* these straight into synapsd documents and read them back via getByUrl.
|
|
902
|
+
*/
|
|
903
|
+
async locations(idOrKey) {
|
|
904
|
+
const meta = this.#index.get(idOrKey);
|
|
905
|
+
return (meta?.locations || []).map(l => this.#describeLocation(l));
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
#describeLocation(l) {
|
|
909
|
+
const backend = this.#backends.get(l.backend);
|
|
910
|
+
// Live backend first (a mount may have been re-detected since indexing),
|
|
911
|
+
// falling back to the snapshot on the location for offline backends.
|
|
912
|
+
const remote = backend ? backend.remote : (l.source?.remote === true);
|
|
913
|
+
return {
|
|
914
|
+
url: `stored://${l.backend}/${l.key}`,
|
|
915
|
+
nativeUrl: backend ? backend.nativeUrl(l.key) : null,
|
|
916
|
+
backend: l.backend,
|
|
917
|
+
key: l.key,
|
|
918
|
+
driver: l.driver,
|
|
919
|
+
synced: l.synced,
|
|
920
|
+
size: l.size,
|
|
921
|
+
remote,
|
|
922
|
+
transport: backend ? backend.transport : (l.source?.transport ?? null),
|
|
923
|
+
source: l.source,
|
|
924
|
+
};
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
// Iterate all indexed metadata entries.
|
|
928
|
+
async *list() {
|
|
929
|
+
for (const [, meta] of this.#index.entries()) yield meta;
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
// Iterate a single backend's native listing (raw keys, not index entries).
|
|
933
|
+
async *listBackend(name, options = {}) {
|
|
934
|
+
const backend = this.#backends.get(name);
|
|
935
|
+
if (backend) yield* backend.list(options);
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
939
|
+
// Scan / Index
|
|
940
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* Index existing objects from one backend (or all). `options` is forwarded
|
|
944
|
+
* verbatim to each backend's `scan()` so each driver defines its own knobs.
|
|
945
|
+
* File backends get a size+mtime skip predicate so unchanged files are not
|
|
946
|
+
* re-hashed. Returns { ok:true, backend, count, files } | { ok:false, reason }.
|
|
947
|
+
*/
|
|
948
|
+
async scan(backendName, options = {}) {
|
|
949
|
+
const { onFile, ...backendOptions } = options;
|
|
950
|
+
const list = backendName
|
|
951
|
+
? [this.#backends.get(backendName)].filter(Boolean)
|
|
952
|
+
: this.#backends.all();
|
|
953
|
+
if (backendName && !list.length) return { ok: false, reason: 'unknown-backend' };
|
|
954
|
+
|
|
955
|
+
const files = [];
|
|
956
|
+
let complete = true;
|
|
957
|
+
let errors = null;
|
|
958
|
+
for (const backend of list) {
|
|
959
|
+
// Per-file ingest into the stored index — idempotent, guarded so the
|
|
960
|
+
// streaming path (backend onFile) and the returned-rows path don't
|
|
961
|
+
// double-process a key.
|
|
962
|
+
const processed = new Set();
|
|
963
|
+
const ingest = (file) => {
|
|
964
|
+
if (processed.has(file.key)) return;
|
|
965
|
+
processed.add(file.key);
|
|
966
|
+
if (!file.checksums) return;
|
|
967
|
+
const id = formatId(file.checksums, this.#config.primaryChecksum);
|
|
968
|
+
const existing = this.#index.get(id);
|
|
969
|
+
const locations = existing?.locations || [];
|
|
970
|
+
const match = locations.find(l => l.backend === file.backend && l.key === file.key);
|
|
971
|
+
if (match) {
|
|
972
|
+
match.size = file.size;
|
|
973
|
+
match.mtime = file.modified;
|
|
974
|
+
if (file.ino != null) { match.ino = file.ino; match.dev = file.dev; }
|
|
975
|
+
} else {
|
|
976
|
+
locations.push(this.#buildLocation(file.backend, file.key, true, {
|
|
977
|
+
size: file.size,
|
|
978
|
+
mtime: file.modified,
|
|
979
|
+
// Inode identity — enables rename matching (same
|
|
980
|
+
// (dev,ino) at a new path is a move, not delete+add).
|
|
981
|
+
...(file.ino != null ? { ino: file.ino, dev: file.dev } : {}),
|
|
982
|
+
}));
|
|
983
|
+
}
|
|
984
|
+
this.#index.put(id, { checksums: file.checksums, size: file.size, mimeType: file.mimeType, locations });
|
|
985
|
+
};
|
|
986
|
+
|
|
987
|
+
// Inode identity snapshot for rename matching: same (dev, ino) at a
|
|
988
|
+
// NEW path with unchanged size+mtime is a move — reuse the indexed
|
|
989
|
+
// checksums instead of rehashing, so a folder rename (thousands of
|
|
990
|
+
// "moves") collapses into cheap URL rewrites.
|
|
991
|
+
const inodeMap = this.#buildInodeMap(backend.name);
|
|
992
|
+
|
|
993
|
+
const result = await backend.scan({
|
|
994
|
+
algorithms: this.#config.checksums,
|
|
995
|
+
knownChecksums: (k, st) =>
|
|
996
|
+
this.#knownIfUnchanged(backend.name, k, st) || this.#knownByInode(inodeMap, k, st),
|
|
997
|
+
...backendOptions,
|
|
998
|
+
// Stream-through: index the file, then hand it to the caller so
|
|
999
|
+
// consumers (workspace doc upserts) see results as the walk runs.
|
|
1000
|
+
onFile: async (file) => {
|
|
1001
|
+
ingest(file);
|
|
1002
|
+
if (typeof onFile === 'function') await onFile(file);
|
|
1003
|
+
},
|
|
1004
|
+
});
|
|
1005
|
+
// Snapshot shape { files, complete, errors } (file driver) or a bare
|
|
1006
|
+
// rows array (legacy/simple drivers — treated as a complete walk).
|
|
1007
|
+
// Backends whose scan() reports via events instead (non-array, no
|
|
1008
|
+
// files[]) are not content-addressable here.
|
|
1009
|
+
const snapshot = Array.isArray(result)
|
|
1010
|
+
? { files: result, complete: true, errors: null }
|
|
1011
|
+
: (Array.isArray(result?.files) ? result : null);
|
|
1012
|
+
if (!snapshot) continue;
|
|
1013
|
+
|
|
1014
|
+
if (snapshot.complete === false) complete = false;
|
|
1015
|
+
if (snapshot.errors) errors = { ...(errors || {}), [backend.name]: snapshot.errors };
|
|
1016
|
+
|
|
1017
|
+
// Backends that ignore onFile still get their rows indexed here.
|
|
1018
|
+
for (const file of snapshot.files) ingest(file);
|
|
1019
|
+
|
|
1020
|
+
// Removal only ever runs against a usable snapshot: a dead/absent
|
|
1021
|
+
// root (errors.root) means the backend is offline — nothing is
|
|
1022
|
+
// removed. Hash-failed rows are still rows, so their keys count as
|
|
1023
|
+
// present; unreadable subtrees carry their prior entries forward.
|
|
1024
|
+
if (snapshot.errors?.root) continue;
|
|
1025
|
+
const presentKeys = new Set(snapshot.files.map(file => file.key));
|
|
1026
|
+
const erroredPrefixes = (snapshot.errors?.dirs || []).map(d => d.prefix).filter(Boolean);
|
|
1027
|
+
this.#removeMissingLocations(backend.name, presentKeys, erroredPrefixes);
|
|
1028
|
+
files.push(...snapshot.files);
|
|
1029
|
+
}
|
|
1030
|
+
return { ok: true, backend: backendName ?? null, count: files.length, files, complete, errors };
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
/**
|
|
1034
|
+
* Cheap structural walk of one backend (dirs + file count, no hashing).
|
|
1035
|
+
* Returns { ok, dirs, files } or { ok:false } when the backend does not
|
|
1036
|
+
* support it (only file backends do).
|
|
1037
|
+
*/
|
|
1038
|
+
async shape(backendName) {
|
|
1039
|
+
const backend = this.#backends.get(backendName);
|
|
1040
|
+
if (!backend) return { ok: false, reason: 'unknown-backend' };
|
|
1041
|
+
if (typeof backend.shape !== 'function') return { ok: false, reason: 'unsupported' };
|
|
1042
|
+
const { dirs, files } = await backend.shape();
|
|
1043
|
+
return { ok: true, dirs, files };
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1047
|
+
// Lifecycle
|
|
1048
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1049
|
+
|
|
1050
|
+
async stop() {
|
|
1051
|
+
if (this.#sweepTimer) { clearInterval(this.#sweepTimer); this.#sweepTimer = null; }
|
|
1052
|
+
await this.#syncQueue.stop();
|
|
1053
|
+
await this.#backends.stopAll();
|
|
1054
|
+
// Flush held unlinks (watchers are stopped — no add can claim them now)
|
|
1055
|
+
// so the index reflects them before it closes.
|
|
1056
|
+
for (const [pendingKey, { data, timer }] of this.#pendingUnlinks) {
|
|
1057
|
+
clearTimeout(timer);
|
|
1058
|
+
this.#pendingUnlinks.delete(pendingKey);
|
|
1059
|
+
this.#processUnlink(data);
|
|
1060
|
+
}
|
|
1061
|
+
this.#index.close();
|
|
1062
|
+
debug('Stopped');
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1066
|
+
// Private — sync result handling
|
|
1067
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1068
|
+
|
|
1069
|
+
#handleSyncResult(id, results) {
|
|
1070
|
+
const meta = this.#index.get(id);
|
|
1071
|
+
if (!meta) return;
|
|
1072
|
+
|
|
1073
|
+
for (const r of results) {
|
|
1074
|
+
if (!r.success) continue;
|
|
1075
|
+
const loc = meta.locations.find(l => l.backend === r.backend);
|
|
1076
|
+
if (loc) loc.synced = true;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
this.#index.put(meta.id, meta);
|
|
1080
|
+
this.emit('synced', { id, results });
|
|
1081
|
+
this.#resolvePendingMoves(id, results);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1085
|
+
// Private — copy / move
|
|
1086
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1087
|
+
|
|
1088
|
+
/**
|
|
1089
|
+
* Validate a copy/move request before a single byte is read: content must
|
|
1090
|
+
* exist, the source must be readable and online, every target must exist, be
|
|
1091
|
+
* writable and be online. Targets that already hold `destKey` are dropped
|
|
1092
|
+
* (a copy onto an existing location is a no-op, not an error).
|
|
1093
|
+
*/
|
|
1094
|
+
async #planTransfer(idOrKey, options = {}) {
|
|
1095
|
+
const meta = this.#index.get(idOrKey);
|
|
1096
|
+
if (!meta) return { ok: false, reason: 'not-found' };
|
|
1097
|
+
|
|
1098
|
+
const onConflict = ['error', 'overwrite', 'rename'].includes(options.onConflict)
|
|
1099
|
+
? options.onConflict
|
|
1100
|
+
: 'error';
|
|
1101
|
+
|
|
1102
|
+
const toNames = (Array.isArray(options.to) ? options.to : [options.to]).filter(Boolean).map(String);
|
|
1103
|
+
if (!toNames.length) return { ok: false, reason: 'no-targets' };
|
|
1104
|
+
|
|
1105
|
+
const sourceLocation = this.#resolveSourceLocation(meta, options.from);
|
|
1106
|
+
if (!sourceLocation) return { ok: false, reason: 'no-source' };
|
|
1107
|
+
const sourceBackend = this.#backends.get(sourceLocation.backend);
|
|
1108
|
+
const sourceLive = await this.#verifyBackendRoot(sourceBackend);
|
|
1109
|
+
if (!sourceLive.ok) {
|
|
1110
|
+
return { ok: false, reason: 'source-offline', backend: sourceLocation.backend, detail: sourceLive.reason };
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
// Normalize the destination key: a template that leaves an empty
|
|
1114
|
+
// segment ('Fotky//2019') writes a file the OS reports back under the
|
|
1115
|
+
// collapsed path, and the watcher would then index it a SECOND time as
|
|
1116
|
+
// a new location. The index must spell keys the way the filesystem does.
|
|
1117
|
+
const destKey = this.#normalizeKey(options.key || sourceLocation.key);
|
|
1118
|
+
if (!destKey) return { ok: false, reason: 'invalid-key' };
|
|
1119
|
+
const conflictKey = options.conflictKey ? this.#normalizeKey(options.conflictKey) : null;
|
|
1120
|
+
const targets = [];
|
|
1121
|
+
for (const name of toNames) {
|
|
1122
|
+
const backend = this.#backends.get(name);
|
|
1123
|
+
if (!backend) return { ok: false, reason: 'unknown-backend', backend: name };
|
|
1124
|
+
if (!backend.canWrite) return { ok: false, reason: 'read-only-target', backend: name };
|
|
1125
|
+
if (name === sourceLocation.backend && destKey === sourceLocation.key) continue;
|
|
1126
|
+
// Already recorded there — but only skip if the bytes are REALLY
|
|
1127
|
+
// there. An index that outlived its file (deleted behind our back,
|
|
1128
|
+
// an unmounted share, a failed sync) must not turn a copy into a
|
|
1129
|
+
// silent no-op that reports success.
|
|
1130
|
+
const ours = (meta.locations || []).some(l => l.backend === name && l.key === destKey);
|
|
1131
|
+
const present = await backend.stat(destKey).catch(() => null);
|
|
1132
|
+
if (ours && present) continue;
|
|
1133
|
+
if (ours) debug(`Stale location ${name}:${destKey} (indexed, no bytes) — re-transferring`);
|
|
1134
|
+
|
|
1135
|
+
const live = await this.#verifyBackendRoot(backend);
|
|
1136
|
+
if (!live.ok) return { ok: false, reason: 'target-offline', backend: name, detail: live.reason };
|
|
1137
|
+
|
|
1138
|
+
// Somebody else's bytes are sitting on the destination key. Writing
|
|
1139
|
+
// over them is silent data loss, so it takes an explicit policy.
|
|
1140
|
+
let key = destKey;
|
|
1141
|
+
let overwrite = false;
|
|
1142
|
+
if (present && !ours && !this.#isSameContent(name, destKey, meta.id)) {
|
|
1143
|
+
if (onConflict === 'error') return { ok: false, reason: 'target-exists', backend: name, key: destKey };
|
|
1144
|
+
if (onConflict === 'rename') {
|
|
1145
|
+
// A caller-chosen conflict name (`… (conflict from laptop …).ext`)
|
|
1146
|
+
// wins when free; otherwise the generic `name-1.ext` ladder.
|
|
1147
|
+
const wanted = conflictKey && conflictKey !== destKey && !(await backend.stat(conflictKey).catch(() => null))
|
|
1148
|
+
? conflictKey : null;
|
|
1149
|
+
key = wanted || await this.#freeKey(backend, destKey);
|
|
1150
|
+
debug(`Conflict on ${name}:${destKey} — using ${key}`);
|
|
1151
|
+
}
|
|
1152
|
+
// 'overwrite' keeps destKey and lets the commit replace it; the
|
|
1153
|
+
// displaced content is reconciled after the bytes landed.
|
|
1154
|
+
if (onConflict === 'overwrite') overwrite = true;
|
|
1155
|
+
}
|
|
1156
|
+
targets.push({ name, backend, key, overwrite });
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
return { ok: true, meta, sourceLocation, sourceBackend, targets, destKey };
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
/** Do the bytes already on `key` happen to BE this content? Then it is not a conflict. */
|
|
1163
|
+
#isSameContent(backendName, key, id) {
|
|
1164
|
+
return this.#index.get(`${backendName}:${key}`)?.id === id;
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
/**
|
|
1168
|
+
* First free variant of `key`: `photo.jpg` → `photo-1.jpg` → `photo-2.jpg`.
|
|
1169
|
+
* Two photos taken in the same second produce the same templated name, and
|
|
1170
|
+
* silently overwriting one with the other is the worst possible outcome.
|
|
1171
|
+
*/
|
|
1172
|
+
async #freeKey(backend, key) {
|
|
1173
|
+
const dot = key.lastIndexOf('.');
|
|
1174
|
+
const slash = key.lastIndexOf('/');
|
|
1175
|
+
const hasExt = dot > slash + 1;
|
|
1176
|
+
const stem = hasExt ? key.slice(0, dot) : key;
|
|
1177
|
+
const ext = hasExt ? key.slice(dot) : '';
|
|
1178
|
+
|
|
1179
|
+
for (let n = 1; n <= 1000; n += 1) {
|
|
1180
|
+
const candidate = `${stem}-${n}${ext}`;
|
|
1181
|
+
if (!(await backend.stat(candidate).catch(() => null))) return candidate;
|
|
1182
|
+
}
|
|
1183
|
+
throw new Error(`No free key for ${key} after 1000 attempts`);
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
/**
|
|
1187
|
+
* Which location do we read from? An explicit `from` (backend name or
|
|
1188
|
+
* `stored://` URL) wins; otherwise the cheapest readable one — local disk
|
|
1189
|
+
* before a network mount, so a copy between two local backends never pulls
|
|
1190
|
+
* the bytes over the LAN just because the NAS location was indexed first.
|
|
1191
|
+
*/
|
|
1192
|
+
#resolveSourceLocation(meta, from) {
|
|
1193
|
+
const locations = meta.locations || [];
|
|
1194
|
+
if (from) {
|
|
1195
|
+
const parsed = this.#parseStoredUrl(from);
|
|
1196
|
+
const match = parsed
|
|
1197
|
+
? locations.find(l => l.backend === parsed.backend && l.key === parsed.key)
|
|
1198
|
+
: locations.find(l => l.backend === from);
|
|
1199
|
+
if (!match) return null;
|
|
1200
|
+
const backend = this.#backends.get(match.backend);
|
|
1201
|
+
return backend && backend.canRead && match.synced ? match : null;
|
|
1202
|
+
}
|
|
1203
|
+
return this.#readableLocations(meta)[0] || null;
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
/**
|
|
1207
|
+
* Liveness gate. For file backends an absent mountpoint must fail the whole
|
|
1208
|
+
* transfer up front: writing into the empty directory where a NAS used to be
|
|
1209
|
+
* silently creates a second, divergent copy — and reading from one yields
|
|
1210
|
+
* "not found" for content that is merely offline.
|
|
1211
|
+
*/
|
|
1212
|
+
async #verifyBackendRoot(backend) {
|
|
1213
|
+
if (!backend) return { ok: false, reason: 'unknown-backend' };
|
|
1214
|
+
if (typeof backend.verifyRoot !== 'function') return { ok: true };
|
|
1215
|
+
return backend.verifyRoot();
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
/**
|
|
1219
|
+
* Stream source bytes onto the targets through one hashing pass, verifying
|
|
1220
|
+
* content identity before committing. The temp file is staged on the first
|
|
1221
|
+
* local target's filesystem so `#commit` can hardlink instead of copying.
|
|
1222
|
+
*/
|
|
1223
|
+
async #streamToTargets(meta, sourceLocation, targets, destKey, options = {}) {
|
|
1224
|
+
const sourceBackend = this.#backends.get(sourceLocation.backend);
|
|
1225
|
+
// One remote target that speaks `putStream`: stream straight to it —
|
|
1226
|
+
// no temp file, no cache entry, no SyncQueue. The hub verifies the
|
|
1227
|
+
// digest (`X-Canvas-Sha256`) and answers the precondition itself.
|
|
1228
|
+
if (targets.length === 1 && targets[0].backend.type === 'remote' && typeof targets[0].backend.putStream === 'function') {
|
|
1229
|
+
return this.#streamToRemote(meta, sourceBackend, sourceLocation, targets[0], destKey, options);
|
|
1230
|
+
}
|
|
1231
|
+
const source = await sourceBackend.get(sourceLocation.key, { stream: true });
|
|
1232
|
+
if (!source) return { ok: false, reason: 'source-unreadable', backend: sourceLocation.backend, key: sourceLocation.key };
|
|
1233
|
+
|
|
1234
|
+
const firstLocal = targets.find(t => t.backend.type === 'local')?.backend;
|
|
1235
|
+
const tempDir = firstLocal ? firstLocal.tempDir : path.join(this.#paths.cache, '.tmp');
|
|
1236
|
+
const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
1237
|
+
await fsp.mkdir(tempDir, { recursive: true });
|
|
1238
|
+
|
|
1239
|
+
try {
|
|
1240
|
+
const { checksums, size } = await this.#hashToFile(source, tempPath);
|
|
1241
|
+
const id = formatId(checksums, this.#config.primaryChecksum);
|
|
1242
|
+
if (id !== meta.id) {
|
|
1243
|
+
debug(`Transfer aborted: ${sourceLocation.backend}:${sourceLocation.key} hashes to ${id}, indexed as ${meta.id}`);
|
|
1244
|
+
return { ok: false, reason: 'checksum-mismatch', expected: meta.id, actual: id };
|
|
1245
|
+
}
|
|
1246
|
+
const { locations, remoteTargets } = await this.#commit(
|
|
1247
|
+
targets, destKey, meta.id, { file: tempPath }, { checksums, size, mimeType: meta.mimeType }, options,
|
|
1248
|
+
);
|
|
1249
|
+
return { ok: true, locations, remoteTargets, size };
|
|
1250
|
+
} catch (err) {
|
|
1251
|
+
return { ok: false, reason: 'transfer-failed', error: err.message };
|
|
1252
|
+
} finally {
|
|
1253
|
+
await fsp.rm(tempPath, { force: true }).catch(() => {});
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// Protocol options a copy/move forwards verbatim to a remote `putStream`.
|
|
1258
|
+
#transferOptions(options = {}) {
|
|
1259
|
+
const out = { mtime: options.mtime };
|
|
1260
|
+
for (const k of ['ifMatch', 'ifNoneMatch', 'origin', 'conflictOf', 'conflictMode', 'baseSha256', 'deviceName']) {
|
|
1261
|
+
if (options[k] != null) out[k] = options[k];
|
|
1262
|
+
}
|
|
1263
|
+
return out;
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
// Typed driver failures → the `{ ok:false, reason }` vocabulary the
|
|
1267
|
+
// keyed-write API already speaks (412 → `precondition-failed` + `current`).
|
|
1268
|
+
#remoteFailure(err, backendName, key) {
|
|
1269
|
+
const code = err?.code || null;
|
|
1270
|
+
const base = { ok: false, code, backend: backendName, key, error: err?.message, status: err?.status ?? null };
|
|
1271
|
+
if (code === 'PRECONDITION_FAILED') return { ...base, reason: 'precondition-failed', current: err.current ?? null };
|
|
1272
|
+
if (code === 'TARGET_EXISTS') return { ...base, reason: 'target-exists' };
|
|
1273
|
+
if (code === 'NOT_FOUND') return { ...base, reason: 'not-found' };
|
|
1274
|
+
if (code === 'OFFLINE') return { ...base, reason: 'target-offline', detail: 'offline' };
|
|
1275
|
+
if (code === 'UNAUTHORIZED') return { ...base, reason: 'target-offline', detail: 'unauthorized' };
|
|
1276
|
+
return { ...base, reason: 'transfer-failed' };
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
async #streamToRemote(meta, sourceBackend, sourceLocation, target, destKey, options = {}) {
|
|
1280
|
+
const targetKey = target.key || destKey;
|
|
1281
|
+
const source = await sourceBackend.get(sourceLocation.key, { stream: true });
|
|
1282
|
+
if (!source) return { ok: false, reason: 'source-unreadable', backend: sourceLocation.backend, key: sourceLocation.key };
|
|
1283
|
+
const sha256 = meta.checksums?.sha256 ?? null;
|
|
1284
|
+
try {
|
|
1285
|
+
const res = await target.backend.putStream(targetKey, source, {
|
|
1286
|
+
...options,
|
|
1287
|
+
sha256,
|
|
1288
|
+
mtime: options.mtime ?? sourceLocation.mtime ?? null,
|
|
1289
|
+
mimeType: meta.mimeType || undefined,
|
|
1290
|
+
size: sourceLocation.size ?? meta.size ?? null,
|
|
1291
|
+
});
|
|
1292
|
+
if (res?.conflict) {
|
|
1293
|
+
// Inbox mode places nothing at the key; rename mode wrote an
|
|
1294
|
+
// ordinary object at the conflict key.
|
|
1295
|
+
const locations = options.conflictMode === 'rename'
|
|
1296
|
+
? [this.#buildLocation(target.name, targetKey, true, {
|
|
1297
|
+
size: sourceLocation.size ?? meta.size, ...(sourceLocation.mtime != null ? { mtime: sourceLocation.mtime } : {}),
|
|
1298
|
+
...(sha256 ? { ino: sha256, dev: target.backend.dev ?? null } : {}),
|
|
1299
|
+
})]
|
|
1300
|
+
: [];
|
|
1301
|
+
return { ok: true, locations, remoteTargets: [], size: meta.size, conflict: res };
|
|
1302
|
+
}
|
|
1303
|
+
if (sha256 && res?.sha256 && res.sha256 !== sha256) {
|
|
1304
|
+
return { ok: false, reason: 'checksum-mismatch', expected: meta.id, actual: `sha256:${res.sha256}` };
|
|
1305
|
+
}
|
|
1306
|
+
const location = this.#buildLocation(target.name, targetKey, true, {
|
|
1307
|
+
size: res?.size ?? sourceLocation.size ?? meta.size,
|
|
1308
|
+
...(res?.mtime != null ? { mtime: res.mtime } : (sourceLocation.mtime != null ? { mtime: sourceLocation.mtime } : {})),
|
|
1309
|
+
...(sha256 ? { ino: sha256, dev: target.backend.dev ?? null } : {}),
|
|
1310
|
+
});
|
|
1311
|
+
return { ok: true, locations: [location], remoteTargets: [], size: location.size, result: res };
|
|
1312
|
+
} catch (err) {
|
|
1313
|
+
source.destroy?.();
|
|
1314
|
+
debug(`Remote transfer to ${target.name}:${targetKey} failed: ${err.code || ''} ${err.message}`);
|
|
1315
|
+
return this.#remoteFailure(err, target.name, targetKey);
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
// Server-side rename on a remote driver; the location keeps its identity.
|
|
1320
|
+
async #tryRemoteRename(backend, sourceLocation, targetKey, options = {}) {
|
|
1321
|
+
try {
|
|
1322
|
+
const r = await backend.rename(sourceLocation.key, targetKey, { ifMatch: options.ifMatch, origin: options.origin });
|
|
1323
|
+
const location = this.#buildLocation(backend.name, targetKey, true, {
|
|
1324
|
+
size: r?.size ?? sourceLocation.size,
|
|
1325
|
+
...((r?.modified ?? sourceLocation.mtime) != null ? { mtime: r?.modified ?? sourceLocation.mtime } : {}),
|
|
1326
|
+
...(sourceLocation.ino != null ? { ino: sourceLocation.ino, dev: sourceLocation.dev } : {}),
|
|
1327
|
+
});
|
|
1328
|
+
return { ok: true, location, result: r };
|
|
1329
|
+
} catch (err) {
|
|
1330
|
+
return this.#remoteFailure(err, backend.name, sourceLocation.key);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
/**
|
|
1335
|
+
* Same-filesystem shortcut: one `rename(2)` instead of reading and rewriting
|
|
1336
|
+
* the whole object. Returns the destination stat on success, or null when
|
|
1337
|
+
* the shortcut does not apply (different filesystems, non-file drivers) so
|
|
1338
|
+
* the caller falls back to a stream copy. No re-hash — the inode is carried
|
|
1339
|
+
* over intact, so the bytes are the same bytes by construction.
|
|
1340
|
+
*/
|
|
1341
|
+
async #tryRename(sourceBackend, sourceKey, targetBackend, destKey) {
|
|
1342
|
+
if (typeof sourceBackend?.resolveKeyPath !== 'function') return null;
|
|
1343
|
+
if (typeof targetBackend?.renameFrom !== 'function' || !targetBackend.root) return null;
|
|
1344
|
+
|
|
1345
|
+
const sourcePath = sourceBackend.resolveKeyPath(sourceKey);
|
|
1346
|
+
const [sourceStat, targetStat] = await Promise.all([
|
|
1347
|
+
fsp.stat(sourcePath).catch(() => null),
|
|
1348
|
+
fsp.stat(targetBackend.root).catch(() => null),
|
|
1349
|
+
]);
|
|
1350
|
+
if (!sourceStat || !targetStat || sourceStat.dev !== targetStat.dev) return null;
|
|
1351
|
+
|
|
1352
|
+
try {
|
|
1353
|
+
return await targetBackend.renameFrom(destKey, sourcePath);
|
|
1354
|
+
} catch (err) {
|
|
1355
|
+
if (err.code === 'EXDEV') return null; // mount table lied; copy instead
|
|
1356
|
+
throw err;
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/**
|
|
1361
|
+
* Complete a move: drop the source bytes (unless a rename already relocated
|
|
1362
|
+
* them), swap the location in the index, emit `object:move`.
|
|
1363
|
+
*
|
|
1364
|
+
* A failed source delete is reported as `source-delete-failed` with
|
|
1365
|
+
* `degradedToCopy` — the content is safely at the destination, there is just
|
|
1366
|
+
* a stale second copy left behind. That is a cleanup problem, never data
|
|
1367
|
+
* loss, and the caller must be able to tell the two apart.
|
|
1368
|
+
*/
|
|
1369
|
+
async #finalizeMove(id, sourceLocation, newLocation, { removeSourceBytes, origin = null, previous = null }) {
|
|
1370
|
+
const release = this.#holdKeys([`${sourceLocation.backend}:${sourceLocation.key}`]);
|
|
1371
|
+
try {
|
|
1372
|
+
if (removeSourceBytes) {
|
|
1373
|
+
const sourceBackend = this.#backends.get(sourceLocation.backend);
|
|
1374
|
+
const deleted = await sourceBackend.delete(sourceLocation.key).catch(() => false);
|
|
1375
|
+
if (!deleted) {
|
|
1376
|
+
return {
|
|
1377
|
+
ok: false,
|
|
1378
|
+
reason: 'source-delete-failed',
|
|
1379
|
+
degradedToCopy: true,
|
|
1380
|
+
id,
|
|
1381
|
+
from: this.#endpoint(sourceLocation),
|
|
1382
|
+
to: this.#endpoint(newLocation),
|
|
1383
|
+
};
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
// Re-read: a pending move completes on the sync callback, long after
|
|
1388
|
+
// the plan was made, and the entry may have gained locations since.
|
|
1389
|
+
const current = this.#index.get(id);
|
|
1390
|
+
if (!current) return { ok: false, reason: 'not-found' };
|
|
1391
|
+
const locations = this.#mergeLocations(current, [newLocation])
|
|
1392
|
+
.filter(l => !(l.backend === sourceLocation.backend && l.key === sourceLocation.key));
|
|
1393
|
+
// Same backend, new key = a rename for the change log (one entry,
|
|
1394
|
+
// `from` carried), not a delete + put pair.
|
|
1395
|
+
const rename = sourceLocation.backend === newLocation.backend
|
|
1396
|
+
? { backend: newLocation.backend, from: sourceLocation.key, to: newLocation.key }
|
|
1397
|
+
: null;
|
|
1398
|
+
const saved = this.#index.put(id, { ...current, locations }, { origin, rename });
|
|
1399
|
+
|
|
1400
|
+
this.#emitObject('move', {
|
|
1401
|
+
id,
|
|
1402
|
+
checksums: saved.checksums,
|
|
1403
|
+
from: this.#endpoint(sourceLocation),
|
|
1404
|
+
to: this.#endpoint(newLocation),
|
|
1405
|
+
location: this.#describeLocation(newLocation),
|
|
1406
|
+
locations: saved.locations.map(l => this.#describeLocation(l)),
|
|
1407
|
+
...(previous ? { previous } : {}),
|
|
1408
|
+
...(origin ? { origin } : {}),
|
|
1409
|
+
});
|
|
1410
|
+
|
|
1411
|
+
return {
|
|
1412
|
+
ok: true,
|
|
1413
|
+
id,
|
|
1414
|
+
state: 'complete',
|
|
1415
|
+
from: this.#endpoint(sourceLocation),
|
|
1416
|
+
to: this.#endpoint(newLocation),
|
|
1417
|
+
locations: saved.locations.map(l => this.#describeLocation(l)),
|
|
1418
|
+
};
|
|
1419
|
+
} finally {
|
|
1420
|
+
release();
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
/** Finish moves that were waiting on a remote destination write. */
|
|
1425
|
+
#resolvePendingMoves(id, results = []) {
|
|
1426
|
+
const pending = this.#pendingMoves.get(id);
|
|
1427
|
+
if (!pending) return;
|
|
1428
|
+
|
|
1429
|
+
const succeeded = new Set(results.filter(r => r.success).map(r => r.backend));
|
|
1430
|
+
const failed = new Set(results.filter(r => !r.success).map(r => r.backend));
|
|
1431
|
+
const remaining = [];
|
|
1432
|
+
|
|
1433
|
+
for (const entry of pending) {
|
|
1434
|
+
if (succeeded.has(entry.target)) {
|
|
1435
|
+
const meta = this.#index.get(id);
|
|
1436
|
+
const location = (meta?.locations || []).find(l => l.backend === entry.target && l.key === entry.destKey);
|
|
1437
|
+
if (location) {
|
|
1438
|
+
this.#finalizeMove(id, entry.sourceLocation, location, { removeSourceBytes: true })
|
|
1439
|
+
.catch(err => this.emit('error', err));
|
|
1440
|
+
}
|
|
1441
|
+
} else if (failed.has(entry.target)) {
|
|
1442
|
+
// Destination write failed — the source stays exactly where it
|
|
1443
|
+
// is. The object is unharmed; the move simply did not happen.
|
|
1444
|
+
this.emit('error', Object.assign(
|
|
1445
|
+
new Error(`Move to ${entry.target} failed; source retained at ${entry.sourceLocation.backend}:${entry.sourceLocation.key}`),
|
|
1446
|
+
{ id, code: 'MOVE_SYNC_FAILED' },
|
|
1447
|
+
));
|
|
1448
|
+
} else {
|
|
1449
|
+
remaining.push(entry);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
if (remaining.length) this.#pendingMoves.set(id, remaining);
|
|
1454
|
+
else this.#pendingMoves.delete(id);
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
/**
|
|
1458
|
+
* After an `onConflict:'overwrite'` transfer landed, reconcile the content
|
|
1459
|
+
* that used to own each overwritten key: strip the stale location from
|
|
1460
|
+
* the displaced entry (drop the entry when that was its last home) and
|
|
1461
|
+
* tell consumers with the same succession vocabulary a watcher-observed
|
|
1462
|
+
* edit uses — `object:unlink` carrying `successor` — so the document
|
|
1463
|
+
* behind the old bytes is re-pointed or orphaned, never left claiming a
|
|
1464
|
+
* path it lost. Returns the first displaced `{ id, checksums }` (for the
|
|
1465
|
+
* `previous` hint on the follow-up add) or null.
|
|
1466
|
+
*/
|
|
1467
|
+
#displaceTargets(targets, destKey, newId, options = {}) {
|
|
1468
|
+
let previous = null;
|
|
1469
|
+
for (const t of targets) {
|
|
1470
|
+
if (!t.overwrite) continue;
|
|
1471
|
+
const displaced = this.#displace(t.name, t.key || destKey, newId, options);
|
|
1472
|
+
if (displaced && !previous) previous = displaced;
|
|
1473
|
+
}
|
|
1474
|
+
return previous;
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
#displace(backendName, key, newId, { origin = null } = {}) {
|
|
1478
|
+
const prev = this.#index.get(`${backendName}:${key}`);
|
|
1479
|
+
if (!prev || prev.id === newId) return null;
|
|
1480
|
+
const remaining = (prev.locations || []).filter(l => !(l.backend === backendName && l.key === key));
|
|
1481
|
+
if (remaining.length === 0) this.#index.delete(prev.id, { origin });
|
|
1482
|
+
else this.#index.put(prev.id, { ...prev, locations: remaining }, { origin });
|
|
1483
|
+
this.#emitObject('unlink', {
|
|
1484
|
+
backend: backendName,
|
|
1485
|
+
key,
|
|
1486
|
+
id: prev.id,
|
|
1487
|
+
checksums: prev.checksums,
|
|
1488
|
+
locations: remaining,
|
|
1489
|
+
successor: { id: newId },
|
|
1490
|
+
reason: 'overwritten',
|
|
1491
|
+
...(origin ? { origin } : {}),
|
|
1492
|
+
});
|
|
1493
|
+
debug(`Displaced ${prev.id.slice(0, 19)}... from ${backendName}:${key}`);
|
|
1494
|
+
return { id: prev.id, checksums: prev.checksums };
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
/** Upsert locations by (backend, key), preserving order. */
|
|
1498
|
+
#mergeLocations(meta, added = []) {
|
|
1499
|
+
const out = [...(meta.locations || [])];
|
|
1500
|
+
for (const loc of added) {
|
|
1501
|
+
const at = out.findIndex(l => l.backend === loc.backend && l.key === loc.key);
|
|
1502
|
+
if (at >= 0) out[at] = loc;
|
|
1503
|
+
else out.push(loc);
|
|
1504
|
+
}
|
|
1505
|
+
return out;
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
#endpoint(location) {
|
|
1509
|
+
return {
|
|
1510
|
+
backend: location.backend,
|
|
1511
|
+
key: location.key,
|
|
1512
|
+
url: `stored://${location.backend}/${location.key}`,
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
#emitLocationEvent(suffix, meta, location, extra = {}) {
|
|
1517
|
+
this.#emitObject(suffix, {
|
|
1518
|
+
id: meta.id,
|
|
1519
|
+
checksums: meta.checksums,
|
|
1520
|
+
backend: location.backend,
|
|
1521
|
+
key: location.key,
|
|
1522
|
+
location: this.#describeLocation(location),
|
|
1523
|
+
locations: (meta.locations || []).map(l => this.#describeLocation(l)),
|
|
1524
|
+
...extra,
|
|
1525
|
+
});
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
/**
|
|
1529
|
+
* Suppress watcher events for keys we are about to write or delete
|
|
1530
|
+
* ourselves. Refcounted (concurrent transfers may overlap) and released on a
|
|
1531
|
+
* delay, because the events arrive well after the syscall returns.
|
|
1532
|
+
*/
|
|
1533
|
+
#holdKeys(allKeys) {
|
|
1534
|
+
// Drivers that already filter their own echoes (a canvas hub feed
|
|
1535
|
+
// stamped with our origin) need no hold — and holding would swallow
|
|
1536
|
+
// a genuine remote change that lands right after our write.
|
|
1537
|
+
const keys = allKeys.filter((pathKey) => {
|
|
1538
|
+
const name = this.#backends.list().filter((n) => pathKey.startsWith(`${n}:`)).sort((a, b) => b.length - a.length)[0];
|
|
1539
|
+
const backend = name ? this.#backends.get(name) : null;
|
|
1540
|
+
return !(backend && backend.suppressEchoes === false);
|
|
1541
|
+
});
|
|
1542
|
+
for (const key of keys) this.#suppressedKeys.set(key, (this.#suppressedKeys.get(key) || 0) + 1);
|
|
1543
|
+
let released = false;
|
|
1544
|
+
return () => {
|
|
1545
|
+
if (released) return;
|
|
1546
|
+
released = true;
|
|
1547
|
+
const timer = setTimeout(() => {
|
|
1548
|
+
for (const key of keys) {
|
|
1549
|
+
const next = (this.#suppressedKeys.get(key) || 1) - 1;
|
|
1550
|
+
if (next <= 0) this.#suppressedKeys.delete(key);
|
|
1551
|
+
else this.#suppressedKeys.set(key, next);
|
|
1552
|
+
}
|
|
1553
|
+
}, MOVE_SUPPRESS_MS);
|
|
1554
|
+
timer.unref?.();
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1559
|
+
// Private — reads
|
|
1560
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1561
|
+
|
|
1562
|
+
/**
|
|
1563
|
+
* Synced locations whose backend is registered and readable, cheapest first:
|
|
1564
|
+
* local disk, then network mounts, then `type:'remote'` protocol backends
|
|
1565
|
+
* (S3/HTTP — a real request, not just a slower filesystem).
|
|
1566
|
+
*/
|
|
1567
|
+
#readableLocations(meta) {
|
|
1568
|
+
const cost = (backend) => (backend.type === 'remote' ? 2 : (backend.remote ? 1 : 0));
|
|
1569
|
+
return (meta?.locations || [])
|
|
1570
|
+
.filter(l => l.synced)
|
|
1571
|
+
.map(l => ({ l, backend: this.#backends.get(l.backend) }))
|
|
1572
|
+
.filter(({ backend }) => backend && backend.canRead)
|
|
1573
|
+
.sort((a, b) => cost(a.backend) - cost(b.backend))
|
|
1574
|
+
.map(({ l }) => l);
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
async #read(idOrKey, stream) {
|
|
1578
|
+
const meta = this.#index.get(idOrKey);
|
|
1579
|
+
if (!meta) return null;
|
|
1580
|
+
|
|
1581
|
+
// 1. Cache by content id.
|
|
1582
|
+
if (stream) {
|
|
1583
|
+
const info = await this.#cache.getInfo(meta.id).catch(() => null);
|
|
1584
|
+
if (info) return this.#cache.getStream(meta.id);
|
|
1585
|
+
} else {
|
|
1586
|
+
try { return (await this.#cache.get(meta.id)).data; } catch { /* miss */ }
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
// 2. Best available backend location — local before remote (see
|
|
1590
|
+
// #readableLocations); a LAN round-trip must never win over a copy
|
|
1591
|
+
// sitting on local disk just because it was indexed first.
|
|
1592
|
+
const location = this.#readableLocations(meta)[0];
|
|
1593
|
+
const backend = location && this.#backends.get(location.backend);
|
|
1594
|
+
if (!backend) return null;
|
|
1595
|
+
|
|
1596
|
+
const data = await backend.get(location.key, { stream });
|
|
1597
|
+
if (!stream && Buffer.isBuffer(data)) this.#cache.put(meta.id, data).catch(() => {});
|
|
1598
|
+
return data;
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1602
|
+
// Private — ingest (streaming + in-memory)
|
|
1603
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1604
|
+
|
|
1605
|
+
// Best-effort extraction hook — bytes are in hand at ingest (works even for
|
|
1606
|
+
// remote/S3-destined blobs, which still pass through the server here). Never
|
|
1607
|
+
// throws; returns {} when disabled/unsupported/failed.
|
|
1608
|
+
async #maybeExtract(source, mimeType, key) {
|
|
1609
|
+
if (!this.#extract) { return {}; }
|
|
1610
|
+
try { return (await this.#extract(source, { mimeType, key })) || {}; }
|
|
1611
|
+
catch (e) { debug(`extract hook failed (${mimeType}): ${e.message}`); return {}; }
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
// Buffer / string: already resident, write directly (no temp file).
|
|
1615
|
+
async #ingestMemory(blob, targets, key, mimeHint, options = {}) {
|
|
1616
|
+
const data = isBuffer(blob) ? blob : Buffer.from(blob);
|
|
1617
|
+
const checksums = checksumBuffer(data, this.#config.checksums);
|
|
1618
|
+
const id = formatId(checksums, this.#config.primaryChecksum);
|
|
1619
|
+
const finalKey = key || this.#generateKey(checksums);
|
|
1620
|
+
const mimeType = mimeHint || await detectMimeType(data);
|
|
1621
|
+
|
|
1622
|
+
const extracted = await this.#maybeExtract({ data }, mimeType, finalKey);
|
|
1623
|
+
const { locations, remoteTargets } = await this.#commit(targets, finalKey, id, { data }, { checksums, size: data.length, mimeType }, options);
|
|
1624
|
+
return { id, finalKey, checksums, size: data.length, mimeType, locations, remoteTargets, extracted };
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
// Path / stream: stream through a hash pass into a temp file on the primary
|
|
1628
|
+
// local backend's filesystem, then commit (hardlink/rename) to targets.
|
|
1629
|
+
async #ingestStream(blob, targets, key, mimeHint, options = {}) {
|
|
1630
|
+
const source = isStream(blob) ? blob : createReadStream(blob);
|
|
1631
|
+
const firstLocal = targets.find(t => t.backend.type === 'local')?.backend;
|
|
1632
|
+
const tempDir = firstLocal ? firstLocal.tempDir : path.join(this.#paths.cache, '.tmp');
|
|
1633
|
+
const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
1634
|
+
await fsp.mkdir(tempDir, { recursive: true });
|
|
1635
|
+
|
|
1636
|
+
let result;
|
|
1637
|
+
try {
|
|
1638
|
+
const { checksums, size, head } = await this.#hashToFile(source, tempPath);
|
|
1639
|
+
const id = formatId(checksums, this.#config.primaryChecksum);
|
|
1640
|
+
const finalKey = key || this.#generateKey(checksums);
|
|
1641
|
+
const mimeType = mimeHint || await detectMimeFromHead(head, finalKey);
|
|
1642
|
+
|
|
1643
|
+
// Extract from the whole temp file (head=4KB is too small for EXIF),
|
|
1644
|
+
// before the finally-block deletes it.
|
|
1645
|
+
const extracted = await this.#maybeExtract({ file: tempPath }, mimeType, finalKey);
|
|
1646
|
+
const { locations, remoteTargets } = await this.#commit(targets, finalKey, id, { file: tempPath }, { checksums, size, mimeType }, options);
|
|
1647
|
+
result = { id, finalKey, checksums, size, mimeType, locations, remoteTargets, extracted };
|
|
1648
|
+
} finally {
|
|
1649
|
+
await fsp.rm(tempPath, { force: true }).catch(() => {});
|
|
1650
|
+
}
|
|
1651
|
+
return result;
|
|
1652
|
+
}
|
|
1653
|
+
|
|
1654
|
+
// Single streaming pass: hash (all algorithms), measure size, peek the head
|
|
1655
|
+
// for mime sniffing, and write to `tempPath`. Never buffers the whole blob.
|
|
1656
|
+
async #hashToFile(source, tempPath) {
|
|
1657
|
+
const hashes = this.#config.checksums.map(algo => ({ algo, hash: crypto.createHash(algo) }));
|
|
1658
|
+
const ws = createWriteStream(tempPath);
|
|
1659
|
+
let size = 0;
|
|
1660
|
+
const headChunks = [];
|
|
1661
|
+
let headLen = 0;
|
|
1662
|
+
|
|
1663
|
+
try {
|
|
1664
|
+
for await (const chunk of source) {
|
|
1665
|
+
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
1666
|
+
for (const { hash } of hashes) hash.update(buf);
|
|
1667
|
+
size += buf.length;
|
|
1668
|
+
if (headLen < HEAD_BYTES) {
|
|
1669
|
+
const slice = buf.subarray(0, HEAD_BYTES - headLen);
|
|
1670
|
+
headChunks.push(slice);
|
|
1671
|
+
headLen += slice.length;
|
|
1672
|
+
}
|
|
1673
|
+
if (!ws.write(buf)) await once(ws, 'drain');
|
|
1674
|
+
}
|
|
1675
|
+
ws.end();
|
|
1676
|
+
await once(ws, 'finish');
|
|
1677
|
+
} catch (err) {
|
|
1678
|
+
ws.destroy();
|
|
1679
|
+
throw err;
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
const checksums = {};
|
|
1683
|
+
for (const { algo, hash } of hashes) checksums[algo] = hash.digest('hex');
|
|
1684
|
+
return { checksums, size, head: Buffer.concat(headChunks) };
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
// Place ingested bytes on every target: local backends get the bytes now
|
|
1688
|
+
// (buffer write or hardlink/copy from temp); remote targets get a cache
|
|
1689
|
+
// entry + a SyncQueue placeholder. Returns { locations, remoteTargets }.
|
|
1690
|
+
async #commit(targets, finalKey, id, source, meta, options = {}) {
|
|
1691
|
+
const locations = [];
|
|
1692
|
+
const remoteTargets = [];
|
|
1693
|
+
|
|
1694
|
+
for (const { name, backend, key, overwrite } of targets) {
|
|
1695
|
+
// A target may carry its own key: conflict resolution can rename per
|
|
1696
|
+
// backend (`photo.jpg` is free on one, taken on another). `put()`
|
|
1697
|
+
// targets never do, so they all land on `finalKey`.
|
|
1698
|
+
const targetKey = key || finalKey;
|
|
1699
|
+
if (backend.type === 'local') {
|
|
1700
|
+
if (overwrite) await this.#retain(name, targetKey, this.#index.get(`${name}:${targetKey}`));
|
|
1701
|
+
if (source.data) await backend.put(targetKey, source.data);
|
|
1702
|
+
else await backend.commit(targetKey, source.file);
|
|
1703
|
+
if (options.mtime != null && typeof backend.utimes === 'function') {
|
|
1704
|
+
await backend.utimes(targetKey, options.mtime).catch(() => {});
|
|
1705
|
+
}
|
|
1706
|
+
// Record the on-disk identity (mtime + inode) the way the
|
|
1707
|
+
// watcher/scan paths do, so a later rename of this file pairs
|
|
1708
|
+
// as a move and a rescan can skip the rehash.
|
|
1709
|
+
const st = typeof backend.stat === 'function' ? await backend.stat(targetKey).catch(() => null) : null;
|
|
1710
|
+
locations.push(this.#buildLocation(name, targetKey, true, {
|
|
1711
|
+
size: st?.size ?? meta.size,
|
|
1712
|
+
...(st?.modified != null ? { mtime: st.modified } : {}),
|
|
1713
|
+
...(st?.ino != null ? { ino: st.ino, dev: st.dev } : {}),
|
|
1714
|
+
}));
|
|
1715
|
+
} else {
|
|
1716
|
+
locations.push(this.#buildLocation(name, targetKey, false, { size: meta.size }));
|
|
1717
|
+
remoteTargets.push({ name, driver: backend.config.driver, root: backend.config.root, key: targetKey });
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
if (remoteTargets.length) {
|
|
1722
|
+
const cacheMeta = { key: finalKey, checksums: meta.checksums, size: meta.size, mimeType: meta.mimeType };
|
|
1723
|
+
if (source.data) await this.#cache.put(id, source.data, cacheMeta);
|
|
1724
|
+
else await pipeline(createReadStream(source.file), this.#cache.putStream(id, cacheMeta));
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
return { locations, remoteTargets };
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
// Skip-hash predicate for scan: returns the cached descriptor when a file's
|
|
1731
|
+
// size+mtime match an existing indexed location, else null.
|
|
1732
|
+
#knownIfUnchanged(backendName, key, stat) {
|
|
1733
|
+
const meta = this.#index.get(`${backendName}:${key}`);
|
|
1734
|
+
const loc = meta?.locations?.find(l => l.backend === backendName && l.key === key);
|
|
1735
|
+
if (!loc || loc.size !== stat.size || loc.mtime !== stat.mtime) return null;
|
|
1736
|
+
return { checksums: meta.checksums, mimeType: meta.mimeType };
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1739
|
+
// One-shot snapshot of a backend's indexed locations keyed by inode
|
|
1740
|
+
// identity, built per scan (the index is LMDB — a per-file reverse lookup
|
|
1741
|
+
// would be O(N²)).
|
|
1742
|
+
#buildInodeMap(backendName) {
|
|
1743
|
+
const map = new Map();
|
|
1744
|
+
for (const [, meta] of this.#index.entries()) {
|
|
1745
|
+
for (const loc of meta.locations || []) {
|
|
1746
|
+
if (loc.backend !== backendName || loc.ino == null) continue;
|
|
1747
|
+
map.set(`${loc.dev}:${loc.ino}`, {
|
|
1748
|
+
key: loc.key,
|
|
1749
|
+
size: loc.size,
|
|
1750
|
+
mtime: loc.mtime,
|
|
1751
|
+
checksums: meta.checksums,
|
|
1752
|
+
mimeType: meta.mimeType,
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
return map;
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
// Rename-match skip-hash: the path is new, but the inode is one we already
|
|
1760
|
+
// indexed and size+mtime are untouched — trust the prior checksums. (Also
|
|
1761
|
+
// covers hardlink twins: same inode ⇒ same bytes by definition.)
|
|
1762
|
+
#knownByInode(inodeMap, key, stat) {
|
|
1763
|
+
if (stat?.ino == null) return null;
|
|
1764
|
+
const prior = inodeMap.get(`${stat.dev}:${stat.ino}`);
|
|
1765
|
+
if (!prior || prior.key === key) return null;
|
|
1766
|
+
if (prior.size !== stat.size || prior.mtime !== stat.mtime) return null;
|
|
1767
|
+
return { checksums: prior.checksums, mimeType: prior.mimeType };
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
#generateKey(checksums) {
|
|
1771
|
+
const hash = checksums[this.#config.primaryChecksum];
|
|
1772
|
+
return `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash}`;
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
#buildLocation(backendName, key, synced, extra = {}) {
|
|
1776
|
+
const backend = this.#backends.get(backendName);
|
|
1777
|
+
const config = backend?.config || {};
|
|
1778
|
+
|
|
1779
|
+
return {
|
|
1780
|
+
backend: backendName,
|
|
1781
|
+
driver: config.driver || null,
|
|
1782
|
+
key,
|
|
1783
|
+
synced,
|
|
1784
|
+
...extra,
|
|
1785
|
+
source: this.#buildSourceDescriptor(backendName, key, config),
|
|
1786
|
+
};
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
#buildSourceDescriptor(backendName, key, config = {}) {
|
|
1790
|
+
const backend = this.#backends.get(backendName);
|
|
1791
|
+
const [providerHint, ...accountHintParts] = String(backendName || '').split(':').filter(Boolean);
|
|
1792
|
+
const provider = config.provider || providerHint || config.driver || 'unknown';
|
|
1793
|
+
const account = config.account
|
|
1794
|
+
|| (accountHintParts.length > 0 ? accountHintParts.join(':') : (providerHint || backendName || 'default'));
|
|
1795
|
+
const container = config.container
|
|
1796
|
+
|| config.bucket
|
|
1797
|
+
|| config.share
|
|
1798
|
+
|| config.folder
|
|
1799
|
+
|| (config.root ? path.basename(path.resolve(config.root)) : 'root');
|
|
1800
|
+
|
|
1801
|
+
// `remote`/`transport` are recorded on the location itself, not just read
|
|
1802
|
+
// off the live backend: a location must stay self-describing after its
|
|
1803
|
+
// backend is unmounted or dropped from config (the offline case is
|
|
1804
|
+
// exactly when a consumer needs to know "this lived on the NAS").
|
|
1805
|
+
// Sparse on purpose — local locations are the overwhelming majority and
|
|
1806
|
+
// carry no extra bytes.
|
|
1807
|
+
const remote = backend ? backend.remote : (config.remote === true);
|
|
1808
|
+
const transport = backend ? backend.transport : (config.transport ?? null);
|
|
1809
|
+
|
|
1810
|
+
return {
|
|
1811
|
+
provider,
|
|
1812
|
+
account,
|
|
1813
|
+
container,
|
|
1814
|
+
path: key,
|
|
1815
|
+
...(remote ? { remote: true } : {}),
|
|
1816
|
+
...(remote && transport ? { transport } : {}),
|
|
1817
|
+
};
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
// `erroredPrefixes`: subtrees the scan could not read (EACCES etc.) — their
|
|
1821
|
+
// prior entries are carried forward, never treated as deleted. Key presence
|
|
1822
|
+
// is compared NFC-normalized: macOS reports NFD, so without this every
|
|
1823
|
+
// accented filename would diff as a perpetual delete+add pair.
|
|
1824
|
+
#removeMissingLocations(backendName, presentKeys, erroredPrefixes = []) {
|
|
1825
|
+
const nfc = (k) => String(k).normalize('NFC');
|
|
1826
|
+
const present = new Set([...presentKeys].map(nfc));
|
|
1827
|
+
const underErroredPrefix = (key) => erroredPrefixes.some((prefix) =>
|
|
1828
|
+
key === prefix || key.startsWith(`${prefix}/`) || key.startsWith(`${prefix}\\`));
|
|
1829
|
+
for (const [id, meta] of this.#index.entries()) {
|
|
1830
|
+
const nextLocations = (meta.locations || []).filter(location =>
|
|
1831
|
+
location.backend !== backendName
|
|
1832
|
+
|| present.has(nfc(location.key))
|
|
1833
|
+
|| underErroredPrefix(location.key)
|
|
1834
|
+
);
|
|
1835
|
+
|
|
1836
|
+
if (nextLocations.length === (meta.locations || []).length) continue;
|
|
1837
|
+
|
|
1838
|
+
if (nextLocations.length === 0) {
|
|
1839
|
+
this.#index.delete(id);
|
|
1840
|
+
continue;
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
this.#index.put(id, { ...meta, locations: nextLocations });
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
// Emit a file event under both its typed name (`file:<suffix>`) and the
|
|
1848
|
+
// generic `object:<suffix>` (kind:'file') so consumers can bind either.
|
|
1849
|
+
#emitObject(suffix, payload) {
|
|
1850
|
+
this.emit(`file:${suffix}`, payload);
|
|
1851
|
+
this.emit(`object:${suffix}`, { kind: 'file', ...payload });
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
// `options.force` bypasses the echo suppression — used when stored itself
|
|
1855
|
+
// wrote the bytes (writeObject) and feeds the resulting stat through the
|
|
1856
|
+
// very same path a watcher-observed change takes.
|
|
1857
|
+
#handleFileEvent(event, data, options = {}) {
|
|
1858
|
+
const pathKey = `${data.backend}:${data.key}`;
|
|
1859
|
+
// Our own copy/move touching a watched root — the index was already
|
|
1860
|
+
// updated by the operation itself. Re-processing the echo would emit a
|
|
1861
|
+
// spurious add (or, worse, an unlink that drops the location we just
|
|
1862
|
+
// moved the object to).
|
|
1863
|
+
if (!options.force && this.#suppressedKeys.has(pathKey)) {
|
|
1864
|
+
// Our own echo carries exactly the bytes we just indexed under the
|
|
1865
|
+
// key; anything else arriving inside the hold is a genuine change
|
|
1866
|
+
// made underneath the transfer and must not be lost. Unlink echoes
|
|
1867
|
+
// (the source of a move) are always ours.
|
|
1868
|
+
const indexed = data.checksums ? this.#index.get(pathKey) : null;
|
|
1869
|
+
const sameBytes = !!indexed && formatId(data.checksums, this.#config.primaryChecksum) === indexed.id;
|
|
1870
|
+
if (event === 'file:unlink') {
|
|
1871
|
+
// Ours (the source of a move, a remove-before-link) — or a
|
|
1872
|
+
// genuine delete landing inside the window. Decide once the
|
|
1873
|
+
// hold is over, from what is actually on disk.
|
|
1874
|
+
debug(`Deferred unlink under in-flight transfer ${pathKey}`);
|
|
1875
|
+
this.#recheckUnlink(data);
|
|
1876
|
+
return;
|
|
1877
|
+
}
|
|
1878
|
+
if (!data.checksums || sameBytes) {
|
|
1879
|
+
debug(`Suppressed ${event} for in-flight transfer ${pathKey}`);
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
debug(`Held key ${pathKey} changed underneath the transfer — processing ${event}`);
|
|
1883
|
+
}
|
|
1884
|
+
const origin = data.origin || null;
|
|
1885
|
+
const location = this.#buildLocation(data.backend, data.key, true, {
|
|
1886
|
+
...(data.size != null ? { size: data.size } : {}),
|
|
1887
|
+
...(data.modified != null ? { mtime: data.modified } : {}),
|
|
1888
|
+
...(data.ino != null ? { ino: data.ino, dev: data.dev } : {}),
|
|
1889
|
+
});
|
|
1890
|
+
// Refresh an already-indexed location in place (size/mtime/inode may
|
|
1891
|
+
// have been unknown — a `put()`-created location learns its inode from
|
|
1892
|
+
// the first watcher event it gets) or append a new one.
|
|
1893
|
+
const upsertLocation = (locations) => {
|
|
1894
|
+
const at = locations.findIndex(l => l.backend === data.backend && l.key === data.key);
|
|
1895
|
+
if (at >= 0) locations[at] = { ...locations[at], ...location };
|
|
1896
|
+
else locations.push(location);
|
|
1897
|
+
return locations;
|
|
1898
|
+
};
|
|
1899
|
+
|
|
1900
|
+
if (event === 'file:add' && data.checksums) {
|
|
1901
|
+
const id = formatId(data.checksums, this.#config.primaryChecksum);
|
|
1902
|
+
// An `add` on a path the index already attributes to other bytes
|
|
1903
|
+
// (chokidar coalesces a write that follows a rename into one add)
|
|
1904
|
+
// is a succession: take the change path so the previous owner is
|
|
1905
|
+
// displaced and never re-claims the key.
|
|
1906
|
+
const occupant = this.#index.get(pathKey);
|
|
1907
|
+
if (occupant && occupant.id !== id) {
|
|
1908
|
+
this.#handleFileEvent('file:change', data, { ...options, force: true });
|
|
1909
|
+
return;
|
|
1910
|
+
}
|
|
1911
|
+
|
|
1912
|
+
// Rename pairing: an add whose inode matches a held unlink is the
|
|
1913
|
+
// second half of a move. Same content → silently drop the old-path
|
|
1914
|
+
// location (no unlink emission, no identity churn) and let the add
|
|
1915
|
+
// land the new path on the same id, as ONE index write so the
|
|
1916
|
+
// change log records a rename. Content changed mid-move → release
|
|
1917
|
+
// the held unlink first; the succession flow handles it.
|
|
1918
|
+
let renamedFrom = null;
|
|
1919
|
+
if (data.ino != null) {
|
|
1920
|
+
const pendingKey = `${data.backend}|${data.dev}:${data.ino}`;
|
|
1921
|
+
const pending = this.#pendingUnlinks.get(pendingKey);
|
|
1922
|
+
if (pending) {
|
|
1923
|
+
clearTimeout(pending.timer);
|
|
1924
|
+
this.#pendingUnlinks.delete(pendingKey);
|
|
1925
|
+
const oldMeta = this.#index.get(`${pending.data.backend}:${pending.data.key}`);
|
|
1926
|
+
if (oldMeta && oldMeta.id === id && pending.data.backend === data.backend) {
|
|
1927
|
+
renamedFrom = pending.data.key;
|
|
1928
|
+
} else {
|
|
1929
|
+
this.#processUnlink(pending.data);
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
const existing = this.#index.get(id);
|
|
1935
|
+
const locations = upsertLocation((existing?.locations || []).filter(l =>
|
|
1936
|
+
!(renamedFrom && l.backend === data.backend && l.key === renamedFrom)
|
|
1937
|
+
));
|
|
1938
|
+
|
|
1939
|
+
this.#index.put(id, {
|
|
1940
|
+
checksums: data.checksums,
|
|
1941
|
+
size: data.size,
|
|
1942
|
+
mimeType: data.mimeType,
|
|
1943
|
+
locations,
|
|
1944
|
+
}, {
|
|
1945
|
+
origin,
|
|
1946
|
+
rename: renamedFrom ? { backend: data.backend, from: renamedFrom, to: data.key } : null,
|
|
1947
|
+
});
|
|
1948
|
+
this.#emitObject('add', { ...data, id, locations, ...(renamedFrom ? { renamedFrom } : {}) });
|
|
1949
|
+
|
|
1950
|
+
} else if (event === 'file:change' && data.checksums) {
|
|
1951
|
+
const newId = formatId(data.checksums, this.#config.primaryChecksum);
|
|
1952
|
+
const oldMeta = this.#index.get(pathKey);
|
|
1953
|
+
// Same path, new bytes = a successor under content identity. Carry
|
|
1954
|
+
// the predecessor's identity on the add event so consumers can
|
|
1955
|
+
// migrate curated placements instead of orphaning them. The two
|
|
1956
|
+
// index writes share one transaction: the change log then holds a
|
|
1957
|
+
// single `put` for the key, never a transient delete.
|
|
1958
|
+
let previous = null;
|
|
1959
|
+
const locations = this.#index.transaction(() => {
|
|
1960
|
+
if (oldMeta && oldMeta.id !== newId) {
|
|
1961
|
+
previous = { id: oldMeta.id, checksums: oldMeta.checksums };
|
|
1962
|
+
oldMeta.locations = oldMeta.locations.filter(l =>
|
|
1963
|
+
!(l.backend === data.backend && l.key === data.key)
|
|
1964
|
+
);
|
|
1965
|
+
if (oldMeta.locations.length === 0) {
|
|
1966
|
+
this.#index.delete(oldMeta.id, { origin });
|
|
1967
|
+
} else {
|
|
1968
|
+
this.#index.put(oldMeta.id, oldMeta, { origin });
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
const existing = this.#index.get(newId);
|
|
1973
|
+
const next = upsertLocation(existing?.locations || []);
|
|
1974
|
+
this.#index.put(newId, {
|
|
1975
|
+
checksums: data.checksums,
|
|
1976
|
+
size: data.size,
|
|
1977
|
+
mimeType: data.mimeType,
|
|
1978
|
+
locations: next,
|
|
1979
|
+
}, { origin });
|
|
1980
|
+
return next;
|
|
1981
|
+
});
|
|
1982
|
+
if (previous) {
|
|
1983
|
+
this.#emitObject('unlink', { ...data, id: previous.id, checksums: previous.checksums, successor: { id: newId, checksums: data.checksums } });
|
|
1984
|
+
}
|
|
1985
|
+
this.#emitObject('add', { ...data, id: newId, locations, previous });
|
|
1986
|
+
|
|
1987
|
+
} else if (event === 'file:unlink') {
|
|
1988
|
+
// Hold the unlink briefly when we know the file's inode — if a
|
|
1989
|
+
// same-inode add arrives inside the window this was a move, not a
|
|
1990
|
+
// deletion, and consumers never see the transient absence.
|
|
1991
|
+
const meta = this.#index.get(pathKey);
|
|
1992
|
+
const loc = meta?.locations?.find(l => l.backend === data.backend && l.key === data.key);
|
|
1993
|
+
if (loc?.ino != null) {
|
|
1994
|
+
const pendingKey = `${data.backend}|${loc.dev}:${loc.ino}`;
|
|
1995
|
+
// A second unlink for the same inode (hardlink twin) processes
|
|
1996
|
+
// the held one immediately — one hold per inode.
|
|
1997
|
+
const prior = this.#pendingUnlinks.get(pendingKey);
|
|
1998
|
+
if (prior) {
|
|
1999
|
+
clearTimeout(prior.timer);
|
|
2000
|
+
this.#pendingUnlinks.delete(pendingKey);
|
|
2001
|
+
this.#processUnlink(prior.data);
|
|
2002
|
+
}
|
|
2003
|
+
const timer = setTimeout(() => {
|
|
2004
|
+
this.#pendingUnlinks.delete(pendingKey);
|
|
2005
|
+
this.#processUnlink(data);
|
|
2006
|
+
}, RENAME_WINDOW_MS);
|
|
2007
|
+
timer.unref?.();
|
|
2008
|
+
this.#pendingUnlinks.set(pendingKey, { data, timer });
|
|
2009
|
+
return;
|
|
2010
|
+
}
|
|
2011
|
+
this.#processUnlink(data);
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
// An unlink observed while its key was held: once the hold expires, a key
|
|
2016
|
+
// the index still claims but the backend no longer has was a real delete
|
|
2017
|
+
// (the user removed the file right after we wrote it) and is processed
|
|
2018
|
+
// through the normal path; the echo of our own remove/rename is a no-op
|
|
2019
|
+
// (the transfer already rewrote the index, or the file is back).
|
|
2020
|
+
#recheckUnlink(data) {
|
|
2021
|
+
const pathKey = `${data.backend}:${data.key}`;
|
|
2022
|
+
const timer = setTimeout(async () => {
|
|
2023
|
+
if (this.#suppressedKeys.has(pathKey)) { this.#recheckUnlink(data); return; }
|
|
2024
|
+
const meta = this.#index.get(pathKey);
|
|
2025
|
+
if (!meta) return;
|
|
2026
|
+
const backend = this.#backends.get(data.backend);
|
|
2027
|
+
if (!backend) return;
|
|
2028
|
+
const present = await backend.stat(data.key).catch(() => null);
|
|
2029
|
+
if (present) return;
|
|
2030
|
+
debug(`Deferred unlink confirmed for ${pathKey}`);
|
|
2031
|
+
this.#handleFileEvent('file:unlink', data);
|
|
2032
|
+
}, MOVE_SUPPRESS_MS + 100);
|
|
2033
|
+
timer.unref?.();
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
// The genuine-deletion path for a file:unlink (immediate when the inode is
|
|
2037
|
+
// unknown, deferred through #pendingUnlinks otherwise).
|
|
2038
|
+
#processUnlink(data) {
|
|
2039
|
+
const pathKey = `${data.backend}:${data.key}`;
|
|
2040
|
+
const meta = this.#index.get(pathKey);
|
|
2041
|
+
const origin = data.origin || null;
|
|
2042
|
+
if (meta) {
|
|
2043
|
+
meta.locations = meta.locations.filter(l =>
|
|
2044
|
+
!(l.backend === data.backend && l.key === data.key)
|
|
2045
|
+
);
|
|
2046
|
+
if (meta.locations.length === 0) {
|
|
2047
|
+
this.#index.delete(meta.id, { origin });
|
|
2048
|
+
} else {
|
|
2049
|
+
this.#index.put(meta.id, meta, { origin });
|
|
2050
|
+
}
|
|
2051
|
+
this.#emitObject('unlink', { ...data, id: meta.id, checksums: meta.checksums, locations: meta.locations });
|
|
2052
|
+
} else {
|
|
2053
|
+
this.#emitObject('unlink', data);
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
export { Mirror, JobQueue, Ledger, BackendManager, Index };
|