@augmentd-labs/canvas-edge 0.6.3 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/canvas-stored/README.md +189 -9
- package/node_modules/canvas-stored/package.json +2 -2
- package/node_modules/canvas-stored/src/backends/BackendManager.js +11 -1
- package/node_modules/canvas-stored/src/backends/StorageBackend.js +24 -12
- package/node_modules/canvas-stored/src/backends/cacache/index.js +49 -6
- package/node_modules/canvas-stored/src/backends/canvas/index.js +48 -6
- package/node_modules/canvas-stored/src/backends/file/index.js +181 -39
- package/node_modules/canvas-stored/src/backends/gdrive/index.js +57 -10
- package/node_modules/canvas-stored/src/backends/http/index.js +3 -1
- package/node_modules/canvas-stored/src/backends/s3/index.js +244 -15
- package/node_modules/canvas-stored/src/cache/collect-garbage.js +47 -0
- package/node_modules/canvas-stored/src/cache/compact-index.js +13 -0
- package/node_modules/canvas-stored/src/cache/index.js +39 -5
- package/node_modules/canvas-stored/src/cache/ingest-stream.js +65 -0
- package/node_modules/canvas-stored/src/index/index.js +5 -3
- package/node_modules/canvas-stored/src/index.js +797 -140
- package/node_modules/canvas-stored/src/runtime/MutationQueue.js +60 -0
- package/node_modules/canvas-stored/src/storage/lmdb.js +62 -0
- package/node_modules/canvas-stored/src/storage/recover-index.js +33 -0
- package/node_modules/canvas-stored/src/storage/verify-worker.js +23 -0
- package/node_modules/canvas-stored/src/sync/Ledger.js +21 -3
- package/node_modules/canvas-stored/src/sync/Mirror.js +335 -25
- package/node_modules/canvas-stored/src/sync/REPLICAS.md +64 -0
- package/node_modules/canvas-stored/src/sync/ReplicaService.js +623 -0
- package/node_modules/canvas-stored/src/sync/SyncQueue.js +23 -103
- package/node_modules/canvas-stored/src/trash.js +219 -0
- package/node_modules/canvas-stored/src/utils/common.js +7 -0
- package/package.json +5 -4
- package/src/binding.js +33 -0
- package/src/control.js +13 -0
- package/src/daemon.js +14 -1
- package/src/mirror-runtime.js +14 -3
|
@@ -1,26 +1,46 @@
|
|
|
1
1
|
import EventEmitter2 from 'eventemitter2';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import crypto from 'crypto';
|
|
4
|
+
import MutationQueue from './runtime/MutationQueue.js';
|
|
4
5
|
import { createReadStream, createWriteStream, promises as fsp } from 'fs';
|
|
5
6
|
import { once } from 'events';
|
|
6
7
|
import { Readable } from 'stream';
|
|
7
8
|
import { pipeline } from 'stream/promises';
|
|
9
|
+
import { setImmediate as yieldToIO } from 'node:timers/promises';
|
|
10
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
8
11
|
import Debug from 'debug';
|
|
9
12
|
import Cache from './cache/index.js';
|
|
10
13
|
import BackendManager from './backends/BackendManager.js';
|
|
14
|
+
export { getBackendCapabilities } from './backends/BackendManager.js';
|
|
11
15
|
import Index from './index/index.js';
|
|
16
|
+
import { prepareIndex } from './storage/recover-index.js';
|
|
12
17
|
import SyncQueue from './sync/SyncQueue.js';
|
|
13
18
|
import JobQueue from './sync/JobQueue.js';
|
|
14
19
|
import Ledger from './sync/Ledger.js';
|
|
15
20
|
import Mirror from './sync/Mirror.js';
|
|
16
|
-
import
|
|
17
|
-
import
|
|
21
|
+
import BackendTrash from './trash.js';
|
|
22
|
+
import ReplicaService from './sync/ReplicaService.js';
|
|
23
|
+
import { isBuffer, isFileAsync, isStream, resolveStoredPaths } from './utils/common.js';
|
|
24
|
+
import { checksumBuffer, checksumStream, formatId } from './utils/checksum.js';
|
|
18
25
|
import { detectMimeType, detectMimeFromHead } from './utils/mime.js';
|
|
19
26
|
|
|
20
27
|
const debug = Debug('stored');
|
|
28
|
+
// Source retirement belongs to the final move event, never a source-loss unlink.
|
|
29
|
+
const MOVE_SOURCE = Symbol('move-source');
|
|
30
|
+
// A remote source already downloaded for this copy: { path, checksums, size, backend, key }.
|
|
31
|
+
const STAGED = Symbol('staged-source');
|
|
21
32
|
|
|
22
33
|
const HEAD_BYTES = 4096;
|
|
23
34
|
|
|
35
|
+
async function readHead(file) {
|
|
36
|
+
const handle = await fsp.open(file, 'r');
|
|
37
|
+
try {
|
|
38
|
+
const buffer = Buffer.alloc(HEAD_BYTES);
|
|
39
|
+
const { bytesRead } = await handle.read(buffer, 0, HEAD_BYTES, 0);
|
|
40
|
+
return buffer.subarray(0, bytesRead);
|
|
41
|
+
} finally { await handle.close(); }
|
|
42
|
+
}
|
|
43
|
+
|
|
24
44
|
// A filesystem move reaches the watcher as unlink(old) + add(new). Unlink
|
|
25
45
|
// processing is held back this long so an add carrying the same inode can
|
|
26
46
|
// claim it as a rename (in-place URL rewrite, no orphan churn). Expired
|
|
@@ -36,6 +56,29 @@ const RENAME_WINDOW_MS = 1000;
|
|
|
36
56
|
const MOVE_SUPPRESS_MS = 3000;
|
|
37
57
|
|
|
38
58
|
export default class Stored extends EventEmitter2 {
|
|
59
|
+
/** Safe startup for persistent indexes. The sync constructor remains for
|
|
60
|
+
* callers that already verified their environment (or create a new one).
|
|
61
|
+
*/
|
|
62
|
+
static async open(config = {}) {
|
|
63
|
+
const databasePath = resolveStoredPaths(config).index;
|
|
64
|
+
config.onStartupStage?.({ stage: 'stored.index.verify', databasePath });
|
|
65
|
+
const recovery = await prepareIndex(databasePath);
|
|
66
|
+
if (recovery) {
|
|
67
|
+
config.onStartupStage?.({ stage: 'stored.index.quarantined', ...recovery });
|
|
68
|
+
console.warn(`Stored index verification failed; preserved at ${recovery.preservedIndexPath}. Opening an empty index; rescan backends to rebuild. Operational records remain in quarantine. Reason: ${recovery.reason}`);
|
|
69
|
+
}
|
|
70
|
+
config.onStartupStage?.({ stage: 'stored.index.open', databasePath });
|
|
71
|
+
return new this(config);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
supportsScopedFolderDeletion = true;
|
|
75
|
+
#mutations;
|
|
76
|
+
#closing = false;
|
|
77
|
+
#stopPromise = null;
|
|
78
|
+
|
|
79
|
+
withStorageMutation(action, operation) { return this.#mutations.run(action, operation); }
|
|
80
|
+
detachedStorageWork(action) { return this.#mutations.detach(action); }
|
|
81
|
+
get mutationDiagnostics() { return this.#mutations.diagnostics; }
|
|
39
82
|
#cache;
|
|
40
83
|
#backends;
|
|
41
84
|
#index;
|
|
@@ -55,11 +98,26 @@ export default class Stored extends EventEmitter2 {
|
|
|
55
98
|
#pendingMoves = new Map();
|
|
56
99
|
#retention = null;
|
|
57
100
|
#retainedDb = null;
|
|
101
|
+
#retentionTasks = new Map();
|
|
102
|
+
#trash;
|
|
58
103
|
#sweepTimer = null;
|
|
104
|
+
#directoryMovesDb = null;
|
|
105
|
+
#directoryMoveTasks = new Map();
|
|
106
|
+
#replicas;
|
|
59
107
|
|
|
60
108
|
constructor(config = {}) {
|
|
61
109
|
// Wildcards (':' delimiter) let consumers bind `object:*` across backends.
|
|
62
110
|
super({ wildcard: true, delimiter: ':', maxListeners: 100, verboseMemoryLeak: false });
|
|
111
|
+
this.#mutations = new MutationQueue({ onStall: config.onStall || (state => debug('Mutation still pending: %o', state)) });
|
|
112
|
+
// API/FUSE mutations and replication share one physical executor.
|
|
113
|
+
// Reentrant calls (rename -> move, replica -> write) keep the lock.
|
|
114
|
+
for (const name of ['put', 'copy', 'move', 'writeObject', 'removeObject', 'removeToTrash', 'renameObject', 'renameDirectory', 'trashDirectory', 'removeDirectory',
|
|
115
|
+
'ensureDirectory', 'removeEmptyDirectory', 'restoreTrash', 'discardTrash', 'restoreRetained', 'deleteByUrl', 'delete']) {
|
|
116
|
+
const method = this[name].bind(this);
|
|
117
|
+
this[name] = name === 'copy'
|
|
118
|
+
? (idOrKey, options = {}) => this.#routeCopy(method, idOrKey, options)
|
|
119
|
+
: (...args) => this.withStorageMutation(() => method(...args), name);
|
|
120
|
+
}
|
|
63
121
|
this.#paths = resolveStoredPaths(config);
|
|
64
122
|
this.#config = {
|
|
65
123
|
checksums: config.checksums || ['sha256'],
|
|
@@ -79,16 +137,19 @@ export default class Stored extends EventEmitter2 {
|
|
|
79
137
|
// entry, after the transaction that produced them committed).
|
|
80
138
|
this.#index = new Index(this.#paths.index, {
|
|
81
139
|
changes: config.changes,
|
|
82
|
-
onChange: (entries) => { for (const entry of entries) this.emit('change', entry); },
|
|
140
|
+
onChange: (entries) => this.detachedStorageWork(() => { for (const entry of entries) this.emit('change', entry); }),
|
|
83
141
|
});
|
|
84
142
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
this.#
|
|
143
|
+
this.#trash = new BackendTrash(this);
|
|
144
|
+
|
|
145
|
+
try { this.#replicas = new ReplicaService(this, config.replication || {}); }
|
|
146
|
+
catch (error) { this.#index.close(); void this.#cache.stop(); throw error; }
|
|
147
|
+
this.#replicas.on('changed', operation => this.detachedStorageWork(() => this.emit('replica:changed', operation)));
|
|
148
|
+
this.#replicas.on('fault', error => this.emit('error', error));
|
|
149
|
+
this.#syncQueue = new SyncQueue({ service: this.#replicas });
|
|
90
150
|
this.#syncQueue.on('synced', ({ id, results }) => this.#handleSyncResult(id, results));
|
|
91
|
-
this.#syncQueue.on('error',
|
|
151
|
+
this.#syncQueue.on('error', error => this.emit('error', error));
|
|
152
|
+
this.#pendingMoves = this.#index.openDB('pending-moves.v1');
|
|
92
153
|
|
|
93
154
|
debug('Stored initialized');
|
|
94
155
|
}
|
|
@@ -103,6 +164,36 @@ export default class Stored extends EventEmitter2 {
|
|
|
103
164
|
get cache() { return this.#cache; }
|
|
104
165
|
get index() { return this.#index; }
|
|
105
166
|
get backends() { return this.#backends; }
|
|
167
|
+
get replicas() { return this.#replicas; }
|
|
168
|
+
|
|
169
|
+
// Called only after the executor has verified the exact target bytes.
|
|
170
|
+
async acknowledgeReplica(backendName, key, id, origin) {
|
|
171
|
+
const meta = this.#index.get(id);
|
|
172
|
+
if (!meta) throw new Error('Replica content is not indexed');
|
|
173
|
+
const backend = this.#backends.get(backendName);
|
|
174
|
+
const stat = await backend.stat(key);
|
|
175
|
+
if (!stat) throw new Error('Verified replica disappeared');
|
|
176
|
+
const location = this.#buildLocation(backendName, key, true, {
|
|
177
|
+
size: stat.size, mtime: stat.modified,
|
|
178
|
+
...(stat.ino != null ? { ino: stat.ino, dev: stat.dev } : {}),
|
|
179
|
+
});
|
|
180
|
+
const saved = this.#index.put(id, { ...meta, locations: this.#mergeLocations(meta, [location]) }, { origin });
|
|
181
|
+
this.#emitLocationEvent('location:add', saved, location, { origin });
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async ensureDirectory(backendName, key) {
|
|
185
|
+
const backend = this.#backends.get(backendName);
|
|
186
|
+
if (!this.#replicas.capabilities(backendName).directories || !this.#isSafeKey(key)) throw new Error('Safe directory creation is unsupported');
|
|
187
|
+
if (!(await this.#verifyBackendRoot(backend)).ok) throw Object.assign(new Error('Backend is offline'), { code: 'BACKEND_UNAVAILABLE', retryable: true });
|
|
188
|
+
return backend.createContainer(key);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async removeEmptyDirectory(backendName, key) {
|
|
192
|
+
const backend = this.#backends.get(backendName);
|
|
193
|
+
if (!this.#replicas.capabilities(backendName).directories || !this.#isSafeKey(key) || typeof backend.removeEmptyContainer !== 'function') throw new Error('Safe directory removal is unsupported');
|
|
194
|
+
if (!(await this.#verifyBackendRoot(backend)).ok) throw Object.assign(new Error('Backend is offline'), { code: 'BACKEND_UNAVAILABLE', retryable: true });
|
|
195
|
+
return backend.removeEmptyContainer(key);
|
|
196
|
+
}
|
|
106
197
|
|
|
107
198
|
// ─────────────────────────────────────────────────────────────────────────
|
|
108
199
|
// Backend Management
|
|
@@ -124,7 +215,13 @@ export default class Stored extends EventEmitter2 {
|
|
|
124
215
|
backend.on('scan:complete', e => this.emit('scan:complete', e));
|
|
125
216
|
backend.on('error', e => this.emit('error', e));
|
|
126
217
|
|
|
127
|
-
|
|
218
|
+
backend.isDirectoryProtected = key => this.#replicas.owners().some(owner => owner.replicas.some(r =>
|
|
219
|
+
r.kind === 'directory' && r.backend === name && (r.key === key || r.key.startsWith(`${key}/`))));
|
|
220
|
+
if (config.watch) backend.watch().catch(error => {
|
|
221
|
+
debug('Backend %s watcher startup failed: %s', name, error.message);
|
|
222
|
+
this.emit('backend:state', { backend: name, watching: false, error: error.message });
|
|
223
|
+
if (this.listenerCount('error')) this.emit('error', error);
|
|
224
|
+
});
|
|
128
225
|
return backend;
|
|
129
226
|
}
|
|
130
227
|
|
|
@@ -139,6 +236,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
139
236
|
|
|
140
237
|
listBackends() { return this.#backends.list(); }
|
|
141
238
|
getBackend(name) { return this.#backends.get(name); }
|
|
239
|
+
getBackendCapabilities(name) { return this.#backends.get(name)?.capabilities || null; }
|
|
142
240
|
|
|
143
241
|
/**
|
|
144
242
|
* Parse a canonical `stored://<backend>/<key>` URL. The backend name may
|
|
@@ -203,8 +301,10 @@ export default class Stored extends EventEmitter2 {
|
|
|
203
301
|
const backend = this.#backends.get(p.backend);
|
|
204
302
|
if (!backend) return { ok: false, reason: 'unknown-backend' };
|
|
205
303
|
if (!backend.canDelete) return { ok: false, reason: 'read-only-backend' };
|
|
206
|
-
await this.#retain(p.backend, p.key, this.#index.get(`${p.backend}:${p.key}`));
|
|
304
|
+
const retained = await this.#retain(p.backend, p.key, this.#index.get(`${p.backend}:${p.key}`), 'delete');
|
|
305
|
+
const trashed = retained?.trashItem;
|
|
207
306
|
const deleted = !!(await backend.delete(p.key));
|
|
307
|
+
if (deleted) this.#trash.finish(trashed);
|
|
208
308
|
if (deleted) this.#dropLocation(p.backend, p.key);
|
|
209
309
|
return { ok: deleted };
|
|
210
310
|
}
|
|
@@ -235,6 +335,11 @@ export default class Stored extends EventEmitter2 {
|
|
|
235
335
|
* Store a blob. Accepts Buffer | string (content) | filesystem path |
|
|
236
336
|
* Readable. Paths and streams are ingested without ever materializing the
|
|
237
337
|
* whole blob in memory. `backends` is required (no implicit fan-out).
|
|
338
|
+
* `adopt: true` with a path hands the file over (writeObject takes the
|
|
339
|
+
* same option): a blob-store target moves
|
|
340
|
+
* it in rather than copying it, and the file is gone afterwards. With
|
|
341
|
+
* `checksums` (hex per configured algorithm, computed by the caller as the
|
|
342
|
+
* bytes arrived) the file is not read again to hash it.
|
|
238
343
|
* Returns { ok:true, id, key, size, mimeType, checksums, locations, ... }
|
|
239
344
|
* or { ok:false, reason:'no-targets'|'unknown-backend' }.
|
|
240
345
|
*/
|
|
@@ -248,18 +353,22 @@ export default class Stored extends EventEmitter2 {
|
|
|
248
353
|
if (!targets.length) return { ok: false, reason: 'unknown-backend' };
|
|
249
354
|
|
|
250
355
|
const ingestOptions = { mtime: options.mtime ?? null };
|
|
251
|
-
const ingest = (isBuffer(blob) || (typeof blob === 'string' && !
|
|
356
|
+
const ingest = (isBuffer(blob) || (typeof blob === 'string' && !await isFileAsync(blob)))
|
|
252
357
|
? await this.#ingestMemory(blob, targets, key, options.mimeType, ingestOptions)
|
|
253
|
-
:
|
|
358
|
+
: (options.adopt && typeof blob === 'string')
|
|
359
|
+
? await this.#ingestAdopted(blob, targets, key, options.mimeType, { ...ingestOptions, checksums: options.checksums })
|
|
360
|
+
: await this.#ingestStream(blob, targets, key, options.mimeType, ingestOptions);
|
|
254
361
|
|
|
255
362
|
// Caller-supplied metadata + inline-extracted (EXIF/GPS/dimensions/media)
|
|
256
363
|
// → persisted under `custom`, returned to caller + on the 'put' event.
|
|
257
364
|
const extracted = ingest.extracted && Object.keys(ingest.extracted).length ? ingest.extracted : null;
|
|
258
|
-
|
|
365
|
+
let meta;
|
|
366
|
+
this.#index.transaction(() => {
|
|
367
|
+
meta = this.#index.put(ingest.id, {
|
|
259
368
|
checksums: ingest.checksums,
|
|
260
369
|
size: ingest.size,
|
|
261
370
|
mimeType: ingest.mimeType,
|
|
262
|
-
locations: ingest.locations,
|
|
371
|
+
locations: this.#mergeLocations(this.#index.get(ingest.id) || {}, ingest.locations),
|
|
263
372
|
custom: extracted ? { ...metadata, ...extracted } : metadata,
|
|
264
373
|
}, { origin: options.origin });
|
|
265
374
|
|
|
@@ -267,6 +376,8 @@ export default class Stored extends EventEmitter2 {
|
|
|
267
376
|
this.#syncQueue.enqueue({ id: ingest.id, cacheRoot: this.#cache.root, cacheKey: ingest.id, targets: ingest.remoteTargets });
|
|
268
377
|
}
|
|
269
378
|
|
|
379
|
+
});
|
|
380
|
+
|
|
270
381
|
this.emit('put', { id: ingest.id, key: ingest.finalKey, metadata: meta });
|
|
271
382
|
debug(`PUT ${ingest.id.slice(0, 19)}... → ${targets.map(t => t.name).join(', ')}`);
|
|
272
383
|
return { ok: true, key: ingest.finalKey, ...meta };
|
|
@@ -293,8 +404,10 @@ export default class Stored extends EventEmitter2 {
|
|
|
293
404
|
const removed = new Set();
|
|
294
405
|
for (const loc of targets) {
|
|
295
406
|
const backend = this.#backends.get(loc.backend);
|
|
296
|
-
|
|
407
|
+
const retained = backend && await this.#retain(loc.backend, loc.key, meta, 'delete');
|
|
408
|
+
const trashed = retained?.trashItem;
|
|
297
409
|
if (backend && await backend.delete(loc.key)) {
|
|
410
|
+
this.#trash.finish(trashed);
|
|
298
411
|
deleted.push(loc.backend);
|
|
299
412
|
removed.add(loc);
|
|
300
413
|
}
|
|
@@ -366,7 +479,9 @@ export default class Stored extends EventEmitter2 {
|
|
|
366
479
|
const backend = this.#backends.get(backendName);
|
|
367
480
|
if (!backend) return { ok: false, reason: 'unknown-backend', backend: backendName };
|
|
368
481
|
if (!backend.canWrite) return { ok: false, reason: 'read-only-target', backend: backendName };
|
|
369
|
-
|
|
482
|
+
const remoteCreate = backend.type === 'remote' && typeof backend.createFrom === 'function'
|
|
483
|
+
&& String(options.ifNoneMatch || '').trim() === '*';
|
|
484
|
+
if ((!remoteCreate && backend.type !== 'local') || typeof backend.commit !== 'function') {
|
|
370
485
|
return { ok: false, reason: 'unsupported-backend', backend: backendName };
|
|
371
486
|
}
|
|
372
487
|
const destKey = this.#normalizeKey(key, { nfc: true });
|
|
@@ -375,22 +490,44 @@ export default class Stored extends EventEmitter2 {
|
|
|
375
490
|
if (!live.ok) return { ok: false, reason: 'target-offline', backend: backendName, detail: live.reason };
|
|
376
491
|
|
|
377
492
|
const pathKey = `${backendName}:${destKey}`;
|
|
493
|
+
// `adopt`: the source path is a finished file the caller hands over
|
|
494
|
+
// (a resumable upload's part). It is placed itself, not copied, and
|
|
495
|
+
// stays where it is if the write does not happen.
|
|
496
|
+
const adopted = options.adopt && typeof source === 'string' && await isFileAsync(source);
|
|
378
497
|
const tempDir = backend.tempDir || path.join(this.#paths.cache, '.tmp');
|
|
379
|
-
const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
380
|
-
await fsp.mkdir(tempDir, { recursive: true });
|
|
498
|
+
const tempPath = adopted ? source : path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
499
|
+
if (!adopted) await fsp.mkdir(tempDir, { recursive: true });
|
|
500
|
+
let ingress = null;
|
|
501
|
+
let accepted = false;
|
|
381
502
|
|
|
382
503
|
try {
|
|
383
|
-
const
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
504
|
+
const known = adopted && options.checksums && this.#config.checksums.every(algo => /^[0-9a-f]+$/.test(options.checksums[algo] || ''));
|
|
505
|
+
const { checksums, size, head } = known
|
|
506
|
+
? { checksums: Object.fromEntries(this.#config.checksums.map(algo => [algo, options.checksums[algo]])), size: (await fsp.stat(tempPath)).size, head: await readHead(tempPath) }
|
|
507
|
+
: adopted
|
|
508
|
+
? await this.#hashFile(tempPath)
|
|
509
|
+
: await this.#hashToFile(isBuffer(source) ? Readable.from([source])
|
|
510
|
+
: (typeof source === 'string' && !await isFileAsync(source)) ? Readable.from([Buffer.from(source)])
|
|
511
|
+
: (isStream(source) ? source : createReadStream(source)), tempPath);
|
|
387
512
|
const id = formatId(checksums, this.#config.primaryChecksum);
|
|
388
513
|
const sha256 = checksums.sha256 ?? null;
|
|
389
514
|
if (options.sha256 && sha256 && String(options.sha256).toLowerCase() !== sha256) {
|
|
390
515
|
return { ok: false, reason: 'checksum-mismatch', expected: String(options.sha256).toLowerCase(), actual: sha256 };
|
|
391
516
|
}
|
|
392
517
|
|
|
393
|
-
const
|
|
518
|
+
const indexed = this.#index.get(pathKey);
|
|
519
|
+
const physical = await backend.stat(destKey);
|
|
520
|
+
let current = physical && indexed?.locations?.some(l => l.backend === backendName && l.key === destKey && l.synced !== false) ? indexed : null;
|
|
521
|
+
if (physical && backend.config.driver === 'file') {
|
|
522
|
+
// A watcher/index can lag behind an external writer. Check
|
|
523
|
+
// actual bytes before preconditions or the unchanged shortcut,
|
|
524
|
+
// and index previously unknown files so overwrites retain them.
|
|
525
|
+
const input = await backend.get(destKey, { stream: true });
|
|
526
|
+
const actual = await checksumStream(Buffer.isBuffer(input) ? Readable.from([input]) : input);
|
|
527
|
+
const after = await backend.stat(destKey);
|
|
528
|
+
if (!after || ['size', 'modified', 'ino', 'dev'].some(k => physical[k] !== after[k])) return { ok: false, reason: 'source-changed' };
|
|
529
|
+
if (!current || current.checksums?.sha256 !== actual.sha256) current = await this.refreshObject(backendName, destKey);
|
|
530
|
+
}
|
|
394
531
|
const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
|
|
395
532
|
const failed = this.#checkPrecondition(current, currentLoc, options);
|
|
396
533
|
if (failed) return failed;
|
|
@@ -413,13 +550,32 @@ export default class Stored extends EventEmitter2 {
|
|
|
413
550
|
}
|
|
414
551
|
|
|
415
552
|
const mimeType = options.mimeType || await detectMimeFromHead(head, destKey);
|
|
553
|
+
if (!String(options.origin || '').startsWith('replica:')) ingress = await this.#replicas.captureIngress(backendName, destKey, id, tempPath);
|
|
416
554
|
const release = this.#holdKeys([pathKey]);
|
|
417
555
|
try {
|
|
418
|
-
|
|
556
|
+
const retained = current ? await this.#retain(backendName, destKey, current) : null;
|
|
419
557
|
let placed = null;
|
|
420
|
-
if (
|
|
558
|
+
if (current && retained && typeof backend.trashFile === 'function') {
|
|
559
|
+
// Capture the actual inode before publishing. A writer
|
|
560
|
+
// outside Stored can race a precondition/retention read;
|
|
561
|
+
// its bytes must survive even if it keeps the inode open.
|
|
562
|
+
const captured = this.#trash.prepare(backendName, destKey, { retained, reason: 'overwrite' });
|
|
563
|
+
this.#trash.quarantine(captured);
|
|
564
|
+
const moved = await backend.trashFile(destKey, captured.id);
|
|
565
|
+
this.#trash.finish(captured);
|
|
566
|
+
if (moved.checksums.sha256 !== current.checksums?.sha256) {
|
|
567
|
+
await this.#trash.restore(backendName, captured.id).catch(() => {});
|
|
568
|
+
return { ok: false, reason: 'source-changed', trashId: captured.id };
|
|
569
|
+
}
|
|
570
|
+
try { placed = await backend.createFrom(destKey, tempPath, { owned: true }); }
|
|
571
|
+
catch (error) {
|
|
572
|
+
await this.#trash.restore(backendName, captured.id).catch(() => {});
|
|
573
|
+
if (error.code === 'EEXIST') return { ok: false, reason: 'precondition-failed', trashId: captured.id };
|
|
574
|
+
throw error;
|
|
575
|
+
}
|
|
576
|
+
} else if (String(options.ifNoneMatch || '').trim() === '*') {
|
|
421
577
|
if (typeof backend.createFrom !== 'function') return { ok: false, reason: 'unsupported-backend' };
|
|
422
|
-
try { placed = await backend.createFrom(destKey, tempPath); }
|
|
578
|
+
try { placed = await backend.createFrom(destKey, tempPath, { owned: true }); }
|
|
423
579
|
catch (error) {
|
|
424
580
|
if (error.code === 'EEXIST') return { ok: false, reason: 'precondition-failed', current: null };
|
|
425
581
|
throw error;
|
|
@@ -453,6 +609,8 @@ export default class Stored extends EventEmitter2 {
|
|
|
453
609
|
const saved = this.#index.get(id);
|
|
454
610
|
const loc = saved?.locations?.find(l => l.backend === backendName && l.key === destKey) || null;
|
|
455
611
|
debug(`WRITE ${pathKey} ← ${id.slice(0, 19)}...${current ? ` (replaces ${current.id.slice(0, 19)}...)` : ''}`);
|
|
612
|
+
this.#replicas.commitIngress(ingress);
|
|
613
|
+
accepted = true;
|
|
456
614
|
return {
|
|
457
615
|
ok: true,
|
|
458
616
|
id,
|
|
@@ -468,7 +626,8 @@ export default class Stored extends EventEmitter2 {
|
|
|
468
626
|
release();
|
|
469
627
|
}
|
|
470
628
|
} finally {
|
|
471
|
-
|
|
629
|
+
if (!accepted) await this.#replicas.discardIngress(ingress);
|
|
630
|
+
if (accepted || !adopted) await fsp.rm(tempPath, { force: true }).catch(() => {});
|
|
472
631
|
}
|
|
473
632
|
}
|
|
474
633
|
|
|
@@ -491,36 +650,93 @@ export default class Stored extends EventEmitter2 {
|
|
|
491
650
|
return this.#retainedDb;
|
|
492
651
|
}
|
|
493
652
|
|
|
494
|
-
async #retain(backendName, key, meta) {
|
|
653
|
+
async #retain(backendName, key, meta, reason = 'version') {
|
|
495
654
|
if (!this.#retention || !meta) return null;
|
|
496
655
|
const backend = this.#backends.get(backendName);
|
|
497
656
|
if (!backend || backend.type !== 'local' || typeof backend.retain !== 'function') return null;
|
|
498
657
|
const sha256 = meta.checksums?.sha256 ? String(meta.checksums.sha256).toLowerCase() : null;
|
|
499
658
|
if (!sha256) return null;
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
659
|
+
return this.#withRetainedLock(sha256, async () => {
|
|
660
|
+
try {
|
|
661
|
+
const kept = await backend.retain(key, sha256);
|
|
662
|
+
if (!kept) return null;
|
|
663
|
+
const now = Date.now();
|
|
664
|
+
const prev = this.#retained.get(sha256) || null;
|
|
665
|
+
const keys = [...new Set([...(prev?.keys || []), `${backendName}:${key}`])];
|
|
666
|
+
const entry = {
|
|
667
|
+
sha256,
|
|
668
|
+
backend: prev?.backend || backendName,
|
|
669
|
+
size: kept.size ?? prev?.size ?? meta.size ?? null,
|
|
670
|
+
mimeType: meta.mimeType ?? prev?.mimeType ?? null,
|
|
671
|
+
id: meta.id ?? prev?.id ?? null,
|
|
672
|
+
keys,
|
|
673
|
+
backends: [...new Set([...(prev?.backends || this.#retainedBackendNames(prev)), backendName])],
|
|
674
|
+
versionKeys: [...new Set([...(prev?.versionKeys || []), ...(reason === 'version' ? [`${backendName}:${key}`] : [])])],
|
|
675
|
+
firstAt: prev?.firstAt ?? now,
|
|
676
|
+
lastAt: now,
|
|
677
|
+
trashVersion: 1,
|
|
678
|
+
legacyKeys: prev?.legacyKeys ?? (prev && !prev.trashVersion ? prev.keys : []),
|
|
679
|
+
legacyAt: prev?.legacyAt ?? prev?.lastAt ?? null,
|
|
680
|
+
};
|
|
681
|
+
this.#retained.putSync(sha256, entry);
|
|
682
|
+
this.#scheduleSweep();
|
|
683
|
+
if (reason === 'delete') {
|
|
684
|
+
// Publish the pending deletion while still holding the
|
|
685
|
+
// digest lock: discard must see this new reference even
|
|
686
|
+
// if the physical unlink has not happened yet.
|
|
687
|
+
const mtime = meta.locations?.find(loc => loc.backend === backendName && loc.key === key)?.mtime ?? null;
|
|
688
|
+
return { ...entry, trashItem: this.#trash.prepare(backendName, key, { retained: entry, mtime }) };
|
|
689
|
+
}
|
|
690
|
+
return entry;
|
|
691
|
+
} catch (err) {
|
|
692
|
+
debug(`retain ${backendName}:${key} failed: ${err.message}`);
|
|
693
|
+
// A failed backup must never turn a recoverable delete/overwrite
|
|
694
|
+
// into permanent data loss (for example when the disk is full).
|
|
695
|
+
throw Object.assign(new Error(`retention failed for ${backendName}:${key}: ${err.message}`, { cause: err }), { code: 'RETAIN_FAILED' });
|
|
696
|
+
}
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
#withRetainedLock(sha256, fn) {
|
|
701
|
+
const previous = this.#retentionTasks.get(sha256) || Promise.resolve();
|
|
702
|
+
const task = previous.catch(() => {}).then(fn);
|
|
703
|
+
this.#retentionTasks.set(sha256, task);
|
|
704
|
+
void task.finally(() => {
|
|
705
|
+
if (this.#retentionTasks.get(sha256) === task) this.#retentionTasks.delete(sha256);
|
|
706
|
+
}).catch(() => {});
|
|
707
|
+
return task;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
#retainedBackendNames(entry) {
|
|
711
|
+
return [...new Set([...(entry?.backends || []), ...(entry?.backend ? [entry.backend] : []),
|
|
712
|
+
...this.#backends.list().filter(name => entry?.keys?.some(key => key.startsWith(`${name}:`)))])];
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
// Called only for a recorded Trash deletion. Other locations and retained
|
|
716
|
+
// overwrite versions keep their shared bytes. Never touch the original key.
|
|
717
|
+
async discardTrashBytes(item, finish) {
|
|
718
|
+
return this.#withRetainedLock(item.sha256, async () => {
|
|
719
|
+
const entry = this.getRetained(item.sha256);
|
|
720
|
+
if (!entry) { finish(); return; }
|
|
721
|
+
const address = `${item.backend}:${item.key}`;
|
|
722
|
+
const shared = this.#trash.hasReference(item.sha256, item.id, address);
|
|
723
|
+
const keys = (entry.keys || []).filter(key => key !== address || shared || entry.versionKeys?.includes(key));
|
|
724
|
+
const backends = this.#retainedBackendNames(entry);
|
|
725
|
+
if (keys.length || this.#trash.hasReference(item.sha256, item.id)) {
|
|
726
|
+
this.#retained.putSync(item.sha256, { ...entry, keys, backends });
|
|
727
|
+
finish();
|
|
728
|
+
return;
|
|
729
|
+
}
|
|
730
|
+
// Leave metadata intact on failure so the user can retry; drops
|
|
731
|
+
// are idempotent if an earlier attempt removed some copies.
|
|
732
|
+
for (const name of backends) {
|
|
733
|
+
const backend = this.#backends.get(name);
|
|
734
|
+
if (!backend?.dropRetained) throw new Error(`Recovery backend unavailable: ${name}`);
|
|
735
|
+
await backend.dropRetained(item.sha256);
|
|
736
|
+
}
|
|
737
|
+
this.#retained.removeSync(item.sha256);
|
|
738
|
+
finish();
|
|
739
|
+
});
|
|
524
740
|
}
|
|
525
741
|
|
|
526
742
|
#scheduleSweep() {
|
|
@@ -549,13 +765,21 @@ export default class Stored extends EventEmitter2 {
|
|
|
549
765
|
}
|
|
550
766
|
|
|
551
767
|
/** Readable stream of retained bytes, or null. */
|
|
552
|
-
async retainedStream(sha256) {
|
|
768
|
+
async retainedStream(sha256, preferredBackend = null) {
|
|
553
769
|
const entry = this.getRetained(sha256);
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
770
|
+
if (!entry) return null;
|
|
771
|
+
// Equal contents can have retained copies on several backends. A
|
|
772
|
+
// missing first backend must not hide another backend's recovery copy.
|
|
773
|
+
const candidates = this.#backends.list().filter(name => name === entry.backend
|
|
774
|
+
|| entry.keys?.some(key => key.startsWith(`${name}:`)));
|
|
775
|
+
if (preferredBackend && candidates.includes(preferredBackend)) candidates.unshift(preferredBackend);
|
|
776
|
+
for (const name of new Set(candidates)) {
|
|
777
|
+
const backend = this.#backends.get(name);
|
|
778
|
+
if (typeof backend.retainedPath !== 'function') continue;
|
|
779
|
+
const p = backend.retainedPath(entry.sha256);
|
|
780
|
+
if (await fsp.stat(p).catch(() => null)) return createReadStream(p);
|
|
781
|
+
}
|
|
782
|
+
return null;
|
|
559
783
|
}
|
|
560
784
|
|
|
561
785
|
/**
|
|
@@ -566,7 +790,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
566
790
|
async restoreRetained(sha256, { backend, key, ifMatch = null, ifNoneMatch = null, origin = null, mtime = null } = {}) {
|
|
567
791
|
const entry = this.getRetained(sha256);
|
|
568
792
|
if (!entry) return { ok: false, reason: 'not-found', sha256 };
|
|
569
|
-
const stream = await this.retainedStream(entry.sha256);
|
|
793
|
+
const stream = await this.retainedStream(entry.sha256, backend);
|
|
570
794
|
if (!stream) return { ok: false, reason: 'not-found', sha256, detail: 'retained bytes missing on disk' };
|
|
571
795
|
const target = backend || entry.backend;
|
|
572
796
|
const targetKey = key || entry.keys?.at(-1)?.slice(target.length + 1) || null;
|
|
@@ -582,16 +806,22 @@ export default class Stored extends EventEmitter2 {
|
|
|
582
806
|
/** Drop entries (and their bytes) displaced longer than the window ago. Returns `{ swept, kept }`. */
|
|
583
807
|
async sweepRetained({ olderThanMs = null, now = Date.now() } = {}) {
|
|
584
808
|
if (!this.#retention) return { swept: 0, kept: 0 };
|
|
809
|
+
await this.#trash.sweep(now);
|
|
585
810
|
const window = olderThanMs ?? this.#retention.days * 86_400_000;
|
|
586
811
|
let swept = 0;
|
|
587
812
|
let kept = 0;
|
|
588
|
-
for (const { key: sha
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
813
|
+
for (const { key: sha } of [...this.#retained.getRange()]) {
|
|
814
|
+
await this.#withRetainedLock(sha, async () => {
|
|
815
|
+
const value = this.#retained.get(sha);
|
|
816
|
+
if (!value?.sha256) return;
|
|
817
|
+
if ((now - (value.lastAt ?? 0)) < window) { kept += 1; return; }
|
|
818
|
+
for (const name of this.#retainedBackendNames(value)) {
|
|
819
|
+
const backend = this.#backends.get(name);
|
|
820
|
+
if (backend?.dropRetained) await backend.dropRetained(value.sha256);
|
|
821
|
+
}
|
|
822
|
+
this.#retained.removeSync(sha);
|
|
823
|
+
swept += 1;
|
|
824
|
+
});
|
|
595
825
|
}
|
|
596
826
|
if (swept) debug(`retention sweep: ${swept} dropped, ${kept} kept`);
|
|
597
827
|
return { swept, kept };
|
|
@@ -606,6 +836,70 @@ export default class Stored extends EventEmitter2 {
|
|
|
606
836
|
return true;
|
|
607
837
|
}
|
|
608
838
|
|
|
839
|
+
listTrash(backend, options = {}) { return this.#trash.list(backend, options); }
|
|
840
|
+
removeToTrash(backend, key, options = {}) { return this.#trash.remove(backend, key, options); }
|
|
841
|
+
getTrashItem(backend, id) { return this.#trash.get(backend, id); }
|
|
842
|
+
restoreTrash(backend, id, options = {}) { return this.#trash.restore(backend, id, options); }
|
|
843
|
+
discardTrash(backend, id) { return this.#trash.discard(backend, id); }
|
|
844
|
+
|
|
845
|
+
/** Delete a native remote folder and unlink only its indexed descendants.
|
|
846
|
+
* Local folders retain the recoverable, whole-directory Trash operation.
|
|
847
|
+
* The driver owns remote deletion policy (Drive uses its native Trash).
|
|
848
|
+
*/
|
|
849
|
+
async removeDirectory(backendName, key, { origin = null } = {}) {
|
|
850
|
+
const backend = this.#backends.get(backendName);
|
|
851
|
+
if (!backend?.canDelete || backend.config.readOnly === true) throw new Error('Backend is read-only or unavailable');
|
|
852
|
+
if (typeof backend.trashContainer === 'function') return this.trashDirectory(backendName, key, { origin });
|
|
853
|
+
if (backend.transport !== 'gdrive') throw new Error('Backend does not support native folder deletion');
|
|
854
|
+
const normalized = this.#normalizeKey(key, { nfc: true });
|
|
855
|
+
if (!normalized || !this.#isSafeKey(normalized)) throw new Error('Invalid folder key');
|
|
856
|
+
this.#replicas.assertIdleEndpoint(backendName, normalized, { directory: true });
|
|
857
|
+
const keys = new Set();
|
|
858
|
+
for (const [, meta] of this.#index.entries()) {
|
|
859
|
+
for (const location of meta.locations || []) {
|
|
860
|
+
if (location.backend === backendName && location.key.startsWith(`${normalized}/`)) keys.add(location.key);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
const release = this.#holdKeys([...keys].map(k => `${backendName}:${k}`));
|
|
864
|
+
try {
|
|
865
|
+
// A missing folder is an idempotent success; a failed remote write
|
|
866
|
+
// must leave every local location intact for retry.
|
|
867
|
+
await backend.deleteContainer(normalized);
|
|
868
|
+
for (const key of keys) this.#processUnlink({ backend: backendName, key, ...(origin ? { origin } : {}) });
|
|
869
|
+
await this.#replicas.abandonUnownedIngress(backendName, normalized, { directory: true });
|
|
870
|
+
return { ok: true, key: normalized, type: 'directory' };
|
|
871
|
+
} finally { release(); }
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/** Move the whole directory into private trash in one filesystem rename.
|
|
875
|
+
* This preserves unindexed/hidden files, empty folders and timestamps too.
|
|
876
|
+
* No recursive rm is permitted when recoverable deletion is enabled.
|
|
877
|
+
*/
|
|
878
|
+
async trashDirectory(backendName, key, { origin = null } = {}) {
|
|
879
|
+
const backend = this.#backends.get(backendName);
|
|
880
|
+
if (!backend?.canDelete || typeof backend.trashContainer !== 'function') throw new Error('Backend does not support recoverable folder deletion');
|
|
881
|
+
const normalized = this.#normalizeKey(key, { nfc: true });
|
|
882
|
+
if (!normalized || !this.#isSafeKey(normalized)) throw new Error('Invalid trash folder key');
|
|
883
|
+
this.#replicas.assertIdleEndpoint(backendName, normalized, { directory: true });
|
|
884
|
+
if (!this.#retention) throw new Error('Enable backend retention before moving folders to Trash');
|
|
885
|
+
const keys = [];
|
|
886
|
+
for (const [, meta] of this.#index.entries()) {
|
|
887
|
+
for (const location of meta.locations || []) {
|
|
888
|
+
if (location.backend === backendName && location.key.startsWith(`${normalized}/`)) keys.push(location.key);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
const item = this.#trash.prepare(backendName, normalized, { directory: true });
|
|
892
|
+
const release = this.#holdKeys(keys.map(k => `${backendName}:${k}`));
|
|
893
|
+
try {
|
|
894
|
+
await backend.trashContainer(normalized, item.id);
|
|
895
|
+
this.#trash.finish(item);
|
|
896
|
+
this.#scheduleSweep();
|
|
897
|
+
for (const key of keys) this.#processUnlink({ backend: backendName, key, ...(origin ? { origin } : {}) });
|
|
898
|
+
await this.#replicas.abandonUnownedIngress(backendName, normalized, { directory: true });
|
|
899
|
+
return { ok: true, id: item.id, key: normalized, type: 'directory' };
|
|
900
|
+
} finally { release(); }
|
|
901
|
+
}
|
|
902
|
+
|
|
609
903
|
/**
|
|
610
904
|
* Delete the bytes at `backend:key` (precondition-checked like
|
|
611
905
|
* writeObject) and process it as the genuine unlink it is: consumers get
|
|
@@ -617,6 +911,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
617
911
|
if (!backend.canDelete) return { ok: false, reason: 'read-only-backend', backend: backendName };
|
|
618
912
|
const destKey = this.#normalizeKey(key, { nfc: true });
|
|
619
913
|
if (!destKey || !this.#isSafeKey(destKey)) return { ok: false, reason: 'invalid-key', key };
|
|
914
|
+
this.#replicas.assertIdleEndpoint(backendName, destKey, { except: String(options.origin || '').startsWith('replica:') ? options.origin.slice(8) : null });
|
|
620
915
|
|
|
621
916
|
const pathKey = `${backendName}:${destKey}`;
|
|
622
917
|
const current = this.#index.get(pathKey);
|
|
@@ -630,9 +925,22 @@ export default class Stored extends EventEmitter2 {
|
|
|
630
925
|
// false = bytes already gone; the location still goes. A remote
|
|
631
926
|
// driver re-evaluates the precondition at the other end (412 →
|
|
632
927
|
// typed error, nothing is unlinked here).
|
|
633
|
-
await this.#retain(backendName, destKey, current);
|
|
634
|
-
|
|
635
|
-
|
|
928
|
+
const retained = await this.#retain(backendName, destKey, current, options.trashItem ? 'version' : 'delete');
|
|
929
|
+
const trashed = options.trashItem || retained?.trashItem;
|
|
930
|
+
if (trashed && typeof backend.trashFile === 'function') {
|
|
931
|
+
this.#trash.quarantine(trashed);
|
|
932
|
+
const moved = await backend.trashFile(destKey, trashed.id);
|
|
933
|
+
this.#trash.finish(trashed);
|
|
934
|
+
if (moved.checksums.sha256 !== current.checksums?.sha256) {
|
|
935
|
+
// A filesystem writer won the race. The captured inode is
|
|
936
|
+
// recoverable even if it is still open in that process.
|
|
937
|
+
await this.#trash.restore(backendName, trashed.id).catch(() => {});
|
|
938
|
+
return { ok: false, reason: 'source-changed', trashId: trashed.id };
|
|
939
|
+
}
|
|
940
|
+
} else await backend.delete(destKey, { ifMatch: options.ifMatch, origin: options.origin });
|
|
941
|
+
this.#trash.finish(trashed);
|
|
942
|
+
if (!options[MOVE_SOURCE]) this.#processUnlink({ backend: backendName, key: destKey, ...(options.origin ? { origin: options.origin } : {}) });
|
|
943
|
+
await this.#replicas.abandonUnownedIngress(backendName, destKey);
|
|
636
944
|
debug(`REMOVE ${pathKey}`);
|
|
637
945
|
return { ok: true, id: current.id, sha256: current.checksums?.sha256 ?? null, seq: this.#index.head() };
|
|
638
946
|
} finally {
|
|
@@ -640,6 +948,19 @@ export default class Stored extends EventEmitter2 {
|
|
|
640
948
|
}
|
|
641
949
|
}
|
|
642
950
|
|
|
951
|
+
async refreshObject(backendName, key, origin = null) {
|
|
952
|
+
const backend = this.#backends.get(backendName);
|
|
953
|
+
const before = await backend.stat(key);
|
|
954
|
+
if (!before) return null;
|
|
955
|
+
const input = await backend.get(key, { stream: true });
|
|
956
|
+
const checksums = await checksumStream(Buffer.isBuffer(input) ? Readable.from([input]) : input);
|
|
957
|
+
const after = await backend.stat(key);
|
|
958
|
+
if (!after || ['size', 'modified', 'ino', 'dev'].some(k => before[k] !== after[k])) throw new Error('File changed during refresh');
|
|
959
|
+
this.#handleFileEvent('file:add', { backend: backendName, key, checksums, ...after, origin,
|
|
960
|
+
path: backend.resolveKeyPath?.(key), mimeType: await detectMimeType(backend.resolveKeyPath(key)) }, { force: true });
|
|
961
|
+
return this.#index.get(`${backendName}:${key}`);
|
|
962
|
+
}
|
|
963
|
+
|
|
643
964
|
/**
|
|
644
965
|
* Rename `backend:from` to `backend:to` on the same backend: one
|
|
645
966
|
* `rename(2)` where the driver supports it, identity and inode intact,
|
|
@@ -654,6 +975,8 @@ export default class Stored extends EventEmitter2 {
|
|
|
654
975
|
if (!fromKey || !this.#isSafeKey(fromKey)) return { ok: false, reason: 'invalid-key', key: from };
|
|
655
976
|
if (!toKey || !this.#isSafeKey(toKey)) return { ok: false, reason: 'invalid-key', key: to };
|
|
656
977
|
if (fromKey === toKey) return { ok: false, reason: 'same-key', key: fromKey };
|
|
978
|
+
this.#replicas.assertIdleEndpoint(backendName, fromKey);
|
|
979
|
+
this.#replicas.assertIdleEndpoint(backendName, toKey);
|
|
657
980
|
|
|
658
981
|
const current = this.#index.get(`${backendName}:${fromKey}`);
|
|
659
982
|
const currentLoc = current?.locations?.find(l => l.backend === backendName && l.key === fromKey) || null;
|
|
@@ -684,6 +1007,133 @@ export default class Stored extends EventEmitter2 {
|
|
|
684
1007
|
};
|
|
685
1008
|
}
|
|
686
1009
|
|
|
1010
|
+
/** One filesystem rename, then an atomic re-key of indexed locations.
|
|
1011
|
+
* The durable operation receipt makes a lost HTTP response safe to retry.
|
|
1012
|
+
* No object streams, checksums or per-file filesystem moves are needed.
|
|
1013
|
+
*/
|
|
1014
|
+
async renameDirectory(backendName, from, to, { operationId, origin = null } = {}) {
|
|
1015
|
+
const backend = this.#backends.get(backendName);
|
|
1016
|
+
if (!backend) return { ok: false, reason: 'unknown-backend' };
|
|
1017
|
+
if (!backend.canWrite) return { ok: false, reason: 'read-only-target' };
|
|
1018
|
+
const remote = typeof backend.renameDirectory === 'function';
|
|
1019
|
+
if (!remote && (!backend.root || typeof backend.resolveKeyPath !== 'function')) return { ok: false, reason: 'unsupported-backend' };
|
|
1020
|
+
const fromKey = this.#normalizeKey(from, { nfc: true });
|
|
1021
|
+
const toKey = this.#normalizeKey(to, { nfc: true });
|
|
1022
|
+
if (!fromKey || !toKey || !this.#isSafeKey(fromKey) || !this.#isSafeKey(toKey)
|
|
1023
|
+
|| backend.isIgnored?.(fromKey) || backend.isIgnored?.(toKey)
|
|
1024
|
+
|| fromKey === toKey || toKey.startsWith(`${fromKey}/`) || fromKey.startsWith(`${toKey}/`)
|
|
1025
|
+
|| typeof operationId !== 'string' || !operationId || operationId.length > 128) {
|
|
1026
|
+
return { ok: false, reason: 'invalid-key' };
|
|
1027
|
+
}
|
|
1028
|
+
const receiptKey = `${backendName}:${operationId}`;
|
|
1029
|
+
this.#replicas.assertIdleEndpoint(backendName, fromKey, { directory: true });
|
|
1030
|
+
this.#replicas.assertIdleEndpoint(backendName, toKey, { directory: true });
|
|
1031
|
+
if (remote) return this.#renameRemoteDirectory(backend, fromKey, toKey, { operationId, origin });
|
|
1032
|
+
const prior = this.#directoryMoveTasks.get(backendName) || Promise.resolve();
|
|
1033
|
+
const task = prior.catch(() => {}).then(async () => {
|
|
1034
|
+
const db = this.#directoryMovesDb ||= this.#index.openDB('directory-moves');
|
|
1035
|
+
let receipt = db.get(receiptKey);
|
|
1036
|
+
if (receipt && (receipt.from !== fromKey || receipt.to !== toKey)) return { ok: false, reason: 'invalid-key' };
|
|
1037
|
+
const result = () => ({ ok: true, directory: true, from: fromKey, to: toKey, state: 'complete', seq: this.#index.head() });
|
|
1038
|
+
if (receipt?.done) return result();
|
|
1039
|
+
const source = backend.resolveKeyPath(fromKey);
|
|
1040
|
+
const target = backend.resolveKeyPath(toKey);
|
|
1041
|
+
const root = await fsp.realpath(backend.root);
|
|
1042
|
+
const contained = (p) => { const rel = path.relative(root, p); return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel)); };
|
|
1043
|
+
const stat = async (p) => { try { return await fsp.lstat(p); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } };
|
|
1044
|
+
const sourceStat = await stat(source);
|
|
1045
|
+
if (!receipt) {
|
|
1046
|
+
if (!sourceStat) return { ok: false, reason: 'not-found' };
|
|
1047
|
+
if (!sourceStat.isDirectory() || !contained(await fsp.realpath(source))) return { ok: false, reason: 'invalid-key' };
|
|
1048
|
+
if (await stat(target)) return { ok: false, reason: 'target-exists' };
|
|
1049
|
+
const rows = this.#index.locationsByBackend(backendName, { prefix: `${fromKey}/`, limit: Number.MAX_SAFE_INTEGER }).objects;
|
|
1050
|
+
if (rows.some(row => backend.isIgnored?.(`${toKey}/${row.key.slice(fromKey.length + 1)}`))) return { ok: false, reason: 'invalid-key' };
|
|
1051
|
+
receipt = { from: fromKey, to: toKey, dev: sourceStat.dev, ino: sourceStat.ino, keys: rows.map(row => row.key) };
|
|
1052
|
+
db.putSync(receiptKey, receipt);
|
|
1053
|
+
}
|
|
1054
|
+
const movedKey = (key) => `${toKey}/${key.slice(fromKey.length + 1)}`;
|
|
1055
|
+
const release = this.#holdKeys(receipt.keys.flatMap(key => [`${backendName}:${key}`, `${backendName}:${movedKey(key)}`]));
|
|
1056
|
+
try {
|
|
1057
|
+
if (sourceStat?.dev === receipt.dev && sourceStat.ino === receipt.ino) {
|
|
1058
|
+
if (!contained(await fsp.realpath(source))) return { ok: false, reason: 'invalid-key' };
|
|
1059
|
+
if (await stat(target)) return { ok: false, reason: 'target-exists' };
|
|
1060
|
+
// Check existing parents before mkdir: a symlink must not create
|
|
1061
|
+
// directories outside this backend, even if the rename fails.
|
|
1062
|
+
let parent = path.dirname(target);
|
|
1063
|
+
while (!await stat(parent)) parent = path.dirname(parent);
|
|
1064
|
+
if (!contained(await fsp.realpath(parent))) return { ok: false, reason: 'invalid-key' };
|
|
1065
|
+
await fsp.mkdir(path.dirname(target), { recursive: true });
|
|
1066
|
+
if (!contained(await fsp.realpath(path.dirname(target)))) return { ok: false, reason: 'invalid-key' };
|
|
1067
|
+
await fsp.rename(source, target); // EXDEV is an error, never a copy fallback.
|
|
1068
|
+
} else {
|
|
1069
|
+
// A recreated source may now have different indexed data.
|
|
1070
|
+
// Leave both trees intact instead of re-keying that data.
|
|
1071
|
+
if (sourceStat) return { ok: false, reason: 'target-exists' };
|
|
1072
|
+
const targetStat = await stat(target);
|
|
1073
|
+
if (targetStat?.dev !== receipt.dev || targetStat.ino !== receipt.ino) return { ok: false, reason: 'target-exists' };
|
|
1074
|
+
}
|
|
1075
|
+
const events = [];
|
|
1076
|
+
this.#index.transaction(() => {
|
|
1077
|
+
for (const key of receipt.keys) {
|
|
1078
|
+
const current = this.#index.get(`${backendName}:${key}`);
|
|
1079
|
+
const old = current?.locations?.find(l => l.backend === backendName && l.key === key);
|
|
1080
|
+
if (!old) continue;
|
|
1081
|
+
const nextKey = movedKey(key);
|
|
1082
|
+
const location = this.#buildLocation(backendName, nextKey, old.synced, { ...old, key: nextKey });
|
|
1083
|
+
const saved = this.#index.put(current.id, {
|
|
1084
|
+
...current,
|
|
1085
|
+
locations: current.locations.map(l => l === old ? location : l),
|
|
1086
|
+
}, { origin, rename: { backend: backendName, from: key, to: nextKey } });
|
|
1087
|
+
events.push({ id: current.id, checksums: saved.checksums,
|
|
1088
|
+
from: this.#endpoint(old), to: this.#endpoint(location),
|
|
1089
|
+
location: this.#describeLocation(location),
|
|
1090
|
+
locations: saved.locations.map(l => this.#describeLocation(l)), origin });
|
|
1091
|
+
}
|
|
1092
|
+
db.putSync(receiptKey, { from: fromKey, to: toKey, done: true });
|
|
1093
|
+
this.#replicas.moveIngress(backendName, fromKey, backendName, toKey, { directory: true });
|
|
1094
|
+
});
|
|
1095
|
+
for (const event of events) this.#emitObject('move', event);
|
|
1096
|
+
return result();
|
|
1097
|
+
} finally { release(); }
|
|
1098
|
+
});
|
|
1099
|
+
this.#directoryMoveTasks.set(backendName, task);
|
|
1100
|
+
try { return await task; }
|
|
1101
|
+
finally { if (this.#directoryMoveTasks.get(backendName) === task) this.#directoryMoveTasks.delete(backendName); }
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
// A backend that renames a directory itself (the canvas driver: one hub
|
|
1105
|
+
// request). The index follows with one re-key of the locations under it.
|
|
1106
|
+
async #renameRemoteDirectory(backend, fromKey, toKey, { operationId, origin }) {
|
|
1107
|
+
const name = backend.name;
|
|
1108
|
+
const prefix = `${fromKey}/`;
|
|
1109
|
+
const rows = this.#index.locationsByBackend(name, { prefix, limit: Number.MAX_SAFE_INTEGER }).objects;
|
|
1110
|
+
const movedKey = (key) => `${toKey}/${key.slice(prefix.length)}`;
|
|
1111
|
+
const release = this.#holdKeys(rows.flatMap(row => [`${name}:${row.key}`, `${name}:${movedKey(row.key)}`]));
|
|
1112
|
+
try {
|
|
1113
|
+
const moved = await backend.renameDirectory(fromKey, toKey, { operationId, origin });
|
|
1114
|
+
const events = [];
|
|
1115
|
+
this.#index.transaction(() => {
|
|
1116
|
+
for (const { key } of rows) {
|
|
1117
|
+
const current = this.#index.get(`${name}:${key}`);
|
|
1118
|
+
const old = current?.locations?.find(l => l.backend === name && l.key === key);
|
|
1119
|
+
if (!old) continue;
|
|
1120
|
+
const nextKey = movedKey(key);
|
|
1121
|
+
const location = this.#buildLocation(name, nextKey, old.synced, { ...old, key: nextKey });
|
|
1122
|
+
const saved = this.#index.put(current.id, {
|
|
1123
|
+
...current,
|
|
1124
|
+
locations: current.locations.map(l => l === old ? location : l),
|
|
1125
|
+
}, { origin, rename: { backend: name, from: key, to: nextKey } });
|
|
1126
|
+
events.push({ id: current.id, checksums: saved.checksums,
|
|
1127
|
+
from: this.#endpoint(old), to: this.#endpoint(location),
|
|
1128
|
+
location: this.#describeLocation(location),
|
|
1129
|
+
locations: saved.locations.map(l => this.#describeLocation(l)), origin });
|
|
1130
|
+
}
|
|
1131
|
+
});
|
|
1132
|
+
for (const event of events) this.#emitObject('move', event);
|
|
1133
|
+
return { ok: true, directory: true, from: fromKey, to: toKey, state: 'complete', seq: moved.seq ?? null, count: events.length };
|
|
1134
|
+
} finally { release(); }
|
|
1135
|
+
}
|
|
1136
|
+
|
|
687
1137
|
// `If-Match` / `If-None-Match` semantics over the index. Returns the typed
|
|
688
1138
|
// failure (with what is actually there) or null when the write may go on.
|
|
689
1139
|
#checkPrecondition(current, currentLoc, { ifMatch = null, ifNoneMatch = null } = {}) {
|
|
@@ -750,6 +1200,51 @@ export default class Stored extends EventEmitter2 {
|
|
|
750
1200
|
* pick the next free `name-1.ext`, or replace it.
|
|
751
1201
|
* @returns {Promise<{ ok:true, id, added:string[], locations } | { ok:false, reason:string }>}
|
|
752
1202
|
*/
|
|
1203
|
+
/**
|
|
1204
|
+
* Network time stays out of the mutation executor, so transfers to and
|
|
1205
|
+
* from a hub run side by side. An upload to a streaming remote touches no
|
|
1206
|
+
* local bytes and records its result in one index transaction, so it runs
|
|
1207
|
+
* unqueued. A download is staged to a temp file first; only the commit
|
|
1208
|
+
* onto the local target is queued.
|
|
1209
|
+
*/
|
|
1210
|
+
async #routeCopy(method, idOrKey, options) {
|
|
1211
|
+
const names = (Array.isArray(options.to) ? options.to : [options.to]).filter(Boolean).map(String);
|
|
1212
|
+
const targets = names.map((name) => this.#backends.get(name));
|
|
1213
|
+
if (targets.length === 1 && targets[0]?.type === 'remote' && typeof targets[0].putStream === 'function') {
|
|
1214
|
+
return method(idOrKey, options);
|
|
1215
|
+
}
|
|
1216
|
+
const staged = this.#closing ? null : await this.#stageRemoteSource(idOrKey, options, targets);
|
|
1217
|
+
if (!staged) return this.withStorageMutation(() => method(idOrKey, options), 'copy');
|
|
1218
|
+
try {
|
|
1219
|
+
return await this.withStorageMutation(() => method(idOrKey, { ...options, [STAGED]: staged }), 'copy');
|
|
1220
|
+
} finally {
|
|
1221
|
+
await fsp.rm(staged.path, { force: true }).catch(() => {});
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
// Download a remote source next to its first local target. Null when the
|
|
1226
|
+
// source is not remote or cannot be read; the queued copy then reports it.
|
|
1227
|
+
async #stageRemoteSource(idOrKey, options, targets) {
|
|
1228
|
+
const meta = this.#index.get(idOrKey);
|
|
1229
|
+
const location = meta ? this.#resolveSourceLocation(meta, options.from) : null;
|
|
1230
|
+
const backend = location ? this.#backends.get(location.backend) : null;
|
|
1231
|
+
if (backend?.type !== 'remote') return null;
|
|
1232
|
+
const firstLocal = targets.find((t) => t?.type === 'local');
|
|
1233
|
+
const tempDir = firstLocal ? firstLocal.tempDir : path.join(this.#paths.cache, '.tmp');
|
|
1234
|
+
const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
1235
|
+
try {
|
|
1236
|
+
const source = await backend.get(location.key, { stream: true });
|
|
1237
|
+
if (!source) return null;
|
|
1238
|
+
await fsp.mkdir(tempDir, { recursive: true });
|
|
1239
|
+
const { checksums, size } = await this.#hashToFile(source, tempPath);
|
|
1240
|
+
return { path: tempPath, checksums, size, backend: location.backend, key: location.key };
|
|
1241
|
+
} catch (err) {
|
|
1242
|
+
debug(`Staging ${location.backend}:${location.key} failed: ${err.message}`);
|
|
1243
|
+
await fsp.rm(tempPath, { force: true }).catch(() => {});
|
|
1244
|
+
return null;
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
|
|
753
1248
|
async copy(idOrKey, options = {}) {
|
|
754
1249
|
const plan = await this.#planTransfer(idOrKey, options);
|
|
755
1250
|
if (!plan.ok) return plan;
|
|
@@ -761,7 +1256,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
761
1256
|
|
|
762
1257
|
const release = this.#holdKeys(targets.map(t => `${t.name}:${t.key || destKey}`));
|
|
763
1258
|
try {
|
|
764
|
-
const transferred = await this.#streamToTargets(meta, sourceLocation, targets, destKey, this.#transferOptions(options));
|
|
1259
|
+
const transferred = await this.#streamToTargets(meta, sourceLocation, targets, destKey, this.#transferOptions(options), options[STAGED]);
|
|
765
1260
|
if (!transferred.ok) return transferred;
|
|
766
1261
|
// A conflict-inbox upload placed nothing at the key: no location.
|
|
767
1262
|
if (transferred.conflict && !transferred.locations.length) {
|
|
@@ -774,11 +1269,9 @@ export default class Stored extends EventEmitter2 {
|
|
|
774
1269
|
const previous = this.#displaceTargets(targets, destKey, meta.id, options);
|
|
775
1270
|
const current = this.#index.get(meta.id) || meta;
|
|
776
1271
|
const saved = this.#index.put(meta.id, { ...current, locations: this.#mergeLocations(current, transferred.locations) }, { origin: options.origin });
|
|
1272
|
+
if (transferred.remoteTargets.length) this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
|
|
777
1273
|
return { saved, previous };
|
|
778
1274
|
});
|
|
779
|
-
if (transferred.remoteTargets.length) {
|
|
780
|
-
this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
|
|
781
|
-
}
|
|
782
1275
|
|
|
783
1276
|
for (const loc of transferred.locations) {
|
|
784
1277
|
this.#emitLocationEvent('location:add', saved, loc, previous ? { previous } : {});
|
|
@@ -819,6 +1312,8 @@ export default class Stored extends EventEmitter2 {
|
|
|
819
1312
|
if (!plan.ok) return plan;
|
|
820
1313
|
|
|
821
1314
|
const { meta, sourceLocation, sourceBackend, targets, destKey } = plan;
|
|
1315
|
+
this.#replicas.assertIdleEndpoint(sourceLocation.backend, sourceLocation.key);
|
|
1316
|
+
for (const target of targets) this.#replicas.assertIdleEndpoint(target.name, destKey);
|
|
822
1317
|
if (!targets.length) {
|
|
823
1318
|
return { ok: true, id: meta.id, state: 'complete', unchanged: true, locations: await this.locations(meta.id) };
|
|
824
1319
|
}
|
|
@@ -874,15 +1369,17 @@ export default class Stored extends EventEmitter2 {
|
|
|
874
1369
|
const previous = this.#displaceTargets(targets, destKey, meta.id, options);
|
|
875
1370
|
const current = this.#index.get(meta.id) || meta;
|
|
876
1371
|
const saved = this.#index.put(meta.id, { ...current, locations: this.#mergeLocations(current, transferred.locations) }, { origin: options.origin });
|
|
1372
|
+
if (transferred.remoteTargets.length) {
|
|
1373
|
+
const pending = this.#pendingMoves.get(meta.id) || [];
|
|
1374
|
+
pending.push({ sourceLocation, target: target.name, destKey: targetKey });
|
|
1375
|
+
this.#pendingMoves.putSync(meta.id, pending);
|
|
1376
|
+
this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
|
|
1377
|
+
}
|
|
877
1378
|
return { saved, previous };
|
|
878
1379
|
});
|
|
879
1380
|
this.#emitLocationEvent('location:add', saved, location, previous ? { previous } : {});
|
|
880
1381
|
|
|
881
1382
|
if (transferred.remoteTargets.length) {
|
|
882
|
-
const pending = this.#pendingMoves.get(meta.id) || [];
|
|
883
|
-
pending.push({ sourceLocation, target: target.name, destKey: targetKey });
|
|
884
|
-
this.#pendingMoves.set(meta.id, pending);
|
|
885
|
-
this.#syncQueue.enqueue({ id: meta.id, cacheRoot: this.#cache.root, cacheKey: meta.id, targets: transferred.remoteTargets });
|
|
886
1383
|
debug(`MOVE (pending sync) ${sourceLocation.backend}:${sourceLocation.key} → ${target.name}:${targetKey}`);
|
|
887
1384
|
return {
|
|
888
1385
|
ok: true,
|
|
@@ -953,7 +1450,10 @@ export default class Stored extends EventEmitter2 {
|
|
|
953
1450
|
* re-hashed. Returns { ok:true, backend, count, files } | { ok:false, reason }.
|
|
954
1451
|
*/
|
|
955
1452
|
async scan(backendName, options = {}) {
|
|
956
|
-
const { onFile, ...backendOptions } = options;
|
|
1453
|
+
const { onFile, isCancelled, ...backendOptions } = options;
|
|
1454
|
+
const checkCancelled = () => {
|
|
1455
|
+
if (isCancelled?.()) throw Object.assign(new Error('Scan cancelled'), { code: 'RESYNC_CANCELLED' });
|
|
1456
|
+
};
|
|
957
1457
|
const list = backendName
|
|
958
1458
|
? [this.#backends.get(backendName)].filter(Boolean)
|
|
959
1459
|
: this.#backends.all();
|
|
@@ -963,6 +1463,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
963
1463
|
let complete = true;
|
|
964
1464
|
let errors = null;
|
|
965
1465
|
for (const backend of list) {
|
|
1466
|
+
checkCancelled();
|
|
966
1467
|
// Per-file ingest into the stored index — idempotent, guarded so the
|
|
967
1468
|
// streaming path (backend onFile) and the returned-rows path don't
|
|
968
1469
|
// double-process a key.
|
|
@@ -972,12 +1473,19 @@ export default class Stored extends EventEmitter2 {
|
|
|
972
1473
|
processed.add(file.key);
|
|
973
1474
|
if (!file.checksums) return;
|
|
974
1475
|
const id = formatId(file.checksums, this.#config.primaryChecksum);
|
|
1476
|
+
const occupant = this.#index.get(`${file.backend}:${file.key}`);
|
|
1477
|
+
// Scans must report the same edit provenance as watcher/API
|
|
1478
|
+
// writes. Otherwise consumers see new content as a new file,
|
|
1479
|
+
// lose the document ID, and later mistake the old row for a
|
|
1480
|
+
// deletion. Keep the hint on both streamed and returned rows.
|
|
1481
|
+
const previous = occupant && occupant.id !== id ? occupant : null;
|
|
1482
|
+
if (previous) file.previous = { id: previous.id, checksums: previous.checksums };
|
|
975
1483
|
const existing = this.#index.get(id);
|
|
976
1484
|
const locations = existing?.locations || [];
|
|
977
1485
|
const match = locations.find(l => l.backend === file.backend && l.key === file.key);
|
|
978
1486
|
// Avoid LMDB/index/change-feed churn on warm scans. Still emit
|
|
979
1487
|
// the row to consumers: their document index may need repair.
|
|
980
|
-
if (match && match.synced === true && match.size === file.size && match.mtime === file.modified
|
|
1488
|
+
if (!previous && match && match.synced === true && match.size === file.size && match.mtime === file.modified
|
|
981
1489
|
&& (file.ino == null || (match.ino === file.ino && match.dev === file.dev))
|
|
982
1490
|
&& existing.size === file.size && existing.mimeType === file.mimeType) return;
|
|
983
1491
|
if (match) {
|
|
@@ -993,14 +1501,22 @@ export default class Stored extends EventEmitter2 {
|
|
|
993
1501
|
...(file.ino != null ? { ino: file.ino, dev: file.dev } : {}),
|
|
994
1502
|
}));
|
|
995
1503
|
}
|
|
996
|
-
this.#index.
|
|
1504
|
+
this.#index.transaction(() => {
|
|
1505
|
+
if (previous) {
|
|
1506
|
+
const remaining = previous.locations.filter(l => !(l.backend === file.backend && l.key === file.key));
|
|
1507
|
+
if (remaining.length) this.#index.put(previous.id, { ...previous, locations: remaining });
|
|
1508
|
+
else this.#index.delete(previous.id);
|
|
1509
|
+
}
|
|
1510
|
+
this.#index.put(id, { checksums: file.checksums, size: file.size, mimeType: file.mimeType, locations });
|
|
1511
|
+
});
|
|
997
1512
|
};
|
|
998
1513
|
|
|
999
1514
|
// Inode identity snapshot for rename matching: same (dev, ino) at a
|
|
1000
1515
|
// NEW path with unchanged size+mtime is a move — reuse the indexed
|
|
1001
1516
|
// checksums instead of rehashing, so a folder rename (thousands of
|
|
1002
1517
|
// "moves") collapses into cheap URL rewrites.
|
|
1003
|
-
const inodeMap = this.#buildInodeMap(backend.name);
|
|
1518
|
+
const inodeMap = await this.#buildInodeMap(backend.name);
|
|
1519
|
+
checkCancelled();
|
|
1004
1520
|
|
|
1005
1521
|
const result = await backend.scan({
|
|
1006
1522
|
algorithms: this.#config.checksums,
|
|
@@ -1010,10 +1526,13 @@ export default class Stored extends EventEmitter2 {
|
|
|
1010
1526
|
// Stream-through: index the file, then hand it to the caller so
|
|
1011
1527
|
// consumers (workspace doc upserts) see results as the walk runs.
|
|
1012
1528
|
onFile: async (file) => {
|
|
1529
|
+
checkCancelled();
|
|
1013
1530
|
ingest(file);
|
|
1014
1531
|
if (typeof onFile === 'function') await onFile(file);
|
|
1532
|
+
if (processed.size % 32 === 0) await yieldToIO();
|
|
1015
1533
|
},
|
|
1016
1534
|
});
|
|
1535
|
+
checkCancelled();
|
|
1017
1536
|
// Snapshot shape { files, complete, errors } (file driver) or a bare
|
|
1018
1537
|
// rows array (legacy/simple drivers — treated as a complete walk).
|
|
1019
1538
|
// Backends whose scan() reports via events instead (non-array, no
|
|
@@ -1021,13 +1540,24 @@ export default class Stored extends EventEmitter2 {
|
|
|
1021
1540
|
const snapshot = Array.isArray(result)
|
|
1022
1541
|
? { files: result, complete: true, errors: null }
|
|
1023
1542
|
: (Array.isArray(result?.files) ? result : null);
|
|
1024
|
-
if (!snapshot)
|
|
1543
|
+
if (!snapshot || result?.ok === false) {
|
|
1544
|
+
complete = false;
|
|
1545
|
+
errors = { ...(errors || {}), [backend.name]: { root: 'scan returned no usable snapshot' } };
|
|
1546
|
+
continue;
|
|
1547
|
+
}
|
|
1025
1548
|
|
|
1026
1549
|
if (snapshot.complete === false) complete = false;
|
|
1027
1550
|
if (snapshot.errors) errors = { ...(errors || {}), [backend.name]: snapshot.errors };
|
|
1028
1551
|
|
|
1029
1552
|
// Backends that ignore onFile still get their rows indexed here.
|
|
1030
|
-
|
|
1553
|
+
let visited = 0;
|
|
1554
|
+
for (const file of snapshot.files) {
|
|
1555
|
+
if (++visited % 128 === 0) { await yieldToIO(); checkCancelled(); }
|
|
1556
|
+
ingest(file);
|
|
1557
|
+
}
|
|
1558
|
+
// Index change listeners may request cancellation synchronously on
|
|
1559
|
+
// the last ingested row, without another file callback or I/O turn.
|
|
1560
|
+
checkCancelled();
|
|
1031
1561
|
|
|
1032
1562
|
// Removal only ever runs against a usable snapshot: a dead/absent
|
|
1033
1563
|
// root (errors.root) means the backend is offline — nothing is
|
|
@@ -1036,7 +1566,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
1036
1566
|
if (snapshot.errors?.root) continue;
|
|
1037
1567
|
const presentKeys = new Set(snapshot.files.map(file => file.key));
|
|
1038
1568
|
const erroredPrefixes = (snapshot.errors?.dirs || []).map(d => d.prefix).filter(Boolean);
|
|
1039
|
-
this.#removeMissingLocations(backend.name, presentKeys, erroredPrefixes);
|
|
1569
|
+
if (snapshot.complete !== false) this.#removeMissingLocations(backend.name, presentKeys, erroredPrefixes);
|
|
1040
1570
|
files.push(...snapshot.files);
|
|
1041
1571
|
}
|
|
1042
1572
|
return { ok: true, backend: backendName ?? null, count: files.length, files, complete, errors };
|
|
@@ -1059,8 +1589,14 @@ export default class Stored extends EventEmitter2 {
|
|
|
1059
1589
|
// Lifecycle
|
|
1060
1590
|
// ─────────────────────────────────────────────────────────────────────────
|
|
1061
1591
|
|
|
1062
|
-
|
|
1592
|
+
stop() {
|
|
1593
|
+
if (!this.#stopPromise) { this.#closing = true; this.#mutations.close(); this.#stopPromise = this.#stop(); }
|
|
1594
|
+
return this.#stopPromise;
|
|
1595
|
+
}
|
|
1596
|
+
async #stop() {
|
|
1063
1597
|
if (this.#sweepTimer) { clearInterval(this.#sweepTimer); this.#sweepTimer = null; }
|
|
1598
|
+
await this.#replicas.stop();
|
|
1599
|
+
await this.#mutations.drain();
|
|
1064
1600
|
await this.#syncQueue.stop();
|
|
1065
1601
|
await this.#backends.stopAll();
|
|
1066
1602
|
// Flush held unlinks (watchers are stopped — no add can claim them now)
|
|
@@ -1070,7 +1606,11 @@ export default class Stored extends EventEmitter2 {
|
|
|
1070
1606
|
this.#pendingUnlinks.delete(pendingKey);
|
|
1071
1607
|
this.#processUnlink(data);
|
|
1072
1608
|
}
|
|
1073
|
-
|
|
1609
|
+
// No stream writers remain at shutdown. Verify reclaims content whose
|
|
1610
|
+
// staging keys were released without racing an ingress stream.
|
|
1611
|
+
await this.#cache.stop();
|
|
1612
|
+
this.#replicas.releaseExecutor();
|
|
1613
|
+
await this.#index.close();
|
|
1074
1614
|
debug('Stopped');
|
|
1075
1615
|
}
|
|
1076
1616
|
|
|
@@ -1084,13 +1624,14 @@ export default class Stored extends EventEmitter2 {
|
|
|
1084
1624
|
|
|
1085
1625
|
for (const r of results) {
|
|
1086
1626
|
if (!r.success) continue;
|
|
1087
|
-
const loc = meta.locations.find(l => l.backend === r.backend);
|
|
1627
|
+
const loc = meta.locations.find(l => l.backend === r.backend && l.key === r.key);
|
|
1088
1628
|
if (loc) loc.synced = true;
|
|
1089
1629
|
}
|
|
1090
1630
|
|
|
1091
1631
|
this.#index.put(meta.id, meta);
|
|
1092
1632
|
this.emit('synced', { id, results });
|
|
1093
|
-
|
|
1633
|
+
// Source retirement is replayed by resumePendingMoves, under the same
|
|
1634
|
+
// executor lock, after the durable destination receipt commits.
|
|
1094
1635
|
}
|
|
1095
1636
|
|
|
1096
1637
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -1140,7 +1681,13 @@ export default class Stored extends EventEmitter2 {
|
|
|
1140
1681
|
// an unmounted share, a failed sync) must not turn a copy into a
|
|
1141
1682
|
// silent no-op that reports success.
|
|
1142
1683
|
const ours = (meta.locations || []).some(l => l.backend === name && l.key === destKey);
|
|
1143
|
-
|
|
1684
|
+
// A streaming remote enforces If-Match / If-None-Match itself, so
|
|
1685
|
+
// the indexed view of the key stands in for a round trip.
|
|
1686
|
+
const preconditioned = backend.type === 'remote' && typeof backend.putStream === 'function'
|
|
1687
|
+
&& (options.ifMatch != null || options.ifNoneMatch != null);
|
|
1688
|
+
const present = preconditioned
|
|
1689
|
+
? (this.#index.get(`${name}:${destKey}`) ? { key: destKey } : null)
|
|
1690
|
+
: await backend.stat(destKey).catch(() => null);
|
|
1144
1691
|
if (ours && present) continue;
|
|
1145
1692
|
if (ours) debug(`Stale location ${name}:${destKey} (indexed, no bytes) — re-transferring`);
|
|
1146
1693
|
|
|
@@ -1232,7 +1779,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
1232
1779
|
* content identity before committing. The temp file is staged on the first
|
|
1233
1780
|
* local target's filesystem so `#commit` can hardlink instead of copying.
|
|
1234
1781
|
*/
|
|
1235
|
-
async #streamToTargets(meta, sourceLocation, targets, destKey, options = {}) {
|
|
1782
|
+
async #streamToTargets(meta, sourceLocation, targets, destKey, options = {}, staged = null) {
|
|
1236
1783
|
const sourceBackend = this.#backends.get(sourceLocation.backend);
|
|
1237
1784
|
// One remote target that speaks `putStream`: stream straight to it —
|
|
1238
1785
|
// no temp file, no cache entry, no SyncQueue. The hub verifies the
|
|
@@ -1240,16 +1787,17 @@ export default class Stored extends EventEmitter2 {
|
|
|
1240
1787
|
if (targets.length === 1 && targets[0].backend.type === 'remote' && typeof targets[0].backend.putStream === 'function') {
|
|
1241
1788
|
return this.#streamToRemote(meta, sourceBackend, sourceLocation, targets[0], destKey, options);
|
|
1242
1789
|
}
|
|
1243
|
-
const
|
|
1244
|
-
|
|
1790
|
+
const prefetched = staged && staged.backend === sourceLocation.backend && staged.key === sourceLocation.key ? staged : null;
|
|
1791
|
+
const source = prefetched ? null : await sourceBackend.get(sourceLocation.key, { stream: true });
|
|
1792
|
+
if (!prefetched && !source) return { ok: false, reason: 'source-unreadable', backend: sourceLocation.backend, key: sourceLocation.key };
|
|
1245
1793
|
|
|
1246
1794
|
const firstLocal = targets.find(t => t.backend.type === 'local')?.backend;
|
|
1247
1795
|
const tempDir = firstLocal ? firstLocal.tempDir : path.join(this.#paths.cache, '.tmp');
|
|
1248
|
-
const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
1249
|
-
await fsp.mkdir(tempDir, { recursive: true });
|
|
1796
|
+
const tempPath = prefetched ? prefetched.path : path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
1797
|
+
if (!prefetched) await fsp.mkdir(tempDir, { recursive: true });
|
|
1250
1798
|
|
|
1251
1799
|
try {
|
|
1252
|
-
const { checksums, size } = await this.#hashToFile(source, tempPath);
|
|
1800
|
+
const { checksums, size } = prefetched || await this.#hashToFile(source, tempPath);
|
|
1253
1801
|
const id = formatId(checksums, this.#config.primaryChecksum);
|
|
1254
1802
|
if (id !== meta.id) {
|
|
1255
1803
|
debug(`Transfer aborted: ${sourceLocation.backend}:${sourceLocation.key} hashes to ${id}, indexed as ${meta.id}`);
|
|
@@ -1383,16 +1931,21 @@ export default class Stored extends EventEmitter2 {
|
|
|
1383
1931
|
try {
|
|
1384
1932
|
if (removeSourceBytes) {
|
|
1385
1933
|
const sourceBackend = this.#backends.get(sourceLocation.backend);
|
|
1386
|
-
const
|
|
1387
|
-
if (!
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1934
|
+
const targetBackend = this.#backends.get(newLocation.backend);
|
|
1935
|
+
if (!sourceBackend || !targetBackend) return { ok: false, reason: 'backend-offline' };
|
|
1936
|
+
const bytes = await targetBackend.get(newLocation.key, { stream: true });
|
|
1937
|
+
if (!bytes || `sha256:${(await checksumStream(Buffer.isBuffer(bytes) ? Readable.from([bytes]) : bytes)).sha256}` !== id) return { ok: false, reason: 'target-changed' };
|
|
1938
|
+
const sourceStat = await sourceBackend.stat(sourceLocation.key);
|
|
1939
|
+
if (sourceStat) {
|
|
1940
|
+
const source = await sourceBackend.get(sourceLocation.key, { stream: true });
|
|
1941
|
+
if (!source || `sha256:${(await checksumStream(Buffer.isBuffer(source) ? Readable.from([source]) : source)).sha256}` !== id) return { ok: false, reason: 'source-changed' };
|
|
1942
|
+
this.#replicas.moveIngress(sourceLocation.backend, sourceLocation.key, newLocation.backend, newLocation.key);
|
|
1943
|
+
const result = sourceBackend.config.driver === 'cacache'
|
|
1944
|
+
? await this.deleteByUrl(`stored://${sourceLocation.backend}/${sourceLocation.key}`)
|
|
1945
|
+
: (this.#retention && sourceBackend.config.driver === 'file')
|
|
1946
|
+
? await this.removeObject(sourceLocation.backend, sourceLocation.key, { ifMatch: id.slice(7), origin, [MOVE_SOURCE]: true })
|
|
1947
|
+
: { ok: false, reason: 'recovery-unavailable' };
|
|
1948
|
+
if (!result.ok) return { ...result, degradedToCopy: true };
|
|
1396
1949
|
}
|
|
1397
1950
|
}
|
|
1398
1951
|
|
|
@@ -1408,6 +1961,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
1408
1961
|
? { backend: newLocation.backend, from: sourceLocation.key, to: newLocation.key }
|
|
1409
1962
|
: null;
|
|
1410
1963
|
const saved = this.#index.put(id, { ...current, locations }, { origin, rename });
|
|
1964
|
+
this.#replicas.moveIngress(sourceLocation.backend, sourceLocation.key, newLocation.backend, newLocation.key);
|
|
1411
1965
|
|
|
1412
1966
|
this.#emitObject('move', {
|
|
1413
1967
|
id,
|
|
@@ -1434,36 +1988,35 @@ export default class Stored extends EventEmitter2 {
|
|
|
1434
1988
|
}
|
|
1435
1989
|
|
|
1436
1990
|
/** Finish moves that were waiting on a remote destination write. */
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
const
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
const
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1991
|
+
async resumePendingMoves() {
|
|
1992
|
+
// Snapshot keys only; each record is re-read under the mutation lock.
|
|
1993
|
+
// Holding the lock over the whole backlog delayed interactive byte
|
|
1994
|
+
// operations behind every remote verification. Awaiting resolved
|
|
1995
|
+
// promises also never lets HTTP/timer callbacks run.
|
|
1996
|
+
let processed = 0;
|
|
1997
|
+
for (const id of [...this.#pendingMoves.getKeys()]) {
|
|
1998
|
+
if (this.#closing) break;
|
|
1999
|
+
await this.withStorageMutation(async () => {
|
|
2000
|
+
const pending = this.#pendingMoves.get(id);
|
|
2001
|
+
if (!pending) return;
|
|
2002
|
+
const remaining = [];
|
|
2003
|
+
for (const entry of pending) {
|
|
2004
|
+
try {
|
|
2005
|
+
const meta = this.#index.get(id);
|
|
2006
|
+
const location = meta?.locations?.find(l => l.backend === entry.target && l.key === entry.destKey && l.synced !== false);
|
|
2007
|
+
if (!location) { remaining.push(entry); continue; }
|
|
2008
|
+
const result = await this.#finalizeMove(id, entry.sourceLocation, location, { removeSourceBytes: true });
|
|
2009
|
+
if (!result.ok) remaining.push({ ...entry, error: result.reason });
|
|
2010
|
+
} catch (error) { remaining.push({ ...entry, error: error.message }); }
|
|
1452
2011
|
}
|
|
1453
|
-
|
|
1454
|
-
//
|
|
1455
|
-
//
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
} else {
|
|
1461
|
-
remaining.push(entry);
|
|
1462
|
-
}
|
|
2012
|
+
// Disabled/offline destinations can leave thousands of moves
|
|
2013
|
+
// pending. Rewriting identical records on every retry caused
|
|
2014
|
+
// one synchronous LMDB commit per item, twice each queue tick.
|
|
2015
|
+
if (!remaining.length) this.#pendingMoves.removeSync(id);
|
|
2016
|
+
else if (!isDeepStrictEqual(pending, remaining)) this.#pendingMoves.putSync(id, remaining);
|
|
2017
|
+
});
|
|
2018
|
+
if (++processed % 32 === 0) await yieldToIO();
|
|
1463
2019
|
}
|
|
1464
|
-
|
|
1465
|
-
if (remaining.length) this.#pendingMoves.set(id, remaining);
|
|
1466
|
-
else this.#pendingMoves.delete(id);
|
|
1467
2020
|
}
|
|
1468
2021
|
|
|
1469
2022
|
/**
|
|
@@ -1640,6 +2193,10 @@ export default class Stored extends EventEmitter2 {
|
|
|
1640
2193
|
// local backend's filesystem, then commit (hardlink/rename) to targets.
|
|
1641
2194
|
async #ingestStream(blob, targets, key, mimeHint, options = {}) {
|
|
1642
2195
|
const source = isStream(blob) ? blob : createReadStream(blob);
|
|
2196
|
+
const blobTarget = targets.find(t => typeof t.backend.ingestStream === 'function');
|
|
2197
|
+
if (blobTarget || targets.every(t => t.backend.type !== 'local')) {
|
|
2198
|
+
return this.#ingestBlobStream(source, targets, blobTarget, key, mimeHint, options);
|
|
2199
|
+
}
|
|
1643
2200
|
const firstLocal = targets.find(t => t.backend.type === 'local')?.backend;
|
|
1644
2201
|
const tempDir = firstLocal ? firstLocal.tempDir : path.join(this.#paths.cache, '.tmp');
|
|
1645
2202
|
const tempPath = path.join(tempDir, `${Date.now()}-${crypto.randomUUID()}`);
|
|
@@ -1663,6 +2220,94 @@ export default class Stored extends EventEmitter2 {
|
|
|
1663
2220
|
return result;
|
|
1664
2221
|
}
|
|
1665
2222
|
|
|
2223
|
+
// Upload directly to the managed blob store. Hash and sniff in-flight;
|
|
2224
|
+
// extraction and additional destinations read the completed immutable blob.
|
|
2225
|
+
// Cacache owns the only temporary payload and atomically publishes it.
|
|
2226
|
+
async #ingestBlobStream(source, targets, target, key, mimeHint, options) {
|
|
2227
|
+
const hashes = this.#config.checksums.map(algo => ({ algo, hash: crypto.createHash(algo) }));
|
|
2228
|
+
const checksums = {};
|
|
2229
|
+
const headChunks = [];
|
|
2230
|
+
let headLen = 0;
|
|
2231
|
+
async function* measured() {
|
|
2232
|
+
for await (const chunk of source) {
|
|
2233
|
+
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
2234
|
+
for (const { hash } of hashes) hash.update(buf);
|
|
2235
|
+
if (headLen < HEAD_BYTES) {
|
|
2236
|
+
const slice = buf.subarray(0, HEAD_BYTES - headLen);
|
|
2237
|
+
headChunks.push(Buffer.from(slice));
|
|
2238
|
+
headLen += slice.length;
|
|
2239
|
+
}
|
|
2240
|
+
yield buf;
|
|
2241
|
+
}
|
|
2242
|
+
for (const { algo, hash } of hashes) checksums[algo] = hash.digest('hex');
|
|
2243
|
+
}
|
|
2244
|
+
let id, finalKey, mimeType, extracted;
|
|
2245
|
+
const sink = target?.backend || this.#cache;
|
|
2246
|
+
const { file, size } = await sink.ingestStream(measured(), async content => {
|
|
2247
|
+
id = formatId(checksums, this.#config.primaryChecksum);
|
|
2248
|
+
finalKey = key || this.#generateKey(checksums);
|
|
2249
|
+
mimeType = mimeHint || await detectMimeFromHead(Buffer.concat(headChunks), finalKey);
|
|
2250
|
+
extracted = await this.#maybeExtract({ file: content.file }, mimeType, finalKey);
|
|
2251
|
+
return target ? finalKey : { key: id, metadata: { key: finalKey, checksums, size: content.size, mimeType } };
|
|
2252
|
+
});
|
|
2253
|
+
const rest = targets.filter(t => t !== target);
|
|
2254
|
+
const { locations, remoteTargets } = await this.#commit(rest, finalKey, id, { file, cached: !target }, { checksums, size, mimeType }, options);
|
|
2255
|
+
if (target) {
|
|
2256
|
+
const stat = await target.backend.stat(finalKey);
|
|
2257
|
+
const location = this.#buildLocation(target.name, finalKey, true, { size, mtime: stat.modified });
|
|
2258
|
+
locations.splice(targets.indexOf(target), 0, location);
|
|
2259
|
+
}
|
|
2260
|
+
return { id, finalKey, checksums, size, mimeType, locations, remoteTargets, extracted };
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
// A file handed over by the caller: moved into the blob store when a
|
|
2264
|
+
// target can adopt it (one rename, no copy); otherwise ingested as a
|
|
2265
|
+
// stream and removed. Hashing reads the file only without caller checksums.
|
|
2266
|
+
async #ingestAdopted(file, targets, key, mimeHint, options) {
|
|
2267
|
+
const target = targets.find(t => typeof t.backend.adoptFile === 'function');
|
|
2268
|
+
if (!target) {
|
|
2269
|
+
try { return await this.#ingestStream(file, targets, key, mimeHint, options); }
|
|
2270
|
+
finally { await fsp.rm(file, { force: true }); }
|
|
2271
|
+
}
|
|
2272
|
+
const known = options.checksums && this.#config.checksums.every(algo => /^[0-9a-f]+$/.test(options.checksums[algo] || ''));
|
|
2273
|
+
const { checksums, size, head } = known
|
|
2274
|
+
? { checksums: Object.fromEntries(this.#config.checksums.map(algo => [algo, options.checksums[algo]])), size: (await fsp.stat(file)).size, head: await readHead(file) }
|
|
2275
|
+
: await this.#hashFile(file);
|
|
2276
|
+
const id = formatId(checksums, this.#config.primaryChecksum);
|
|
2277
|
+
const finalKey = key || this.#generateKey(checksums);
|
|
2278
|
+
const mimeType = mimeHint || await detectMimeFromHead(head, finalKey);
|
|
2279
|
+
let extracted;
|
|
2280
|
+
const adopted = await target.backend.adoptFile(file, { sha256: checksums.sha256, size }, async content => {
|
|
2281
|
+
extracted = await this.#maybeExtract({ file: content.file }, mimeType, finalKey);
|
|
2282
|
+
return finalKey;
|
|
2283
|
+
});
|
|
2284
|
+
const rest = targets.filter(t => t !== target);
|
|
2285
|
+
const { locations, remoteTargets } = await this.#commit(rest, finalKey, id, { file: adopted.file, cached: false }, { checksums, size, mimeType }, options);
|
|
2286
|
+
const stat = await target.backend.stat(finalKey);
|
|
2287
|
+
locations.splice(targets.indexOf(target), 0, this.#buildLocation(target.name, finalKey, true, { size, mtime: stat.modified }));
|
|
2288
|
+
return { id, finalKey, checksums, size, mimeType, locations, remoteTargets, extracted };
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2291
|
+
// One read pass over a file: checksums (all algorithms), size and head.
|
|
2292
|
+
async #hashFile(file) {
|
|
2293
|
+
const hashes = this.#config.checksums.map(algo => ({ algo, hash: crypto.createHash(algo) }));
|
|
2294
|
+
let size = 0;
|
|
2295
|
+
const headChunks = [];
|
|
2296
|
+
let headLen = 0;
|
|
2297
|
+
for await (const chunk of createReadStream(file)) {
|
|
2298
|
+
for (const { hash } of hashes) hash.update(chunk);
|
|
2299
|
+
size += chunk.length;
|
|
2300
|
+
if (headLen < HEAD_BYTES) {
|
|
2301
|
+
const slice = chunk.subarray(0, HEAD_BYTES - headLen);
|
|
2302
|
+
headChunks.push(slice);
|
|
2303
|
+
headLen += slice.length;
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
const checksums = {};
|
|
2307
|
+
for (const { algo, hash } of hashes) checksums[algo] = hash.digest('hex');
|
|
2308
|
+
return { checksums, size, head: Buffer.concat(headChunks) };
|
|
2309
|
+
}
|
|
2310
|
+
|
|
1666
2311
|
// Single streaming pass: hash (all algorithms), measure size, peek the head
|
|
1667
2312
|
// for mime sniffing, and write to `tempPath`. Never buffers the whole blob.
|
|
1668
2313
|
async #hashToFile(source, tempPath) {
|
|
@@ -1686,6 +2331,8 @@ export default class Stored extends EventEmitter2 {
|
|
|
1686
2331
|
}
|
|
1687
2332
|
ws.end();
|
|
1688
2333
|
await once(ws, 'finish');
|
|
2334
|
+
const handle = await fsp.open(tempPath, 'r');
|
|
2335
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
1689
2336
|
} catch (err) {
|
|
1690
2337
|
ws.destroy();
|
|
1691
2338
|
throw err;
|
|
@@ -1730,7 +2377,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
1730
2377
|
}
|
|
1731
2378
|
}
|
|
1732
2379
|
|
|
1733
|
-
if (remoteTargets.length) {
|
|
2380
|
+
if (remoteTargets.length && !source.cached) {
|
|
1734
2381
|
const cacheMeta = { key: finalKey, checksums: meta.checksums, size: meta.size, mimeType: meta.mimeType };
|
|
1735
2382
|
if (source.data) await this.#cache.put(id, source.data, cacheMeta);
|
|
1736
2383
|
else await pipeline(createReadStream(source.file), this.#cache.putStream(id, cacheMeta));
|
|
@@ -1751,9 +2398,11 @@ export default class Stored extends EventEmitter2 {
|
|
|
1751
2398
|
// One-shot snapshot of a backend's indexed locations keyed by inode
|
|
1752
2399
|
// identity, built per scan (the index is LMDB — a per-file reverse lookup
|
|
1753
2400
|
// would be O(N²)).
|
|
1754
|
-
#buildInodeMap(backendName) {
|
|
2401
|
+
async #buildInodeMap(backendName) {
|
|
1755
2402
|
const map = new Map();
|
|
2403
|
+
let visited = 0;
|
|
1756
2404
|
for (const [, meta] of this.#index.entries()) {
|
|
2405
|
+
if (++visited % 128 === 0) await yieldToIO();
|
|
1757
2406
|
for (const loc of meta.locations || []) {
|
|
1758
2407
|
if (loc.backend !== backendName || loc.ino == null) continue;
|
|
1759
2408
|
map.set(`${loc.dev}:${loc.ino}`, {
|
|
@@ -1859,8 +2508,10 @@ export default class Stored extends EventEmitter2 {
|
|
|
1859
2508
|
// Emit a file event under both its typed name (`file:<suffix>`) and the
|
|
1860
2509
|
// generic `object:<suffix>` (kind:'file') so consumers can bind either.
|
|
1861
2510
|
#emitObject(suffix, payload) {
|
|
1862
|
-
this.
|
|
1863
|
-
|
|
2511
|
+
this.detachedStorageWork(() => {
|
|
2512
|
+
this.emit(`file:${suffix}`, payload);
|
|
2513
|
+
this.emit(`object:${suffix}`, { kind: 'file', ...payload });
|
|
2514
|
+
});
|
|
1864
2515
|
}
|
|
1865
2516
|
|
|
1866
2517
|
// `options.force` bypasses the echo suppression — used when stored itself
|
|
@@ -2002,8 +2653,13 @@ export default class Stored extends EventEmitter2 {
|
|
|
2002
2653
|
// deletion, and consumers never see the transient absence.
|
|
2003
2654
|
const meta = this.#index.get(pathKey);
|
|
2004
2655
|
const loc = meta?.locations?.find(l => l.backend === data.backend && l.key === data.key);
|
|
2005
|
-
|
|
2006
|
-
|
|
2656
|
+
// A driver that names the inode in the event (the canvas driver:
|
|
2657
|
+
// the content digest) is believed over the indexed location, which
|
|
2658
|
+
// a put() created without one.
|
|
2659
|
+
const ino = data.ino ?? loc?.ino ?? null;
|
|
2660
|
+
const dev = data.ino != null ? data.dev : loc?.dev;
|
|
2661
|
+
if (loc && ino != null) {
|
|
2662
|
+
const pendingKey = `${data.backend}|${dev}:${ino}`;
|
|
2007
2663
|
// A second unlink for the same inode (hardlink twin) processes
|
|
2008
2664
|
// the held one immediately — one hold per inode.
|
|
2009
2665
|
const prior = this.#pendingUnlinks.get(pendingKey);
|
|
@@ -2032,6 +2688,7 @@ export default class Stored extends EventEmitter2 {
|
|
|
2032
2688
|
#recheckUnlink(data) {
|
|
2033
2689
|
const pathKey = `${data.backend}:${data.key}`;
|
|
2034
2690
|
const timer = setTimeout(async () => {
|
|
2691
|
+
if (this.#closing) return;
|
|
2035
2692
|
if (this.#suppressedKeys.has(pathKey)) { this.#recheckUnlink(data); return; }
|
|
2036
2693
|
const meta = this.#index.get(pathKey);
|
|
2037
2694
|
if (!meta) return;
|