@augmentd-labs/canvas-edge 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/canvas-stored/README.md +189 -9
- package/node_modules/canvas-stored/package.json +2 -2
- package/node_modules/canvas-stored/src/backends/BackendManager.js +11 -1
- package/node_modules/canvas-stored/src/backends/StorageBackend.js +24 -12
- package/node_modules/canvas-stored/src/backends/cacache/index.js +49 -6
- package/node_modules/canvas-stored/src/backends/canvas/index.js +41 -4
- package/node_modules/canvas-stored/src/backends/file/index.js +181 -39
- package/node_modules/canvas-stored/src/backends/gdrive/index.js +57 -10
- package/node_modules/canvas-stored/src/backends/http/index.js +3 -1
- package/node_modules/canvas-stored/src/backends/s3/index.js +244 -15
- package/node_modules/canvas-stored/src/cache/collect-garbage.js +47 -0
- package/node_modules/canvas-stored/src/cache/compact-index.js +13 -0
- package/node_modules/canvas-stored/src/cache/index.js +39 -5
- package/node_modules/canvas-stored/src/cache/ingest-stream.js +65 -0
- package/node_modules/canvas-stored/src/index/index.js +5 -3
- package/node_modules/canvas-stored/src/index.js +733 -132
- package/node_modules/canvas-stored/src/runtime/MutationQueue.js +60 -0
- package/node_modules/canvas-stored/src/storage/lmdb.js +62 -0
- package/node_modules/canvas-stored/src/storage/recover-index.js +33 -0
- package/node_modules/canvas-stored/src/storage/verify-worker.js +23 -0
- package/node_modules/canvas-stored/src/sync/Ledger.js +21 -3
- package/node_modules/canvas-stored/src/sync/Mirror.js +334 -25
- package/node_modules/canvas-stored/src/sync/REPLICAS.md +64 -0
- package/node_modules/canvas-stored/src/sync/ReplicaService.js +623 -0
- package/node_modules/canvas-stored/src/sync/SyncQueue.js +23 -103
- package/node_modules/canvas-stored/src/trash.js +219 -0
- package/node_modules/canvas-stored/src/utils/common.js +7 -0
- package/package.json +5 -4
- package/src/binding.js +33 -0
- package/src/control.js +13 -0
- package/src/daemon.js +14 -1
- package/src/mirror-runtime.js +14 -3
|
@@ -27,7 +27,8 @@ export default class FileBackend extends StorageBackend {
|
|
|
27
27
|
#tempDir;
|
|
28
28
|
#tempRel = null;
|
|
29
29
|
#watcher = null;
|
|
30
|
-
#
|
|
30
|
+
#watchEpoch = 0;
|
|
31
|
+
#rootReady;
|
|
31
32
|
#ignored;
|
|
32
33
|
#ignoreMatcher = null;
|
|
33
34
|
#defaultAlgorithms = ['sha256'];
|
|
@@ -42,7 +43,6 @@ export default class FileBackend extends StorageBackend {
|
|
|
42
43
|
this.#tempDir = config.tempDir ? path.resolve(this.#root, String(config.tempDir)) : path.join(this.#root, TMP_DIR);
|
|
43
44
|
const tempRel = path.relative(this.#root, this.#tempDir).split(path.sep).join('/');
|
|
44
45
|
this.#tempRel = tempRel && !tempRel.startsWith('..') && !path.isAbsolute(tempRel) ? tempRel : null;
|
|
45
|
-
this.#watchEnabled = config.watch ?? false;
|
|
46
46
|
this.#ignored = config.ignored || null;
|
|
47
47
|
this.#ignoreMatcher = this.#buildIgnoreMatcher(this.#ignored);
|
|
48
48
|
this.#defaultAlgorithms = config.algorithms || ['sha256'];
|
|
@@ -52,7 +52,12 @@ export default class FileBackend extends StorageBackend {
|
|
|
52
52
|
// creating an empty dir at an unmounted mountpoint makes an absent
|
|
53
53
|
// drive indistinguishable from an empty one — verifyRoot() + the
|
|
54
54
|
// resync liveness gate rely on the root being genuinely missing.
|
|
55
|
-
|
|
55
|
+
// A stat/mkdir on an unresponsive NAS can block the whole Node thread.
|
|
56
|
+
// Registration stays synchronous; filesystem users await readiness.
|
|
57
|
+
this.#rootReady = config.createRoot !== false ? fs.ensureDir(this.#root) : Promise.resolve();
|
|
58
|
+
// Keep registration fire-and-forget safe. ready()/operations still
|
|
59
|
+
// observe and propagate the original failure.
|
|
60
|
+
this.#rootReady.catch(() => {});
|
|
56
61
|
|
|
57
62
|
// Network mount detection. A CIFS/NFS share reached through a POSIX path
|
|
58
63
|
// is indistinguishable from local disk to every fs call we make, so the
|
|
@@ -69,10 +74,13 @@ export default class FileBackend extends StorageBackend {
|
|
|
69
74
|
get root() { return this.#root; }
|
|
70
75
|
get tempDir() { return this.#tempDir; }
|
|
71
76
|
get watching() { return !!this.#watcher; }
|
|
72
|
-
|
|
77
|
+
static getCapabilities(config = {}) {
|
|
78
|
+
return { ...super.getCapabilities(config), canEnumerate: true, paths: true, mutableContainers: config.readOnly !== true, upload: config.readOnly !== true };
|
|
79
|
+
}
|
|
73
80
|
get remote() { return this.#remote; }
|
|
74
81
|
get transport() { return this.#transport; }
|
|
75
82
|
get mountSource() { return this.#mountSource; }
|
|
83
|
+
ready() { return this.#rootReady; }
|
|
76
84
|
|
|
77
85
|
/**
|
|
78
86
|
* Absolute on-disk path for `key`. Public because cross-backend copy/move
|
|
@@ -152,6 +160,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
152
160
|
const snapshot = expected || this.config.fsid || null;
|
|
153
161
|
let stats;
|
|
154
162
|
try {
|
|
163
|
+
await this.ready();
|
|
155
164
|
stats = await fs.stat(this.#root);
|
|
156
165
|
} catch {
|
|
157
166
|
return { ok: false, reason: 'root-missing', fsid: null };
|
|
@@ -164,6 +173,15 @@ export default class FileBackend extends StorageBackend {
|
|
|
164
173
|
if (snapshot?.ino != null && Number(snapshot.ino) !== stats.ino) {
|
|
165
174
|
return { ok: false, reason: 'root-replaced', fsid };
|
|
166
175
|
}
|
|
176
|
+
// `config.marker` ({ file, id }) binds the folder to whoever keeps
|
|
177
|
+
// history about it: an empty directory standing in for an unmounted
|
|
178
|
+
// drive, or another folder at the same path, does not carry the id.
|
|
179
|
+
const marker = this.config.marker;
|
|
180
|
+
if (marker?.file) {
|
|
181
|
+
let found = null;
|
|
182
|
+
try { found = (await fs.readFile(path.join(this.#root, marker.file), 'utf8')).trim(); } catch { /* absent */ }
|
|
183
|
+
if (found !== String(marker.id)) return { ok: false, reason: found == null ? 'marker-missing' : 'marker-mismatch', fsid };
|
|
184
|
+
}
|
|
167
185
|
return { ok: true, fsid };
|
|
168
186
|
}
|
|
169
187
|
|
|
@@ -190,6 +208,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
190
208
|
// backends reflect the byte changes; empty directories are surfaced by the
|
|
191
209
|
// caller (Workspace) inserting a tree node.
|
|
192
210
|
async createContainer(key) {
|
|
211
|
+
await this.ready();
|
|
193
212
|
const dir = this.#safeResolve(key);
|
|
194
213
|
await fs.ensureDir(dir);
|
|
195
214
|
debug(`MKDIR ${key}`);
|
|
@@ -197,6 +216,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
197
216
|
}
|
|
198
217
|
|
|
199
218
|
async deleteContainer(key) {
|
|
219
|
+
await this.ready();
|
|
200
220
|
const dir = this.#safeResolve(key);
|
|
201
221
|
if (!await fs.pathExists(dir)) return false;
|
|
202
222
|
await fs.remove(dir);
|
|
@@ -204,7 +224,87 @@ export default class FileBackend extends StorageBackend {
|
|
|
204
224
|
return true;
|
|
205
225
|
}
|
|
206
226
|
|
|
227
|
+
async removeEmptyContainer(key) {
|
|
228
|
+
await this.ready();
|
|
229
|
+
const dir = await this.#trashTarget(key);
|
|
230
|
+
try { await fs.rmdir(dir); return { removed: true }; }
|
|
231
|
+
catch (error) {
|
|
232
|
+
if (error.code === 'ENOENT') return { removed: false };
|
|
233
|
+
// Preserve unindexed and independently owned children.
|
|
234
|
+
if (error.code === 'ENOTEMPTY' || error.code === 'EEXIST') return { removed: false, reason: 'not-empty' };
|
|
235
|
+
throw error;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async statContainer(key) {
|
|
240
|
+
await this.ready();
|
|
241
|
+
const target = await this.#trashTarget(key);
|
|
242
|
+
try { const stat = await fs.lstat(target); return stat.isDirectory() ? { kind: 'directory', dev: stat.dev, ino: stat.ino } : null; }
|
|
243
|
+
catch (error) { if (error.code === 'ENOENT') return null; throw error; }
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
#trashPath(id) {
|
|
247
|
+
if (!/^[a-f0-9-]{36}$/.test(id)) throw new Error('Invalid trash item id');
|
|
248
|
+
return path.join(this.#tempDir, 'trash', id);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async #trashTarget(key) {
|
|
252
|
+
const target = this.#safeResolve(key);
|
|
253
|
+
// Never traverse a symlink when deleting or restoring an original path.
|
|
254
|
+
let current = this.#root;
|
|
255
|
+
for (const part of path.relative(this.#root, target).split(path.sep)) {
|
|
256
|
+
current = path.join(current, part);
|
|
257
|
+
const stat = await fs.lstat(current).catch(error => {
|
|
258
|
+
if (error.code === 'ENOENT') return null;
|
|
259
|
+
throw error;
|
|
260
|
+
});
|
|
261
|
+
if (stat?.isSymbolicLink()) throw new Error('Trash paths cannot traverse symbolic links');
|
|
262
|
+
}
|
|
263
|
+
return target;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
async trashContainer(key, id) {
|
|
267
|
+
await this.ready();
|
|
268
|
+
const source = await this.#trashTarget(key);
|
|
269
|
+
if (!(await fs.lstat(source)).isDirectory()) throw new Error('Trash source is not a directory');
|
|
270
|
+
const destination = this.#trashPath(id);
|
|
271
|
+
await fs.ensureDir(path.dirname(destination));
|
|
272
|
+
// EXDEV fails safely with the source intact (a custom staging directory
|
|
273
|
+
// on another volume must not trigger copy-and-delete of a live tree).
|
|
274
|
+
await fs.rename(source, destination);
|
|
275
|
+
await this.#syncDirectory(path.dirname(source));
|
|
276
|
+
await this.#syncDirectory(path.dirname(destination));
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
async trashFile(key, id) {
|
|
280
|
+
await this.ready();
|
|
281
|
+
const source = await this.#trashTarget(key);
|
|
282
|
+
if (!(await fs.lstat(source)).isFile()) throw new Error('Trash source is not a regular file');
|
|
283
|
+
const destination = this.#trashPath(id);
|
|
284
|
+
await fs.ensureDir(path.dirname(destination));
|
|
285
|
+
await fs.rename(source, destination);
|
|
286
|
+
await this.#syncDirectory(path.dirname(source));
|
|
287
|
+
await this.#syncDirectory(path.dirname(destination));
|
|
288
|
+
return { checksums: await checksumFile(destination, ['sha256']) };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async validateTrashTarget(key) { await this.#trashTarget(key); }
|
|
292
|
+
|
|
293
|
+
async hasTrashedDirectory(id) { return fs.pathExists(this.#trashPath(id)); }
|
|
294
|
+
async dropTrashedDirectory(id) { await fs.remove(this.#trashPath(id)); }
|
|
295
|
+
|
|
296
|
+
async restoreTrashedDirectory(id, key) {
|
|
297
|
+
await this.ready();
|
|
298
|
+
const destination = await this.#trashTarget(key);
|
|
299
|
+
if (await fs.pathExists(destination)) return { ok: false, reason: 'target-exists' };
|
|
300
|
+
if (!await this.hasTrashedDirectory(id)) return { ok: false, reason: 'not-found' };
|
|
301
|
+
await fs.ensureDir(path.dirname(destination));
|
|
302
|
+
await fs.move(this.#trashPath(id), destination, { overwrite: false });
|
|
303
|
+
return { ok: true };
|
|
304
|
+
}
|
|
305
|
+
|
|
207
306
|
async renameContainer(fromKey, toKey) {
|
|
307
|
+
await this.ready();
|
|
208
308
|
const from = this.#safeResolve(fromKey);
|
|
209
309
|
const to = this.#safeResolve(toKey);
|
|
210
310
|
if (!await fs.pathExists(from)) throw new Error(`Container not found: ${fromKey}`);
|
|
@@ -221,11 +321,18 @@ export default class FileBackend extends StorageBackend {
|
|
|
221
321
|
return path.join(this.#tempDir, `${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2)}`);
|
|
222
322
|
}
|
|
223
323
|
|
|
324
|
+
async #syncDirectory(directory) {
|
|
325
|
+
for (let current = directory;; current = path.dirname(current)) {
|
|
326
|
+
const handle = await fsp.open(current, 'r');
|
|
327
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
328
|
+
if (current === this.#root || !current.startsWith(`${this.#root}${path.sep}`)) break;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
224
332
|
// Every placement ends with rename(2) over the destination: a reader sees
|
|
225
333
|
// either the previous bytes or the complete new ones, never a partial
|
|
226
|
-
// file
|
|
227
|
-
//
|
|
228
|
-
// inode is never written to in place, it is displaced.
|
|
334
|
+
// file. Retention uses independent inodes because external writers may
|
|
335
|
+
// keep modifying a displaced inode through an open handle.
|
|
229
336
|
//
|
|
230
337
|
// The staging dir normally shares the destination's filesystem, but a
|
|
231
338
|
// configured `tempDir` can sit on another mount (a container's /state next
|
|
@@ -234,8 +341,11 @@ export default class FileBackend extends StorageBackend {
|
|
|
234
341
|
// destination — same directory, same mount by construction — and the
|
|
235
342
|
// rename from there. Siblings are ignored by the watcher/scan like the dir.
|
|
236
343
|
async #finish(tmp, dest) {
|
|
344
|
+
const handle = await fsp.open(tmp, 'r');
|
|
345
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
237
346
|
try {
|
|
238
347
|
await fs.rename(tmp, dest);
|
|
348
|
+
await this.#syncDirectory(path.dirname(dest));
|
|
239
349
|
return;
|
|
240
350
|
} catch (err) {
|
|
241
351
|
if (err.code !== 'EXDEV') throw err;
|
|
@@ -243,13 +353,17 @@ export default class FileBackend extends StorageBackend {
|
|
|
243
353
|
const sibling = path.join(path.dirname(dest), `${TMP_DIR}-${process.pid}-${Math.random().toString(16).slice(2)}`);
|
|
244
354
|
try {
|
|
245
355
|
await fs.copyFile(tmp, sibling);
|
|
356
|
+
const handle = await fsp.open(sibling, 'r');
|
|
357
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
246
358
|
await fs.rename(sibling, dest);
|
|
359
|
+
await this.#syncDirectory(path.dirname(dest));
|
|
247
360
|
} finally {
|
|
248
361
|
await fs.remove(sibling).catch(() => {});
|
|
249
362
|
}
|
|
250
363
|
}
|
|
251
364
|
|
|
252
365
|
async put(key, data) {
|
|
366
|
+
await this.ready();
|
|
253
367
|
const filePath = this.#resolvePath(key);
|
|
254
368
|
await fs.ensureDir(path.dirname(filePath));
|
|
255
369
|
const tmp = await this.#stagingPath();
|
|
@@ -265,20 +379,15 @@ export default class FileBackend extends StorageBackend {
|
|
|
265
379
|
return { key, size: stats.size };
|
|
266
380
|
}
|
|
267
381
|
|
|
268
|
-
//
|
|
269
|
-
//
|
|
270
|
-
// across filesystems), then rename over the destination.
|
|
382
|
+
// Clone to a separate inode before publishing. The input may be an
|
|
383
|
+
// immutable cache pin; writable files must never share its inode.
|
|
271
384
|
async commit(key, srcPath) {
|
|
385
|
+
await this.ready();
|
|
272
386
|
const dest = this.#resolvePath(key);
|
|
273
387
|
await fs.ensureDir(path.dirname(dest));
|
|
274
388
|
const tmp = await this.#stagingPath();
|
|
275
389
|
try {
|
|
276
|
-
|
|
277
|
-
await fs.link(srcPath, tmp);
|
|
278
|
-
} catch (err) {
|
|
279
|
-
if (!['EXDEV', 'EPERM', 'EMLINK', 'ENOTSUP', 'EOPNOTSUPP'].includes(err.code)) throw err;
|
|
280
|
-
await fs.copyFile(srcPath, tmp);
|
|
281
|
-
}
|
|
390
|
+
await fs.copyFile(srcPath, tmp, fs.constants.COPYFILE_FICLONE);
|
|
282
391
|
await this.#finish(tmp, dest);
|
|
283
392
|
} finally {
|
|
284
393
|
await fs.remove(tmp).catch(() => {});
|
|
@@ -290,22 +399,36 @@ export default class FileBackend extends StorageBackend {
|
|
|
290
399
|
|
|
291
400
|
// Create-only placement: link(2) fails with EEXIST even for an unindexed
|
|
292
401
|
// file, directory or dangling symlink. Never use rename-over for this path.
|
|
293
|
-
|
|
402
|
+
// `owned`: srcPath is the caller's private file, removed after this call,
|
|
403
|
+
// so it is linked into place itself; otherwise (or across filesystems) a
|
|
404
|
+
// copy gets its own inode first.
|
|
405
|
+
async createFrom(key, srcPath, { owned = false } = {}) {
|
|
406
|
+
await this.ready();
|
|
294
407
|
const dest = this.#resolvePath(key);
|
|
295
408
|
await fs.ensureDir(path.dirname(dest));
|
|
296
|
-
|
|
297
|
-
await
|
|
298
|
-
|
|
299
|
-
if (error.code !== 'EXDEV') throw error;
|
|
300
|
-
// Stage on the destination filesystem, then publish atomically.
|
|
301
|
-
const sibling = path.join(path.dirname(dest), `${TMP_DIR}-${process.pid}-${Math.random().toString(16).slice(2)}`);
|
|
409
|
+
if (owned) {
|
|
410
|
+
const handle = await fsp.open(srcPath, 'r');
|
|
411
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
302
412
|
try {
|
|
303
|
-
await fs.
|
|
304
|
-
await
|
|
305
|
-
|
|
306
|
-
|
|
413
|
+
await fs.link(srcPath, dest);
|
|
414
|
+
await this.#syncDirectory(path.dirname(dest));
|
|
415
|
+
const stat = await this.stat(key);
|
|
416
|
+
if (!stat) throw new Error(`Upload destination disappeared: ${key}`);
|
|
417
|
+
return stat;
|
|
418
|
+
} catch (error) {
|
|
419
|
+
if (error.code !== 'EXDEV') throw error;
|
|
307
420
|
}
|
|
308
421
|
}
|
|
422
|
+
const sibling = path.join(path.dirname(dest), `${TMP_DIR}-${process.pid}-${Math.random().toString(16).slice(2)}`);
|
|
423
|
+
try {
|
|
424
|
+
await fs.copyFile(srcPath, sibling, fs.constants.COPYFILE_FICLONE);
|
|
425
|
+
const handle = await fsp.open(sibling, 'r');
|
|
426
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
427
|
+
await fs.link(sibling, dest);
|
|
428
|
+
await this.#syncDirectory(path.dirname(dest));
|
|
429
|
+
} finally {
|
|
430
|
+
await fs.remove(sibling).catch(() => {});
|
|
431
|
+
}
|
|
309
432
|
const stat = await this.stat(key);
|
|
310
433
|
if (!stat) throw new Error(`Upload destination disappeared: ${key}`);
|
|
311
434
|
return stat;
|
|
@@ -315,10 +438,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
315
438
|
//
|
|
316
439
|
// Bytes an overwrite or delete is about to displace are kept for a while
|
|
317
440
|
// under `<tempDir>/retained/<sha256>` (Stored decides when; the window is
|
|
318
|
-
// its config).
|
|
319
|
-
// until the placement above renames a new inode over it — placements are
|
|
320
|
-
// never in-place writes, so the retained copy stays what it was. On
|
|
321
|
-
// filesystems without hardlinks the bytes are copied.
|
|
441
|
+
// its config). Reflink or copy gives each version an independent inode.
|
|
322
442
|
|
|
323
443
|
get retainDir() { return path.join(this.#tempDir, 'retained'); }
|
|
324
444
|
retainedPath(sha256) { return path.join(this.retainDir, String(sha256).toLowerCase()); }
|
|
@@ -328,16 +448,23 @@ export default class FileBackend extends StorageBackend {
|
|
|
328
448
|
if (!sha256 || !await fs.pathExists(src)) return null;
|
|
329
449
|
const dest = this.retainedPath(sha256);
|
|
330
450
|
const existing = await fs.stat(dest).catch(() => null);
|
|
331
|
-
if (existing)
|
|
451
|
+
if (existing) {
|
|
452
|
+
if ((await checksumFile(dest, ['sha256'])).sha256 !== sha256) throw new Error('Retained content failed verification');
|
|
453
|
+
return { path: dest, size: existing.size, existing: true };
|
|
454
|
+
}
|
|
332
455
|
await fs.ensureDir(this.retainDir);
|
|
456
|
+
// A network-share writer can keep an inode open and modify it in
|
|
457
|
+
// place. A hardlink is therefore not a recoverable version.
|
|
458
|
+
const tmp = await this.#stagingPath();
|
|
333
459
|
try {
|
|
334
|
-
await fs.
|
|
335
|
-
|
|
336
|
-
if (
|
|
337
|
-
const
|
|
338
|
-
try { await
|
|
339
|
-
|
|
340
|
-
|
|
460
|
+
await fs.copyFile(src, tmp, fs.constants.COPYFILE_FICLONE);
|
|
461
|
+
const checksums = await checksumFile(tmp, ['sha256']);
|
|
462
|
+
if (checksums.sha256 !== sha256) throw Object.assign(new Error('File changed before retention'), { code: 'SOURCE_CHANGED' });
|
|
463
|
+
const handle = await fsp.open(tmp, 'r');
|
|
464
|
+
try { await handle.sync(); } finally { await handle.close(); }
|
|
465
|
+
await fs.rename(tmp, dest);
|
|
466
|
+
await this.#syncDirectory(path.dirname(dest));
|
|
467
|
+
} finally { await fs.remove(tmp).catch(() => {}); }
|
|
341
468
|
const stats = await fs.stat(dest);
|
|
342
469
|
debug(`RETAIN ${key} → ${sha256.slice(0, 12)} (${stats.size} bytes)`);
|
|
343
470
|
return { path: dest, size: stats.size, existing: false };
|
|
@@ -357,6 +484,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
357
484
|
* EXDEV otherwise, which callers treat as "fall back to a stream copy".
|
|
358
485
|
*/
|
|
359
486
|
async renameFrom(key, srcPath) {
|
|
487
|
+
await this.ready();
|
|
360
488
|
const dest = this.#resolvePath(key);
|
|
361
489
|
await fs.ensureDir(path.dirname(dest));
|
|
362
490
|
await fs.rename(srcPath, dest);
|
|
@@ -366,6 +494,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
366
494
|
}
|
|
367
495
|
|
|
368
496
|
async get(key, options = {}) {
|
|
497
|
+
await this.ready();
|
|
369
498
|
const filePath = this.#resolvePath(key);
|
|
370
499
|
if (!await fs.pathExists(filePath)) return null;
|
|
371
500
|
return options.stream ? fs.createReadStream(filePath) : fs.readFile(filePath);
|
|
@@ -374,12 +503,14 @@ export default class FileBackend extends StorageBackend {
|
|
|
374
503
|
// Ranged read for HTTP Range/streaming. `end` is inclusive (matches HTTP
|
|
375
504
|
// Range and createReadStream). Null on miss → caller falls back to full read.
|
|
376
505
|
async getRange(key, { start, end }) {
|
|
506
|
+
await this.ready();
|
|
377
507
|
const filePath = this.#resolvePath(key);
|
|
378
508
|
if (!await fs.pathExists(filePath)) return null;
|
|
379
509
|
return fs.createReadStream(filePath, { start, end });
|
|
380
510
|
}
|
|
381
511
|
|
|
382
512
|
async delete(key) {
|
|
513
|
+
await this.ready();
|
|
383
514
|
const filePath = this.#resolvePath(key);
|
|
384
515
|
if (!await fs.pathExists(filePath)) return false;
|
|
385
516
|
await fs.remove(filePath);
|
|
@@ -398,6 +529,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
398
529
|
async pruneEmptyParents(key) {
|
|
399
530
|
let rel = path.posix.dirname(String(key).split(path.sep).join('/'));
|
|
400
531
|
while (rel && rel !== '.' && rel !== '/') {
|
|
532
|
+
if (this.isDirectoryProtected?.(rel)) return;
|
|
401
533
|
if (this.#isTempPath(rel) || (this.#tempRel && this.#tempRel.startsWith(`${rel}/`))) return;
|
|
402
534
|
const dir = this.#resolvePath(rel);
|
|
403
535
|
if (dir === this.#root) return;
|
|
@@ -413,6 +545,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
413
545
|
* on "when" without a second hash.
|
|
414
546
|
*/
|
|
415
547
|
async utimes(key, mtime) {
|
|
548
|
+
await this.ready();
|
|
416
549
|
const filePath = this.#resolvePath(key);
|
|
417
550
|
const when = mtime instanceof Date ? mtime : new Date(typeof mtime === 'string' ? Date.parse(mtime) : Number(mtime));
|
|
418
551
|
if (Number.isNaN(when.getTime())) throw new Error(`Invalid mtime: ${mtime}`);
|
|
@@ -421,6 +554,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
421
554
|
}
|
|
422
555
|
|
|
423
556
|
async stat(key) {
|
|
557
|
+
await this.ready();
|
|
424
558
|
const filePath = this.#resolvePath(key);
|
|
425
559
|
let stats;
|
|
426
560
|
try { stats = await fs.stat(filePath); } catch (error) {
|
|
@@ -437,6 +571,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
437
571
|
// is reported and skipped instead of aborting the walk — one denied subtree
|
|
438
572
|
// must not kill (or worse, truncate) a whole-mount enumeration.
|
|
439
573
|
async *list(options = {}) {
|
|
574
|
+
await this.ready();
|
|
440
575
|
const { prefix = '', recursive = true, onError = null } = options;
|
|
441
576
|
const searchPath = this.#resolvePath(prefix);
|
|
442
577
|
|
|
@@ -488,6 +623,11 @@ export default class FileBackend extends StorageBackend {
|
|
|
488
623
|
return false;
|
|
489
624
|
}
|
|
490
625
|
|
|
626
|
+
const epoch = this.#watchEpoch;
|
|
627
|
+
await this.ready();
|
|
628
|
+
if (epoch !== this.#watchEpoch) return false;
|
|
629
|
+
if (this.#watcher) return true;
|
|
630
|
+
|
|
491
631
|
// `stabilityThreshold`: how long a file must stop growing before it is
|
|
492
632
|
// hashed (a mirror bumps it — a multi-GB copy from a USB stick must
|
|
493
633
|
// not be hashed mid-write). `followSymlinks`: chokidar's default is
|
|
@@ -660,6 +800,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
660
800
|
* folder skeleton (and size a progress bar) before the expensive scan.
|
|
661
801
|
*/
|
|
662
802
|
async shape(prefix = '') {
|
|
803
|
+
await this.ready();
|
|
663
804
|
const dirs = [];
|
|
664
805
|
let files = 0;
|
|
665
806
|
const walk = async (rel) => {
|
|
@@ -680,6 +821,7 @@ export default class FileBackend extends StorageBackend {
|
|
|
680
821
|
}
|
|
681
822
|
|
|
682
823
|
async stop() {
|
|
824
|
+
this.#watchEpoch++;
|
|
683
825
|
if (this.#watcher) {
|
|
684
826
|
await this.#watcher.close();
|
|
685
827
|
this.#watcher = null;
|
|
@@ -62,11 +62,17 @@ const splitKey = (key) => {
|
|
|
62
62
|
};
|
|
63
63
|
|
|
64
64
|
class DriveError extends Error {
|
|
65
|
-
constructor(message, { code = 'unreachable', status = null } = {}) {
|
|
66
|
-
|
|
65
|
+
constructor(message, { code = 'unreachable', status = null, cause = undefined } = {}) {
|
|
66
|
+
const transportCode = cause?.cause?.code || cause?.code;
|
|
67
|
+
// InvalidArgumentError's message names the rejected option/header.
|
|
68
|
+
// Keep it visible rather than reducing it to fetch's generic wrapper.
|
|
69
|
+
const argumentDetail = transportCode === 'UND_ERR_INVALID_ARG' ? cause?.cause?.message : null;
|
|
70
|
+
super(transportCode ? `${message} (${transportCode}${argumentDetail ? `: ${argumentDetail}` : ''})` : message, { cause });
|
|
67
71
|
this.name = 'DriveError';
|
|
68
72
|
this.code = code;
|
|
69
73
|
this.status = status;
|
|
74
|
+
this.statusCode = code === 'invalid-request' ? 500 : 502;
|
|
75
|
+
if (transportCode) this.transportCode = transportCode;
|
|
70
76
|
}
|
|
71
77
|
}
|
|
72
78
|
|
|
@@ -89,6 +95,7 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
89
95
|
|
|
90
96
|
#pollTimer = null;
|
|
91
97
|
#polling = false;
|
|
98
|
+
#creates = new Map();
|
|
92
99
|
#pageToken = null;
|
|
93
100
|
|
|
94
101
|
constructor(name, config = {}) {
|
|
@@ -100,7 +107,9 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
100
107
|
debug(`GdriveBackend "${name}" initialized (folder=${this.#rootRef})`);
|
|
101
108
|
}
|
|
102
109
|
|
|
103
|
-
|
|
110
|
+
static getCapabilities(config = {}) {
|
|
111
|
+
return { ...super.getCapabilities(config), canEnumerate: true, remote: true, paths: true, mutableContainers: config.readOnly !== true, upload: config.readOnly !== true };
|
|
112
|
+
}
|
|
104
113
|
get remote() { return true; }
|
|
105
114
|
get transport() { return DEV; }
|
|
106
115
|
get watching() { return !!this.#pollTimer; }
|
|
@@ -134,12 +143,17 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
134
143
|
}),
|
|
135
144
|
});
|
|
136
145
|
} catch (err) {
|
|
137
|
-
throw new DriveError(`gdrive token endpoint unreachable: ${err.message}`, { code: 'unreachable' });
|
|
146
|
+
throw new DriveError(`gdrive token endpoint unreachable: ${err.message}`, { code: 'unreachable', cause: err });
|
|
138
147
|
}
|
|
139
148
|
const json = await res.json().catch(() => null);
|
|
140
149
|
if (!res.ok || !json?.access_token) {
|
|
141
|
-
const detail = json?.
|
|
142
|
-
|
|
150
|
+
const detail = [json?.error, json?.error_description].filter(Boolean).join(': ') || `HTTP ${res.status}`;
|
|
151
|
+
const hint = json?.error === 'invalid_grant'
|
|
152
|
+
? ' Reconnect this Google account and save the new refresh token.'
|
|
153
|
+
: json?.error === 'invalid_client'
|
|
154
|
+
? ' Check the OAuth client id and client secret.'
|
|
155
|
+
: '';
|
|
156
|
+
throw new DriveError(`gdrive token refresh failed: ${detail}${hint}`, { code: 'unauthorized', status: res.status });
|
|
143
157
|
}
|
|
144
158
|
this.#accessToken = json.access_token;
|
|
145
159
|
this.#tokenExpiresAt = Date.now() + (Number(json.expires_in) || 3600) * 1000;
|
|
@@ -159,8 +173,11 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
159
173
|
try {
|
|
160
174
|
res = await this.#fetch(url, { ...init, headers: { Authorization: `Bearer ${token}`, ...(init.headers || {}) } });
|
|
161
175
|
} catch (err) {
|
|
176
|
+
if ((err.cause?.code || err.code) === 'UND_ERR_INVALID_ARG') {
|
|
177
|
+
throw new DriveError(`gdrive ${init.method || 'GET'} request arguments rejected: ${err.message}`, { code: 'invalid-request', cause: err });
|
|
178
|
+
}
|
|
162
179
|
if (attempt < RETRY_DELAYS_MS.length) { await sleep(RETRY_DELAYS_MS[attempt]); continue; }
|
|
163
|
-
throw new DriveError(`gdrive request failed: ${err.message}`, { code: 'unreachable' });
|
|
180
|
+
throw new DriveError(`gdrive ${init.method || 'GET'} ${new URL(url).origin}${new URL(url).pathname} request failed: ${err.message}`, { code: 'unreachable', cause: err });
|
|
164
181
|
}
|
|
165
182
|
if (res.ok || okStatuses.includes(res.status)) return res;
|
|
166
183
|
if (res.status === 404 && allow404) return res;
|
|
@@ -509,7 +526,19 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
509
526
|
const { dir, name, key: clean } = splitKey(key);
|
|
510
527
|
if (!clean) throw new Error('gdrive put: key is required');
|
|
511
528
|
const parentId = await this.#folderId(dir, { create: true });
|
|
512
|
-
|
|
529
|
+
let existing = null;
|
|
530
|
+
if (options.createOnly) {
|
|
531
|
+
// Query Drive, including folders and native documents: the local
|
|
532
|
+
// index and watcher may not have seen this name yet.
|
|
533
|
+
const found = await this.#json('/files', { query: {
|
|
534
|
+
q: `'${qEscape(parentId)}' in parents and name = '${qEscape(name)}' and trashed = false`,
|
|
535
|
+
fields: 'files(id)', pageSize: 1,
|
|
536
|
+
supportsAllDrives: true, includeItemsFromAllDrives: true,
|
|
537
|
+
} });
|
|
538
|
+
if (found?.files?.length) throw Object.assign(new Error(`File already exists: ${clean}`), { code: 'EEXIST' });
|
|
539
|
+
// Always POST. Drive permits duplicate names, so an external
|
|
540
|
+
// concurrent creator may produce a sibling, but is never patched.
|
|
541
|
+
} else existing = await this.#lookup(clean);
|
|
513
542
|
const mimeType = options.mimeType || 'application/octet-stream';
|
|
514
543
|
const source = Buffer.isBuffer(data) ? data : (typeof data === 'string' ? Buffer.from(data) : data);
|
|
515
544
|
const size = Buffer.isBuffer(source) ? source.length : (Number.isFinite(options.size) ? options.size : null);
|
|
@@ -544,7 +573,12 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
544
573
|
: `bytes */${last ? total : '*'}`;
|
|
545
574
|
const res = await this.#call(session, {
|
|
546
575
|
method: 'PUT',
|
|
547
|
-
|
|
576
|
+
// Drive uses 308 as an acknowledgement of an incomplete upload,
|
|
577
|
+
// not an instruction to replay this chunk at Location.
|
|
578
|
+
redirect: 'manual',
|
|
579
|
+
// fetch derives Content-Length from this Buffer. Supplying it
|
|
580
|
+
// separately is redundant and can conflict with fetch's framing.
|
|
581
|
+
headers: { 'Content-Range': range, 'Content-Type': mimeType },
|
|
548
582
|
body: buf,
|
|
549
583
|
}, { okStatuses: [308] });
|
|
550
584
|
if (last) {
|
|
@@ -587,6 +621,19 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
587
621
|
return this.put(key, createReadStream(srcPath), { size });
|
|
588
622
|
}
|
|
589
623
|
|
|
624
|
+
async createFrom(key, srcPath) {
|
|
625
|
+
const clean = splitKey(key).key;
|
|
626
|
+
const previous = this.#creates.get(clean) || Promise.resolve();
|
|
627
|
+
const task = previous.catch(() => {}).then(async () => {
|
|
628
|
+
const { size } = await fsp.stat(srcPath);
|
|
629
|
+
await this.put(clean, createReadStream(srcPath), { size, createOnly: true });
|
|
630
|
+
return this.stat(clean);
|
|
631
|
+
});
|
|
632
|
+
this.#creates.set(clean, task);
|
|
633
|
+
try { return await task; }
|
|
634
|
+
finally { if (this.#creates.get(clean) === task) this.#creates.delete(clean); }
|
|
635
|
+
}
|
|
636
|
+
|
|
590
637
|
/**
|
|
591
638
|
* Remove the bytes at `key`. Trashes by default — a user's Drive is shared
|
|
592
639
|
* with their other tools and the trash is their safety net; set
|
|
@@ -738,7 +785,7 @@ export default class GdriveBackend extends StorageBackend {
|
|
|
738
785
|
}
|
|
739
786
|
|
|
740
787
|
for (const row of walk.files.values()) {
|
|
741
|
-
let checksums
|
|
788
|
+
let checksums;
|
|
742
789
|
const have = row.checksums || {};
|
|
743
790
|
if (algorithms.every((a) => have[a])) {
|
|
744
791
|
checksums = Object.fromEntries(algorithms.map((a) => [a, have[a]]));
|
|
@@ -22,7 +22,9 @@ export default class HttpBackend extends StorageBackend {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
// Read-only source: bytes can't be removed remotely, only the reference dropped.
|
|
25
|
-
|
|
25
|
+
static getCapabilities(config = {}) {
|
|
26
|
+
return { ...super.getCapabilities(config), write: false, delete: false, remote: true };
|
|
27
|
+
}
|
|
26
28
|
|
|
27
29
|
nativeUrl(key) {
|
|
28
30
|
try { return this.#baseUrl ? new URL(key, this.#baseUrl).href : key; }
|