@augmentd-labs/canvas-edge 0.6.3 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/node_modules/canvas-stored/README.md +189 -9
  2. package/node_modules/canvas-stored/package.json +2 -2
  3. package/node_modules/canvas-stored/src/backends/BackendManager.js +11 -1
  4. package/node_modules/canvas-stored/src/backends/StorageBackend.js +24 -12
  5. package/node_modules/canvas-stored/src/backends/cacache/index.js +49 -6
  6. package/node_modules/canvas-stored/src/backends/canvas/index.js +41 -4
  7. package/node_modules/canvas-stored/src/backends/file/index.js +181 -39
  8. package/node_modules/canvas-stored/src/backends/gdrive/index.js +57 -10
  9. package/node_modules/canvas-stored/src/backends/http/index.js +3 -1
  10. package/node_modules/canvas-stored/src/backends/s3/index.js +244 -15
  11. package/node_modules/canvas-stored/src/cache/collect-garbage.js +47 -0
  12. package/node_modules/canvas-stored/src/cache/compact-index.js +13 -0
  13. package/node_modules/canvas-stored/src/cache/index.js +39 -5
  14. package/node_modules/canvas-stored/src/cache/ingest-stream.js +65 -0
  15. package/node_modules/canvas-stored/src/index/index.js +5 -3
  16. package/node_modules/canvas-stored/src/index.js +733 -132
  17. package/node_modules/canvas-stored/src/runtime/MutationQueue.js +60 -0
  18. package/node_modules/canvas-stored/src/storage/lmdb.js +62 -0
  19. package/node_modules/canvas-stored/src/storage/recover-index.js +33 -0
  20. package/node_modules/canvas-stored/src/storage/verify-worker.js +23 -0
  21. package/node_modules/canvas-stored/src/sync/Ledger.js +21 -3
  22. package/node_modules/canvas-stored/src/sync/Mirror.js +334 -25
  23. package/node_modules/canvas-stored/src/sync/REPLICAS.md +64 -0
  24. package/node_modules/canvas-stored/src/sync/ReplicaService.js +623 -0
  25. package/node_modules/canvas-stored/src/sync/SyncQueue.js +23 -103
  26. package/node_modules/canvas-stored/src/trash.js +219 -0
  27. package/node_modules/canvas-stored/src/utils/common.js +7 -0
  28. package/package.json +5 -4
  29. package/src/binding.js +33 -0
  30. package/src/control.js +13 -0
  31. package/src/daemon.js +14 -1
  32. package/src/mirror-runtime.js +14 -3
@@ -27,7 +27,8 @@ export default class FileBackend extends StorageBackend {
27
27
  #tempDir;
28
28
  #tempRel = null;
29
29
  #watcher = null;
30
- #watchEnabled;
30
+ #watchEpoch = 0;
31
+ #rootReady;
31
32
  #ignored;
32
33
  #ignoreMatcher = null;
33
34
  #defaultAlgorithms = ['sha256'];
@@ -42,7 +43,6 @@ export default class FileBackend extends StorageBackend {
42
43
  this.#tempDir = config.tempDir ? path.resolve(this.#root, String(config.tempDir)) : path.join(this.#root, TMP_DIR);
43
44
  const tempRel = path.relative(this.#root, this.#tempDir).split(path.sep).join('/');
44
45
  this.#tempRel = tempRel && !tempRel.startsWith('..') && !path.isAbsolute(tempRel) ? tempRel : null;
45
- this.#watchEnabled = config.watch ?? false;
46
46
  this.#ignored = config.ignored || null;
47
47
  this.#ignoreMatcher = this.#buildIgnoreMatcher(this.#ignored);
48
48
  this.#defaultAlgorithms = config.algorithms || ['sha256'];
@@ -52,7 +52,12 @@ export default class FileBackend extends StorageBackend {
52
52
  // creating an empty dir at an unmounted mountpoint makes an absent
53
53
  // drive indistinguishable from an empty one — verifyRoot() + the
54
54
  // resync liveness gate rely on the root being genuinely missing.
55
- if (config.createRoot !== false) fs.ensureDirSync(this.#root);
55
+ // A stat/mkdir on an unresponsive NAS can block the whole Node thread.
56
+ // Registration stays synchronous; filesystem users await readiness.
57
+ this.#rootReady = config.createRoot !== false ? fs.ensureDir(this.#root) : Promise.resolve();
58
+ // Keep registration fire-and-forget safe. ready()/operations still
59
+ // observe and propagate the original failure.
60
+ this.#rootReady.catch(() => {});
56
61
 
57
62
  // Network mount detection. A CIFS/NFS share reached through a POSIX path
58
63
  // is indistinguishable from local disk to every fs call we make, so the
@@ -69,10 +74,13 @@ export default class FileBackend extends StorageBackend {
69
74
  get root() { return this.#root; }
70
75
  get tempDir() { return this.#tempDir; }
71
76
  get watching() { return !!this.#watcher; }
72
- get capabilities() { return { ...super.capabilities, canEnumerate: true }; }
77
+ static getCapabilities(config = {}) {
78
+ return { ...super.getCapabilities(config), canEnumerate: true, paths: true, mutableContainers: config.readOnly !== true, upload: config.readOnly !== true };
79
+ }
73
80
  get remote() { return this.#remote; }
74
81
  get transport() { return this.#transport; }
75
82
  get mountSource() { return this.#mountSource; }
83
+ ready() { return this.#rootReady; }
76
84
 
77
85
  /**
78
86
  * Absolute on-disk path for `key`. Public because cross-backend copy/move
@@ -152,6 +160,7 @@ export default class FileBackend extends StorageBackend {
152
160
  const snapshot = expected || this.config.fsid || null;
153
161
  let stats;
154
162
  try {
163
+ await this.ready();
155
164
  stats = await fs.stat(this.#root);
156
165
  } catch {
157
166
  return { ok: false, reason: 'root-missing', fsid: null };
@@ -164,6 +173,15 @@ export default class FileBackend extends StorageBackend {
164
173
  if (snapshot?.ino != null && Number(snapshot.ino) !== stats.ino) {
165
174
  return { ok: false, reason: 'root-replaced', fsid };
166
175
  }
176
+ // `config.marker` ({ file, id }) binds the folder to whoever keeps
177
+ // history about it: an empty directory standing in for an unmounted
178
+ // drive, or another folder at the same path, does not carry the id.
179
+ const marker = this.config.marker;
180
+ if (marker?.file) {
181
+ let found = null;
182
+ try { found = (await fs.readFile(path.join(this.#root, marker.file), 'utf8')).trim(); } catch { /* absent */ }
183
+ if (found !== String(marker.id)) return { ok: false, reason: found == null ? 'marker-missing' : 'marker-mismatch', fsid };
184
+ }
167
185
  return { ok: true, fsid };
168
186
  }
169
187
 
@@ -190,6 +208,7 @@ export default class FileBackend extends StorageBackend {
190
208
  // backends reflect the byte changes; empty directories are surfaced by the
191
209
  // caller (Workspace) inserting a tree node.
192
210
  async createContainer(key) {
211
+ await this.ready();
193
212
  const dir = this.#safeResolve(key);
194
213
  await fs.ensureDir(dir);
195
214
  debug(`MKDIR ${key}`);
@@ -197,6 +216,7 @@ export default class FileBackend extends StorageBackend {
197
216
  }
198
217
 
199
218
  async deleteContainer(key) {
219
+ await this.ready();
200
220
  const dir = this.#safeResolve(key);
201
221
  if (!await fs.pathExists(dir)) return false;
202
222
  await fs.remove(dir);
@@ -204,7 +224,87 @@ export default class FileBackend extends StorageBackend {
204
224
  return true;
205
225
  }
206
226
 
227
+ async removeEmptyContainer(key) {
228
+ await this.ready();
229
+ const dir = await this.#trashTarget(key);
230
+ try { await fs.rmdir(dir); return { removed: true }; }
231
+ catch (error) {
232
+ if (error.code === 'ENOENT') return { removed: false };
233
+ // Preserve unindexed and independently owned children.
234
+ if (error.code === 'ENOTEMPTY' || error.code === 'EEXIST') return { removed: false, reason: 'not-empty' };
235
+ throw error;
236
+ }
237
+ }
238
+
239
+ async statContainer(key) {
240
+ await this.ready();
241
+ const target = await this.#trashTarget(key);
242
+ try { const stat = await fs.lstat(target); return stat.isDirectory() ? { kind: 'directory', dev: stat.dev, ino: stat.ino } : null; }
243
+ catch (error) { if (error.code === 'ENOENT') return null; throw error; }
244
+ }
245
+
246
+ #trashPath(id) {
247
+ if (!/^[a-f0-9-]{36}$/.test(id)) throw new Error('Invalid trash item id');
248
+ return path.join(this.#tempDir, 'trash', id);
249
+ }
250
+
251
+ async #trashTarget(key) {
252
+ const target = this.#safeResolve(key);
253
+ // Never traverse a symlink when deleting or restoring an original path.
254
+ let current = this.#root;
255
+ for (const part of path.relative(this.#root, target).split(path.sep)) {
256
+ current = path.join(current, part);
257
+ const stat = await fs.lstat(current).catch(error => {
258
+ if (error.code === 'ENOENT') return null;
259
+ throw error;
260
+ });
261
+ if (stat?.isSymbolicLink()) throw new Error('Trash paths cannot traverse symbolic links');
262
+ }
263
+ return target;
264
+ }
265
+
266
+ async trashContainer(key, id) {
267
+ await this.ready();
268
+ const source = await this.#trashTarget(key);
269
+ if (!(await fs.lstat(source)).isDirectory()) throw new Error('Trash source is not a directory');
270
+ const destination = this.#trashPath(id);
271
+ await fs.ensureDir(path.dirname(destination));
272
+ // EXDEV fails safely with the source intact (a custom staging directory
273
+ // on another volume must not trigger copy-and-delete of a live tree).
274
+ await fs.rename(source, destination);
275
+ await this.#syncDirectory(path.dirname(source));
276
+ await this.#syncDirectory(path.dirname(destination));
277
+ }
278
+
279
+ async trashFile(key, id) {
280
+ await this.ready();
281
+ const source = await this.#trashTarget(key);
282
+ if (!(await fs.lstat(source)).isFile()) throw new Error('Trash source is not a regular file');
283
+ const destination = this.#trashPath(id);
284
+ await fs.ensureDir(path.dirname(destination));
285
+ await fs.rename(source, destination);
286
+ await this.#syncDirectory(path.dirname(source));
287
+ await this.#syncDirectory(path.dirname(destination));
288
+ return { checksums: await checksumFile(destination, ['sha256']) };
289
+ }
290
+
291
+ async validateTrashTarget(key) { await this.#trashTarget(key); }
292
+
293
+ async hasTrashedDirectory(id) { return fs.pathExists(this.#trashPath(id)); }
294
+ async dropTrashedDirectory(id) { await fs.remove(this.#trashPath(id)); }
295
+
296
+ async restoreTrashedDirectory(id, key) {
297
+ await this.ready();
298
+ const destination = await this.#trashTarget(key);
299
+ if (await fs.pathExists(destination)) return { ok: false, reason: 'target-exists' };
300
+ if (!await this.hasTrashedDirectory(id)) return { ok: false, reason: 'not-found' };
301
+ await fs.ensureDir(path.dirname(destination));
302
+ await fs.move(this.#trashPath(id), destination, { overwrite: false });
303
+ return { ok: true };
304
+ }
305
+
207
306
  async renameContainer(fromKey, toKey) {
307
+ await this.ready();
208
308
  const from = this.#safeResolve(fromKey);
209
309
  const to = this.#safeResolve(toKey);
210
310
  if (!await fs.pathExists(from)) throw new Error(`Container not found: ${fromKey}`);
@@ -221,11 +321,18 @@ export default class FileBackend extends StorageBackend {
221
321
  return path.join(this.#tempDir, `${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2)}`);
222
322
  }
223
323
 
324
+ async #syncDirectory(directory) {
325
+ for (let current = directory;; current = path.dirname(current)) {
326
+ const handle = await fsp.open(current, 'r');
327
+ try { await handle.sync(); } finally { await handle.close(); }
328
+ if (current === this.#root || !current.startsWith(`${this.#root}${path.sep}`)) break;
329
+ }
330
+ }
331
+
224
332
  // Every placement ends with rename(2) over the destination: a reader sees
225
333
  // either the previous bytes or the complete new ones, never a partial
226
- // file, and a crash mid-write leaves only a staging file behind. This is
227
- // also what makes hardlink-based retention safe (see retain()): the old
228
- // inode is never written to in place, it is displaced.
334
+ // file. Retention uses independent inodes because external writers may
335
+ // keep modifying a displaced inode through an open handle.
229
336
  //
230
337
  // The staging dir normally shares the destination's filesystem, but a
231
338
  // configured `tempDir` can sit on another mount (a container's /state next
@@ -234,8 +341,11 @@ export default class FileBackend extends StorageBackend {
234
341
  // destination — same directory, same mount by construction — and the
235
342
  // rename from there. Siblings are ignored by the watcher/scan like the dir.
236
343
  async #finish(tmp, dest) {
344
+ const handle = await fsp.open(tmp, 'r');
345
+ try { await handle.sync(); } finally { await handle.close(); }
237
346
  try {
238
347
  await fs.rename(tmp, dest);
348
+ await this.#syncDirectory(path.dirname(dest));
239
349
  return;
240
350
  } catch (err) {
241
351
  if (err.code !== 'EXDEV') throw err;
@@ -243,13 +353,17 @@ export default class FileBackend extends StorageBackend {
243
353
  const sibling = path.join(path.dirname(dest), `${TMP_DIR}-${process.pid}-${Math.random().toString(16).slice(2)}`);
244
354
  try {
245
355
  await fs.copyFile(tmp, sibling);
356
+ const handle = await fsp.open(sibling, 'r');
357
+ try { await handle.sync(); } finally { await handle.close(); }
246
358
  await fs.rename(sibling, dest);
359
+ await this.#syncDirectory(path.dirname(dest));
247
360
  } finally {
248
361
  await fs.remove(sibling).catch(() => {});
249
362
  }
250
363
  }
251
364
 
252
365
  async put(key, data) {
366
+ await this.ready();
253
367
  const filePath = this.#resolvePath(key);
254
368
  await fs.ensureDir(path.dirname(filePath));
255
369
  const tmp = await this.#stagingPath();
@@ -265,20 +379,15 @@ export default class FileBackend extends StorageBackend {
265
379
  return { key, size: stats.size };
266
380
  }
267
381
 
268
- // Place an already-written file (e.g. a streamed temp) at `key`: hardlink
269
- // it to a staging name (zero copy on the same filesystem; a byte copy
270
- // across filesystems), then rename over the destination.
382
+ // Clone to a separate inode before publishing. The input may be an
383
+ // immutable cache pin; writable files must never share its inode.
271
384
  async commit(key, srcPath) {
385
+ await this.ready();
272
386
  const dest = this.#resolvePath(key);
273
387
  await fs.ensureDir(path.dirname(dest));
274
388
  const tmp = await this.#stagingPath();
275
389
  try {
276
- try {
277
- await fs.link(srcPath, tmp);
278
- } catch (err) {
279
- if (!['EXDEV', 'EPERM', 'EMLINK', 'ENOTSUP', 'EOPNOTSUPP'].includes(err.code)) throw err;
280
- await fs.copyFile(srcPath, tmp);
281
- }
390
+ await fs.copyFile(srcPath, tmp, fs.constants.COPYFILE_FICLONE);
282
391
  await this.#finish(tmp, dest);
283
392
  } finally {
284
393
  await fs.remove(tmp).catch(() => {});
@@ -290,22 +399,36 @@ export default class FileBackend extends StorageBackend {
290
399
 
291
400
  // Create-only placement: link(2) fails with EEXIST even for an unindexed
292
401
  // file, directory or dangling symlink. Never use rename-over for this path.
293
- async createFrom(key, srcPath) {
402
+ // `owned`: srcPath is the caller's private file, removed after this call,
403
+ // so it is linked into place itself; otherwise (or across filesystems) a
404
+ // copy gets its own inode first.
405
+ async createFrom(key, srcPath, { owned = false } = {}) {
406
+ await this.ready();
294
407
  const dest = this.#resolvePath(key);
295
408
  await fs.ensureDir(path.dirname(dest));
296
- try {
297
- await fs.link(srcPath, dest);
298
- } catch (error) {
299
- if (error.code !== 'EXDEV') throw error;
300
- // Stage on the destination filesystem, then publish atomically.
301
- const sibling = path.join(path.dirname(dest), `${TMP_DIR}-${process.pid}-${Math.random().toString(16).slice(2)}`);
409
+ if (owned) {
410
+ const handle = await fsp.open(srcPath, 'r');
411
+ try { await handle.sync(); } finally { await handle.close(); }
302
412
  try {
303
- await fs.copyFile(srcPath, sibling);
304
- await fs.link(sibling, dest);
305
- } finally {
306
- await fs.remove(sibling).catch(() => {});
413
+ await fs.link(srcPath, dest);
414
+ await this.#syncDirectory(path.dirname(dest));
415
+ const stat = await this.stat(key);
416
+ if (!stat) throw new Error(`Upload destination disappeared: ${key}`);
417
+ return stat;
418
+ } catch (error) {
419
+ if (error.code !== 'EXDEV') throw error;
307
420
  }
308
421
  }
422
+ const sibling = path.join(path.dirname(dest), `${TMP_DIR}-${process.pid}-${Math.random().toString(16).slice(2)}`);
423
+ try {
424
+ await fs.copyFile(srcPath, sibling, fs.constants.COPYFILE_FICLONE);
425
+ const handle = await fsp.open(sibling, 'r');
426
+ try { await handle.sync(); } finally { await handle.close(); }
427
+ await fs.link(sibling, dest);
428
+ await this.#syncDirectory(path.dirname(dest));
429
+ } finally {
430
+ await fs.remove(sibling).catch(() => {});
431
+ }
309
432
  const stat = await this.stat(key);
310
433
  if (!stat) throw new Error(`Upload destination disappeared: ${key}`);
311
434
  return stat;
@@ -315,10 +438,7 @@ export default class FileBackend extends StorageBackend {
315
438
  //
316
439
  // Bytes an overwrite or delete is about to displace are kept for a while
317
440
  // under `<tempDir>/retained/<sha256>` (Stored decides when; the window is
318
- // its config). A hardlink shares the inode with the visible file only
319
- // until the placement above renames a new inode over it — placements are
320
- // never in-place writes, so the retained copy stays what it was. On
321
- // filesystems without hardlinks the bytes are copied.
441
+ // its config). Reflink or copy gives each version an independent inode.
322
442
 
323
443
  get retainDir() { return path.join(this.#tempDir, 'retained'); }
324
444
  retainedPath(sha256) { return path.join(this.retainDir, String(sha256).toLowerCase()); }
@@ -328,16 +448,23 @@ export default class FileBackend extends StorageBackend {
328
448
  if (!sha256 || !await fs.pathExists(src)) return null;
329
449
  const dest = this.retainedPath(sha256);
330
450
  const existing = await fs.stat(dest).catch(() => null);
331
- if (existing) return { path: dest, size: existing.size, existing: true };
451
+ if (existing) {
452
+ if ((await checksumFile(dest, ['sha256'])).sha256 !== sha256) throw new Error('Retained content failed verification');
453
+ return { path: dest, size: existing.size, existing: true };
454
+ }
332
455
  await fs.ensureDir(this.retainDir);
456
+ // A network-share writer can keep an inode open and modify it in
457
+ // place. A hardlink is therefore not a recoverable version.
458
+ const tmp = await this.#stagingPath();
333
459
  try {
334
- await fs.link(src, dest);
335
- } catch (err) {
336
- if (!['EXDEV', 'EPERM', 'EMLINK', 'ENOTSUP', 'EOPNOTSUPP'].includes(err.code)) throw err;
337
- const tmp = await this.#stagingPath();
338
- try { await fs.copyFile(src, tmp); await fs.rename(tmp, dest); }
339
- finally { await fs.remove(tmp).catch(() => {}); }
340
- }
460
+ await fs.copyFile(src, tmp, fs.constants.COPYFILE_FICLONE);
461
+ const checksums = await checksumFile(tmp, ['sha256']);
462
+ if (checksums.sha256 !== sha256) throw Object.assign(new Error('File changed before retention'), { code: 'SOURCE_CHANGED' });
463
+ const handle = await fsp.open(tmp, 'r');
464
+ try { await handle.sync(); } finally { await handle.close(); }
465
+ await fs.rename(tmp, dest);
466
+ await this.#syncDirectory(path.dirname(dest));
467
+ } finally { await fs.remove(tmp).catch(() => {}); }
341
468
  const stats = await fs.stat(dest);
342
469
  debug(`RETAIN ${key} → ${sha256.slice(0, 12)} (${stats.size} bytes)`);
343
470
  return { path: dest, size: stats.size, existing: false };
@@ -357,6 +484,7 @@ export default class FileBackend extends StorageBackend {
357
484
  * EXDEV otherwise, which callers treat as "fall back to a stream copy".
358
485
  */
359
486
  async renameFrom(key, srcPath) {
487
+ await this.ready();
360
488
  const dest = this.#resolvePath(key);
361
489
  await fs.ensureDir(path.dirname(dest));
362
490
  await fs.rename(srcPath, dest);
@@ -366,6 +494,7 @@ export default class FileBackend extends StorageBackend {
366
494
  }
367
495
 
368
496
  async get(key, options = {}) {
497
+ await this.ready();
369
498
  const filePath = this.#resolvePath(key);
370
499
  if (!await fs.pathExists(filePath)) return null;
371
500
  return options.stream ? fs.createReadStream(filePath) : fs.readFile(filePath);
@@ -374,12 +503,14 @@ export default class FileBackend extends StorageBackend {
374
503
  // Ranged read for HTTP Range/streaming. `end` is inclusive (matches HTTP
375
504
  // Range and createReadStream). Null on miss → caller falls back to full read.
376
505
  async getRange(key, { start, end }) {
506
+ await this.ready();
377
507
  const filePath = this.#resolvePath(key);
378
508
  if (!await fs.pathExists(filePath)) return null;
379
509
  return fs.createReadStream(filePath, { start, end });
380
510
  }
381
511
 
382
512
  async delete(key) {
513
+ await this.ready();
383
514
  const filePath = this.#resolvePath(key);
384
515
  if (!await fs.pathExists(filePath)) return false;
385
516
  await fs.remove(filePath);
@@ -398,6 +529,7 @@ export default class FileBackend extends StorageBackend {
398
529
  async pruneEmptyParents(key) {
399
530
  let rel = path.posix.dirname(String(key).split(path.sep).join('/'));
400
531
  while (rel && rel !== '.' && rel !== '/') {
532
+ if (this.isDirectoryProtected?.(rel)) return;
401
533
  if (this.#isTempPath(rel) || (this.#tempRel && this.#tempRel.startsWith(`${rel}/`))) return;
402
534
  const dir = this.#resolvePath(rel);
403
535
  if (dir === this.#root) return;
@@ -413,6 +545,7 @@ export default class FileBackend extends StorageBackend {
413
545
  * on "when" without a second hash.
414
546
  */
415
547
  async utimes(key, mtime) {
548
+ await this.ready();
416
549
  const filePath = this.#resolvePath(key);
417
550
  const when = mtime instanceof Date ? mtime : new Date(typeof mtime === 'string' ? Date.parse(mtime) : Number(mtime));
418
551
  if (Number.isNaN(when.getTime())) throw new Error(`Invalid mtime: ${mtime}`);
@@ -421,6 +554,7 @@ export default class FileBackend extends StorageBackend {
421
554
  }
422
555
 
423
556
  async stat(key) {
557
+ await this.ready();
424
558
  const filePath = this.#resolvePath(key);
425
559
  let stats;
426
560
  try { stats = await fs.stat(filePath); } catch (error) {
@@ -437,6 +571,7 @@ export default class FileBackend extends StorageBackend {
437
571
  // is reported and skipped instead of aborting the walk — one denied subtree
438
572
  // must not kill (or worse, truncate) a whole-mount enumeration.
439
573
  async *list(options = {}) {
574
+ await this.ready();
440
575
  const { prefix = '', recursive = true, onError = null } = options;
441
576
  const searchPath = this.#resolvePath(prefix);
442
577
 
@@ -488,6 +623,11 @@ export default class FileBackend extends StorageBackend {
488
623
  return false;
489
624
  }
490
625
 
626
+ const epoch = this.#watchEpoch;
627
+ await this.ready();
628
+ if (epoch !== this.#watchEpoch) return false;
629
+ if (this.#watcher) return true;
630
+
491
631
  // `stabilityThreshold`: how long a file must stop growing before it is
492
632
  // hashed (a mirror bumps it — a multi-GB copy from a USB stick must
493
633
  // not be hashed mid-write). `followSymlinks`: chokidar's default is
@@ -660,6 +800,7 @@ export default class FileBackend extends StorageBackend {
660
800
  * folder skeleton (and size a progress bar) before the expensive scan.
661
801
  */
662
802
  async shape(prefix = '') {
803
+ await this.ready();
663
804
  const dirs = [];
664
805
  let files = 0;
665
806
  const walk = async (rel) => {
@@ -680,6 +821,7 @@ export default class FileBackend extends StorageBackend {
680
821
  }
681
822
 
682
823
  async stop() {
824
+ this.#watchEpoch++;
683
825
  if (this.#watcher) {
684
826
  await this.#watcher.close();
685
827
  this.#watcher = null;
@@ -62,11 +62,17 @@ const splitKey = (key) => {
62
62
  };
63
63
 
64
64
  class DriveError extends Error {
65
- constructor(message, { code = 'unreachable', status = null } = {}) {
66
- super(message);
65
+ constructor(message, { code = 'unreachable', status = null, cause = undefined } = {}) {
66
+ const transportCode = cause?.cause?.code || cause?.code;
67
+ // InvalidArgumentError's message names the rejected option/header.
68
+ // Keep it visible rather than reducing it to fetch's generic wrapper.
69
+ const argumentDetail = transportCode === 'UND_ERR_INVALID_ARG' ? cause?.cause?.message : null;
70
+ super(transportCode ? `${message} (${transportCode}${argumentDetail ? `: ${argumentDetail}` : ''})` : message, { cause });
67
71
  this.name = 'DriveError';
68
72
  this.code = code;
69
73
  this.status = status;
74
+ this.statusCode = code === 'invalid-request' ? 500 : 502;
75
+ if (transportCode) this.transportCode = transportCode;
70
76
  }
71
77
  }
72
78
 
@@ -89,6 +95,7 @@ export default class GdriveBackend extends StorageBackend {
89
95
 
90
96
  #pollTimer = null;
91
97
  #polling = false;
98
+ #creates = new Map();
92
99
  #pageToken = null;
93
100
 
94
101
  constructor(name, config = {}) {
@@ -100,7 +107,9 @@ export default class GdriveBackend extends StorageBackend {
100
107
  debug(`GdriveBackend "${name}" initialized (folder=${this.#rootRef})`);
101
108
  }
102
109
 
103
- get capabilities() { return { read: true, write: true, delete: true, canEnumerate: true, remote: true }; }
110
+ static getCapabilities(config = {}) {
111
+ return { ...super.getCapabilities(config), canEnumerate: true, remote: true, paths: true, mutableContainers: config.readOnly !== true, upload: config.readOnly !== true };
112
+ }
104
113
  get remote() { return true; }
105
114
  get transport() { return DEV; }
106
115
  get watching() { return !!this.#pollTimer; }
@@ -134,12 +143,17 @@ export default class GdriveBackend extends StorageBackend {
134
143
  }),
135
144
  });
136
145
  } catch (err) {
137
- throw new DriveError(`gdrive token endpoint unreachable: ${err.message}`, { code: 'unreachable' });
146
+ throw new DriveError(`gdrive token endpoint unreachable: ${err.message}`, { code: 'unreachable', cause: err });
138
147
  }
139
148
  const json = await res.json().catch(() => null);
140
149
  if (!res.ok || !json?.access_token) {
141
- const detail = json?.error_description || json?.error || `HTTP ${res.status}`;
142
- throw new DriveError(`gdrive token refresh failed: ${detail}`, { code: 'unauthorized', status: res.status });
150
+ const detail = [json?.error, json?.error_description].filter(Boolean).join(': ') || `HTTP ${res.status}`;
151
+ const hint = json?.error === 'invalid_grant'
152
+ ? ' Reconnect this Google account and save the new refresh token.'
153
+ : json?.error === 'invalid_client'
154
+ ? ' Check the OAuth client id and client secret.'
155
+ : '';
156
+ throw new DriveError(`gdrive token refresh failed: ${detail}${hint}`, { code: 'unauthorized', status: res.status });
143
157
  }
144
158
  this.#accessToken = json.access_token;
145
159
  this.#tokenExpiresAt = Date.now() + (Number(json.expires_in) || 3600) * 1000;
@@ -159,8 +173,11 @@ export default class GdriveBackend extends StorageBackend {
159
173
  try {
160
174
  res = await this.#fetch(url, { ...init, headers: { Authorization: `Bearer ${token}`, ...(init.headers || {}) } });
161
175
  } catch (err) {
176
+ if ((err.cause?.code || err.code) === 'UND_ERR_INVALID_ARG') {
177
+ throw new DriveError(`gdrive ${init.method || 'GET'} request arguments rejected: ${err.message}`, { code: 'invalid-request', cause: err });
178
+ }
162
179
  if (attempt < RETRY_DELAYS_MS.length) { await sleep(RETRY_DELAYS_MS[attempt]); continue; }
163
- throw new DriveError(`gdrive request failed: ${err.message}`, { code: 'unreachable' });
180
+ throw new DriveError(`gdrive ${init.method || 'GET'} ${new URL(url).origin}${new URL(url).pathname} request failed: ${err.message}`, { code: 'unreachable', cause: err });
164
181
  }
165
182
  if (res.ok || okStatuses.includes(res.status)) return res;
166
183
  if (res.status === 404 && allow404) return res;
@@ -509,7 +526,19 @@ export default class GdriveBackend extends StorageBackend {
509
526
  const { dir, name, key: clean } = splitKey(key);
510
527
  if (!clean) throw new Error('gdrive put: key is required');
511
528
  const parentId = await this.#folderId(dir, { create: true });
512
- const existing = await this.#lookup(clean);
529
+ let existing = null;
530
+ if (options.createOnly) {
531
+ // Query Drive, including folders and native documents: the local
532
+ // index and watcher may not have seen this name yet.
533
+ const found = await this.#json('/files', { query: {
534
+ q: `'${qEscape(parentId)}' in parents and name = '${qEscape(name)}' and trashed = false`,
535
+ fields: 'files(id)', pageSize: 1,
536
+ supportsAllDrives: true, includeItemsFromAllDrives: true,
537
+ } });
538
+ if (found?.files?.length) throw Object.assign(new Error(`File already exists: ${clean}`), { code: 'EEXIST' });
539
+ // Always POST. Drive permits duplicate names, so an external
540
+ // concurrent creator may produce a sibling, but is never patched.
541
+ } else existing = await this.#lookup(clean);
513
542
  const mimeType = options.mimeType || 'application/octet-stream';
514
543
  const source = Buffer.isBuffer(data) ? data : (typeof data === 'string' ? Buffer.from(data) : data);
515
544
  const size = Buffer.isBuffer(source) ? source.length : (Number.isFinite(options.size) ? options.size : null);
@@ -544,7 +573,12 @@ export default class GdriveBackend extends StorageBackend {
544
573
  : `bytes */${last ? total : '*'}`;
545
574
  const res = await this.#call(session, {
546
575
  method: 'PUT',
547
- headers: { 'Content-Length': String(buf.length), 'Content-Range': range, 'Content-Type': mimeType },
576
+ // Drive uses 308 as an acknowledgement of an incomplete upload,
577
+ // not an instruction to replay this chunk at Location.
578
+ redirect: 'manual',
579
+ // fetch derives Content-Length from this Buffer. Supplying it
580
+ // separately is redundant and can conflict with fetch's framing.
581
+ headers: { 'Content-Range': range, 'Content-Type': mimeType },
548
582
  body: buf,
549
583
  }, { okStatuses: [308] });
550
584
  if (last) {
@@ -587,6 +621,19 @@ export default class GdriveBackend extends StorageBackend {
587
621
  return this.put(key, createReadStream(srcPath), { size });
588
622
  }
589
623
 
624
+ async createFrom(key, srcPath) {
625
+ const clean = splitKey(key).key;
626
+ const previous = this.#creates.get(clean) || Promise.resolve();
627
+ const task = previous.catch(() => {}).then(async () => {
628
+ const { size } = await fsp.stat(srcPath);
629
+ await this.put(clean, createReadStream(srcPath), { size, createOnly: true });
630
+ return this.stat(clean);
631
+ });
632
+ this.#creates.set(clean, task);
633
+ try { return await task; }
634
+ finally { if (this.#creates.get(clean) === task) this.#creates.delete(clean); }
635
+ }
636
+
590
637
  /**
591
638
  * Remove the bytes at `key`. Trashes by default — a user's Drive is shared
592
639
  * with their other tools and the trash is their safety net; set
@@ -738,7 +785,7 @@ export default class GdriveBackend extends StorageBackend {
738
785
  }
739
786
 
740
787
  for (const row of walk.files.values()) {
741
- let checksums = null;
788
+ let checksums;
742
789
  const have = row.checksums || {};
743
790
  if (algorithms.every((a) => have[a])) {
744
791
  checksums = Object.fromEntries(algorithms.map((a) => [a, have[a]]));
@@ -22,7 +22,9 @@ export default class HttpBackend extends StorageBackend {
22
22
  }
23
23
 
24
24
  // Read-only source: bytes can't be removed remotely, only the reference dropped.
25
- get capabilities() { return { read: true, write: false, delete: false, canEnumerate: false }; }
25
+ static getCapabilities(config = {}) {
26
+ return { ...super.getCapabilities(config), write: false, delete: false, remote: true };
27
+ }
26
28
 
27
29
  nativeUrl(key) {
28
30
  try { return this.#baseUrl ? new URL(key, this.#baseUrl).href : key; }