@augmentd-labs/canvas-edge 0.7.1 → 0.7.2

package/node_modules/canvas-stored/README.md

@@ -317,7 +317,7 @@ await mirror.stop();            // drains running jobs
 
 - **View**: Stored's index holds both sides — local keys through the file watcher/scan, hub keys through the `canvas` driver's feed poller. **Base ledger** (`Ledger`, LMDB sub-db `mirror`): `base/<mirror>/<key>` → `{ sha256, size, mtime, remoteSeq }`, `cursor/<mirror>`, `skip/<mirror>/<key>`, `state/<mirror>`. Keys + digests are the identity; hub document ids are never stored.
 - **Decision**: per dirty key, the protocol's three-way table (`decide(L, B, R)`, exported from `src/sync/Mirror.js`); R is confirmed with `HEAD` before a push or a conflict.
-- **Jobs** (`JobQueue`, LMDB sub-dbs `jobs` + `jobs.dedupe`): `push`, `pull`, `delete-remote`, `trash-local`, `rename-remote`, `rename-remote-dir`, `rename-local`, `conflict`; dedupe per `<kind>|<key>`, backoff `min(60s·2^n, 1h)`, running→pending recovery on open, one job per key in flight, concurrency 4 (a brand-new key skips the pre-push HEAD; the PUT's If-None-Match:* guards it). Hub requests abort after `requestTimeoutMs` (60 s) plus an upload allowance (`minUploadBytesPerSec`, 50 kB/s) so a dropped connection never pins a worker; a directory rename gets 250 ms more per file under the folder. Transfers to and from the hub run outside the storage mutation queue (an upload streams unqueued, a download is staged and only its commit is queued), so the four workers overlap on the network. Every byte move goes through `stored.copy`/`move`/`renameObject`/`removeObject`, so locations, succession and echo suppression are the ordinary ones. Base is written only after the operation succeeded; the feed cursor after a whole batch reconciled.
+- **Jobs** (`JobQueue`, LMDB sub-dbs `jobs` + `jobs.dedupe`): `push`, `pull`, `delete-remote`, `trash-local`, `rename-remote`, `rename-remote-dir`, `rename-local`, `conflict`; dedupe per `<kind>|<key>`, backoff `min(60s·2^n, 1h)`, running→pending recovery on open, one job per key in flight, concurrency 4 (a brand-new key skips the pre-push HEAD; the PUT's If-None-Match:* guards it). Hub requests abort after `requestTimeoutMs` (60 s) plus an upload allowance (`minUploadBytesPerSec`, 50 kB/s) so a dropped connection never pins a worker. A directory rename the hub answers `202` (still running) is requested again with the same `operationId` until the hub reports the result. Requests to the hub run outside the storage mutation queue (an upload streams unqueued, a download is staged and only its commit is queued, a delete or a single-file rename is one request and one index transaction), so the four workers overlap on the network. Every byte move goes through `stored.copy`/`move`/`renameObject`/`removeObject`, so locations, succession and echo suppression are the ordinary ones. Base is written only after the operation succeeded; the feed cursor after a whole batch reconciled.
 - **Conflicts** (`prompt`): the device version is copied to the `conflicts` backend as `<stem> (conflict from <device> <YYYY-MM-DD HHmm>).<ext>`, uploaded with `X-Canvas-Conflict-Of` (inbox), the hub version is pulled into place, base := R. `rename`: the upload is an ordinary object at the conflict-copy key (the hub tags it); the copy then arrives on every device like any other object.
 - **Renames**: a local `mv` (paired by Stored's inode window, or by content after downtime) → `POST objects/rename`. Renames are collected for `moveSettleMs` (1.5 s); when a whole folder went (the source folder is gone from the device, the target is new to the hub) they are sent as one request with `directory: true`, whatever the number of files. An inferred local deletion waits `deleteDelayMs` (5 s) before it is sent, so the second half of a move can still claim it. A hub rename → local `rename(2)` when the local copy is clean. **Hub delete** → local copy to the `trash` backend. **Offline** is a state: jobs stay queued, the engine retries with backoff and catches up (`poll`, or a full listing after `410`).
 - **Keys**: `src/sync/keys.js` — `normalizeKey` (NFC, `/`), `validateKey` (traversal, NUL, Windows reserved names, `<>:"|?*`, trailing dot/space, length) → skipped visibly (`skip` event, `status().skips`), `isIgnored(key, patterns)` (picomatch, `dot:true`), `matchesPrefixes`, `conflictKey`, `MIRROR_IGNORE_DEFAULTS`.

package/node_modules/canvas-stored/package.json

@@ -1,6 +1,6 @@
 {
   "name": "canvas-stored",
-  "version": "1.19.0",
+  "version": "1.20.0",
   "description": "",
   "main": "./src/index.js",
   "type": "module",

package/node_modules/canvas-stored/src/backends/canvas/index.js

@@ -42,8 +42,8 @@ const RETRY_DELAYS_MS = [500, 1500, 4000];
 // large file on a slow uplink is never cut short.
 const REQUEST_TIMEOUT_MS = 60_000;
 const MIN_UPLOAD_BYTES_PER_SEC = 50_000;
-// Extra time a directory rename is given per file the hub has under it.
-const DIRECTORY_RENAME_MS_PER_FILE = 250;
+// Pause between two requests for a directory rename the hub is still running.
+const DIRECTORY_RENAME_POLL_MS = [500, 1000, 2000, 5000];
 const JSON_HEADERS = { Accept: 'application/json' };
 
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -198,10 +198,10 @@ export default class CanvasBackend extends StorageBackend {
      * body is sent once and a retryable status surfaces as `RETRYABLE`.
      * `okStatuses` widens success (304, 404 for stat-like calls).
      */
-    async #call(url, init = {}, { okStatuses = [], key = null, uploadBytes = null, extraMs = 0 } = {}) {
+    async #call(url, init = {}, { okStatuses = [], key = null, uploadBytes = null } = {}) {
         const { body: bodySpec, ...rest } = init;
         const streaming = bodySpec != null && typeof bodySpec !== 'function' && !Buffer.isBuffer(bodySpec) && typeof bodySpec !== 'string';
-        const timeoutMs = this.#requestTimeoutMs + extraMs + (uploadBytes > 0 ? Math.ceil(uploadBytes / this.#minUploadBytesPerSec) * 1000 : 0);
+        const timeoutMs = this.#requestTimeoutMs + (uploadBytes > 0 ? Math.ceil(uploadBytes / this.#minUploadBytesPerSec) * 1000 : 0);
         let last;
         for (let attempt = 0; ; attempt += 1) {
             const body = typeof bodySpec === 'function' ? bodySpec() : bodySpec;
@@ -570,8 +570,10 @@ export default class CanvasBackend extends StorageBackend {
 
     /**
      * `POST objects/rename` with `directory: true`: the hub moves the folder
-     * and everything under it in one operation. `operationId` makes a retry
-     * after a lost response return the first result.
+     * and everything under it in one operation. A large folder is answered
+     * `202` while the hub works; the same request is repeated until it
+     * reports the result. `operationId` makes every repeat, and a retry
+     * after a lost response, return the first result.
      */
     async renameDirectory(from, to, { operationId, origin = null } = {}) {
         const fromKey = normalizeKey(from);
@@ -579,15 +581,18 @@ export default class CanvasBackend extends StorageBackend {
         if (!fromKey || !toKey || !operationId) throw new CanvasHubError('canvas directory rename: from, to and operationId are required', { code: 'REFUSED' });
         const body = { from: fromKey, to: toKey, directory: true, operationId };
         if (origin || this.#deviceId) body.origin = origin || this.#deviceId;
-        // The hub re-indexes every file under the folder before it answers.
         const prefix = `${fromKey}/`;
-        let files = 0;
-        for (const key of this.#files.keys()) if (key.startsWith(prefix)) files += 1;
-        const { payload } = await this.#json(`${this.#backendUrl()}/objects/rename`, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify(body),
-        }, { key: fromKey, extraMs: files * DIRECTORY_RENAME_MS_PER_FILE });
+        let payload;
+        for (let attempt = 0; ; attempt += 1) {
+            const answer = await this.#json(`${this.#backendUrl()}/objects/rename`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(body),
+            }, { key: fromKey });
+            payload = answer.payload;
+            if (answer.status !== 202) break;
+            await sleep(DIRECTORY_RENAME_POLL_MS[Math.min(attempt, DIRECTORY_RENAME_POLL_MS.length - 1)]);
+        }
         if (payload?.directory !== true) throw new CanvasHubError('the hub did not acknowledge a directory rename', { code: 'REFUSED', hubCode: 'NOT_IMPLEMENTED', key: fromKey });
         for (const [key, row] of [...this.#files]) {
             if (!key.startsWith(prefix)) continue;

package/node_modules/canvas-stored/src/index.js

@@ -103,6 +103,7 @@ export default class Stored extends EventEmitter2 {
     #sweepTimer = null;
     #directoryMovesDb = null;
     #directoryMoveTasks = new Map();
+    #moveUnqueued;
     #replicas;
 
     constructor(config = {}) {
@@ -114,9 +115,15 @@ export default class Stored extends EventEmitter2 {
         for (const name of ['put', 'copy', 'move', 'writeObject', 'removeObject', 'removeToTrash', 'renameObject', 'renameDirectory', 'trashDirectory', 'removeDirectory',
             'ensureDirectory', 'removeEmptyDirectory', 'restoreTrash', 'discardTrash', 'restoreRetained', 'deleteByUrl', 'delete']) {
             const method = this[name].bind(this);
-            this[name] = name === 'copy'
-                ? (idOrKey, options = {}) => this.#routeCopy(method, idOrKey, options)
-                : (...args) => this.withStorageMutation(() => method(...args), name);
+            if (name === 'move') this.#moveUnqueued = method;
+            if (name === 'copy') this[name] = (idOrKey, options = {}) => this.#routeCopy(method, idOrKey, options);
+            // A delete or rename on a streaming remote is one request and one
+            // index transaction, so several run side by side.
+            else if (name === 'removeObject' || name === 'renameObject') {
+                this[name] = (backendName, ...rest) => (this.#streamsRemote(backendName)
+                    ? method(backendName, ...rest)
+                    : this.withStorageMutation(() => method(backendName, ...rest), name));
+            } else this[name] = (...args) => this.withStorageMutation(() => method(...args), name);
         }
         this.#paths = resolveStoredPaths(config);
         this.#config = {
@@ -987,7 +994,8 @@ export default class Stored extends EventEmitter2 {
             return { ok: false, reason: 'target-exists', key: toKey };
         }
 
-        const result = await this.move(`${backendName}:${fromKey}`, {
+        const move = this.#streamsRemote(backendName) ? this.#moveUnqueued : this.move;
+        const result = await move(`${backendName}:${fromKey}`, {
             to: backendName,
             key: toKey,
             from: `stored://${backendName}/${fromKey}`,
@@ -1210,7 +1218,7 @@ export default class Stored extends EventEmitter2 {
     async #routeCopy(method, idOrKey, options) {
         const names = (Array.isArray(options.to) ? options.to : [options.to]).filter(Boolean).map(String);
         const targets = names.map((name) => this.#backends.get(name));
-        if (targets.length === 1 && targets[0]?.type === 'remote' && typeof targets[0].putStream === 'function') {
+        if (names.length === 1 && this.#streamsRemote(names[0])) {
             return method(idOrKey, options);
         }
         const staged = this.#closing ? null : await this.#stageRemoteSource(idOrKey, options, targets);
@@ -1222,6 +1230,13 @@ export default class Stored extends EventEmitter2 {
         }
     }
 
+    // A remote that takes writes directly (the canvas driver): it holds no
+    // local bytes, so its requests need no place in the mutation queue.
+    #streamsRemote(backendName) {
+        const backend = this.#backends.get(String(backendName));
+        return backend?.type === 'remote' && typeof backend.putStream === 'function';
+    }
+
     // Download a remote source next to its first local target. Null when the
     // source is not remote or cannot be read; the queued copy then reports it.
     async #stageRemoteSource(idOrKey, options, targets) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@augmentd-labs/canvas-edge",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "description": "Canvas device runtime: folder mirrors (canvas-stored Mirror engine) + edge tunnel client (dist artifact)",
   "license": "AGPL-3.0-or-later",
   "type": "module",
@@ -21,7 +21,7 @@
   },
   "dependencies": {
     "@augmentd-labs/canvas-protocol": "^0.5.0",
-    "canvas-stored": "1.19.0",
+    "canvas-stored": "1.20.0",
     "@aws-sdk/client-s3": "^3.1149.0",
     "cacache": "^20.0.4",
     "chokidar": "^5.0.0",
@@ -48,6 +48,6 @@
   "bundleDependencies": [
     "canvas-stored"
   ],
-  "canvasRev": "a499f3e",
+  "canvasRev": "8758ca3",
   "canvasSource": "runtimes/edge"
 }