@augmentd-labs/canvas-edge 0.6.2 → 0.7.0

package/node_modules/canvas-stored/src/runtime/MutationQueue.js

@@ -0,0 +1,60 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { performance } from 'node:perf_hooks';
+import { setImmediate as yieldToIO } from 'node:timers/promises';
+
+/** One physical mutation executor. Nested operations join their owner's scope;
+ * observers must detach before scheduling independent work. Diagnostics never
+ * abandon a write or release its lock early.
+ */
+export default class MutationQueue {
+    #context = new AsyncLocalStorage();
+    #tail = Promise.resolve();
+    #waiting = 0;
+    #active = null;
+    #closed = false;
+    #timer = null;
+    #report;
+    #threshold;
+    #completed = 0;
+    constructor({ onStall = null, stallMs = 5000 } = {}) {
+        this.#report = onStall;
+        this.#threshold = Math.max(1, stallMs);
+    }
+    get diagnostics() {
+        return { queued: this.#waiting, active: this.#active
+            ? { operation: this.#active.operation, elapsedMs: Math.round(performance.now() - this.#active.started) } : null };
+    }
+    detach(action) { return this.#context.exit(action); }
+    close() { this.#closed = true; }
+    drain() { return this.#tail.catch(() => {}); }
+    run(action, operation = 'mutation') {
+        if (this.#context.getStore()?.active) return action();
+        if (this.#closed) return Promise.reject(Object.assign(new Error('Stored is stopping'), { code: 'STOPPED' }));
+        this.#waiting++;
+        if (!this.#timer && this.#report) {
+            this.#timer = setInterval(() => {
+                const state = this.diagnostics;
+                if (state.active?.elapsedMs >= this.#threshold) {
+                    try { this.#report(state); } catch { /* diagnostic only */ }
+                }
+            }, this.#threshold);
+            this.#timer.unref?.();
+        }
+        const next = this.#tail.catch(() => {}).then(async () => {
+            // Make progress on HTTP, cancellation and watcher callbacks even
+            // when thousands of queued mutations resolve from memory.
+            if (++this.#completed % 32 === 0) await yieldToIO();
+            this.#waiting--;
+            this.#active = { operation, started: performance.now() };
+            const scope = { active: true };
+            try { return await this.#context.run(scope, action); }
+            finally {
+                scope.active = false;
+                this.#active = null;
+                if (!this.#waiting) { clearInterval(this.#timer); this.#timer = null; }
+            }
+        });
+        this.#tail = next;
+        return next;
+    }
+}

package/node_modules/canvas-stored/src/storage/lmdb.js

@@ -0,0 +1,62 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { open } from 'lmdb';
+
+const VERIFY_WORKER = fileURLToPath(new URL('./verify-worker.js', import.meta.url));
+const VERIFY_TIMEOUT_MS = 30 * 60 * 1000;
+
+/** The LMDB environment behind the index and ledgers (compressed, named sub-dbs). */
+export function openKeyValueStore({ path: dbPath, name, compression = true, maxDbs = 64 }) {
+    if (!dbPath) throw new Error('LMDB database path not provided');
+    return open({ path: dbPath, name, compression, maxDbs, cache: false, strictAsyncOrder: false });
+}
+
+/**
+ * Check an existing environment in a child process before opening it in this
+ * one: a corrupt map can SIGBUS, which try/catch cannot contain. Walks every
+ * named database read-only. Does not repair, truncate, replace or create
+ * data.mdb. Returns null when there is no database yet.
+ */
+export async function verifyDatabase(directory) {
+    const databasePath = path.resolve(directory);
+    const file = path.join(databasePath, 'data.mdb');
+    try {
+        let stat;
+        try { stat = await fs.stat(file); }
+        catch (error) { if (error.code === 'ENOENT') return null; throw error; }
+        if (!stat.isFile()) throw new Error('Expected data.mdb to be a regular file');
+        if (stat.size === 0) throw Object.assign(new Error('Empty data.mdb'), { canQuarantine: true });
+        return await runVerifyWorker(databasePath);
+    } catch (cause) {
+        throw Object.assign(new Error(`Database verification failed for ${file}: ${cause.message}. No repair or replacement was attempted.`, { cause }),
+            { code: 'DATABASE_INTEGRITY_FAILED', databasePath: file, canQuarantine: cause.canQuarantine === true, statusCode: 500 });
+    }
+}
+
+function runVerifyWorker(databasePath) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(process.execPath, [VERIFY_WORKER, databasePath], {
+            stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, NODE_OPTIONS: '' },
+        });
+        let output = '';
+        let errors = '';
+        child.stdout.on('data', data => { output += data; });
+        child.stderr.on('data', data => { errors = (errors + data).slice(-4096); });
+        let timedOut = false;
+        const timeout = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, VERIFY_TIMEOUT_MS);
+        child.on('error', error => { clearTimeout(timeout); reject(error); });
+        child.on('close', (code, signal) => {
+            clearTimeout(timeout);
+            if (timedOut) { reject(new Error('Database verification timed out')); return; }
+            if (code !== 0) {
+                const canQuarantine = ['SIGABRT', 'SIGBUS', 'SIGSEGV'].includes(signal)
+                    || /MDB_(CORRUPTED|PAGE_NOTFOUND|INVALID)\b|Unreadable entry in |Incomplete dataset: /.test(errors);
+                reject(Object.assign(new Error(`verify worker ${signal || `exit ${code}`}: ${errors.trim()}`), { canQuarantine }));
+                return;
+            }
+            try { resolve(JSON.parse(output)); } catch { reject(new Error('Invalid verify worker response')); }
+        });
+    });
+}

package/node_modules/canvas-stored/src/storage/recover-index.js

@@ -0,0 +1,33 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { verifyDatabase } from './lmdb.js';
+
+/** Prepare the Stored index before opening LMDB. Only an unreadable environment
+ * is quarantined; infrastructure failures must not discard a usable index.
+ * The entire environment is retained, including operational sub-databases.
+ */
+export async function prepareIndex(directory, { verify = verifyDatabase } = {}) {
+    const databasePath = path.resolve(directory);
+    try {
+        await verify(databasePath);
+        return null;
+    } catch (error) {
+        if (error.code !== 'DATABASE_INTEGRITY_FAILED' || error.canQuarantine !== true) throw error;
+        // Renaming a symlink would leave its actual environment in place.
+        if (!(await fs.lstat(databasePath)).isDirectory()) throw error;
+        const createdAt = new Date().toISOString();
+        const quarantinePath = await fs.mkdtemp(`${databasePath}.quarantine-${createdAt.replaceAll(':', '-')}-`);
+        const recovery = {
+            databasePath,
+            quarantinePath,
+            preservedIndexPath: path.join(quarantinePath, 'index'),
+            createdAt,
+            reason: error.cause?.message || error.message,
+        };
+        // Write the report before moving anything. A rename failure propagates:
+        // never open a fresh index unless the original was preserved successfully.
+        await fs.writeFile(path.join(quarantinePath, 'recovery.json'), JSON.stringify(recovery, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
+        await fs.rename(databasePath, recovery.preservedIndexPath);
+        return recovery;
+    }
+}

package/node_modules/canvas-stored/src/storage/verify-worker.js

@@ -0,0 +1,23 @@
+// Native faults stay in this disposable process, never in the caller.
+import { open } from 'lmdb';
+
+const db = open({ path: process.argv[2], readOnly: true, create: false, maxDbs: 256,
+    compression: false, encoding: 'binary', keyEncoding: 'binary' });
+try {
+    const datasets = {};
+    // The main database holds the named-database descriptors; discover them
+    // rather than miss a store. Finish the root cursor before openDB, which
+    // renews the read transaction.
+    const names = Array.from(db.getKeys(), key => key.toString('utf8').replace(/\0$/, ''));
+    for (const name of names) {
+        const store = db.openDB(name, { create: false, encoding: 'binary', keyEncoding: 'binary' });
+        if (!store) throw new Error(`Missing dataset: ${name}`);
+        let count = 0;
+        for (const { key, value } of store.getRange()) { if (key == null || value == null) throw new Error(`Unreadable entry in ${name}`); count++; }
+        if (count !== store.getStats().entryCount) throw new Error(`Incomplete dataset: ${name}`);
+        datasets[name] = { count };
+    }
+    process.stdout.write(JSON.stringify({ datasets }));
+} finally {
+    await db.close();
+}

package/node_modules/canvas-stored/src/sync/Ledger.js

@@ -1,4 +1,4 @@
-import { open } from 'lmdb';
+import { openKeyValueStore } from '../storage/lmdb.js';
 import Debug from 'debug';
 
 const debug = Debug('stored:sync:ledger');
@@ -29,7 +29,7 @@ export default class Ledger {
         if (index && typeof index.openDB === 'function') {
             this.#db = index.openDB(name);
         } else if (path) {
-            this.#own = open({ path, name, compression: true });
+            this.#own = openKeyValueStore({ path, name, compression: true });
             this.#db = this.#own;
         } else {
             throw new Error('Ledger requires { index } or { path }');
@@ -71,6 +71,24 @@ export default class Ledger {
         });
     }
 
+    /**
+     * Re-key every base under `from/` to `to/` in one transaction (a folder
+     * rename). `patch(toKey)` may add fields. Returns the new keys.
+     */
+    moveBases(from, to, patch = () => ({})) {
+        const rows = [...this.bases(`${from}/`)];
+        return this.#db.transactionSync(() => {
+            const moved = [];
+            for (const [key, base] of rows) {
+                const next = `${to}/${key.slice(from.length + 1)}`;
+                this.#db.removeSync(this.#k('base', key));
+                this.#db.putSync(this.#k('base', next), { ...base, ...patch(next), at: Date.now() });
+                moved.push(next);
+            }
+            return moved;
+        });
+    }
+
     /** All bases in key order: yields `[key, base]`. */
     *bases(prefix = '') {
         const start = this.#k('base', prefix);
@@ -133,6 +151,6 @@ export default class Ledger {
     }
 
     close() {
-        if (this.#own) { this.#own.close(); this.#own = null; }
+        if (this.#own) { const closing = this.#own.close(); this.#own = null; return closing; }
     }
 }