@augmentd-labs/canvas-edge 0.0.0-stage → 0.4.1

package/node_modules/canvas-stored/src/backends/StorageBackend.js

@@ -0,0 +1,72 @@
+import { EventEmitter } from 'events';
+
+/**
+ * Base class for storage backends.
+ * All backends must implement: put, get, delete, stat, list
+ * Optional: watch, scan, stop
+ */
+export default class StorageBackend extends EventEmitter {
+    constructor(name, config = {}) {
+        super();
+        this.name = name;
+        this.type = 'base';
+        this.config = config;
+    }
+
+    /**
+     * Capability flags consumed by the Destroy/lifecycle logic.
+     * Default is fully read-write; read-only backends (e.g. HTTP) override.
+     * `delete:false` means a location on this backend can only be dropped as a
+     * reference — its bytes cannot be removed by us.
+     * `canEnumerate:true` means the backend implements the async-generator
+     * `list()` (and usually `scan()`) so callers can mirror its whole tree;
+     * drivers without real enumeration (http, s3 skeleton) leave it false.
+     * `remote` mirrors the getter below — carried here so a single capabilities
+     * snapshot is enough for consumers deciding how expensive an op will be.
+     */
+    get capabilities() { return { read: true, write: true, delete: true, canEnumerate: false, remote: this.remote }; }
+    get canDelete() { return this.capabilities.delete !== false; }
+    get canWrite() { return this.capabilities.write !== false; }
+    get canRead() { return this.capabilities.read !== false; }
+
+    /**
+     * Is the data behind this backend reached over the network?
+     *
+     * Distinct from `type` on purpose: `type: 'local'|'remote'` decides the
+     * WRITE path (synchronous put vs cache + SyncQueue), and an NFS/CIFS mount
+     * is still a synchronous POSIX write target — `type:'local'`, `remote:true`.
+     * This flag is about latency and trust: no reliable inotify, expensive
+     * full-file hashing, and a root that can disappear wholesale.
+     */
+    get remote() { return this.config.remote ?? (this.type === 'remote'); }
+
+    /** Transport serving the bytes ('cifs', 'nfs4', 'fuse.sshfs', 's3', …) or null. */
+    get transport() { return this.config.transport ?? null; }
+
+    // The real, protocol-native URL for `key` (e.g. https://…, s3://…, smb://…,
+    // file://…), used for provenance/UI. `stored://<backend>/<key>` remains the
+    // canonical fetch form. Returns null when the backend has no meaningful one.
+    nativeUrl(key) { return null; }
+
+    // Required methods - must be implemented by subclasses
+    async put(key, data) { throw new Error('Not implemented'); }
+    // Place an already-written file at `key` (streaming put commit). Local
+    // backends implement this; remote backends are fed via the cache + SyncQueue.
+    async commit(key, srcPath) { throw new Error('Not implemented'); }
+    async get(key, options = {}) { throw new Error('Not implemented'); }
+    async delete(key) { throw new Error('Not implemented'); }
+    async stat(key) { throw new Error('Not implemented'); }
+    async *list(options = {}) { throw new Error('Not implemented'); }
+
+    // Optional methods
+    async watch() { return false; }
+    async scan() { return []; }
+    async stop() { }
+
+    // Optional container (directory/folder) mutation — only backends with a real
+    // hierarchical namespace (the file driver) implement these. Advertised via
+    // the Workspace `mutableContainers` capability; callers must gate on it.
+    async createContainer(key) { throw new Error('Container ops not supported by this backend'); }
+    async deleteContainer(key) { throw new Error('Container ops not supported by this backend'); }
+    async renameContainer(fromKey, toKey) { throw new Error('Container ops not supported by this backend'); }
+}

package/node_modules/canvas-stored/src/backends/cacache/index.js

@@ -0,0 +1,130 @@
+import fs from 'fs-extra';
+import path from 'path';
+import { createReadStream } from 'fs';
+import { pipeline } from 'stream/promises';
+import cacache from 'cacache';
+import contentPath from 'cacache/lib/content/path.js';
+import Debug from 'debug';
+import StorageBackend from '../StorageBackend.js';
+
+const debug = Debug('stored:backend:cacache');
+
+// memoize:false — this store fronts blobs up to ~20GB across 1M+ entries; never
+// let cacache's in-memory LRU hold content. All bytes flow through streams.
+const CACHE_OPTS = { memoize: false };
+
+/**
+ * cacache storage backend — a content-addressable local blob store.
+ *
+ * Same key/value CRUD surface as FileBackend, but bytes live in a cacache
+ * content store (sha-keyed, deduped, integrity-checked) instead of a plain
+ * directory tree. Keys are arbitrary `stored://<backend>/<key>` keys mapped to
+ * content by cacache's index. `type = 'local'` so the Stored ingest path writes
+ * bytes immediately (no SyncQueue).
+ *
+ * There is no meaningful protocol-native URL (the store is internal), so
+ * `nativeUrl` is null and `stored://<backend>/<key>` is the only address.
+ *
+ * Not watched/scanned: it is a managed write target, not an external source —
+ * `watch`/`scan` inherit the no-op base behaviour.
+ *
+ * `config`: { root } — the cacache store directory ("the data route").
+ */
+export default class CacacheBackend extends StorageBackend {
+    #root;
+    #algorithms;
+
+    constructor(name, config = {}) {
+        super(name, config);
+        if (!config.root) throw new Error('CacacheBackend requires root path');
+        this.#root = path.resolve(config.root);
+        this.#algorithms = config.algorithms || ['sha256'];
+        this.type = 'local';
+        fs.ensureDirSync(this.#root);
+        debug(`CacacheBackend "${name}" initialized at ${this.#root}`);
+    }
+
+    get root() { return this.#root; }
+    // list() streams the cacache index, so full-key enumeration works even
+    // though the store is opaque byte-wise.
+    get capabilities() { return { ...super.capabilities, canEnumerate: true }; }
+    // Staging dir for the Stored streaming-ingest hash pass (shares a filesystem
+    // with the store so the temp file is cheap to read back on commit).
+    get tempDir() { return path.join(this.#root, 'tmp'); }
+
+    // Internal blob store: no protocol-native URL to surface.
+    nativeUrl(key) { return null; }
+
+    // ─────────────────────────────────────────────────────────────────────────
+    // CRUD Operations
+    // ─────────────────────────────────────────────────────────────────────────
+
+    async put(key, data) {
+        await cacache.put(this.#root, key, data, { algorithms: this.#algorithms, ...CACHE_OPTS });
+        const size = Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
+        debug(`PUT ${key} (${size} bytes)`);
+        return { key, size };
+    }
+
+    // Commit an already-written file (the Stored streaming-ingest temp) at `key`
+    // by streaming it into the content store.
+    async commit(key, srcPath) {
+        await pipeline(createReadStream(srcPath), cacache.put.stream(this.#root, key, { algorithms: this.#algorithms, ...CACHE_OPTS }));
+        const info = await cacache.get.info(this.#root, key);
+        const size = info?.size ?? 0;
+        debug(`COMMIT ${key} (${size} bytes)`);
+        return { key, size };
+    }
+
+    // Ranged read for HTTP Range/streaming: cacache has no ranged API, so read a
+    // byte window straight off the content-addressed blob file. `end` is
+    // inclusive (matches HTTP Range and fs.createReadStream). Returns null on a
+    // cache miss so the caller can fall back to a full read.
+    async getRange(key, { start, end }) {
+        const info = await cacache.get.info(this.#root, key);
+        if (!info) return null;
+        return createReadStream(contentPath(this.#root, info.integrity), { start, end });
+    }
+
+    async get(key, options = {}) {
+        if (options.stream) {
+            // get.stream emits ENOENT on miss; surface null instead of throwing.
+            const exists = await cacache.get.info(this.#root, key);
+            return exists ? cacache.get.stream(this.#root, key, CACHE_OPTS) : null;
+        }
+        try {
+            const { data } = await cacache.get(this.#root, key, CACHE_OPTS);
+            return data;
+        } catch (err) {
+            if (err.code === 'ENOENT') return null;
+            throw err;
+        }
+    }
+
+    async delete(key) {
+        const info = await cacache.get.info(this.#root, key);
+        if (!info) return false;
+        await cacache.rm.entry(this.#root, key, { removeFully: true });
+        debug(`DELETE ${key}`);
+        return true;
+    }
+
+    async stat(key) {
+        const info = await cacache.get.info(this.#root, key);
+        if (!info) return null;
+        return { key, size: info.size, modified: info.time, created: info.time };
+    }
+
+    async *list(options = {}) {
+        const { prefix = '' } = options;
+        // Stream the index so a 1M+ entry store is walked without ever building
+        // the full key→info map in memory.
+        for await (const info of cacache.ls.stream(this.#root)) {
+            if (prefix && !info.key.startsWith(prefix)) continue;
+            yield { key: info.key, size: info.size, modified: info.time, created: info.time };
+        }
+    }
+
+    // Drop the whole content store (index + content). Used by Destroy/reset paths.
+    async clear() { return cacache.rm.all(this.#root); }
+}