@augmentd-labs/canvas-edge 0.0.0-stage → 0.4.1

package/node_modules/canvas-stored/src/backends/http/index.js

@@ -0,0 +1,47 @@
+import Debug from 'debug';
+import StorageBackend from '../StorageBackend.js';
+
+const debug = Debug('stored:backend:http');
+
+/**
+ * HTTP(S) storage backend — SKELETON.
+ *
+ * Registered so `stored://http:<account>/<key>` URLs parse and dispatch.
+ * `get` has a minimal read-only fetch implementation (HTTP sources are
+ * commonly read-only); `put`/`delete`/`list` inherit `Not implemented`.
+ * `config` carries { baseUrl, account, headers? }.
+ */
+export default class HttpBackend extends StorageBackend {
+    #baseUrl;
+
+    constructor(name, config = {}) {
+        super(name, config);
+        this.type = 'remote';
+        this.#baseUrl = config.baseUrl || '';
+        debug(`HttpBackend "${name}" registered (skeleton; baseUrl=${this.#baseUrl || '?'})`);
+    }
+
+    // Read-only source: bytes can't be removed remotely, only the reference dropped.
+    get capabilities() { return { read: true, write: false, delete: false, canEnumerate: false }; }
+
+    nativeUrl(key) {
+        try { return this.#baseUrl ? new URL(key, this.#baseUrl).href : key; }
+        catch { return null; }
+    }
+
+    async get(key, options = {}) {
+        const url = this.#baseUrl ? new URL(key, this.#baseUrl).toString() : key;
+        const res = await fetch(url, { headers: this.config.headers || {} });
+        if (!res.ok) return null;
+        if (options.stream) return res.body;
+        return Buffer.from(await res.arrayBuffer());
+    }
+
+    async stat(key) {
+        const url = this.#baseUrl ? new URL(key, this.#baseUrl).toString() : key;
+        const res = await fetch(url, { method: 'HEAD', headers: this.config.headers || {} });
+        if (!res.ok) return null;
+        const size = Number(res.headers.get('content-length'));
+        return { key, size: Number.isFinite(size) ? size : null, modified: null, created: null };
+    }
+}

package/node_modules/canvas-stored/src/backends/s3/index.js

@@ -0,0 +1,25 @@
+import Debug from 'debug';
+import StorageBackend from '../StorageBackend.js';
+
+const debug = Debug('stored:backend:s3');
+
+/**
+ * S3 storage backend — SKELETON.
+ *
+ * Registered so `stored://s3:<account>/<key>` URLs parse and dispatch, but the
+ * CRUD methods are not yet implemented (they inherit `Not implemented` throws
+ * from StorageBackend). Wire up an S3 client (`@aws-sdk/client-s3`) in a
+ * follow-up: `config` carries { bucket, region, account, prefix?, credentials? }.
+ */
+export default class S3Backend extends StorageBackend {
+    constructor(name, config = {}) {
+        super(name, config);
+        this.type = 'remote';
+        debug(`S3Backend "${name}" registered (skeleton; bucket=${config.bucket ?? '?'})`);
+    }
+
+    nativeUrl(key) {
+        const { bucket, prefix = '' } = this.config;
+        return bucket ? `s3://${bucket}/${prefix}${key}` : null;
+    }
+}

package/node_modules/canvas-stored/src/cache/index.js

@@ -0,0 +1,65 @@
+import cacache from 'cacache';
+import Debug from 'debug';
+
+const debug = Debug('stored:cache');
+
+// memoize:false on every read/write — the cache fronts blobs up to ~20GB and is
+// hit by 1M+ entries; cacache's in-memory LRU memoizer would defeat the point of
+// a disk-backed store and risk OOM. Bytes always flow through the content store.
+const CACHE_OPTS = { memoize: false };
+
+export default class Cache {
+    #root;
+    #algorithms;
+
+    constructor(config) {
+        if (!config?.path) throw new Error('Cache path required');
+        this.#root = config.path;
+        this.#algorithms = config.algorithms || ['sha256'];
+        debug(`Cache initialized at "${this.#root}"`);
+    }
+
+    get root() { return this.#root; }
+
+    // Materializes the entire index into one object — fine for small stores, but
+    // prefer listStream() at scale (1M+ entries) to avoid the in-RAM map.
+    list() { return cacache.ls(this.#root); }
+
+    // Streaming index walk: yields one entry at a time, never building the whole
+    // index in memory. Use this for large stores.
+    listStream() { return cacache.ls.stream(this.#root); }
+
+    has(key) { return cacache.get.info(this.#root, key); }
+
+    put(key, data, metadata = {}) {
+        return cacache.put(this.#root, key, data, { algorithms: this.#algorithms, metadata, ...CACHE_OPTS });
+    }
+
+    putStream(key, metadata = {}) {
+        return cacache.put.stream(this.#root, key, { algorithms: this.#algorithms, metadata, ...CACHE_OPTS });
+    }
+
+    get(key) { return cacache.get(this.#root, key, CACHE_OPTS); }
+
+    getStream(key) { return cacache.get.stream(this.#root, key, CACHE_OPTS); }
+
+    getInfo(key) { return cacache.get.info(this.#root, key); }
+
+    delete(key) { return cacache.rm.entry(this.#root, key, { removeFully: true }); }
+
+    clear() { return cacache.rm.all(this.#root); }
+
+    verify() { return cacache.verify(this.#root); }
+
+    // Stream the index so a 1M+ entry store is tallied without ever holding the
+    // full key set in memory.
+    async stats() {
+        let entries = 0;
+        let size = 0;
+        for await (const entry of cacache.ls.stream(this.#root)) {
+            entries += 1;
+            size += entry.size || 0;
+        }
+        return { entries, size };
+    }
+}

package/node_modules/canvas-stored/src/extractors/audio.js

@@ -0,0 +1,53 @@
+'use strict';
+
+import debugInstance from 'debug';
+const debug = debugInstance('canvas:stored:extract:audio');
+
+let _mm;
+let _loaded = false;
+async function load() {
+    if (_loaded) { return; }
+    _loaded = true;
+    try { _mm = await import('music-metadata'); } catch { debug('music-metadata not installed — skipping audio tags'); }
+}
+
+function clean(obj) {
+    const out = {};
+    for (const [k, v] of Object.entries(obj)) {
+        if (v !== undefined && v !== null && v !== '') { out[k] = v; }
+    }
+    return out;
+}
+
+/**
+ * Extract audio tags (ID3/Vorbis) + format info → { media }. Never throws.
+ */
+export async function extractAudio(source, mimeType) {
+    await load();
+    if (!_mm) { return {}; }
+    try {
+        const meta = source.data
+            ? await _mm.parseBuffer(source.data, { mimeType })
+            : await _mm.parseFile(source.file);
+        const c = meta.common || {};
+        const f = meta.format || {};
+        const media = clean({
+            title: c.title,
+            artist: c.artist,
+            album: c.album,
+            year: c.year,
+            genre: Array.isArray(c.genre) ? c.genre[0] : c.genre,
+            track: c.track?.no ?? undefined,
+            duration: f.duration,
+            bitrate: f.bitrate,
+            sampleRate: f.sampleRate,
+            codec: f.codec,
+        });
+        return Object.keys(media).length ? { media } : {};
+    } catch (e) {
+        debug(`audio parse: ${e.message}`);
+        return {};
+    }
+}
+
+export default extractAudio;

package/node_modules/canvas-stored/src/extractors/image.js

@@ -0,0 +1,90 @@
+'use strict';
+
+import { readFile } from 'fs/promises';
+import debugInstance from 'debug';
+const debug = debugInstance('canvas:stored:extract:image');
+
+// Lazy, optional deps — resolved once. If absent, that sub-extraction no-ops.
+let _exifr;
+let _imageSize;
+let _loaded = false;
+async function load() {
+    if (_loaded) { return; }
+    _loaded = true;
+    try { _exifr = (await import('exifr')).default; } catch { debug('exifr not installed — skipping EXIF/GPS'); }
+    try {
+        const m = await import('image-size');
+        _imageSize = m.imageSize || m.default; // v2 named export / v1 default
+    } catch { debug('image-size not installed — skipping dimensions'); }
+}
+
+function clean(obj) {
+    const out = {};
+    for (const [k, v] of Object.entries(obj)) {
+        if (v !== undefined && v !== null && v !== '') { out[k] = v; }
+    }
+    return out;
+}
+
+/**
+ * Extract image dimensions + EXIF (incl. GPS geo + capture date). Returns
+ * { dimensions?, geo?, exif? }. Never throws.
+ */
+export async function extractImage(source, _mimeType) {
+    await load();
+    let buf = source.data;
+    if (!buf && source.file) {
+        try { buf = await readFile(source.file); } catch (e) { debug(`read failed: ${e.message}`); return {}; }
+    }
+    if (!buf) { return {}; }
+
+    const out = {};
+
+    if (_imageSize) {
+        try {
+            const d = _imageSize(buf);
+            if (d?.width && d?.height) {
+                out.dimensions = clean({ width: d.width, height: d.height, type: d.type, orientation: d.orientation });
+            }
+        } catch (e) { debug(`dimensions: ${e.message}`); }
+    }
+
+    if (_exifr) {
+        try {
+            const ex = await _exifr.parse(buf, { gps: true, tiff: true, exif: true, ifd0: true });
+            if (ex) {
+                if (typeof ex.latitude === 'number' && typeof ex.longitude === 'number') {
+                    // `source` marks provenance so consumers can reconcile this
+                    // against device/manual geo instead of last-writer-wins.
+                    // `accuracy` = GPSHPositioningError, the camera's own
+                    // horizontal error radius in metres — the thing that explains
+                    // a pin sitting a block away from where the photo was taken.
+                    out.geo = clean({
+                        lat: ex.latitude,
+                        lon: ex.longitude,
+                        alt: typeof ex.GPSAltitude === 'number' ? ex.GPSAltitude : undefined,
+                        accuracy: typeof ex.GPSHPositioningError === 'number' ? ex.GPSHPositioningError : undefined,
+                        source: 'exif',
+                    });
+                }
+                const date = ex.DateTimeOriginal || ex.CreateDate || ex.ModifyDate;
+                const exif = clean({
+                    make: ex.Make,
+                    model: ex.Model,
+                    lensModel: ex.LensModel,
+                    orientation: ex.Orientation,
+                    iso: ex.ISO,
+                    fNumber: ex.FNumber,
+                    exposureTime: ex.ExposureTime,
+                    focalLength: ex.FocalLength,
+                    capturedAt: date instanceof Date ? date.toISOString() : (typeof date === 'string' ? date : undefined),
+                });
+                if (Object.keys(exif).length) { out.exif = exif; }
+            }
+        } catch (e) { debug(`exif: ${e.message}`); }
+    }
+
+    return out;
+}
+
+export default extractImage;

package/node_modules/canvas-stored/src/extractors/index.js

@@ -0,0 +1,41 @@
+'use strict';
+
+import debugInstance from 'debug';
+const debug = debugInstance('canvas:stored:extract');
+
+import { extractImage } from './image.js';
+import { extractAudio } from './audio.js';
+import { extractText } from './text.js';
+
+/**
+ * Best-effort metadata extraction from blob bytes at ingest, dispatched by mime.
+ *
+ * Contract: NEVER throws — extraction failure must never fail a store. Parser
+ * libs (exifr, image-size, music-metadata) are lazy + OPTIONAL; if one isn't
+ * installed, that modality silently returns {}. Returns a flat metadata object
+ * ({ geo, exif, dimensions, media, text }) to be merged into the blob's `custom`
+ * record — never file bytes.
+ *
+ * `text` is the one derived FIELD rather than a fact about the container: a
+ * bounded head of a text blob, so the bytes are findable by what they SAY. A
+ * File is otherwise indexed by its name alone, which makes every note-shaped
+ * thing that arrived as a file invisible to a keyword search.
+ *
+ * @param {{file?:string, data?:Buffer}} source  temp-file path OR in-memory buffer
+ * @param {{mimeType?:string, key?:string}} [opts]
+ * @returns {Promise<object>} extracted metadata ({} when nothing/unsupported)
+ */
+export async function extract(source, { mimeType = '', key } = {}) {
+    try {
+        if (!source || (!source.file && !source.data)) { return {}; }
+        if (/^image\//.test(mimeType)) { return (await extractImage(source, mimeType)) || {}; }
+        if (/^audio\//.test(mimeType)) { return (await extractAudio(source, mimeType)) || {}; }
+        if (/^text\//.test(mimeType)) { return (await extractText(source, mimeType)) || {}; }
+        return {};
+    } catch (e) {
+        debug(`extract failed (${mimeType}, ${key}): ${e.message}`);
+        return {};
+    }
+}
+
+export default extract;

package/node_modules/canvas-stored/src/extractors/text.js

@@ -0,0 +1,65 @@
+'use strict';
+
+import fs from 'fs';
+
+/**
+ * How much of a text blob becomes searchable text.
+ *
+ * This is a search excerpt, not the file: it rides in the document's metadata,
+ * which every listing and every mount carries, so it has to stay small enough
+ * that a folder of source files is still cheap to list. 8 KB covers a note, a
+ * config, or a source file whole; a long log is indexed by its head and says so
+ * (`truncated`), which is honest about what a search can and cannot find.
+ */
+const MAX_TEXT_BYTES = 8192;
+
+// Text that is really bytes: a NUL in the first block means someone's mime says
+// text/* and means it (a mislabelled binary, a UTF-16 file). Indexing that
+// yields noise, so it is skipped rather than guessed at.
+function looksBinary(buffer) {
+    return buffer.includes(0);
+}
+
+/**
+ * The head of a text blob, as searchable text.
+ *
+ * Reads at most MAX_TEXT_BYTES — a temp file is opened and partially read
+ * rather than slurped, so extracting from a 2 GB log costs one block.
+ */
+export async function extractText(source, mimeType) {
+    let head;
+    if (source.data) {
+        head = source.data.subarray(0, MAX_TEXT_BYTES);
+    } else {
+        const handle = await fs.promises.open(source.file, 'r');
+        try {
+            const buffer = Buffer.allocUnsafe(MAX_TEXT_BYTES);
+            const { bytesRead } = await handle.read(buffer, 0, MAX_TEXT_BYTES, 0);
+            head = buffer.subarray(0, bytesRead);
+        } finally {
+            await handle.close();
+        }
+    }
+
+    if (head.length === 0 || looksBinary(head)) { return {}; }
+
+    // `fatal: false` + stream:true so a multi-byte character split by the read
+    // boundary is dropped rather than becoming a replacement char in the index.
+    const decoded = new TextDecoder('utf-8', { fatal: false }).decode(head, { stream: true });
+    const content = decoded.replace(/�/g, '').trim();
+    if (!content) { return {}; }
+
+    const size = source.data
+        ? source.data.length
+        : await fs.promises.stat(source.file).then((s) => s.size).catch(() => head.length);
+
+    return {
+        text: {
+            content,
+            truncated: size > head.length,
+            mimeType: mimeType || 'text/plain',
+        },
+    };
+}
+
+export default extractText;