@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

package/src/services/blobs/blob-manifest.ts

@@ -0,0 +1,227 @@
+import { RecordId } from 'surrealdb';
+import type { LocalStore, Row } from '../database/cache-engine';
+import type { BlobKey } from './blob-store';
+import { blobKeyId } from './blob-store';
+
+/**
+ * Metadata index for the blob cache, persisted in the `_00_blob` table of the
+ * local engine.
+ *
+ * The manifest is deliberately NOT the source of truth for existence — OPFS is.
+ * Several existing recovery paths destroy the local store while leaving OPFS
+ * intact (the SQLite leader's wipe-on-pool-open, the memory fallback when OPFS
+ * refuses to open for SQLite, IndexedDB corruption recovery on the SurrealDB
+ * engine). Treating the manifest as an index that `reconcile()` can rebuild
+ * from a directory walk means those paths cost metadata, not the offline cache.
+ *
+ * Runtime reads hit the in-memory map; writes are batched. `lastAccess` changes
+ * on every single cache hit, so flushing each one would turn an image render
+ * into a local-DB write.
+ */
+
+export const BLOB_TABLE = '_00_blob';
+
+export interface BlobEntry {
+  /** `${bucket}/${path}` — also the `_00_blob` row id. */
+  id: string;
+  bucket: string;
+  path: string;
+  size: number;
+  contentType: string;
+  createdAt: number;
+  lastAccess: number;
+  hits: number;
+  /** Exempt from pressure eviction. Never expires on its own. */
+  pinned: boolean;
+}
+
+export function entryKey(entry: BlobEntry): BlobKey {
+  return { bucket: entry.bucket, path: entry.path };
+}
+
+function rowToEntry(id: string, row: Row): BlobEntry | null {
+  const bucket = typeof row.bucket === 'string' ? row.bucket : '';
+  const path = typeof row.path === 'string' ? row.path : '';
+  if (!bucket || !path) return null;
+  return {
+    id,
+    bucket,
+    path,
+    size: Number(row.size) || 0,
+    contentType: typeof row.contentType === 'string' ? row.contentType : '',
+    createdAt: Number(row.createdAt) || 0,
+    lastAccess: Number(row.lastAccess) || 0,
+    hits: Number(row.hits) || 0,
+    pinned: row.pinned === true,
+  };
+}
+
+function entryToRow(entry: BlobEntry): Row {
+  return {
+    bucket: entry.bucket,
+    path: entry.path,
+    size: entry.size,
+    contentType: entry.contentType,
+    createdAt: entry.createdAt,
+    lastAccess: entry.lastAccess,
+    hits: entry.hits,
+    pinned: entry.pinned,
+  };
+}
+
+export class BlobManifest {
+  private entries = new Map<string, BlobEntry>();
+  /** Ids whose in-memory state has not been written back yet. */
+  private dirty = new Set<string>();
+  private removed = new Set<string>();
+  private flushing: Promise<void> | null = null;
+
+  constructor(private local: LocalStore) {}
+
+  /**
+   * Hydrate from the rows matching `keys`. Ids come from the OPFS listing, so
+   * this never needs a full-table scan (and therefore never needs a QueryPlan).
+   * Any read failure yields an empty manifest: reconcile then rebuilds every
+   * row from disk, which is exactly the desired degradation.
+   */
+  async load(ids: string[]): Promise<void> {
+    this.entries.clear();
+    this.dirty.clear();
+    this.removed.clear();
+    if (ids.length === 0) return;
+    let rows: Row[] = [];
+    try {
+      rows = await this.local.selectByIds(
+        BLOB_TABLE,
+        ids.map((id) => new RecordId(BLOB_TABLE, id))
+      );
+    } catch {
+      return;
+    }
+    for (const row of rows) {
+      const id = readRowId(row);
+      if (!id) continue;
+      const entry = rowToEntry(id, row);
+      if (entry) this.entries.set(id, entry);
+    }
+  }
+
+  get(key: BlobKey): BlobEntry | undefined {
+    return this.entries.get(blobKeyId(key));
+  }
+
+  getById(id: string): BlobEntry | undefined {
+    return this.entries.get(id);
+  }
+
+  all(): BlobEntry[] {
+    return [...this.entries.values()];
+  }
+
+  totalBytes(): number {
+    let total = 0;
+    for (const entry of this.entries.values()) total += entry.size;
+    return total;
+  }
+
+  pinnedBytes(): number {
+    let total = 0;
+    for (const entry of this.entries.values()) if (entry.pinned) total += entry.size;
+    return total;
+  }
+
+  put(entry: BlobEntry): void {
+    this.entries.set(entry.id, entry);
+    this.removed.delete(entry.id);
+    this.dirty.add(entry.id);
+  }
+
+  touch(id: string, now: number): void {
+    const entry = this.entries.get(id);
+    if (!entry) return;
+    entry.lastAccess = now;
+    entry.hits += 1;
+    this.dirty.add(id);
+  }
+
+  setPinned(id: string, pinned: boolean): boolean {
+    const entry = this.entries.get(id);
+    if (!entry || entry.pinned === pinned) return false;
+    entry.pinned = pinned;
+    this.dirty.add(id);
+    return true;
+  }
+
+  remove(id: string): void {
+    if (!this.entries.delete(id)) return;
+    this.dirty.delete(id);
+    this.removed.add(id);
+  }
+
+  /** Forget everything without scheduling deletes — for a bucket switch, where
+   *  the rows belong to the store we are leaving and must stay put. */
+  reset(): void {
+    this.entries.clear();
+    this.dirty.clear();
+    this.removed.clear();
+  }
+
+  hasPendingWrites(): boolean {
+    return this.dirty.size > 0 || this.removed.size > 0;
+  }
+
+  /**
+   * Write back pending changes. Serialized: a second concurrent flush awaits
+   * the first rather than racing it into the same rows. Failures are swallowed
+   * on purpose — a lost metadata write costs an LRU timestamp, and the entry is
+   * rebuilt from disk on the next reconcile.
+   */
+  async flush(): Promise<void> {
+    if (this.flushing) return this.flushing;
+    if (!this.hasPendingWrites()) return;
+    const run = this.doFlush();
+    this.flushing = run;
+    try {
+      await run;
+    } finally {
+      this.flushing = null;
+    }
+  }
+
+  private async doFlush(): Promise<void> {
+    const dirty = [...this.dirty];
+    const removed = [...this.removed];
+    this.dirty.clear();
+    this.removed.clear();
+    for (const id of dirty) {
+      const entry = this.entries.get(id);
+      if (!entry) continue;
+      try {
+        // A RecordId, never a bare string: the SurrealDB engine binds the id
+        // verbatim and `UPSERT <string>` is an InternalError, so a string id
+        // silently never lands (see the same note on `_00_preload`).
+        await this.local.upsert(BLOB_TABLE, new RecordId(BLOB_TABLE, id), entryToRow(entry), 'replace');
+      } catch {
+        /* metadata only — rebuilt from disk on the next reconcile */
+      }
+    }
+    for (const id of removed) {
+      try {
+        await this.local.delete(BLOB_TABLE, new RecordId(BLOB_TABLE, id));
+      } catch {
+        /* a surviving row with no file is dropped by the next reconcile */
+      }
+    }
+  }
+}
+
+/** Row ids come back as a `RecordId` (SurrealDB) or a `table:id` string (SQLite). */
+function readRowId(row: Row): string | null {
+  const raw = row.id;
+  if (raw instanceof RecordId) return String(raw.id);
+  if (typeof raw === 'string') {
+    const prefix = `${BLOB_TABLE}:`;
+    return raw.startsWith(prefix) ? raw.slice(prefix.length) : raw;
+  }
+  return null;
+}

package/src/services/blobs/blob-store.test.ts

@@ -0,0 +1,77 @@
+import { describe, it, expect } from 'vitest';
+import {
+  BlobKeyError,
+  MemoryBlobStore,
+  blobKeyId,
+  decodeSegment,
+  encodeSegment,
+  keySegments,
+  pathSegments,
+} from './blob-store';
+
+describe('key encoding', () => {
+  it('round-trips segments that are illegal or ambiguous in a filename', () => {
+    for (const raw of ['avatar.png', 'a/b', 'hello world', 'ünïcode.jpg', '..', '%2F', "quote'"]) {
+      expect(decodeSegment(encodeSegment(raw))).toBe(raw);
+    }
+  });
+
+  it('never emits a literal dot', () => {
+    // The `.part-` torn-write marker is only unambiguous because committed
+    // names cannot contain one. If this stops holding, a real file named
+    // `x.part-1` gets swept as a half-written file on the next boot.
+    for (const raw of ['avatar.png', 'a.part-1', '...', 'x.tar.gz']) {
+      expect(encodeSegment(raw)).not.toContain('.');
+    }
+  });
+
+  it('rejects a segment too long for the filesystem', () => {
+    expect(() => encodeSegment('x'.repeat(201))).toThrow(BlobKeyError);
+  });
+
+  it('strips traversal and empty segments so a crafted path cannot escape', () => {
+    expect(pathSegments('../../etc/passwd')).toEqual(['etc', 'passwd']);
+    expect(pathSegments('a//b/./c')).toEqual(['a', 'b', 'c']);
+  });
+
+  it('maps a key to a bucket directory plus the path chain', () => {
+    expect(keySegments({ bucket: 'files', path: 'a/b/c.png' })).toEqual([
+      'files',
+      'a',
+      'b',
+      'c%2Epng',
+    ]);
+  });
+
+  it('rejects a key with no filename', () => {
+    expect(() => keySegments({ bucket: 'files', path: '/' })).toThrow(BlobKeyError);
+  });
+
+  it('normalizes the manifest row id', () => {
+    expect(blobKeyId({ bucket: 'files', path: '/a//b.png' })).toBe('files/a/b.png');
+  });
+});
+
+describe('MemoryBlobStore', () => {
+  it('round-trips bytes and lists them back under the right key', async () => {
+    const store = new MemoryBlobStore('user-1');
+    const key = { bucket: 'files', path: 'nested/dir/photo.jpg' };
+
+    await store.write(key, new Blob(['hello']));
+
+    expect(await (await store.read(key))!.text()).toBe('hello');
+    expect(await store.list()).toEqual([{ key, size: 5, mtime: expect.any(Number) }]);
+  });
+
+  it('isolates namespaces', async () => {
+    const store = new MemoryBlobStore('user-1');
+    const key = { bucket: 'files', path: 'photo.jpg' };
+    await store.write(key, new Blob(['hello']));
+
+    store.setNamespace('user-2');
+    expect(await store.read(key)).toBeNull();
+
+    store.setNamespace('user-1');
+    expect(await store.read(key)).not.toBeNull();
+  });
+});

package/src/services/blobs/blob-store.ts

@@ -0,0 +1,359 @@
+/**
+ * Byte storage for cached bucket files.
+ *
+ * The default implementation is OPFS. Bucket files never arrive over HTTP in
+ * this client — `BucketHandle.get()` is a SurrealQL RPC on the sync socket — so
+ * neither the browser's HTTP cache nor the Cache API can hold them. We persist
+ * the bytes ourselves, and OPFS is the cheapest place to put them: a read is
+ * `getFile()` → a disk-backed lazy `File` that `URL.createObjectURL` can serve
+ * without ever moving the bytes through the JS heap.
+ *
+ * Layout is real nested directories rather than one hashed filename:
+ *
+ *   sp00ky-blobs/<namespace>/<bucket>/<...path segments>
+ *
+ * That costs a `getDirectoryHandle` per segment on write, and buys the property
+ * the whole orphan story rests on: the full `(bucket, path)` key is recoverable
+ * from a directory walk alone. The `_00_blob` manifest can therefore be wiped
+ * (memory fallback, SQLite pool wipe, IndexedDB corruption recovery) and be
+ * rebuilt from disk instead of taking the cached bytes down with it.
+ */
+
+/** Identifies one cached file: the bucket it lives in and its path within. */
+export interface BlobKey {
+  bucket: string;
+  path: string;
+}
+
+/** What a directory walk can tell us about a stored file, with no manifest. */
+export interface BlobStat {
+  key: BlobKey;
+  size: number;
+  /** File mtime. Seeds `lastAccess` when a manifest row has to be rebuilt. */
+  mtime: number;
+}
+
+export interface BlobStore {
+  /** False for {@link MemoryBlobStore} and for OPFS-less environments: the
+   *  cache still dedupes and serves within a tab, but nothing survives reload. */
+  readonly persistent: boolean;
+  /** Namespace (the local bucketId) all keys are resolved under. */
+  readonly namespace: string;
+
+  read(key: BlobKey): Promise<Blob | null>;
+  /** Returns the number of bytes written. Throws on quota exhaustion. */
+  write(key: BlobKey, bytes: Blob): Promise<number>;
+  remove(key: BlobKey): Promise<void>;
+  /** Every committed file under the current namespace. Sweeps torn writes. */
+  list(): Promise<BlobStat[]>;
+  /** Drop the whole namespace (sign-out with `clearOnSignOut`, or a reset). */
+  clear(): Promise<void>;
+  /** Point at another namespace. Does not touch the bytes of the old one. */
+  setNamespace(namespace: string): void;
+}
+
+export const BLOB_ROOT_DIR = 'sp00ky-blobs';
+
+/**
+ * Marks a half-written file. Committed names can never contain a literal `.`
+ * (see {@link encodeSegment}), so this suffix is unambiguous — anything wearing
+ * it at walk time is a write that died before commit, and is swept.
+ */
+const PART_MARKER = '.part-';
+
+/** OPFS names are capped around 255 bytes; stay well clear. */
+const MAX_SEGMENT_LENGTH = 200;
+/** Walk guards, mirroring `walkOpfs` in the DevTools storage report. */
+const MAX_WALK_DEPTH = 12;
+const MAX_WALK_ENTRIES = 5000;
+
+/** A path segment we refuse to store — the caller degrades to no persistence. */
+export class BlobKeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'BlobKeyError';
+  }
+}
+
+/**
+ * Percent-encode a path segment, and escape `.` on top of that. Escaping the
+ * dot is what makes {@link PART_MARKER} safe: a real file called `x.part-1`
+ * would otherwise be swept as a torn write on the next boot.
+ */
+export function encodeSegment(segment: string): string {
+  const encoded = encodeURIComponent(segment).replace(/\./g, '%2E');
+  if (encoded.length > MAX_SEGMENT_LENGTH) {
+    throw new BlobKeyError(`path segment too long to store (${encoded.length} > ${MAX_SEGMENT_LENGTH})`);
+  }
+  return encoded;
+}
+
+export function decodeSegment(segment: string): string {
+  return decodeURIComponent(segment);
+}
+
+/** Path segments, `..`/`.`/empty stripped so a crafted path can't escape the root. */
+export function pathSegments(path: string): string[] {
+  return path.split('/').filter((s) => s.length > 0 && s !== '.' && s !== '..');
+}
+
+/** `<bucket>/<...path>` — the directory chain plus filename for a key. */
+export function keySegments(key: BlobKey): string[] {
+  const parts = pathSegments(key.path);
+  if (parts.length === 0) throw new BlobKeyError(`empty file path for bucket "${key.bucket}"`);
+  return [key.bucket, ...parts].map(encodeSegment);
+}
+
+/** Stable manifest row id. Mirrors the on-disk layout, decoded. */
+export function blobKeyId(key: BlobKey): string {
+  return `${key.bucket}/${pathSegments(key.path).join('/')}`;
+}
+
+/** Whether OPFS is usable for read AND write in this environment. Writes need
+ *  `createWritable`, which arrived late in Safari; without it we run
+ *  memory-only rather than dragging in a sync-access-handle worker. */
+export function opfsWritableSupported(): boolean {
+  if (typeof navigator === 'undefined' || !navigator.storage?.getDirectory) return false;
+  const proto = (globalThis as { FileSystemFileHandle?: { prototype?: unknown } }).FileSystemFileHandle
+    ?.prototype as { createWritable?: unknown } | undefined;
+  return typeof proto?.createWritable === 'function';
+}
+
+function supportsHandleMove(): boolean {
+  const proto = (globalThis as { FileSystemFileHandle?: { prototype?: unknown } }).FileSystemFileHandle
+    ?.prototype as { move?: unknown } | undefined;
+  return typeof proto?.move === 'function';
+}
+
+/** Minimal structural view of the OPFS handles we touch. */
+interface DirHandle {
+  kind: 'directory';
+  name: string;
+  getDirectoryHandle(name: string, opts?: { create?: boolean }): Promise<DirHandle>;
+  getFileHandle(name: string, opts?: { create?: boolean }): Promise<FileHandle>;
+  removeEntry(name: string, opts?: { recursive?: boolean }): Promise<void>;
+  entries(): AsyncIterableIterator<[string, DirHandle | FileHandle]>;
+}
+
+interface FileHandle {
+  kind: 'file';
+  name: string;
+  getFile(): Promise<File>;
+  createWritable(): Promise<{ write(data: Blob): Promise<void>; close(): Promise<void>; abort?(): Promise<void> }>;
+  move?(name: string): Promise<void>;
+}
+
+function isNotFound(err: unknown): boolean {
+  return err instanceof Error && err.name === 'NotFoundError';
+}
+
+/** OPFS-backed byte storage. All calls are async — nothing blocks the main thread. */
+export class OpfsBlobStore implements BlobStore {
+  readonly persistent = true;
+  private ns: string;
+  private readonly useTempCommit = supportsHandleMove();
+  private tempCounter = 0;
+  private readonly tempToken =
+    typeof crypto !== 'undefined' && crypto.randomUUID ? crypto.randomUUID().slice(0, 8) : 'tmp';
+
+  constructor(namespace: string) {
+    this.ns = namespace;
+  }
+
+  get namespace(): string {
+    return this.ns;
+  }
+
+  setNamespace(namespace: string): void {
+    this.ns = namespace;
+  }
+
+  private async root(create: boolean): Promise<DirHandle | null> {
+    const opfs = (await navigator.storage.getDirectory()) as unknown as DirHandle;
+    return this.resolveDir(opfs, [BLOB_ROOT_DIR, encodeSegment(this.ns)], create);
+  }
+
+  private async resolveDir(from: DirHandle, segments: string[], create: boolean): Promise<DirHandle | null> {
+    let dir = from;
+    for (const segment of segments) {
+      try {
+        dir = await dir.getDirectoryHandle(segment, { create });
+      } catch (err) {
+        if (!create && isNotFound(err)) return null;
+        throw err;
+      }
+    }
+    return dir;
+  }
+
+  async read(key: BlobKey): Promise<Blob | null> {
+    const segments = keySegments(key);
+    const root = await this.root(false);
+    if (!root) return null;
+    const dir = await this.resolveDir(root, segments.slice(0, -1), false);
+    if (!dir) return null;
+    try {
+      const handle = await dir.getFileHandle(segments[segments.length - 1]!);
+      return await handle.getFile();
+    } catch (err) {
+      if (isNotFound(err)) return null;
+      throw err;
+    }
+  }
+
+  async write(key: BlobKey, bytes: Blob): Promise<number> {
+    const segments = keySegments(key);
+    const root = await this.root(true);
+    if (!root) throw new Error('OPFS root unavailable');
+    const dir = (await this.resolveDir(root, segments.slice(0, -1), true))!;
+    const name = segments[segments.length - 1]!;
+
+    // With `move()` we write to a temp name and commit by rename, so a crash
+    // mid-write leaves a sweepable `.part-` file rather than a truncated real
+    // one. Without it (Firefox), write in place: a torn file is caught on read
+    // by the manifest size check, which we need anyway for cross-tab races.
+    const target = this.useTempCommit ? `${name}${PART_MARKER}${this.tempToken}-${this.tempCounter++}` : name;
+    const handle = await dir.getFileHandle(target, { create: true });
+    const writable = await handle.createWritable();
+    try {
+      await writable.write(bytes);
+      await writable.close();
+    } catch (err) {
+      await writable.abort?.().catch(() => {});
+      await dir.removeEntry(target).catch(() => {});
+      throw err;
+    }
+    if (this.useTempCommit) await handle.move!(name);
+    return bytes.size;
+  }
+
+  async remove(key: BlobKey): Promise<void> {
+    const segments = keySegments(key);
+    const root = await this.root(false);
+    if (!root) return;
+    const dir = await this.resolveDir(root, segments.slice(0, -1), false);
+    if (!dir) return;
+    try {
+      await dir.removeEntry(segments[segments.length - 1]!);
+    } catch (err) {
+      if (!isNotFound(err)) throw err;
+    }
+    // Directories are left behind deliberately: pruning them would race a
+    // concurrent write into the same folder for no measurable space saving.
+  }
+
+  async list(): Promise<BlobStat[]> {
+    const root = await this.root(false);
+    if (!root) return [];
+    const out: BlobStat[] = [];
+    await this.walk(root, [], out, 0);
+    return out;
+  }
+
+  private async walk(dir: DirHandle, trail: string[], out: BlobStat[], depth: number): Promise<void> {
+    if (depth > MAX_WALK_DEPTH || out.length >= MAX_WALK_ENTRIES) return;
+    for await (const [name, handle] of dir.entries()) {
+      if (out.length >= MAX_WALK_ENTRIES) return;
+      if (handle.kind === 'directory') {
+        await this.walk(handle as DirHandle, [...trail, name], out, depth + 1);
+        continue;
+      }
+      // Torn write from a process that died between create and commit.
+      if (name.includes(PART_MARKER)) {
+        await dir.removeEntry(name).catch(() => {});
+        continue;
+      }
+      // A file directly under the namespace root has no bucket segment.
+      if (trail.length === 0) continue;
+      try {
+        const file = await (handle as FileHandle).getFile();
+        out.push({
+          key: {
+            bucket: decodeSegment(trail[0]!),
+            path: [...trail.slice(1), name].map(decodeSegment).join('/'),
+          },
+          size: file.size,
+          mtime: file.lastModified,
+        });
+      } catch {
+        // Locked or vanished mid-walk: skip. Reconcile is best-effort.
+      }
+    }
+  }
+
+  async clear(): Promise<void> {
+    const opfs = (await navigator.storage.getDirectory()) as unknown as DirHandle;
+    const root = await this.resolveDir(opfs, [BLOB_ROOT_DIR], false);
+    if (!root) return;
+    try {
+      await root.removeEntry(encodeSegment(this.ns), { recursive: true });
+    } catch (err) {
+      if (!isNotFound(err)) throw err;
+    }
+  }
+}
+
+/**
+ * In-memory byte store. Used when OPFS is unavailable (the cache still dedupes
+ * and serves within the tab, matching the pre-existing behaviour) and as the
+ * test double for {@link BlobCache} — `blob-cache.test.ts` runs in node.
+ */
+export class MemoryBlobStore implements BlobStore {
+  readonly persistent = false;
+  private ns: string;
+  private readonly files = new Map<string, Map<string, { blob: Blob; mtime: number }>>();
+  private clock = 0;
+
+  constructor(namespace = 'anon') {
+    this.ns = namespace;
+  }
+
+  get namespace(): string {
+    return this.ns;
+  }
+
+  setNamespace(namespace: string): void {
+    this.ns = namespace;
+  }
+
+  private bucketFiles(): Map<string, { blob: Blob; mtime: number }> {
+    let ns = this.files.get(this.ns);
+    if (!ns) {
+      ns = new Map();
+      this.files.set(this.ns, ns);
+    }
+    return ns;
+  }
+
+  async read(key: BlobKey): Promise<Blob | null> {
+    keySegments(key);
+    return this.bucketFiles().get(blobKeyId(key))?.blob ?? null;
+  }
+
+  async write(key: BlobKey, bytes: Blob): Promise<number> {
+    keySegments(key);
+    this.bucketFiles().set(blobKeyId(key), { blob: bytes, mtime: ++this.clock });
+    return bytes.size;
+  }
+
+  async remove(key: BlobKey): Promise<void> {
+    this.bucketFiles().delete(blobKeyId(key));
+  }
+
+  async list(): Promise<BlobStat[]> {
+    const out: BlobStat[] = [];
+    for (const [id, entry] of this.bucketFiles()) {
+      const slash = id.indexOf('/');
+      out.push({
+        key: { bucket: id.slice(0, slash), path: id.slice(slash + 1) },
+        size: entry.blob.size,
+        mtime: entry.mtime,
+      });
+    }
+    return out;
+  }
+
+  async clear(): Promise<void> {
+    this.files.delete(this.ns);
+  }
+}