@trestleinc/replicate 1.1.1 → 1.1.2-preview.0

package/src/client/persistence/indexeddb.ts

@@ -1,110 +1,137 @@
-/**
- * IndexedDB persistence implementation for browser environments.
- *
- * Uses y-indexeddb for Y.Doc persistence and browser-level for key-value storage.
- * browser-level is an abstract-level database backed by IndexedDB.
- */
-import type * as Y from 'yjs';
-import { IndexeddbPersistence } from 'y-indexeddb';
-import { BrowserLevel } from 'browser-level';
-import type { Persistence, PersistenceProvider, KeyValueStore } from './types.js';
-
-/**
- * browser-level backed key-value store.
- *
- * Uses the Level ecosystem for consistent API across browser and React Native.
- */
-class BrowserLevelKeyValueStore implements KeyValueStore {
-  private db: BrowserLevel<string, string>;
-
-  constructor(dbName: string) {
-    this.db = new BrowserLevel(dbName);
-  }
+import * as Y from "yjs";
+import type { Persistence, PersistenceProvider, KeyValueStore } from "./types.js";
+
+const UPDATES_STORE = "updates";
+const SNAPSHOTS_STORE = "snapshots";
+const KV_STORE = "kv";
 
-  async get<T>(key: string): Promise<T | undefined> {
-    try {
-      const value = await this.db.get(key);
-      if (value === undefined) {
-        return undefined;
+function openDatabase(dbName: string): Promise<IDBDatabase> {
+  return new Promise((resolve, reject) => {
+    const request = indexedDB.open(`replicate-${dbName}`, 1);
+    request.onerror = () => reject(request.error ?? new Error("IndexedDB open failed"));
+    request.onsuccess = () => resolve(request.result);
+    request.onupgradeneeded = () => {
+      const db = request.result;
+      if (!db.objectStoreNames.contains(UPDATES_STORE)) {
+        db.createObjectStore(UPDATES_STORE, { autoIncrement: true });
       }
-      return JSON.parse(value) as T;
-    } catch (err: any) {
-      // Level throws LEVEL_NOT_FOUND error for missing keys
-      if (err.code === 'LEVEL_NOT_FOUND') {
-        return undefined;
+      if (!db.objectStoreNames.contains(SNAPSHOTS_STORE)) {
+        db.createObjectStore(SNAPSHOTS_STORE);
       }
-      throw err;
-    }
-  }
+      if (!db.objectStoreNames.contains(KV_STORE)) {
+        db.createObjectStore(KV_STORE);
+      }
+    };
+  });
+}
+
+class IDBKeyValueStore implements KeyValueStore {
+  constructor(private db: IDBDatabase) {}
 
-  async set<T>(key: string, value: T): Promise<void> {
-    await this.db.put(key, JSON.stringify(value));
+  get<T>(key: string): Promise<T | undefined> {
+    return new Promise((resolve, reject) => {
+      const tx = this.db.transaction(KV_STORE, "readonly");
+      const store = tx.objectStore(KV_STORE);
+      const request = store.get(key);
+      request.onerror = () => reject(request.error ?? new Error("IndexedDB get failed"));
+      request.onsuccess = () => resolve(request.result as T | undefined);
+    });
   }
 
-  async del(key: string): Promise<void> {
-    try {
-      await this.db.del(key);
-    } catch (err: any) {
-      // Ignore not found errors on delete
-      if (err.code !== 'LEVEL_NOT_FOUND') {
-        throw err;
-      }
-    }
+  set<T>(key: string, value: T): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const tx = this.db.transaction(KV_STORE, "readwrite");
+      const store = tx.objectStore(KV_STORE);
+      const request = store.put(value, key);
+      request.onerror = () => reject(request.error ?? new Error("IndexedDB set failed"));
+      request.onsuccess = () => resolve();
+    });
   }
 
-  async close(): Promise<void> {
-    await this.db.close();
+  del(key: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const tx = this.db.transaction(KV_STORE, "readwrite");
+      const store = tx.objectStore(KV_STORE);
+      const request = store.delete(key);
+      request.onerror = () => reject(request.error ?? new Error("IndexedDB delete failed"));
+      request.onsuccess = () => resolve();
+    });
   }
 }
 
-/**
- * IndexedDB persistence provider wrapping y-indexeddb.
- */
-class IndexedDBPersistenceProvider implements PersistenceProvider {
-  private persistence: IndexeddbPersistence;
+class IDBPersistenceProvider implements PersistenceProvider {
+  private updateHandler: (update: Uint8Array, origin: unknown) => void;
   readonly whenSynced: Promise<void>;
 
-  constructor(collection: string, ydoc: Y.Doc) {
-    this.persistence = new IndexeddbPersistence(collection, ydoc);
-
-    // Handle race: check synced state before attaching listener
-    // If database is empty or fast, synced event may fire before we attach
-    this.whenSynced = new Promise((resolve) => {
-      if (this.persistence.synced) {
-        // Already synced - resolve immediately
-        resolve();
-      } else {
-        // Not yet synced - wait for event (use once to prevent listener accumulation)
-        this.persistence.once('synced', () => resolve());
+  constructor(
+    private db: IDBDatabase,
+    private collection: string,
+    private ydoc: Y.Doc,
+  ) {
+    this.whenSynced = this.loadState();
+
+    this.updateHandler = (update: Uint8Array, origin: unknown) => {
+      if (origin !== "idb") {
+        void this.saveUpdate(update);
       }
+    };
+    this.ydoc.on("update", this.updateHandler);
+  }
+
+  private loadState(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const tx = this.db.transaction([SNAPSHOTS_STORE, UPDATES_STORE], "readonly");
+
+      const snapshotStore = tx.objectStore(SNAPSHOTS_STORE);
+      const snapshotRequest = snapshotStore.get(this.collection);
+
+      snapshotRequest.onsuccess = () => {
+        if (snapshotRequest.result) {
+          Y.applyUpdate(this.ydoc, snapshotRequest.result, "idb");
+        }
+
+        const updatesStore = tx.objectStore(UPDATES_STORE);
+        const updatesRequest = updatesStore.getAll();
+
+        updatesRequest.onsuccess = () => {
+          const updates = updatesRequest.result as Uint8Array[];
+          for (const update of updates) {
+            Y.applyUpdate(this.ydoc, update, "idb");
+          }
+          resolve();
+        };
+
+        updatesRequest.onerror = () =>
+          reject(updatesRequest.error ?? new Error("IndexedDB updates load failed"));
+      };
+
+      snapshotRequest.onerror = () =>
+        reject(snapshotRequest.error ?? new Error("IndexedDB snapshot load failed"));
+    });
+  }
+
+  private saveUpdate(update: Uint8Array): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const tx = this.db.transaction(UPDATES_STORE, "readwrite");
+      const store = tx.objectStore(UPDATES_STORE);
+      const request = store.add(update);
+      request.onerror = () => reject(request.error ?? new Error("IndexedDB save update failed"));
+      request.onsuccess = () => resolve();
     });
   }
 
   destroy(): void {
-    this.persistence.destroy();
+    this.ydoc.off("update", this.updateHandler);
   }
 }
 
-/**
- * Create an IndexedDB persistence factory.
- *
- * Uses y-indexeddb for Y.Doc persistence and browser-level for metadata storage.
- *
- * @param dbName - Name for the LevelDB database (default: 'replicate-kv')
- *
- * @example
- * ```typescript
- * convexCollectionOptions<Task>({
- *   // ... other options
- *   persistence: indexeddbPersistence(),
- * });
- * ```
- */
-export function indexeddbPersistence(dbName = 'replicate-kv'): Persistence {
-  const kv = new BrowserLevelKeyValueStore(dbName);
+export async function createIndexedDBPersistence(dbName: string): Promise<Persistence> {
+  const db = await openDatabase(dbName);
+  const kv = new IDBKeyValueStore(db);
+
   return {
     createDocPersistence: (collection: string, ydoc: Y.Doc) =>
-      new IndexedDBPersistenceProvider(collection, ydoc),
+      new IDBPersistenceProvider(db, collection, ydoc),
     kv,
   };
 }

package/src/client/persistence/memory.ts

@@ -3,8 +3,8 @@
  *
  * State is not persisted across sessions - useful for tests and development.
  */
-import type * as Y from 'yjs';
-import type { Persistence, PersistenceProvider, KeyValueStore } from './types.js';
+import type * as Y from "yjs";
+import type { Persistence, PersistenceProvider, KeyValueStore } from "./types.js";
 
 /**
  * In-memory key-value store.
@@ -55,7 +55,7 @@ class MemoryPersistenceProvider implements PersistenceProvider {
 export function memoryPersistence(): Persistence {
   const kv = new MemoryKeyValueStore();
   return {
-    createDocPersistence: (_collection: string, _ydoc: Y.Doc) => new MemoryPersistenceProvider(),
+    createDocPersistence: (_: string, __: Y.Doc) => new MemoryPersistenceProvider(),
     kv,
   };
 }

package/src/client/persistence/sqlite/browser.ts

@@ -0,0 +1,168 @@
+import { initSchema, createPersistenceFromExecutor, type Executor } from "./schema.js";
+import type { Persistence } from "../types.js";
+
+interface SqlJsDatabase {
+  run(sql: string, params?: unknown): unknown;
+  prepare(sql: string): {
+    bind(params?: unknown): void;
+    step(): boolean;
+    getAsObject(): Record<string, unknown>;
+    free(): void;
+  };
+  export(): Uint8Array;
+  close(): void;
+}
+
+interface SqlJsStatic {
+  Database: new (data?: ArrayLike<number> | Buffer | null) => SqlJsDatabase;
+}
+
+function hasOPFS(): boolean {
+  return typeof navigator !== "undefined"
+    && "storage" in navigator
+    && "getDirectory" in navigator.storage;
+}
+
+async function loadFromOPFS(dbName: string): Promise<Uint8Array | null> {
+  try {
+    const root = await navigator.storage.getDirectory();
+    const handle = await root.getFileHandle(`${dbName}.sqlite`);
+    const file = await handle.getFile();
+    return new Uint8Array(await file.arrayBuffer());
+  }
+  catch {
+    return null;
+  }
+}
+
+async function saveToOPFS(dbName: string, data: Uint8Array): Promise<void> {
+  const root = await navigator.storage.getDirectory();
+  const handle = await root.getFileHandle(`${dbName}.sqlite`, { create: true });
+  const writable = await handle.createWritable();
+  await writable.write(new Uint8Array(data));
+  await writable.close();
+}
+
+const IDB_STORE = "sqlite-db";
+
+function openIDB(dbName: string): Promise<IDBDatabase> {
+  return new Promise((resolve, reject) => {
+    const request = indexedDB.open(`replicate-sqlite-${dbName}`, 1);
+    request.onerror = () => reject(request.error ?? new Error("IndexedDB open failed"));
+    request.onsuccess = () => resolve(request.result);
+    request.onupgradeneeded = () => {
+      request.result.createObjectStore(IDB_STORE);
+    };
+  });
+}
+
+async function loadFromIDB(dbName: string): Promise<Uint8Array | null> {
+  try {
+    const db = await openIDB(dbName);
+    return new Promise((resolve) => {
+      const tx = db.transaction(IDB_STORE, "readonly");
+      const request = tx.objectStore(IDB_STORE).get("data");
+      request.onsuccess = () => {
+        db.close();
+        resolve(request.result ?? null);
+      };
+      request.onerror = () => {
+        db.close();
+        resolve(null);
+      };
+    });
+  }
+  catch {
+    return null;
+  }
+}
+
+async function saveToIDB(dbName: string, data: Uint8Array): Promise<void> {
+  const db = await openIDB(dbName);
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(IDB_STORE, "readwrite");
+    const request = tx.objectStore(IDB_STORE).put(data, "data");
+    request.onsuccess = () => {
+      db.close();
+      resolve();
+    };
+    request.onerror = () => {
+      db.close();
+      reject(request.error ?? new Error("IndexedDB put failed"));
+    };
+  });
+}
+
+interface StorageBackend {
+  load(): Promise<Uint8Array | null>;
+  save(data: Uint8Array): Promise<void>;
+}
+
+function createStorageBackend(dbName: string): StorageBackend {
+  if (hasOPFS()) {
+    return {
+      load: () => loadFromOPFS(dbName),
+      save: data => saveToOPFS(dbName, data),
+    };
+  }
+  return {
+    load: () => loadFromIDB(dbName),
+    save: data => saveToIDB(dbName, data),
+  };
+}
+
+class SqlJsExecutor implements Executor {
+  constructor(
+    private db: SqlJsDatabase,
+    private storage: StorageBackend,
+  ) {}
+
+  async execute(sql: string, params?: unknown[]): Promise<{ rows: Record<string, unknown>[] }> {
+    const rows: Record<string, unknown>[] = [];
+    const trimmed = sql.trim().toUpperCase();
+
+    const isWrite = trimmed.startsWith("CREATE")
+      || trimmed.startsWith("INSERT")
+      || trimmed.startsWith("UPDATE")
+      || trimmed.startsWith("DELETE")
+      || trimmed.startsWith("BEGIN")
+      || trimmed.startsWith("COMMIT")
+      || trimmed.startsWith("ROLLBACK");
+
+    if (isWrite) {
+      this.db.run(sql, params);
+      await this.storage.save(this.db.export());
+      return { rows };
+    }
+
+    const stmt = this.db.prepare(sql);
+    if (params?.length) {
+      stmt.bind(params);
+    }
+
+    while (stmt.step()) {
+      rows.push(stmt.getAsObject());
+    }
+    stmt.free();
+
+    return { rows };
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}
+
+export async function createBrowserSqlitePersistence(
+  SQL: SqlJsStatic,
+  dbName: string,
+): Promise<Persistence> {
+  const storage = createStorageBackend(dbName);
+  const existingData = await storage.load();
+  const db = existingData ? new SQL.Database(existingData) : new SQL.Database();
+  const executor = new SqlJsExecutor(db, storage);
+
+  await initSchema(executor);
+
+  return createPersistenceFromExecutor(executor);
+}

package/src/client/persistence/sqlite/native.ts

@@ -0,0 +1,29 @@
+import { initSchema, createPersistenceFromExecutor, type Executor } from "./schema.js";
+import type { Persistence } from "../types.js";
+
+interface OPSQLiteDatabase {
+  execute(sql: string, params?: unknown[]): Promise<{ rows: Record<string, unknown>[] }>;
+  close(): void;
+}
+
+class OPSqliteExecutor implements Executor {
+  constructor(private db: OPSQLiteDatabase) {}
+
+  async execute(sql: string, params?: unknown[]): Promise<{ rows: Record<string, unknown>[] }> {
+    const result = await this.db.execute(sql, params);
+    return { rows: result.rows || [] };
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}
+
+export async function createNativeSqlitePersistence(
+  db: OPSQLiteDatabase,
+  _dbName: string,
+): Promise<Persistence> {
+  const executor = new OPSqliteExecutor(db);
+  await initSchema(executor);
+  return createPersistenceFromExecutor(executor);
+}

package/src/client/persistence/sqlite/schema.ts

@@ -0,0 +1,124 @@
+import * as Y from "yjs";
+import type { Persistence, PersistenceProvider, KeyValueStore } from "../types.js";
+
+export interface Executor {
+  execute(sql: string, params?: unknown[]): Promise<{ rows: Record<string, unknown>[] }>;
+  close(): void;
+}
+
+export async function initSchema(executor: Executor): Promise<void> {
+  await executor.execute(`
+    CREATE TABLE IF NOT EXISTS snapshots (
+      collection TEXT PRIMARY KEY,
+      data BLOB NOT NULL,
+      state_vector BLOB,
+      seq INTEGER DEFAULT 0
+    )
+  `);
+
+  await executor.execute(`
+    CREATE TABLE IF NOT EXISTS updates (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      collection TEXT NOT NULL,
+      data BLOB NOT NULL
+    )
+  `);
+
+  await executor.execute(`
+    CREATE INDEX IF NOT EXISTS updates_collection_idx ON updates (collection)
+  `);
+
+  await executor.execute(`
+    CREATE TABLE IF NOT EXISTS kv (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    )
+  `);
+}
+
+class SqliteKeyValueStore implements KeyValueStore {
+  constructor(private executor: Executor) {}
+
+  async get<T>(key: string): Promise<T | undefined> {
+    const result = await this.executor.execute(
+      "SELECT value FROM kv WHERE key = ?",
+      [key],
+    );
+    if (result.rows.length === 0) return undefined;
+    return JSON.parse(result.rows[0].value as string) as T;
+  }
+
+  async set<T>(key: string, value: T): Promise<void> {
+    await this.executor.execute(
+      "INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)",
+      [key, JSON.stringify(value)],
+    );
+  }
+
+  async del(key: string): Promise<void> {
+    await this.executor.execute("DELETE FROM kv WHERE key = ?", [key]);
+  }
+}
+
+class SqlitePersistenceProvider implements PersistenceProvider {
+  private updateHandler: (update: Uint8Array, origin: unknown) => void;
+  readonly whenSynced: Promise<void>;
+
+  constructor(
+    private executor: Executor,
+    private collection: string,
+    private ydoc: Y.Doc,
+  ) {
+    this.whenSynced = this.loadState();
+
+    this.updateHandler = (update: Uint8Array, origin: unknown) => {
+      if (origin !== "sqlite") {
+        void this.saveUpdate(update);
+      }
+    };
+    this.ydoc.on("update", this.updateHandler);
+  }
+
+  private async loadState(): Promise<void> {
+    const snapshotResult = await this.executor.execute(
+      "SELECT data FROM snapshots WHERE collection = ?",
+      [this.collection],
+    );
+
+    if (snapshotResult.rows.length > 0) {
+      const raw = snapshotResult.rows[0].data;
+      const snapshotData = raw instanceof Uint8Array ? raw : new Uint8Array(raw as ArrayBuffer);
+      Y.applyUpdate(this.ydoc, snapshotData, "sqlite");
+    }
+
+    const updatesResult = await this.executor.execute(
+      "SELECT data FROM updates WHERE collection = ? ORDER BY id ASC",
+      [this.collection],
+    );
+
+    for (const row of updatesResult.rows) {
+      const raw = row.data;
+      const updateData = raw instanceof Uint8Array ? raw : new Uint8Array(raw as ArrayBuffer);
+      Y.applyUpdate(this.ydoc, updateData, "sqlite");
+    }
+  }
+
+  private async saveUpdate(update: Uint8Array): Promise<void> {
+    await this.executor.execute(
+      "INSERT INTO updates (collection, data) VALUES (?, ?)",
+      [this.collection, update],
+    );
+  }
+
+  destroy(): void {
+    this.ydoc.off("update", this.updateHandler);
+  }
+}
+
+export function createPersistenceFromExecutor(executor: Executor): Persistence {
+  return {
+    createDocPersistence: (collection: string, ydoc: Y.Doc) =>
+      new SqlitePersistenceProvider(executor, collection, ydoc),
+    kv: new SqliteKeyValueStore(executor),
+  };
+}

package/src/client/persistence/types.ts

@@ -1,49 +1,53 @@
-/**
- * Persistence layer types for swappable storage backends.
- *
- * Supports IndexedDB (browser), SQLite (React Native), and in-memory (testing).
- */
-import type * as Y from 'yjs';
+import type * as Y from "yjs";
 
 /**
- * Provider that persists Y.Doc state to storage.
+ * Low-level storage adapter for custom backends (Chrome extension, localStorage, cloud).
+ * For SQLite, use `persistence.sqlite()` directly.
  *
- * This wraps providers like y-indexeddb or y-op-sqlite, normalizing their APIs.
+ * @example
+ * ```typescript
+ * class ChromeStorageAdapter implements StorageAdapter {
+ *   async get(key: string) {
+ *     const result = await chrome.storage.local.get(key);
+ *     return result[key] ? new Uint8Array(result[key]) : undefined;
+ *   }
+ *   async set(key: string, value: Uint8Array) {
+ *     await chrome.storage.local.set({ [key]: Array.from(value) });
+ *   }
+ *   async delete(key: string) {
+ *     await chrome.storage.local.remove(key);
+ *   }
+ *   async keys(prefix: string) {
+ *     const all = await chrome.storage.local.get(null);
+ *     return Object.keys(all).filter(k => k.startsWith(prefix));
+ *   }
+ * }
+ * ```
  */
+export interface StorageAdapter {
+  get(key: string): Promise<Uint8Array | undefined>;
+  set(key: string, value: Uint8Array): Promise<void>;
+  delete(key: string): Promise<void>;
+  keys(prefix: string): Promise<string[]>;
+  close?(): void;
+}
+
 export interface PersistenceProvider {
-  /** Promise that resolves when initial sync from storage completes */
   readonly whenSynced: Promise<void>;
-
-  /** Clean up resources (stop observing, close connections) */
   destroy(): void;
 }
 
 /**
- * Factory that creates persistence providers.
- *
- * Each persistence implementation (IndexedDB, SQLite, memory) exports a
- * factory function that returns this interface.
+ * High-level persistence interface for collections.
+ * Create via `persistence.sqlite()`, `persistence.memory()`, or `persistence.custom()`.
  */
 export interface Persistence {
-  /** Create a Y.Doc persistence provider for a collection */
   createDocPersistence(collection: string, ydoc: Y.Doc): PersistenceProvider;
-
-  /** Key-value store for metadata (checkpoints, clientID) */
   readonly kv: KeyValueStore;
 }
 
-/**
- * Simple key-value storage interface.
- *
- * Used for storing metadata like checkpoints and Yjs client IDs.
- */
 export interface KeyValueStore {
-  /** Get a value by key */
   get<T>(key: string): Promise<T | undefined>;
-
-  /** Set a value by key */
   set<T>(key: string, value: T): Promise<void>;
-
-  /** Delete a value by key */
   del(key: string): Promise<void>;
 }