@foretag/tanstack-db-surrealdb 0.2.2 → 0.3.0

package/README.md

@@ -17,14 +17,6 @@ npm install @foretag/tanstack-db-surrealdb
 bun install @foretag/tanstack-db-surrealdb
 ```
 
-### JSR
-```sh
-# NPM
-npx jsr add @foretag/tanstack-db-surrealdb
-# Bun
-bunx jsr add @foretag/tanstack-db-surrealdb
-```
-
 ## Usage
 ```ts
 // db.ts
@@ -60,6 +52,8 @@ export const products = createCollection(
 )
 ```
 
+For syncModes, please see [Example](https://github.com/ForetagInc/tanstack-db-surrealdb/blob/master/examples/syncMode.ts)
+
 ## Vite / Next.JS
 
 ### Vite

package/dist/index.d.mts

@@ -9,23 +9,35 @@ type SyncedTable<T> = WithId<T & {
     sync_deleted?: boolean;
     updated_at?: Date;
 }>;
-type TableOptions<T> = {
-    name: string;
-    fields?: (keyof T)[];
-    where?: ExprLike;
-};
+type SyncMode = 'eager' | 'on-demand' | 'progressive';
 type SurrealCollectionConfig<T extends {
     id: string | RecordId;
 }> = {
     id?: string;
     db: Surreal;
     table: TableOptions<T>;
+    syncMode?: SyncMode;
     useLoro?: boolean;
     onError?: (e: unknown) => void;
 };
+type Field<I> = keyof I | (string & {});
+type FieldList<I> = '*' | ReadonlyArray<Field<I>>;
+type TableOptions<T> = {
+    name: string;
+    fields?: FieldList<T>;
+    where?: ExprLike;
+    pageSize?: number;
+    initialPageSize?: number;
+    onProgress?: (info: {
+        table: string;
+        loaded: number;
+        lastBatch: number;
+        done: boolean;
+    }) => void;
+};
 
 declare function surrealCollectionOptions<T extends SyncedTable<object>, S extends Record<string, Container> = {
     [k: string]: never;
-}>({ id, useLoro, onError, db, ...config }: SurrealCollectionConfig<T>): CollectionConfig<T, string | number, never, UtilsRecord>;
+}>({ id, useLoro, onError, db, syncMode, ...config }: SurrealCollectionConfig<T>): CollectionConfig<T, string | number, never, UtilsRecord>;
 
 export { surrealCollectionOptions };

package/dist/index.d.ts

@@ -9,23 +9,35 @@ type SyncedTable<T> = WithId<T & {
     sync_deleted?: boolean;
     updated_at?: Date;
 }>;
-type TableOptions<T> = {
-    name: string;
-    fields?: (keyof T)[];
-    where?: ExprLike;
-};
+type SyncMode = 'eager' | 'on-demand' | 'progressive';
 type SurrealCollectionConfig<T extends {
     id: string | RecordId;
 }> = {
     id?: string;
     db: Surreal;
     table: TableOptions<T>;
+    syncMode?: SyncMode;
     useLoro?: boolean;
     onError?: (e: unknown) => void;
 };
+type Field<I> = keyof I | (string & {});
+type FieldList<I> = '*' | ReadonlyArray<Field<I>>;
+type TableOptions<T> = {
+    name: string;
+    fields?: FieldList<T>;
+    where?: ExprLike;
+    pageSize?: number;
+    initialPageSize?: number;
+    onProgress?: (info: {
+        table: string;
+        loaded: number;
+        lastBatch: number;
+        done: boolean;
+    }) => void;
+};
 
 declare function surrealCollectionOptions<T extends SyncedTable<object>, S extends Record<string, Container> = {
     [k: string]: never;
-}>({ id, useLoro, onError, db, ...config }: SurrealCollectionConfig<T>): CollectionConfig<T, string | number, never, UtilsRecord>;
+}>({ id, useLoro, onError, db, syncMode, ...config }: SurrealCollectionConfig<T>): CollectionConfig<T, string | number, never, UtilsRecord>;
 
 export { surrealCollectionOptions };

package/dist/index.js

@@ -27,74 +27,162 @@ var import_surrealdb2 = require("surrealdb");
 
 // src/table.ts
 var import_surrealdb = require("surrealdb");
-function manageTable(db, useLoro, { name, ...args }) {
-  const fields = args.fields ?? "*";
-  const listAll = async () => {
+function manageTable(db, useLoro, { name, ...args }, syncMode = "eager") {
+  const rawFields = args.fields ?? "*";
+  const fields = rawFields === "*" ? ["*"] : [...rawFields];
+  const cache = /* @__PURE__ */ new Map();
+  let fullyLoaded = false;
+  const pageSize = args.pageSize ?? 100;
+  const initialPageSize = args.initialPageSize ?? Math.min(50, pageSize);
+  let cursor = 0;
+  let progressiveTask = null;
+  const idKey = (id) => typeof id === "string" ? id : id.toString();
+  const upsertCache = (rows) => {
+    for (const row of rows) cache.set(idKey(row.id), row);
+  };
+  const removeFromCache = (id) => {
+    cache.delete(idKey(id));
+  };
+  const listCached = () => Array.from(cache.values());
+  const buildWhere = () => {
+    if (!useLoro) return args.where;
+    return args.where ? (0, import_surrealdb.and)(args.where, (0, import_surrealdb.eq)("sync_deleted", false)) : (0, import_surrealdb.eq)("sync_deleted", false);
+  };
+  const buildQuery = () => {
     let q = db.select(new import_surrealdb.Table(name));
-    if (args.where) q = q.where(args.where);
-    return await q.fields(...fields);
+    const cond = buildWhere();
+    if (cond) q = q.where(cond);
+    return q;
+  };
+  const applyPaging = (q, start, limit) => {
+    if (typeof start === "number" && q.start) q = q.start(start);
+    if (typeof limit === "number" && q.limit) q = q.limit(limit);
+    return q;
+  };
+  const fetchAll = async () => {
+    const rows = await buildQuery().fields(...fields);
+    upsertCache(rows);
+    fullyLoaded = true;
+    return rows;
+  };
+  const fetchPage = async (opts) => {
+    const q = applyPaging(
+      buildQuery(),
+      opts?.start ?? 0,
+      opts?.limit ?? pageSize
+    );
+    const rows = await q.fields(...fields);
+    upsertCache(rows);
+    if (rows.length < (opts?.limit ?? pageSize)) fullyLoaded = true;
+    return rows;
   };
+  const fetchById = async (id) => {
+    const key = idKey(id);
+    const cached = cache.get(key);
+    if (cached) return cached;
+    const res = await db.select(id);
+    const row = Array.isArray(res) ? res[0] : res;
+    if (!row) return null;
+    if (useLoro && row.sync_deleted)
+      return null;
+    cache.set(key, row);
+    return row;
+  };
+  const loadMore = async (limit = pageSize) => {
+    if (fullyLoaded) return { rows: [], done: true };
+    const rows = await fetchPage({ start: cursor, limit });
+    cursor += rows.length;
+    const done = fullyLoaded || rows.length < limit;
+    args.onProgress?.({
+      table: name,
+      loaded: cache.size,
+      lastBatch: rows.length,
+      done
+    });
+    return { rows, done };
+  };
+  const startProgressive = () => {
+    if (progressiveTask || fullyLoaded) return;
+    progressiveTask = (async () => {
+      if (cache.size === 0) await loadMore(initialPageSize);
+      while (!fullyLoaded) {
+        const { done } = await loadMore(pageSize);
+        if (done) break;
+      }
+    })().finally(() => {
+      progressiveTask = null;
+    });
+  };
+  const listAll = () => fetchAll();
   const listActive = async () => {
-    if (!useLoro) return listAll();
-    return await db.select(new import_surrealdb.Table(name)).where((0, import_surrealdb.and)(args.where, (0, import_surrealdb.eq)("sync_deleted", false))).fields(...fields);
+    if (syncMode === "eager") return fetchAll();
+    if (syncMode === "progressive") {
+      if (cache.size === 0) await loadMore(initialPageSize);
+      startProgressive();
+      return listCached();
+    }
+    return listCached();
   };
   const create = async (data) => {
     await db.create(new import_surrealdb.Table(name)).content(data);
   };
   const update = async (id, data) => {
     if (!useLoro) {
-      await db.update(id).merge({
-        ...data
-      });
+      await db.update(id).merge(data);
       return;
     }
-    try {
-      await db.update(id).merge({
-        ...data,
-        sync_deleted: false,
-        updated_at: Date.now()
-      });
-    } catch (error) {
-      console.warn(
-        `Please ensure the table ${name} has sync_deleted and updated_at fields defined`
-      );
-      console.error(
-        "Failed to update record with Loro (CRDTs) with: ",
-        error
-      );
-    }
+    await db.update(id).merge({
+      ...data,
+      sync_deleted: false,
+      updated_at: /* @__PURE__ */ new Date()
+    });
   };
   const remove = async (id) => {
     await db.delete(id);
+    removeFromCache(id);
   };
   const softDelete = async (id) => {
     if (!useLoro) {
       await db.delete(id);
+      removeFromCache(id);
       return;
     }
     await db.upsert(id).merge({
       sync_deleted: true,
-      updated_at: Date.now()
+      updated_at: /* @__PURE__ */ new Date()
     });
+    removeFromCache(id);
   };
   const subscribe = (cb) => {
     let killed = false;
     let live;
-    const on = ({ action, value }) => {
-      console.debug("[Surreal Live]", name, action, value);
+    const on = (msg) => {
+      const { action, value } = msg;
       if (action === "KILLED") return;
-      if (action === "CREATE") cb({ type: "insert", row: value });
-      else if (action === "UPDATE")
+      if (action === "CREATE") {
+        upsertCache([value]);
+        cb({ type: "insert", row: value });
+        return;
+      }
+      if (action === "UPDATE") {
+        if (useLoro && value.sync_deleted) {
+          removeFromCache(value.id);
+          cb({ type: "delete", row: { id: value.id } });
+          return;
+        }
+        upsertCache([value]);
         cb({ type: "update", row: value });
-      else if (action === "DELETE")
+        return;
+      }
+      if (action === "DELETE") {
+        removeFromCache(value.id);
         cb({ type: "delete", row: { id: value.id } });
+      }
     };
     const start = async () => {
-      const isLiveSupported = db.isFeatureSupported(import_surrealdb.Features.LiveQueries);
-      if (isLiveSupported) {
-        live = await db.live(new import_surrealdb.Table(name)).where(args.where);
-        live.subscribe(on);
-      }
+      if (!db.isFeatureSupported(import_surrealdb.Features.LiveQueries)) return;
+      live = await db.live(new import_surrealdb.Table(name)).where(args.where);
+      live.subscribe(on);
     };
     void start();
     return () => {
@@ -106,25 +194,61 @@ function manageTable(db, useLoro, { name, ...args }) {
   return {
     listAll,
     listActive,
+    listCached,
+    fetchPage,
+    fetchById,
+    loadMore,
     create,
     update,
     remove,
     softDelete,
-    subscribe
+    subscribe,
+    get isFullyLoaded() {
+      return fullyLoaded;
+    },
+    get cachedCount() {
+      return cache.size;
+    }
   };
 }
 
 // src/index.ts
+var DEFAULT_INITIAL_PAGE_SIZE = 50;
+var LOCAL_ID_VERIFY_CHUNK = 500;
+var stableStringify = (value) => {
+  const toJson = (v) => {
+    if (v === null) return null;
+    if (typeof v === "string" || typeof v === "number" || typeof v === "boolean")
+      return v;
+    if (v instanceof Date) return v.toISOString();
+    if (Array.isArray(v)) return v.map(toJson);
+    if (typeof v === "object") {
+      const o = v;
+      const keys = Object.keys(o).sort();
+      const out = {};
+      for (const k of keys) out[k] = toJson(o[k]);
+      return out;
+    }
+    return String(v);
+  };
+  return JSON.stringify(toJson(value));
+};
+var chunk = (arr, size) => {
+  const out = [];
+  for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
+  return out;
+};
 function surrealCollectionOptions({
   id,
   useLoro = false,
   onError,
   db,
+  syncMode,
   ...config
 }) {
   let loro;
   if (useLoro) loro = { doc: new import_loro_crdt.LoroDoc(), key: id };
-  const keyOf = (id2) => typeof id2 === "string" ? id2 : id2.toString();
+  const keyOf = (rid) => typeof rid === "string" ? rid : rid.toString();
   const getKey = (row) => keyOf(row.id);
   const loroKey = loro?.key ?? id ?? "surreal";
   const loroMap = useLoro ? loro?.doc?.getMap?.(loroKey) ?? null : null;
@@ -133,16 +257,17 @@ function surrealCollectionOptions({
     const json = loroMap.toJSON?.() ?? {};
     return Object.values(json);
   };
-  const loroPut = (row) => {
-    if (!loroMap) return;
-    loroMap.set(getKey(row), row);
+  const loroPutMany = (rows) => {
+    if (!loroMap || rows.length === 0) return;
+    for (const row of rows) loroMap.set(getKey(row), row);
     loro?.doc?.commit?.();
   };
-  const loroRemove = (id2) => {
-    if (!loroMap) return;
-    loroMap.delete(id2);
+  const loroRemoveMany = (ids) => {
+    if (!loroMap || ids.length === 0) return;
+    for (const id2 of ids) loroMap.delete(id2);
     loro?.doc?.commit?.();
   };
+  const loroRemove = (id2) => loroRemoveMany([id2]);
   const pushQueue = [];
   const enqueuePush = (op) => {
     if (!useLoro) return;
@@ -154,35 +279,73 @@ function surrealCollectionOptions({
     for (const op of ops) {
       if (op.kind === "create") {
         await table.create(op.row);
-      }
-      if (op.kind === "update") {
+      } else if (op.kind === "update") {
         const rid = new import_surrealdb2.RecordId(
           config.table.name,
           op.row.id.toString()
         );
         await table.update(rid, op.row);
-      }
-      if (op.kind === "delete") {
-        const rid = new import_surrealdb2.RecordId(config.table.name, op.id.toString());
+      } else {
+        const rid = new import_surrealdb2.RecordId(config.table.name, op.id);
         await table.softDelete(rid);
       }
     }
   };
   const newer = (a, b) => (a?.getTime() ?? -1) > (b?.getTime() ?? -1);
-  const reconcileBoot = (serverRows, write) => {
-    if (!useLoro) {
-      diffAndEmit(serverRows, write);
-      return;
+  const fetchServerByLocalIds = async (ids) => {
+    if (ids.length === 0) return [];
+    const tableName = config.table.name;
+    const parts = chunk(ids, LOCAL_ID_VERIFY_CHUNK);
+    const out = [];
+    for (const p of parts) {
+      const [res] = await db.query(
+        "SELECT * FROM type::table($table) WHERE id IN $ids",
+        {
+          table: tableName,
+          ids: p.map((x) => new import_surrealdb2.RecordId(tableName, x))
+        }
+      );
+      if (res) out.push(...res);
+    }
+    return out;
+  };
+  const dedupeById = (rows) => {
+    const m = /* @__PURE__ */ new Map();
+    for (const r of rows) m.set(getKey(r), r);
+    return Array.from(m.values());
+  };
+  let prevById = /* @__PURE__ */ new Map();
+  const buildMap = (rows) => new Map(rows.map((r) => [getKey(r), r]));
+  const same = (a, b) => {
+    if (useLoro) {
+      return (a.sync_deleted ?? false) === (b.sync_deleted ?? false) && (a.updated_at?.getTime() ?? 0) === (b.updated_at?.getTime() ?? 0);
+    }
+    return stableStringify(a) === stableStringify(b);
+  };
+  const diffAndEmit = (currentRows, write) => {
+    const currById = buildMap(currentRows);
+    for (const [id2, row] of currById) {
+      const prev = prevById.get(id2);
+      if (!prev) write({ type: "insert", value: row });
+      else if (!same(prev, row)) write({ type: "update", value: row });
     }
+    for (const [id2, prev] of prevById) {
+      if (!currById.has(id2)) write({ type: "delete", value: prev });
+    }
+    prevById = currById;
+  };
+  const reconcileBoot = (serverRows, write) => {
     const localRows = loroToArray();
     const serverById = new Map(serverRows.map((r) => [getKey(r), r]));
     const localById = new Map(localRows.map((r) => [getKey(r), r]));
     const ids = /* @__PURE__ */ new Set([...serverById.keys(), ...localById.keys()]);
     const current = [];
+    const toRemove = [];
+    const toPut = [];
     const applyLocal = (row) => {
       if (!row) return;
-      if (row.sync_deleted) loroRemove(getKey(row));
-      else loroPut(row);
+      if (row.sync_deleted) toRemove.push(getKey(row));
+      else toPut.push(row);
     };
     for (const id2 of ids) {
       const s = serverById.get(id2);
@@ -242,46 +405,11 @@ function surrealCollectionOptions({
         }
       }
     }
+    loroRemoveMany(toRemove);
+    loroPutMany(toPut);
     diffAndEmit(current, write);
   };
-  let prevById = /* @__PURE__ */ new Map();
-  const buildMap = (rows) => new Map(rows.map((r) => [getKey(r), r]));
-  const same = (a, b) => {
-    if (useLoro) {
-      const aUpdated = a.updated_at;
-      const bUpdated = b.updated_at;
-      const aDeleted = a.sync_deleted ?? false;
-      const bDeleted = b.sync_deleted ?? false;
-      return aDeleted === bDeleted && (aUpdated?.getTime() ?? 0) === (bUpdated?.getTime() ?? 0) && JSON.stringify({
-        ...a,
-        updated_at: void 0,
-        sync_deleted: void 0
-      }) === JSON.stringify({
-        ...b,
-        updated_at: void 0,
-        sync_deleted: void 0
-      });
-    }
-    return JSON.stringify(a) === JSON.stringify(b);
-  };
-  const diffAndEmit = (currentRows, write) => {
-    const currById = buildMap(currentRows);
-    for (const [id2, row] of currById) {
-      const prev = prevById.get(id2);
-      if (!prev) {
-        write({ type: "insert", value: row });
-      } else if (!same(prev, row)) {
-        write({ type: "update", value: row });
-      }
-    }
-    for (const [id2, prev] of prevById) {
-      if (!currById.has(id2)) {
-        write({ type: "delete", value: prev });
-      }
-    }
-    prevById = currById;
-  };
-  const table = manageTable(db, useLoro, config.table);
+  const table = manageTable(db, useLoro, config.table, syncMode);
   const now = () => /* @__PURE__ */ new Date();
   const sync = ({
     begin,
@@ -295,6 +423,11 @@ function surrealCollectionOptions({
       };
     }
     let offLive = null;
+    let work = Promise.resolve();
+    const enqueueWork = (fn) => {
+      work = work.then(fn).catch((e) => onError?.(e));
+      return work;
+    };
     const makeTombstone = (id2) => ({
       id: new import_surrealdb2.RecordId(config.table.name, id2).toString(),
       updated_at: now(),
@@ -302,43 +435,84 @@ function surrealCollectionOptions({
     });
     const start = async () => {
       try {
-        const serverRows = await table.listAll();
-        begin();
-        if (useLoro) reconcileBoot(serverRows, write);
-        else diffAndEmit(serverRows, write);
-        commit();
-        markReady();
+        let serverRows;
+        if (syncMode === "eager") {
+          serverRows = await table.listAll();
+        } else if (syncMode === "progressive") {
+          const first = await table.loadMore(
+            config.table.initialPageSize ?? DEFAULT_INITIAL_PAGE_SIZE
+          );
+          serverRows = first.rows;
+        } else {
+          serverRows = await table.listActive();
+        }
+        await enqueueWork(async () => {
+          begin();
+          if (useLoro) {
+            const localIds = loroToArray().map(getKey);
+            const verifiedServerRows = syncMode === "eager" ? serverRows : dedupeById([
+              ...serverRows,
+              ...await fetchServerByLocalIds(
+                localIds
+              )
+            ]);
+            reconcileBoot(verifiedServerRows, write);
+          } else {
+            diffAndEmit(serverRows, write);
+          }
+          commit();
+          markReady();
+        });
+        if (syncMode === "progressive") {
+          void (async () => {
+            while (!table.isFullyLoaded) {
+              const { rows } = await table.loadMore();
+              if (rows.length === 0) break;
+              await enqueueWork(async () => {
+                begin();
+                try {
+                  if (useLoro) loroPutMany(rows);
+                  diffAndEmit(rows, write);
+                } finally {
+                  commit();
+                }
+              });
+            }
+          })().catch((e) => onError?.(e));
+        }
         await flushPushQueue();
         offLive = table.subscribe((evt) => {
-          begin();
-          try {
-            if (evt.type === "insert" || evt.type === "update") {
-              const row = evt.row;
-              const deleted = useLoro ? row.sync_deleted ?? false : false;
-              if (deleted) {
-                if (useLoro) loroRemove(getKey(row));
-                const prev = prevById.get(getKey(row)) ?? makeTombstone(getKey(row));
-                write({ type: "delete", value: prev });
-                prevById.delete(getKey(row));
+          void enqueueWork(async () => {
+            begin();
+            try {
+              if (evt.type === "insert" || evt.type === "update") {
+                const row = evt.row;
+                const deleted = useLoro ? row.sync_deleted ?? false : false;
+                if (deleted) {
+                  if (useLoro) loroRemove(getKey(row));
+                  const prev = prevById.get(getKey(row)) ?? makeTombstone(getKey(row));
+                  write({ type: "delete", value: prev });
+                  prevById.delete(getKey(row));
+                } else {
+                  if (useLoro) loroPutMany([row]);
+                  const had = prevById.has(getKey(row));
+                  write({
+                    type: had ? "update" : "insert",
+                    value: row
+                  });
+                  prevById.set(getKey(row), row);
+                }
               } else {
-                if (useLoro) loroPut(row);
-                const had = prevById.has(getKey(row));
-                write({
-                  type: had ? "update" : "insert",
-                  value: row
-                });
-                prevById.set(getKey(row), row);
+                const rid = getKey(evt.row);
+                if (useLoro) loroRemove(rid);
+                const prev = prevById.get(rid) ?? makeTombstone(rid);
+                write({ type: "delete", value: prev });
+                prevById.delete(rid);
               }
-            } else if (evt.type === "delete") {
-              const id2 = getKey(evt.row);
-              if (useLoro) loroRemove(id2);
-              const prev = prevById.get(id2) ?? makeTombstone(id2);
-              write({ type: "delete", value: prev });
-              prevById.delete(id2);
+            } finally {
+              commit();
             }
-          } finally {
-            commit();
-          }
+          });
         });
       } catch (e) {
         onError?.(e);
@@ -355,12 +529,8 @@ function surrealCollectionOptions({
     for (const m of p.transaction.mutations) {
       if (m.type !== "insert") continue;
       const base = { ...m.modified };
-      const row = useLoro ? {
-        ...base,
-        updated_at: now(),
-        sync_deleted: false
-      } : base;
-      if (useLoro) loroPut(row);
+      const row = useLoro ? { ...base, updated_at: now(), sync_deleted: false } : base;
+      if (useLoro) loroPutMany([row]);
       await table.create(row);
       resultRows.push(row);
     }
@@ -372,11 +542,8 @@ function surrealCollectionOptions({
       if (m.type !== "update") continue;
       const id2 = m.key;
       const base = { ...m.modified, id: id2 };
-      const merged = useLoro ? {
-        ...base,
-        updated_at: now()
-      } : base;
-      if (useLoro) loroPut(merged);
+      const merged = useLoro ? { ...base, updated_at: now() } : base;
+      if (useLoro) loroPutMany([merged]);
       const rid = new import_surrealdb2.RecordId(config.table.name, keyOf(id2));
       await table.update(rid, merged);
       resultRows.push(merged);

package/dist/index.mjs

@@ -9,74 +9,162 @@ import {
   Features,
   Table
 } from "surrealdb";
-function manageTable(db, useLoro, { name, ...args }) {
-  const fields = args.fields ?? "*";
-  const listAll = async () => {
+function manageTable(db, useLoro, { name, ...args }, syncMode = "eager") {
+  const rawFields = args.fields ?? "*";
+  const fields = rawFields === "*" ? ["*"] : [...rawFields];
+  const cache = /* @__PURE__ */ new Map();
+  let fullyLoaded = false;
+  const pageSize = args.pageSize ?? 100;
+  const initialPageSize = args.initialPageSize ?? Math.min(50, pageSize);
+  let cursor = 0;
+  let progressiveTask = null;
+  const idKey = (id) => typeof id === "string" ? id : id.toString();
+  const upsertCache = (rows) => {
+    for (const row of rows) cache.set(idKey(row.id), row);
+  };
+  const removeFromCache = (id) => {
+    cache.delete(idKey(id));
+  };
+  const listCached = () => Array.from(cache.values());
+  const buildWhere = () => {
+    if (!useLoro) return args.where;
+    return args.where ? and(args.where, eq("sync_deleted", false)) : eq("sync_deleted", false);
+  };
+  const buildQuery = () => {
     let q = db.select(new Table(name));
-    if (args.where) q = q.where(args.where);
-    return await q.fields(...fields);
+    const cond = buildWhere();
+    if (cond) q = q.where(cond);
+    return q;
+  };
+  const applyPaging = (q, start, limit) => {
+    if (typeof start === "number" && q.start) q = q.start(start);
+    if (typeof limit === "number" && q.limit) q = q.limit(limit);
+    return q;
+  };
+  const fetchAll = async () => {
+    const rows = await buildQuery().fields(...fields);
+    upsertCache(rows);
+    fullyLoaded = true;
+    return rows;
+  };
+  const fetchPage = async (opts) => {
+    const q = applyPaging(
+      buildQuery(),
+      opts?.start ?? 0,
+      opts?.limit ?? pageSize
+    );
+    const rows = await q.fields(...fields);
+    upsertCache(rows);
+    if (rows.length < (opts?.limit ?? pageSize)) fullyLoaded = true;
+    return rows;
   };
+  const fetchById = async (id) => {
+    const key = idKey(id);
+    const cached = cache.get(key);
+    if (cached) return cached;
+    const res = await db.select(id);
+    const row = Array.isArray(res) ? res[0] : res;
+    if (!row) return null;
+    if (useLoro && row.sync_deleted)
+      return null;
+    cache.set(key, row);
+    return row;
+  };
+  const loadMore = async (limit = pageSize) => {
+    if (fullyLoaded) return { rows: [], done: true };
+    const rows = await fetchPage({ start: cursor, limit });
+    cursor += rows.length;
+    const done = fullyLoaded || rows.length < limit;
+    args.onProgress?.({
+      table: name,
+      loaded: cache.size,
+      lastBatch: rows.length,
+      done
+    });
+    return { rows, done };
+  };
+  const startProgressive = () => {
+    if (progressiveTask || fullyLoaded) return;
+    progressiveTask = (async () => {
+      if (cache.size === 0) await loadMore(initialPageSize);
+      while (!fullyLoaded) {
+        const { done } = await loadMore(pageSize);
+        if (done) break;
+      }
+    })().finally(() => {
+      progressiveTask = null;
+    });
+  };
+  const listAll = () => fetchAll();
   const listActive = async () => {
-    if (!useLoro) return listAll();
-    return await db.select(new Table(name)).where(and(args.where, eq("sync_deleted", false))).fields(...fields);
+    if (syncMode === "eager") return fetchAll();
+    if (syncMode === "progressive") {
+      if (cache.size === 0) await loadMore(initialPageSize);
+      startProgressive();
+      return listCached();
+    }
+    return listCached();
   };
   const create = async (data) => {
     await db.create(new Table(name)).content(data);
   };
   const update = async (id, data) => {
     if (!useLoro) {
-      await db.update(id).merge({
-        ...data
-      });
+      await db.update(id).merge(data);
       return;
     }
-    try {
-      await db.update(id).merge({
-        ...data,
-        sync_deleted: false,
-        updated_at: Date.now()
-      });
-    } catch (error) {
-      console.warn(
-        `Please ensure the table ${name} has sync_deleted and updated_at fields defined`
-      );
-      console.error(
-        "Failed to update record with Loro (CRDTs) with: ",
-        error
-      );
-    }
+    await db.update(id).merge({
+      ...data,
+      sync_deleted: false,
+      updated_at: /* @__PURE__ */ new Date()
+    });
   };
   const remove = async (id) => {
     await db.delete(id);
+    removeFromCache(id);
   };
   const softDelete = async (id) => {
     if (!useLoro) {
       await db.delete(id);
+      removeFromCache(id);
       return;
     }
     await db.upsert(id).merge({
       sync_deleted: true,
-      updated_at: Date.now()
+      updated_at: /* @__PURE__ */ new Date()
     });
+    removeFromCache(id);
   };
   const subscribe = (cb) => {
     let killed = false;
     let live;
-    const on = ({ action, value }) => {
-      console.debug("[Surreal Live]", name, action, value);
+    const on = (msg) => {
+      const { action, value } = msg;
       if (action === "KILLED") return;
-      if (action === "CREATE") cb({ type: "insert", row: value });
-      else if (action === "UPDATE")
+      if (action === "CREATE") {
+        upsertCache([value]);
+        cb({ type: "insert", row: value });
+        return;
+      }
+      if (action === "UPDATE") {
+        if (useLoro && value.sync_deleted) {
+          removeFromCache(value.id);
+          cb({ type: "delete", row: { id: value.id } });
+          return;
+        }
+        upsertCache([value]);
         cb({ type: "update", row: value });
-      else if (action === "DELETE")
+        return;
+      }
+      if (action === "DELETE") {
+        removeFromCache(value.id);
         cb({ type: "delete", row: { id: value.id } });
+      }
     };
     const start = async () => {
-      const isLiveSupported = db.isFeatureSupported(Features.LiveQueries);
-      if (isLiveSupported) {
-        live = await db.live(new Table(name)).where(args.where);
-        live.subscribe(on);
-      }
+      if (!db.isFeatureSupported(Features.LiveQueries)) return;
+      live = await db.live(new Table(name)).where(args.where);
+      live.subscribe(on);
     };
     void start();
     return () => {
@@ -88,25 +176,61 @@ function manageTable(db, useLoro, { name, ...args }) {
   return {
     listAll,
     listActive,
+    listCached,
+    fetchPage,
+    fetchById,
+    loadMore,
     create,
     update,
     remove,
     softDelete,
-    subscribe
+    subscribe,
+    get isFullyLoaded() {
+      return fullyLoaded;
+    },
+    get cachedCount() {
+      return cache.size;
+    }
   };
 }
 
 // src/index.ts
+var DEFAULT_INITIAL_PAGE_SIZE = 50;
+var LOCAL_ID_VERIFY_CHUNK = 500;
+var stableStringify = (value) => {
+  const toJson = (v) => {
+    if (v === null) return null;
+    if (typeof v === "string" || typeof v === "number" || typeof v === "boolean")
+      return v;
+    if (v instanceof Date) return v.toISOString();
+    if (Array.isArray(v)) return v.map(toJson);
+    if (typeof v === "object") {
+      const o = v;
+      const keys = Object.keys(o).sort();
+      const out = {};
+      for (const k of keys) out[k] = toJson(o[k]);
+      return out;
+    }
+    return String(v);
+  };
+  return JSON.stringify(toJson(value));
+};
+var chunk = (arr, size) => {
+  const out = [];
+  for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
+  return out;
+};
 function surrealCollectionOptions({
   id,
   useLoro = false,
   onError,
   db,
+  syncMode,
   ...config
 }) {
   let loro;
   if (useLoro) loro = { doc: new LoroDoc(), key: id };
-  const keyOf = (id2) => typeof id2 === "string" ? id2 : id2.toString();
+  const keyOf = (rid) => typeof rid === "string" ? rid : rid.toString();
   const getKey = (row) => keyOf(row.id);
   const loroKey = loro?.key ?? id ?? "surreal";
   const loroMap = useLoro ? loro?.doc?.getMap?.(loroKey) ?? null : null;
@@ -115,16 +239,17 @@ function surrealCollectionOptions({
     const json = loroMap.toJSON?.() ?? {};
     return Object.values(json);
   };
-  const loroPut = (row) => {
-    if (!loroMap) return;
-    loroMap.set(getKey(row), row);
+  const loroPutMany = (rows) => {
+    if (!loroMap || rows.length === 0) return;
+    for (const row of rows) loroMap.set(getKey(row), row);
     loro?.doc?.commit?.();
   };
-  const loroRemove = (id2) => {
-    if (!loroMap) return;
-    loroMap.delete(id2);
+  const loroRemoveMany = (ids) => {
+    if (!loroMap || ids.length === 0) return;
+    for (const id2 of ids) loroMap.delete(id2);
     loro?.doc?.commit?.();
   };
+  const loroRemove = (id2) => loroRemoveMany([id2]);
   const pushQueue = [];
   const enqueuePush = (op) => {
     if (!useLoro) return;
@@ -136,35 +261,73 @@ function surrealCollectionOptions({
     for (const op of ops) {
       if (op.kind === "create") {
         await table.create(op.row);
-      }
-      if (op.kind === "update") {
+      } else if (op.kind === "update") {
         const rid = new RecordId(
           config.table.name,
           op.row.id.toString()
         );
         await table.update(rid, op.row);
-      }
-      if (op.kind === "delete") {
-        const rid = new RecordId(config.table.name, op.id.toString());
+      } else {
+        const rid = new RecordId(config.table.name, op.id);
         await table.softDelete(rid);
       }
     }
   };
   const newer = (a, b) => (a?.getTime() ?? -1) > (b?.getTime() ?? -1);
-  const reconcileBoot = (serverRows, write) => {
-    if (!useLoro) {
-      diffAndEmit(serverRows, write);
-      return;
+  const fetchServerByLocalIds = async (ids) => {
+    if (ids.length === 0) return [];
+    const tableName = config.table.name;
+    const parts = chunk(ids, LOCAL_ID_VERIFY_CHUNK);
+    const out = [];
+    for (const p of parts) {
+      const [res] = await db.query(
+        "SELECT * FROM type::table($table) WHERE id IN $ids",
+        {
+          table: tableName,
+          ids: p.map((x) => new RecordId(tableName, x))
+        }
+      );
+      if (res) out.push(...res);
+    }
+    return out;
+  };
+  const dedupeById = (rows) => {
+    const m = /* @__PURE__ */ new Map();
+    for (const r of rows) m.set(getKey(r), r);
+    return Array.from(m.values());
+  };
+  let prevById = /* @__PURE__ */ new Map();
+  const buildMap = (rows) => new Map(rows.map((r) => [getKey(r), r]));
+  const same = (a, b) => {
+    if (useLoro) {
+      return (a.sync_deleted ?? false) === (b.sync_deleted ?? false) && (a.updated_at?.getTime() ?? 0) === (b.updated_at?.getTime() ?? 0);
+    }
+    return stableStringify(a) === stableStringify(b);
+  };
+  const diffAndEmit = (currentRows, write) => {
+    const currById = buildMap(currentRows);
+    for (const [id2, row] of currById) {
+      const prev = prevById.get(id2);
+      if (!prev) write({ type: "insert", value: row });
+      else if (!same(prev, row)) write({ type: "update", value: row });
     }
+    for (const [id2, prev] of prevById) {
+      if (!currById.has(id2)) write({ type: "delete", value: prev });
+    }
+    prevById = currById;
+  };
+  const reconcileBoot = (serverRows, write) => {
     const localRows = loroToArray();
     const serverById = new Map(serverRows.map((r) => [getKey(r), r]));
     const localById = new Map(localRows.map((r) => [getKey(r), r]));
     const ids = /* @__PURE__ */ new Set([...serverById.keys(), ...localById.keys()]);
     const current = [];
+    const toRemove = [];
+    const toPut = [];
     const applyLocal = (row) => {
       if (!row) return;
-      if (row.sync_deleted) loroRemove(getKey(row));
-      else loroPut(row);
+      if (row.sync_deleted) toRemove.push(getKey(row));
+      else toPut.push(row);
     };
     for (const id2 of ids) {
       const s = serverById.get(id2);
@@ -224,46 +387,11 @@ function surrealCollectionOptions({
         }
       }
     }
+    loroRemoveMany(toRemove);
+    loroPutMany(toPut);
     diffAndEmit(current, write);
   };
-  let prevById = /* @__PURE__ */ new Map();
-  const buildMap = (rows) => new Map(rows.map((r) => [getKey(r), r]));
-  const same = (a, b) => {
-    if (useLoro) {
-      const aUpdated = a.updated_at;
-      const bUpdated = b.updated_at;
-      const aDeleted = a.sync_deleted ?? false;
-      const bDeleted = b.sync_deleted ?? false;
-      return aDeleted === bDeleted && (aUpdated?.getTime() ?? 0) === (bUpdated?.getTime() ?? 0) && JSON.stringify({
-        ...a,
-        updated_at: void 0,
-        sync_deleted: void 0
-      }) === JSON.stringify({
-        ...b,
-        updated_at: void 0,
-        sync_deleted: void 0
-      });
-    }
-    return JSON.stringify(a) === JSON.stringify(b);
-  };
-  const diffAndEmit = (currentRows, write) => {
-    const currById = buildMap(currentRows);
-    for (const [id2, row] of currById) {
-      const prev = prevById.get(id2);
-      if (!prev) {
-        write({ type: "insert", value: row });
-      } else if (!same(prev, row)) {
-        write({ type: "update", value: row });
-      }
-    }
-    for (const [id2, prev] of prevById) {
-      if (!currById.has(id2)) {
-        write({ type: "delete", value: prev });
-      }
-    }
-    prevById = currById;
-  };
-  const table = manageTable(db, useLoro, config.table);
+  const table = manageTable(db, useLoro, config.table, syncMode);
   const now = () => /* @__PURE__ */ new Date();
   const sync = ({
     begin,
@@ -277,6 +405,11 @@ function surrealCollectionOptions({
       };
     }
     let offLive = null;
+    let work = Promise.resolve();
+    const enqueueWork = (fn) => {
+      work = work.then(fn).catch((e) => onError?.(e));
+      return work;
+    };
     const makeTombstone = (id2) => ({
       id: new RecordId(config.table.name, id2).toString(),
       updated_at: now(),
@@ -284,43 +417,84 @@ function surrealCollectionOptions({
     });
     const start = async () => {
       try {
-        const serverRows = await table.listAll();
-        begin();
-        if (useLoro) reconcileBoot(serverRows, write);
-        else diffAndEmit(serverRows, write);
-        commit();
-        markReady();
+        let serverRows;
+        if (syncMode === "eager") {
+          serverRows = await table.listAll();
+        } else if (syncMode === "progressive") {
+          const first = await table.loadMore(
+            config.table.initialPageSize ?? DEFAULT_INITIAL_PAGE_SIZE
+          );
+          serverRows = first.rows;
+        } else {
+          serverRows = await table.listActive();
+        }
+        await enqueueWork(async () => {
+          begin();
+          if (useLoro) {
+            const localIds = loroToArray().map(getKey);
+            const verifiedServerRows = syncMode === "eager" ? serverRows : dedupeById([
+              ...serverRows,
+              ...await fetchServerByLocalIds(
+                localIds
+              )
+            ]);
+            reconcileBoot(verifiedServerRows, write);
+          } else {
+            diffAndEmit(serverRows, write);
+          }
+          commit();
+          markReady();
+        });
+        if (syncMode === "progressive") {
+          void (async () => {
+            while (!table.isFullyLoaded) {
+              const { rows } = await table.loadMore();
+              if (rows.length === 0) break;
+              await enqueueWork(async () => {
+                begin();
+                try {
+                  if (useLoro) loroPutMany(rows);
+                  diffAndEmit(rows, write);
+                } finally {
+                  commit();
+                }
+              });
+            }
+          })().catch((e) => onError?.(e));
+        }
         await flushPushQueue();
         offLive = table.subscribe((evt) => {
-          begin();
-          try {
-            if (evt.type === "insert" || evt.type === "update") {
-              const row = evt.row;
-              const deleted = useLoro ? row.sync_deleted ?? false : false;
-              if (deleted) {
-                if (useLoro) loroRemove(getKey(row));
-                const prev = prevById.get(getKey(row)) ?? makeTombstone(getKey(row));
-                write({ type: "delete", value: prev });
-                prevById.delete(getKey(row));
+          void enqueueWork(async () => {
+            begin();
+            try {
+              if (evt.type === "insert" || evt.type === "update") {
+                const row = evt.row;
+                const deleted = useLoro ? row.sync_deleted ?? false : false;
+                if (deleted) {
+                  if (useLoro) loroRemove(getKey(row));
+                  const prev = prevById.get(getKey(row)) ?? makeTombstone(getKey(row));
+                  write({ type: "delete", value: prev });
+                  prevById.delete(getKey(row));
+                } else {
+                  if (useLoro) loroPutMany([row]);
+                  const had = prevById.has(getKey(row));
+                  write({
+                    type: had ? "update" : "insert",
+                    value: row
+                  });
+                  prevById.set(getKey(row), row);
+                }
               } else {
-                if (useLoro) loroPut(row);
-                const had = prevById.has(getKey(row));
-                write({
-                  type: had ? "update" : "insert",
-                  value: row
-                });
-                prevById.set(getKey(row), row);
+                const rid = getKey(evt.row);
+                if (useLoro) loroRemove(rid);
+                const prev = prevById.get(rid) ?? makeTombstone(rid);
+                write({ type: "delete", value: prev });
+                prevById.delete(rid);
               }
-            } else if (evt.type === "delete") {
-              const id2 = getKey(evt.row);
-              if (useLoro) loroRemove(id2);
-              const prev = prevById.get(id2) ?? makeTombstone(id2);
-              write({ type: "delete", value: prev });
-              prevById.delete(id2);
+            } finally {
+              commit();
             }
-          } finally {
-            commit();
-          }
+          });
         });
       } catch (e) {
         onError?.(e);
@@ -337,12 +511,8 @@ function surrealCollectionOptions({
     for (const m of p.transaction.mutations) {
       if (m.type !== "insert") continue;
       const base = { ...m.modified };
-      const row = useLoro ? {
-        ...base,
-        updated_at: now(),
-        sync_deleted: false
-      } : base;
-      if (useLoro) loroPut(row);
+      const row = useLoro ? { ...base, updated_at: now(), sync_deleted: false } : base;
+      if (useLoro) loroPutMany([row]);
       await table.create(row);
       resultRows.push(row);
     }
@@ -354,11 +524,8 @@ function surrealCollectionOptions({
       if (m.type !== "update") continue;
       const id2 = m.key;
       const base = { ...m.modified, id: id2 };
-      const merged = useLoro ? {
-        ...base,
-        updated_at: now()
-      } : base;
-      if (useLoro) loroPut(merged);
+      const merged = useLoro ? { ...base, updated_at: now() } : base;
+      if (useLoro) loroPutMany([merged]);
       const rid = new RecordId(config.table.name, keyOf(id2));
       await table.update(rid, merged);
       resultRows.push(merged);

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@foretag/tanstack-db-surrealdb",
 	"description": "Add Offline / Local First Caching & Syncing to your SurrealDB app with TanstackDB and Loro (CRDTs)",
-	"version": "0.2.2",
+	"version": "0.3.0",
 	"files": [
 		"dist"
 	],