@sqlite-sync/core 0.0.1

package/dist/index.js

@@ -0,0 +1,710 @@
+import {
+  SQLiteDbWrapper,
+  SqliteDriver,
+  applyMemoryDbSchema,
+  applyWorkerDbSchema,
+  broadcastChannelNames,
+  createBroadcastChannels,
+  createCrdtSyncRemoteSource,
+  createSQLiteKysely,
+  createSyncDbMigrations,
+  createSyncDbMigrator,
+  introspectDb,
+  isWorkerInitMessage,
+  isWorkerInitResponse,
+  isWorkerNotificationMessage,
+  isWorkerRequestMessage,
+  isWorkerResponseMessage,
+  startPerformanceLogger,
+  syncDbWorkerLockName
+} from "./chunk-LK5FJCUD.js";
+import {
+  TypedBroadcastChannel,
+  TypedEvent,
+  applyCrdtEventMutations,
+  crdtSchema,
+  createAsyncAutoFlushBuffer,
+  createAutoFlushBuffer,
+  createCrdtStorage,
+  createCrdtSyncProducer,
+  createDeferredPromise,
+  createSyncIdCounter,
+  createTypedEventTarget,
+  dummyKysely,
+  ensureSingletonExecution,
+  generateId,
+  jsonSafeParse,
+  orderBy,
+  registerCrdtFunctions
+} from "./chunk-YLXMST5Z.js";
+
+// src/hlc.ts
+var HLCCounter = class {
+  timestamp;
+  counter;
+  nodeId;
+  getTimestamp;
+  constructor(nodeId, getTimestamp) {
+    this.timestamp = getTimestamp();
+    this.counter = 0;
+    this.nodeId = nodeId;
+    this.getTimestamp = getTimestamp;
+  }
+  getCurrentHLC() {
+    return {
+      timestamp: this.timestamp,
+      counter: this.counter,
+      nodeId: this.nodeId
+    };
+  }
+  getNextHLC() {
+    const now = this.getTimestamp();
+    if (now > this.timestamp) {
+      this.timestamp = now;
+      this.counter = 0;
+      return this.getCurrentHLC();
+    }
+    this.counter++;
+    return this.getCurrentHLC();
+  }
+  mergeHLC(hlc) {
+    if (this.timestamp === hlc.timestamp) {
+      this.counter = Math.max(this.counter, hlc.counter) + 1;
+    } else if (this.timestamp > hlc.timestamp) {
+      this.counter++;
+    } else {
+      this.timestamp = hlc.timestamp;
+      this.counter = hlc.counter + 1;
+    }
+  }
+};
+function serializeHLC(hlc) {
+  return hlc.timestamp.toString().padStart(15, "0") + ":" + hlc.counter.toString(36).padStart(5, "0") + ":" + hlc.nodeId;
+}
+function deserializeHLC(serialized) {
+  const [ts, count, ...node] = serialized.split(":");
+  return {
+    timestamp: parseInt(ts),
+    counter: parseInt(count, 36),
+    nodeId: node.join(":")
+  };
+}
+function compareHLC(one, two) {
+  if (one.timestamp == two.timestamp) {
+    if (one.counter === two.counter) {
+      if (one.nodeId === two.nodeId) {
+        return 0;
+      }
+      return one.nodeId < two.nodeId ? -1 : 1;
+    }
+    return one.counter - two.counter;
+  }
+  return one.timestamp - two.timestamp;
+}
+
+// src/worker-db/db-worker-client.ts
+var createWorkerDbClient = ({
+  broadcastChannels
+}) => {
+  const eventTarget = createTypedEventTarget();
+  const workerRequestsMap = /* @__PURE__ */ new Map();
+  const queryWorker = (method, args) => {
+    const requestId = crypto.randomUUID();
+    const promise = createDeferredPromise();
+    workerRequestsMap.set(requestId, promise);
+    const request = {
+      type: "request",
+      requestId,
+      method,
+      args
+    };
+    broadcastChannels.requests.postMessage(request);
+    return promise.promise;
+  };
+  const handleWorkerResponse = (message) => {
+    const promise = workerRequestsMap.get(message.requestId);
+    if (!promise) {
+      return;
+    }
+    promise.resolve(message.data);
+    workerRequestsMap.delete(message.requestId);
+  };
+  broadcastChannels.responses.onmessage = (event) => {
+    const message = event.data;
+    if (isWorkerResponseMessage(message)) {
+      handleWorkerResponse(message);
+    } else if (isWorkerNotificationMessage(message)) {
+      eventTarget.dispatchEvent("new-notification", message);
+    }
+  };
+  const rpc = {
+    execute: (query) => queryWorker("execute", [query]),
+    getSnapshot: () => queryWorker("getSnapshot", []),
+    pushTabEvents: (request) => queryWorker("pushTabEvents", [request]),
+    pullEvents: (params) => queryWorker("pullEvents", [params]),
+    postInitReady: () => queryWorker("postInitReady", [])
+  };
+  return {
+    ...rpc,
+    addEventListener: eventTarget.addEventListener,
+    removeEventListener: eventTarget.removeEventListener
+  };
+};
+function initializeWorkerDb({
+  worker,
+  broadcastChannels,
+  config
+}) {
+  const promise = createDeferredPromise();
+  broadcastChannels.responses.onmessage = (event) => {
+    const message = event.data;
+    if (!isWorkerInitResponse(message)) {
+      return;
+    }
+    promise.resolve();
+    worker.onmessage = null;
+  };
+  const configMessage = {
+    type: "init",
+    config
+  };
+  worker.postMessage(configMessage);
+  broadcastChannels.requests.postMessage({
+    type: "request",
+    requestId: crypto.randomUUID(),
+    method: "postInitReady",
+    args: []
+  });
+  return promise.promise;
+}
+
+// src/memory-db/sqlite-reactive-db.ts
+import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
+var sqliteModule = null;
+function createSQLiteReactiveDb(opts) {
+  return SQLiteReactiveDb.create(opts);
+}
+var defaultLogger = (type, message, level = "info") => {
+  const logMessage = `[${type}] ${message}`;
+  switch (level) {
+    case "info":
+      console.log(logMessage);
+      break;
+    case "warning":
+      console.warn(logMessage);
+      break;
+    case "error":
+      console.error(logMessage);
+      break;
+    case "trace":
+      console.trace(logMessage);
+      break;
+  }
+};
+var SQLiteReactiveDb = class _SQLiteReactiveDb {
+  db;
+  sqlite3;
+  logger;
+  tablesUsedStatement = null;
+  eventTarget = createTypedEventTarget();
+  constructor(sqlite3, logger) {
+    this.sqlite3 = sqlite3;
+    this.logger = logger;
+    this.db = new SQLiteDbWrapper({
+      db: new sqlite3.oo1.DB({ filename: ":memory:" }),
+      logger: this.logger,
+      loggerPrefix: "memory",
+      sqlite3
+    });
+  }
+  static async create(opts) {
+    const logger = opts.logger ?? defaultLogger;
+    const perf = startPerformanceLogger(logger);
+    if (!sqliteModule) {
+      sqliteModule = await sqlite3InitModule();
+    }
+    const db = new _SQLiteReactiveDb(sqliteModule, logger);
+    if (opts.snapshot) {
+      db.useSnapshot(opts.snapshot);
+    }
+    db.registerDbHooks();
+    perf.logEnd("createSQLiteMemoryDb", "success", "info");
+    return db;
+  }
+  createLiveQuery(query) {
+    const fetchRows = () => this.db.execute({
+      sql: query.sql,
+      parameters: query.parameters ?? []
+    }).rows;
+    let rows = null;
+    const getRows = () => {
+      if (!rows) {
+        rows = fetchRows();
+      }
+      return rows;
+    };
+    let subscriber = null;
+    const refresh = () => {
+      rows = fetchRows();
+      subscriber?.();
+    };
+    const subscribe = (onchange) => {
+      if (subscriber) {
+        throw new Error("Subscriber already exists");
+      }
+      subscriber = onchange;
+      const subscription = this.subscribeToQueryChanges({
+        sql: query.sql,
+        onDataChange: refresh
+      });
+      return () => {
+        subscription.unsubscribe();
+        subscriber = null;
+      };
+    };
+    return { getRows, refresh, subscribe };
+  }
+  subscribeToQueryChanges(params) {
+    const { sql: sql2, onDataChange } = params;
+    const tables = this.getTablesUsed(sql2);
+    const readTables = /* @__PURE__ */ new Set();
+    for (const table of tables) {
+      if (!readTables.has(table.name)) {
+        readTables.add(table.name);
+      } else if (table.isWrite) {
+        throw new Error(
+          "This query writes and reads from the same table. This may cause infinite loops."
+        );
+      }
+    }
+    const notifyDataChange = createDebouncedCallback(() => {
+      onDataChange();
+    }, 30);
+    for (const table of readTables) {
+      this.eventTarget.addEventListener(`table:${table}`, notifyDataChange);
+    }
+    this.eventTarget.addEventListener("any-table-changed", notifyDataChange);
+    return {
+      unsubscribe: () => {
+        for (const table of readTables) {
+          this.eventTarget.removeEventListener(
+            `table:${table}`,
+            notifyDataChange
+          );
+          this.eventTarget.removeEventListener(
+            "any-table-changed",
+            notifyDataChange
+          );
+        }
+      }
+    };
+  }
+  subscribeToTableChanges(table, onChanges) {
+    this.eventTarget.addEventListener(`table:${table}`, onChanges);
+    this.eventTarget.addEventListener("any-table-changed", onChanges);
+    return {
+      unsubscribe: () => {
+        this.eventTarget.removeEventListener(`table:${table}`, onChanges);
+        this.eventTarget.removeEventListener("any-table-changed", onChanges);
+      }
+    };
+  }
+  getTablesUsed(query) {
+    if (!this.tablesUsedStatement) {
+      this.tablesUsedStatement = this.db.prepare(
+        "select t.tbl_name as name, u.wr as isWrite from tables_used(?) as u inner join sqlite_master as t on t.name = u.name where u.schema = 'main'"
+      );
+    }
+    const tables = this.tablesUsedStatement.execute([query]);
+    if (tables.length == 0 && query.toLowerCase().includes("delete")) {
+      tables.push(...this.getClearedTables(query));
+    }
+    return tables;
+  }
+  getClearedTables(query) {
+    const operations = this.db.execute(`EXPLAIN ${query.split(";")[0]}`).rows;
+    const clearedTablesRootPages = /* @__PURE__ */ new Set();
+    for (const operation of operations) {
+      if (operation.opcode === "Clear" && operation.p2 === 0) {
+        clearedTablesRootPages.add(operation.p1);
+      }
+    }
+    if (clearedTablesRootPages.size === 0) {
+      return [];
+    }
+    const tableNames = this.db.execute(
+      `select t.tbl_name as name, true as isWrite from sqlite_master as t where t.rootpage in (${Array.from(
+        clearedTablesRootPages
+      ).join(",")})`
+    ).rows;
+    return tableNames;
+  }
+  addEventListener(type, listener) {
+    this.eventTarget.addEventListener(type, listener);
+  }
+  removeEventListener(type, listener) {
+    this.eventTarget.removeEventListener(type, listener);
+  }
+  notifyTableSubscribers(tables = []) {
+    if (tables.length === 0) {
+      this.eventTarget.dispatchEvent("any-table-changed", void 0);
+      return;
+    }
+    for (const table of tables) {
+      this.eventTarget.dispatchEvent(`table:${table}`, void 0);
+    }
+  }
+  registerDbHooks() {
+    const updateQueue = /* @__PURE__ */ new Set();
+    this.sqlite3.capi.sqlite3_update_hook(
+      this.db.ensureDb,
+      (_ctx, _opId, _db, table) => {
+        updateQueue.add(table);
+      },
+      0
+    );
+    this.sqlite3.capi.sqlite3_rollback_hook(
+      this.db.ensureDb,
+      () => {
+        if (updateQueue.size === 0) {
+          return 0;
+        }
+        updateQueue.clear();
+        this.eventTarget.dispatchEvent("transaction-rolled-back", void 0);
+        return 0;
+      },
+      0
+    );
+    this.sqlite3.capi.sqlite3_commit_hook(
+      this.db.ensureDb,
+      () => {
+        if (updateQueue.size === 0) {
+          return 0;
+        }
+        const tables = Array.from(updateQueue);
+        updateQueue.clear();
+        this.eventTarget.dispatchEvent("transaction-committed", void 0);
+        queueMicrotask(() => {
+          this.notifyTableSubscribers(tables);
+        });
+        return 0;
+      },
+      0
+    );
+  }
+  createSnapshot() {
+    const perf = startPerformanceLogger(this.logger);
+    const snapshot = this.sqlite3.capi.sqlite3_js_db_export(this.db.ensureDb);
+    perf.logEnd(
+      "createSnapshot",
+      `snapshot size: ${snapshot.byteLength}`,
+      "info"
+    );
+    return snapshot;
+  }
+  useSnapshot(snapshot) {
+    this.db.useSnapshot(snapshot);
+    this.notifyTableSubscribers();
+  }
+};
+function createDebouncedCallback(callback, delay) {
+  let timeout = null;
+  let shouldCallWithoutDelay = true;
+  return (...args) => {
+    if (shouldCallWithoutDelay) {
+      callback(...args);
+      shouldCallWithoutDelay = false;
+      return;
+    }
+    const effect = () => {
+      timeout = null;
+      shouldCallWithoutDelay = true;
+      return callback(...args);
+    };
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+    timeout = setTimeout(effect, delay);
+  };
+}
+
+// src/memory-db/memory-db.ts
+import { sql } from "kysely";
+
+// src/sqlite-crdt/make-crdt-table.ts
+function makeCrdtTable({
+  db,
+  baseTableName,
+  crdtTableName
+}) {
+  const tableSchema = db.dbSchema[baseTableName];
+  if (!tableSchema) {
+    throw new Error(`Table ${baseTableName} not found`);
+  }
+  db.execute(`
+create view ${crdtTableName} as
+select * from ${baseTableName}
+where tombstone = 0;`);
+  const allColumnNames = tableSchema.columns.map((column) => column.name);
+  const jsonPayload = (from) => "'{'||" + allColumnNames.map((col) => `'"${col}":'||json_quote(${from}.${col})`).join("||','||") + "||'}'";
+  db.execute(`
+create trigger ${crdtTableName}_created
+instead of insert on ${crdtTableName}
+for each row
+begin
+select handle_item_created('${baseTableName}', ${jsonPayload("new")});
+end;
+`);
+  db.execute(`
+create trigger ${crdtTableName}_updated
+instead of update on ${crdtTableName}
+for each row
+begin
+select handle_item_updated(
+  '${baseTableName}',
+  ${jsonPayload("old")},
+  ${jsonPayload("new")}
+);
+end;
+`);
+  db.execute(`
+create trigger ${crdtTableName}_deleted
+instead of delete on ${crdtTableName}
+for each row
+when old.tombstone = 0
+begin
+select handle_item_deleted('${baseTableName}', old.id);
+end;
+`);
+}
+
+// src/memory-db/memory-db.ts
+async function createMemoryDb({
+  reactiveDb: _reactiveDb,
+  hlcCounter,
+  tabId,
+  crdtTables
+}) {
+  const reactiveDb = _reactiveDb;
+  const db = reactiveDb.db;
+  applyMemoryDbSchema(db);
+  for (const table of crdtTables) {
+    makeCrdtTable({
+      db,
+      baseTableName: table.baseTableName,
+      crdtTableName: table.crdtTableName
+    });
+  }
+  const localSyncId = createSyncIdCounter({
+    initialSyncId: 0
+  });
+  const pendingLocalEvents = [];
+  registerCrdtFunctions({
+    db,
+    getTableSchema: (dataset) => db.dbSchema[dataset],
+    getNextTimestamp: () => serializeHLC(hlcCounter.getNextHLC()),
+    updateLogTableName: "crdt_update_log",
+    onEventApplied: (event) => {
+      const persistedEvent = {
+        ...event,
+        origin: tabId,
+        sync_id: ++localSyncId.current,
+        status: "applied"
+      };
+      enqueueCrdtEvent(db, persistedEvent);
+      pendingLocalEvents.push(persistedEvent);
+    }
+  });
+  db.createScalarFunction({
+    name: "gen_id",
+    callback: () => generateId(),
+    deterministic: false,
+    directOnly: false,
+    innocuous: true
+  });
+  reactiveDb.addEventListener("transaction-rolled-back", () => {
+    pendingLocalEvents.length = 0;
+  });
+  reactiveDb.addEventListener("transaction-committed", () => {
+    const appliedEvents = pendingLocalEvents.splice(0);
+    queueMicrotask(() => {
+      for (const event of appliedEvents) {
+        crdtStorage.dispatchEvent("event-applied", event);
+      }
+      crdtStorage.dispatchEvent("event-processing-done", void 0);
+    });
+  });
+  const crdtStorage = createCrdtStorage({
+    syncId: localSyncId,
+    persistEvents: (events) => persistEvents(db, events),
+    popPendingEventsBatch: () => popPendingEventsBatch(db, 50),
+    applyCrdtEventMutations: (event) => applyCrdtEventMutations({
+      db,
+      event,
+      updateLogTableName: "crdt_update_log"
+    }),
+    updateEventStatus: (syncId, status) => updateEventStatus(db, syncId, status)
+  });
+  return {
+    crdtStorage
+  };
+}
+function enqueueCrdtEvent(db, event) {
+  db.executePrepared(
+    "enqueue-crdt-events",
+    event,
+    (db2, params) => db2.insertInto("persisted_crdt_events").values({
+      status: params("status"),
+      sync_id: params("sync_id"),
+      type: params("type"),
+      timestamp: params("timestamp"),
+      dataset: params("dataset"),
+      item_id: params("item_id"),
+      payload: params("payload"),
+      origin: params("origin")
+    })
+  );
+}
+function persistEvents(db, events) {
+  db.executeTransaction((db2) => {
+    const chunkSize = 100;
+    for (let i = 0; i < events.length; i += chunkSize) {
+      const chunk = events.slice(i, i + chunkSize);
+      db2.executeKysely(
+        (db3) => db3.insertInto("persisted_crdt_events").values(chunk)
+      );
+    }
+  });
+}
+function popPendingEventsBatch(db, limit) {
+  const events = db.executePrepared(
+    "pop-enqueued-crdt-events",
+    {
+      limit
+    },
+    (db2, param) => db2.selectFrom("persisted_crdt_events").where("status", "=", sql.lit("pending")).limit(param("limit")).orderBy("sync_id", "asc").selectAll()
+  );
+  return {
+    events,
+    hasMore: events.length === limit
+  };
+}
+function updateEventStatus(db, syncId, status) {
+  db.executePrepared(
+    "update-crdt-event-status",
+    { syncId, status },
+    (db2, params) => db2.updateTable("persisted_crdt_events").set({ status: params("status") }).where("sync_id", "=", params("syncId"))
+  );
+}
+
+// src/sync-db.ts
+async function createSyncedDb(options) {
+  if (!options.dbPath.startsWith("/")) {
+    throw new Error("dbPath must be an absolute path");
+  }
+  const tabId = generateId();
+  const broadcastChannels = createBroadcastChannels();
+  await initializeWorkerDb({
+    worker: options.worker,
+    broadcastChannels,
+    config: {
+      clientId: generateId(),
+      dbPath: options.dbPath,
+      clearOnInit: options.clearOnInit,
+      syncServer: {
+        host: "",
+        room: ""
+      }
+    }
+  });
+  const workerClient = createWorkerDbClient({
+    broadcastChannels
+  });
+  const hlcCounter = new HLCCounter(tabId, () => Date.now());
+  const workerClientSnapshot = await workerClient.getSnapshot();
+  const reactiveDb = await createSQLiteReactiveDb({
+    snapshot: workerClientSnapshot.file
+  });
+  const { crdtStorage } = await createMemoryDb({
+    reactiveDb,
+    hlcCounter,
+    tabId,
+    crdtTables: options.crdtTables
+  });
+  const remoteSyncId = createSyncIdCounter({
+    initialSyncId: workerClientSnapshot.syncId
+  });
+  const remoteSyncSource = createCrdtSyncRemoteSource({
+    bufferSize: 100,
+    syncId: remoteSyncId,
+    storage: crdtStorage,
+    nodeId: tabId,
+    pullEvents: (request) => workerClient.pullEvents(request),
+    pushEvents: (request) => workerClient.pushTabEvents(request)
+  });
+  workerClient.addEventListener("new-notification", (event) => {
+    const notification = event.payload;
+    if (notification.notificationType === "new-event-chunk-applied" && notification.newSyncId > remoteSyncId.current) {
+      remoteSyncSource.pullEvents();
+    }
+  });
+  crdtStorage.addEventListener("event-applied", (event) => {
+    if (event.payload.origin === "remote") {
+      hlcCounter.mergeHLC(deserializeHLC(event.payload.timestamp));
+    }
+  });
+  return {
+    db: reactiveDb.db,
+    reactiveDb,
+    workerDb: workerClient
+  };
+}
+export {
+  HLCCounter,
+  SQLiteDbWrapper,
+  SQLiteReactiveDb,
+  SqliteDriver,
+  TypedBroadcastChannel,
+  TypedEvent,
+  applyCrdtEventMutations,
+  applyMemoryDbSchema,
+  applyWorkerDbSchema,
+  broadcastChannelNames,
+  compareHLC,
+  crdtSchema,
+  createAsyncAutoFlushBuffer,
+  createAutoFlushBuffer,
+  createBroadcastChannels,
+  createCrdtStorage,
+  createCrdtSyncProducer,
+  createCrdtSyncRemoteSource,
+  createDeferredPromise,
+  createMemoryDb,
+  createSQLiteKysely,
+  createSQLiteReactiveDb,
+  createSyncDbMigrations,
+  createSyncDbMigrator,
+  createSyncIdCounter,
+  createSyncedDb,
+  createTypedEventTarget,
+  createWorkerDbClient,
+  deserializeHLC,
+  dummyKysely,
+  ensureSingletonExecution,
+  generateId,
+  initializeWorkerDb,
+  introspectDb,
+  isWorkerInitMessage,
+  isWorkerInitResponse,
+  isWorkerNotificationMessage,
+  isWorkerRequestMessage,
+  isWorkerResponseMessage,
+  jsonSafeParse,
+  makeCrdtTable,
+  orderBy,
+  registerCrdtFunctions,
+  serializeHLC,
+  startPerformanceLogger,
+  syncDbWorkerLockName
+};
+//# sourceMappingURL=index.js.map