@sqlite-sync/core 0.6.0 → 0.7.1

package/dist/{chunk-3L4QG3BZ.js → chunk-LBYL5L4O.js}

@@ -171,6 +171,8 @@ function introspectDb(_db) {
             isNullable: !col.notnull,
             isAutoIncrementing: col.name === autoIncrementCol,
             hasDefaultValue: col.dflt_value != null,
+            defaultValueSql: col.dflt_value == null ? null : String(col.dflt_value),
+            isPrimaryKey: col.pk > 0,
             comment: void 0
           }))
         }
@@ -892,6 +894,276 @@ function applyMemoryDbSchema(db) {
   );
 }
 
+// src/schema/validate-crdt-event.ts
+var CrdtEventValidationError = class extends Error {
+  constructor(errors) {
+    super(`Invalid CRDT events: ${errors.join("; ")}`);
+    this.errors = errors;
+    this.name = "CrdtEventValidationError";
+  }
+};
+var crdtEventTypes = ["item-created", "item-updated", "item-deleted"];
+function validateNewCrdtEvent(schema, input) {
+  const errors = [];
+  if (!isRecord(input)) {
+    return { success: false, errors: ["Event must be an object"] };
+  }
+  const type = input.type;
+  const dataset = input.dataset;
+  const itemId = input.item_id;
+  let payload = input.payload;
+  let payloadParseFailed = false;
+  if (typeof payload === "string") {
+    try {
+      payload = JSON.parse(payload);
+    } catch {
+      payloadParseFailed = true;
+      errors.push(`Event payload is not valid JSON`);
+    }
+  }
+  if (!isCrdtEventType(type)) {
+    errors.push(`Invalid event type "${String(type)}"`);
+  }
+  if (typeof dataset !== "string") {
+    errors.push(`Event dataset must be a string`);
+  }
+  if (typeof itemId !== "string") {
+    errors.push(`Event item_id must be a string`);
+  }
+  if (!payloadParseFailed && !isRecord(payload)) {
+    errors.push(`Event payload must be an object`);
+  }
+  const datasetMatch = typeof dataset === "string" ? findTableForDataset(schema, dataset) : null;
+  if (typeof dataset === "string" && !datasetMatch) {
+    errors.push(`Unknown dataset "${dataset}"`);
+  }
+  if (isCrdtEventType(type) && datasetMatch && isRecord(payload)) {
+    switch (type) {
+      case "item-created": {
+        const result = datasetMatch.table.validatePayload(payload, { event: "item-created" });
+        if (!result.success) {
+          errors.push(...result.errors.map((error) => `payload: ${error}`));
+        }
+        if (typeof itemId === "string" && "id" in payload && payload.id !== itemId) {
+          errors.push(`payload: Column "id" must match item_id "${itemId}"`);
+        }
+        break;
+      }
+      case "item-updated": {
+        const result = datasetMatch.table.validatePayload(payload, { event: "item-updated" });
+        if (!result.success) {
+          errors.push(...result.errors.map((error) => `payload: ${error}`));
+        }
+        break;
+      }
+      case "item-deleted":
+        if (Object.keys(payload).length > 0) {
+          errors.push(`payload: item-deleted events must have an empty payload`);
+        }
+        break;
+    }
+  }
+  if (errors.length > 0) {
+    return { success: false, errors };
+  }
+  return {
+    success: true,
+    event: {
+      type,
+      dataset: datasetMatch?.baseTableName ?? dataset,
+      item_id: itemId,
+      payload
+    }
+  };
+}
+function findTableForDataset(schema, dataset) {
+  const config = schema.tablesConfig.find(
+    (tableConfig) => tableConfig.baseTableName === dataset || tableConfig.crdtTableName === dataset
+  );
+  if (config) {
+    const table2 = schema.tables[config.crdtTableName];
+    return table2 ? { table: table2, baseTableName: config.baseTableName } : null;
+  }
+  const table = schema.tables[dataset];
+  return table ? { table, baseTableName: dataset } : null;
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isCrdtEventType(value) {
+  return typeof value === "string" && crdtEventTypes.includes(value);
+}
+
+// src/schema/verify-sync-schema.ts
+import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
+var acceptedDataTypes = {
+  text: ["TEXT"],
+  integer: ["INTEGER", "INT"],
+  real: ["REAL", "DOUBLE", "FLOAT"],
+  boolean: ["BOOLEAN", "INTEGER", "INT"],
+  enum: ["TEXT"]
+};
+var sqliteModule = null;
+async function verifySyncSchema(schema) {
+  if (!sqliteModule) {
+    sqliteModule = await sqlite3InitModule();
+  }
+  const sqlite3 = sqliteModule;
+  const db = new SQLiteDbWrapper({
+    db: () => new sqlite3.oo1.DB({ filename: ":memory:" }),
+    logger: () => {
+    },
+    loggerPrefix: "schema-verify",
+    sqlite3
+  });
+  try {
+    const migrator = createMigrator({
+      migrations: schema.migrations,
+      schemaVersion: createStoredValue({ initialValue: -1 })
+    });
+    try {
+      migrator.migrateDbToLatest({
+        startTransaction: (callback) => {
+          db.executeTransaction(
+            (tx) => callback({ execute: (sql4, parameters, meta) => tx.execute({ sql: sql4, parameters }, meta) })
+          );
+        }
+      });
+    } catch (error) {
+      return [
+        {
+          table: "(migrations)",
+          message: `migration replay failed on an empty database: ${error instanceof Error ? error.message : String(error)}`
+        }
+      ];
+    }
+    const introspection = introspectDb(db);
+    const issues = [];
+    for (const { crdtTableName, baseTableName } of schema.tablesConfig) {
+      const declared = schema.tables[crdtTableName];
+      if (!declared) continue;
+      verifyTable(db, baseTableName, declared.columns, introspection[baseTableName], issues);
+    }
+    return issues;
+  } finally {
+    db.close();
+  }
+}
+function formatSchemaVerificationIssues(issues) {
+  const lines = issues.map(
+    (issue) => ` - table "${issue.table}"${issue.column ? ` column "${issue.column}"` : ""}: ${issue.message}`
+  );
+  return `sync schema verification failed (${issues.length} issue${issues.length === 1 ? "" : "s"}):
+${lines.join("\n")}`;
+}
+function verifyTable(db, table, declaredColumns, actual, issues) {
+  if (!actual) {
+    issues.push({
+      table,
+      message: "declared in the schema but missing after migrations \u2014 add a createTable step in a new version"
+    });
+    return;
+  }
+  if (actual.isView) {
+    issues.push({ table, message: "expected a base table, but migrations created a view with this name" });
+    return;
+  }
+  const actualColumns = new Map(actual.columns.map((column) => [column.name, column]));
+  for (const [name, meta] of Object.entries(declaredColumns)) {
+    const column = actualColumns.get(name);
+    if (!column) {
+      issues.push({
+        table,
+        column: name,
+        message: "declared in the schema but missing after migrations \u2014 add an addColumn step in a new version"
+      });
+      continue;
+    }
+    const accepted = acceptedDataTypes[meta.kind];
+    if (!accepted.includes(column.dataType.toUpperCase())) {
+      issues.push({
+        table,
+        column: name,
+        message: `type mismatch \u2014 schema declares ${meta.kind} (${accepted.join("/")}), migrated column is ${column.dataType.toUpperCase()}`
+      });
+    }
+    if (column.isNullable !== meta.nullable) {
+      issues.push({
+        table,
+        column: name,
+        message: meta.nullable ? "schema declares the column nullable, but the migrated column is NOT NULL" : "schema declares the column NOT NULL, but the migrated column is nullable"
+      });
+    }
+    if (name === "id" && !column.isPrimaryKey) {
+      issues.push({ table, column: name, message: "the id column must be the PRIMARY KEY" });
+    }
+    verifyDefault(db, table, name, meta, column.defaultValueSql, issues);
+  }
+  for (const column of actual.columns) {
+    if (!(column.name in declaredColumns)) {
+      issues.push({
+        table,
+        column: column.name,
+        message: "created by migrations but not declared in the schema \u2014 sync event payloads include every column, so declare it (or drop it in a new version)"
+      });
+    }
+  }
+}
+function verifyDefault(db, table, column, meta, defaultValueSql, issues) {
+  if (meta.hasDefault && meta.defaultValue !== null && defaultValueSql === null) {
+    issues.push({
+      table,
+      column,
+      message: `schema declares default ${formatExpected(meta)}, but the migrated column has no default`
+    });
+    return;
+  }
+  if (!meta.hasDefault && defaultValueSql !== null) {
+    issues.push({
+      table,
+      column,
+      message: `migrated column has default ${defaultValueSql}, but the schema declares none`
+    });
+    return;
+  }
+  if (!meta.hasDefault || defaultValueSql === null) {
+    return;
+  }
+  let actual;
+  try {
+    actual = db.execute({
+      sql: `SELECT (${defaultValueSql}) AS value`,
+      parameters: []
+    }).rows[0]?.value;
+  } catch {
+    actual = void 0;
+  }
+  if (actual !== expectedDefaultValue(meta)) {
+    issues.push({
+      table,
+      column,
+      message: `default mismatch \u2014 schema declares ${formatExpected(meta)}, migrated column default is ${defaultValueSql}`
+    });
+  }
+}
+function expectedDefaultValue(meta) {
+  if (meta.defaultValue === null) return null;
+  switch (meta.kind) {
+    case "boolean":
+      return meta.defaultValue ? 1 : 0;
+    case "integer":
+    case "real":
+      return Number(meta.defaultValue);
+    case "text":
+    case "enum":
+      return String(meta.defaultValue);
+  }
+}
+function formatExpected(meta) {
+  const expected = expectedDefaultValue(meta);
+  return typeof expected === "string" ? `'${expected}'` : String(expected);
+}
+
 // src/sqlite-crdt/apply-crdt-event.ts
 var createSQLiteCrdtApplyFunction = ({
   db,
@@ -1084,6 +1356,16 @@ function createCrdtStorage(storage) {
   let localSyncId = storage.initialLocalSyncId;
   const db = storage.db;
   const crdtEventsTable = storage.dbConfig.eventsTable.fullIdentifier;
+  const getInitialSequentialSyncId = () => {
+    const [firstPendingEvent] = db.executePrepared(
+      "get-first-pending-event",
+      { status: "pending" },
+      (db2, params) => db2.selectFrom(crdtEventsTable).select("sync_id").where("status", "=", params("status")).orderBy("sync_id", "asc").limit(sql3.lit(1)),
+      { loggerLevel: "system" }
+    );
+    return firstPendingEvent ? firstPendingEvent.sync_id - 1 : localSyncId;
+  };
+  let sequentialSyncId = getInitialSequentialSyncId();
   const eventTarget = createTypedEventTarget();
   const persistEvent = (tx, event) => {
     tx.executePrepared(
@@ -1138,38 +1420,77 @@ function createCrdtStorage(storage) {
   };
   const notifyEventApplied = (event) => {
     if (event.status === "applied") {
-      storage.onEventApplied?.(event);
+      queueMicrotask(() => {
+        storage.onEventApplied?.(event);
+      });
+    }
+  };
+  const applyOwnEvents = (events) => {
+    const schema = storage.schema;
+    if (!schema) {
+      throw new Error("applyOwnEvents requires a sync schema; none was provided to createCrdtStorage");
+    }
+    const validatedEvents = [];
+    const errors = [];
+    for (const [index, event] of events.entries()) {
+      const result = validateNewCrdtEvent(schema, {
+        type: event.type,
+        dataset: event.dataset,
+        item_id: event.item_id,
+        payload: event.payload
+      });
+      if (!result.success) {
+        errors.push(...result.errors.map((error) => `[${index}] ${error}`));
+        continue;
+      }
+      validatedEvents.push(result.event);
     }
+    if (errors.length > 0) {
+      throw new CrdtEventValidationError(errors);
+    }
+    db.executeTransaction((tx) => {
+      for (const event of validatedEvents) {
+        const persistedEvent = {
+          schema_version: storage.migrator.currentSchemaVersion,
+          timestamp: serializeHLC(storage.hlc.getNextHLC()),
+          type: event.type,
+          dataset: event.dataset,
+          item_id: event.item_id,
+          origin: "own-applied",
+          source_node_id: storage.nodeId,
+          payload: JSON.stringify(event.payload),
+          sync_id: ++localSyncId,
+          status: "pending"
+        };
+        persistEvent(tx, persistedEvent);
+        applyCrdtEvent(persistedEvent);
+      }
+    });
+    void processEnqueuedEvents();
   };
-  const applyOwnEvent = (event, { wrapInTransaction } = {}) => {
+  const applyOwnEventFromTransaction = (tx, event) => {
     const persistedEvent = {
       schema_version: storage.migrator.currentSchemaVersion,
       timestamp: serializeHLC(storage.hlc.getNextHLC()),
       type: event.type,
       dataset: event.dataset,
       item_id: event.item_id,
-      origin: "own",
+      origin: "own-applied",
       source_node_id: storage.nodeId,
       payload: event.payload,
       sync_id: ++localSyncId,
       status: "pending"
     };
-    if (wrapInTransaction) {
-      db.executeTransaction((tx) => {
-        persistEvent(tx, persistedEvent);
-        processPersistedEvent(tx, persistedEvent);
-        persistEventHlcAccumulator();
-      });
-    } else {
-      persistEvent(db, persistedEvent);
-      processPersistedEvent(db, persistedEvent);
-      persistEventHlcAccumulator();
-    }
+    persistEvent(tx, persistedEvent);
+    applyCrdtEvent(persistedEvent);
   };
-  const dispatchEventsApplied = (syncId = localSyncId) => {
+  const dispatchEventsApplied = (previousSequentialSyncId) => {
+    if (sequentialSyncId <= previousSequentialSyncId) {
+      return;
+    }
     eventTarget.dispatchEvent("events-applied", {
-      syncId,
-      eventHlcSum: eventHlcAccumulator?.current ?? null
+      syncId: sequentialSyncId,
+      eventHlcSum: sequentialSyncId === localSyncId ? eventHlcAccumulator?.current ?? null : null
     });
   };
   const hasPendingEvents = () => {
@@ -1188,7 +1509,8 @@ function createCrdtStorage(storage) {
       status: options.status ?? null,
       afterSyncId: options.afterSyncId ?? null,
       excludeOrigin: options.excludeOrigin ?? null,
-      excludeNodeId: options.excludeNodeId ?? null
+      excludeNodeId: options.excludeNodeId ?? null,
+      maxSyncId: options.status === "applied" ? sequentialSyncId : localSyncId
     };
     const filterKeys = [
       queryParams.excludeNodeId ? "nodeid" : "no-nodeid",
@@ -1197,17 +1519,22 @@ function createCrdtStorage(storage) {
     const events = db.executePrepared(
       `get-events-batch-${filterKeys.join("-")}`,
       queryParams,
-      (db2, params) => db2.selectFrom(crdtEventsTable).where("sync_id", ">", params("afterSyncId")).where("status", "=", params("status")).$if(!!queryParams.excludeNodeId, (qb) => qb.where("source_node_id", "!=", params("excludeNodeId"))).$if(!!queryParams.excludeOrigin, (qb) => qb.where("origin", "!=", params("excludeOrigin"))).selectAll().limit(params("limit")).orderBy("sync_id", "asc"),
+      (db2, params) => db2.selectFrom(crdtEventsTable).where("sync_id", ">", params("afterSyncId")).where("sync_id", "<=", params("maxSyncId")).where("status", "=", params("status")).$if(!!queryParams.excludeNodeId, (qb) => qb.where("source_node_id", "!=", params("excludeNodeId"))).$if(!!queryParams.excludeOrigin, (qb) => qb.where("origin", "!=", params("excludeOrigin"))).selectAll().limit(params("limit")).orderBy("sync_id", "asc"),
       { loggerLevel: "system" }
     );
     const hasMore = events.length > limit;
     if (hasMore) {
       events.pop();
     }
+    const lastReturnedSyncId = events[events.length - 1]?.sync_id ?? options.afterSyncId ?? 0;
+    let nextSyncId = lastReturnedSyncId;
+    if (!hasMore && options.status === "applied") {
+      nextSyncId = Math.max(lastReturnedSyncId, queryParams.maxSyncId);
+    }
     return {
       events,
       hasMore,
-      nextSyncId: events[events.length - 1]?.sync_id ?? options.afterSyncId ?? 0
+      nextSyncId
     };
   };
   const checkIsQuiescent = (pushedSyncId) => {
@@ -1278,6 +1605,10 @@ function createCrdtStorage(storage) {
       throw new Error(`Event ${event.sync_id} is not pending`);
     }
     try {
+      if (event.origin === "own-applied") {
+        event.status = "applied";
+        return event;
+      }
       if (hasAcceptedEventWithTimestamp(tx, event)) {
         event.status = "deduped";
         return event;
@@ -1288,7 +1619,6 @@ function createCrdtStorage(storage) {
       if (isNoOpCrdtEventPayload(event.payload)) {
         applyCrdtEvent(event);
         event.status = "applied";
-        eventHlcAccumulator?.add(event.timestamp);
         return event;
       }
       const migratedEvent = storage.migrator.migrateEvent(event, storage.migrator.latestSchemaVersion);
@@ -1297,7 +1627,6 @@ function createCrdtStorage(storage) {
         event.payload = CRDT_EVENT_NO_OP_PAYLOAD;
         applyCrdtEvent(event);
         event.status = "applied";
-        eventHlcAccumulator?.add(event.timestamp);
         return event;
       }
       event.schema_version = migratedEvent.schema_version;
@@ -1307,7 +1636,6 @@ function createCrdtStorage(storage) {
       event.payload = migratedEvent.payload;
       applyCrdtEvent(event);
       event.status = "applied";
-      eventHlcAccumulator?.add(event.timestamp);
     } catch (error) {
       console.error("Error applying enqueued CRDT event", error);
       event.status = "failed";
@@ -1345,26 +1673,28 @@ function createCrdtStorage(storage) {
       if (hasMore) {
         events.pop();
       }
+      const previousSequentialSyncId = sequentialSyncId;
       if (events.length === 0) {
+        sequentialSyncId = localSyncId;
+        dispatchEventsApplied(previousSequentialSyncId);
         break;
       }
-      let appliedSyncId = null;
       const failedRemoteSyncIds = [];
       db.executeTransaction((tx) => {
         for (const event of events) {
           processPersistedEvent(tx, event);
-          notifyEventApplied(event);
           if (event.status === "applied") {
-            appliedSyncId = event.sync_id;
-          } else if (event.status === "failed" && event.origin === "remote") {
+            eventHlcAccumulator?.add(event.timestamp);
+          }
+          notifyEventApplied(event);
+          if (event.status === "failed" && event.origin === "remote") {
             failedRemoteSyncIds.push(event.sync_id);
           }
         }
         persistEventHlcAccumulator();
       });
-      if (appliedSyncId !== null) {
-        dispatchEventsApplied(appliedSyncId);
-      }
+      sequentialSyncId = hasMore ? events[events.length - 1]?.sync_id ?? sequentialSyncId : localSyncId;
+      dispatchEventsApplied(previousSequentialSyncId);
       for (const syncId of failedRemoteSyncIds) {
         eventTarget.dispatchEvent("remote-event-apply-failed", { syncId });
       }
@@ -1377,12 +1707,15 @@ function createCrdtStorage(storage) {
     enqueueLocalEvents,
     enqueueOwnEvents,
     enqueueRemoteEvents,
-    applyOwnEvent,
-    dispatchEventsApplied,
+    applyOwnEvents,
     checkIsQuiescent,
     getEventHlcAccumulator: () => eventHlcAccumulator?.current ?? null,
     addEventListener: eventTarget.addEventListener,
-    removeEventListener: eventTarget.removeEventListener
+    removeEventListener: eventTarget.removeEventListener,
+    internal: {
+      applyOwnEventFromTransaction,
+      processEnqueuedEvents
+    }
   };
 }
 
@@ -1850,6 +2183,10 @@ export {
   runSystemMigrations,
   applyWorkerDbSchema,
   applyMemoryDbSchema,
+  CrdtEventValidationError,
+  validateNewCrdtEvent,
+  verifySyncSchema,
+  formatSchemaVerificationIssues,
   createSQLiteCrdtApplyFunction,
   createCrdtApplyFunction,
   createCrdtStorage,
@@ -1868,4 +2205,4 @@ export {
   createResetStateStore,
   createReloadRequestHandler
 };
-//# sourceMappingURL=chunk-3L4QG3BZ.js.map
+//# sourceMappingURL=chunk-LBYL5L4O.js.map