@evolu/common 7.2.3 → 7.4.0

package/src/local-first/Db.ts

@@ -1,4 +1,9 @@
-import { isNonEmptyArray, NonEmptyReadonlyArray } from "../Array.js";
+import {
+  firstInArray,
+  isNonEmptyArray,
+  NonEmptyReadonlyArray,
+} from "../Array.js";
+import { assertNonEmptyReadonlyArray } from "../Assert.js";
 import { CallbackId } from "../Callbacks.js";
 import { ConsoleConfig, ConsoleDep } from "../Console.js";
 import {
@@ -13,7 +18,6 @@ import { ok, Result } from "../Result.js";
 import {
   createSqlite,
   CreateSqliteDriverDep,
-  explainSqliteQueryPlan,
   sql,
   SqliteDep,
   SqliteError,
@@ -26,9 +30,9 @@ import {
   MessageHandlers,
   Worker,
 } from "../Worker.js";
-import { makePatches, QueryPatches } from "./Diff.js";
 import {
   AppOwner,
+  AppOwnerDep,
   createAppOwner,
   createOwnerSecret,
   createOwnerWebSocketTransport,
@@ -40,11 +44,11 @@ import {
 } from "./Owner.js";
 import { ProtocolError, protocolVersion } from "./Protocol.js";
 import {
-  createQueryRowsCache,
-  deserializeQuery,
-  emptyRows,
+  createGetQueryRowsCache,
+  GetQueryRowsCacheDep,
+  loadQueries,
   Query,
-  QueryRowsCache,
+  QueryPatches,
 } from "./Query.js";
 import {
   DbSchema,
@@ -60,6 +64,7 @@ import {
   createSync,
   SyncDep,
   SyncOwner,
+  tryApplyQuarantinedMessages,
 } from "./Sync.js";
 import {
   Timestamp,
@@ -311,24 +316,16 @@ type DbWorkerDeps = Omit<
   DbWorkerPlatformDeps,
   keyof CreateSqliteDriverDep | keyof CreateWebSocketDep
 > &
+  AppOwnerDep &
   GetQueryRowsCacheDep &
   PostMessageDep &
   SqliteDep &
-  SyncDep &
-  AppOwnerDep;
-
-interface GetQueryRowsCacheDep {
-  readonly getQueryRowsCache: (tabId: Id) => QueryRowsCache;
-}
+  SyncDep;
 
 export interface PostMessageDep {
   readonly postMessage: (message: DbWorkerOutput) => void;
 }
 
-export interface AppOwnerDep {
-  readonly appOwner: AppOwner;
-}
-
 export const createDbWorkerForPlatform = (
   platformDeps: DbWorkerPlatformDeps,
 ): DbWorker =>
@@ -337,146 +334,141 @@ export const createDbWorkerForPlatform = (
     DbWorkerOutput,
     DbWorkerDeps
   >({
-    init: createInit(platformDeps),
-    handlers,
-  });
-
-const createInit =
-  (platformDeps: DbWorkerPlatformDeps) =>
-  async (
-    initMessage: Extract<DbWorkerInput, { type: "init" }>,
-    postMessage: (msg: DbWorkerOutput) => void,
-  ): Promise<DbWorkerDeps | null> => {
-    platformDeps.console.enabled = initMessage.config.enableLogging ?? false;
-
-    const sqliteResult = await createSqlite(platformDeps)(
-      initMessage.config.name,
-      {
-        memory: initMessage.config.inMemory ?? false,
-        encryptionKey: initMessage.config.encryptionKey ?? undefined,
-      },
-    );
-    if (!sqliteResult.ok) {
-      postMessage({ type: "onError", error: sqliteResult.error });
-      return null;
-    }
-    const sqlite = sqliteResult.value;
-
-    const deps = sqlite.transaction(() => {
-      const currentDbSchema = getDbSchema({ sqlite })();
-      if (!currentDbSchema.ok) return currentDbSchema;
+    init: async (initMessage, postMessage) => {
+      platformDeps.console.enabled = initMessage.config.enableLogging ?? false;
 
-      let appOwner: AppOwner;
-      let clock: Clock;
-
-      const dbIsInitialized = currentDbSchema.value.tables.some(
-        (table) => table.name === "evolu_version",
+      const deps = await createDbWorkerDeps(
+        platformDeps,
+        initMessage,
+        postMessage,
       );
 
-      if (dbIsInitialized) {
-        const currentVersion = sqlite.exec<{
-          protocolVersion: number;
-        }>(sql`select protocolVersion from evolu_version limit 1;`);
-        if (!currentVersion.ok) return currentVersion;
-
-        const configResult = sqlite.exec<{
-          clock: TimestampBytes;
-          appOwnerId: OwnerId;
-          appOwnerEncryptionKey: OwnerEncryptionKey;
-          appOwnerWriteKey: OwnerWriteKey;
-          appOwnerMnemonic: Mnemonic | null;
-        }>(sql`
-          select
-            clock,
-            appOwnerId,
-            appOwnerEncryptionKey,
-            appOwnerWriteKey,
-            appOwnerMnemonic
-          from evolu_config
-          limit 1;
-        `);
-        if (!configResult.ok) return configResult;
-
-        const [config] = configResult.value.rows;
+      if (!deps.ok) {
+        postMessage({ type: "onError", error: deps.error });
+        return null;
+      }
 
-        appOwner = {
-          type: "AppOwner",
-          id: config.appOwnerId,
-          encryptionKey: config.appOwnerEncryptionKey,
-          writeKey: config.appOwnerWriteKey,
-          mnemonic: config.appOwnerMnemonic,
-        };
+      return deps.value;
+    },
+    handlers,
+  });
 
-        clock = createClock({ ...platformDeps, sqlite })(
-          timestampBytesToTimestamp(config.clock),
-        );
-      } else {
-        appOwner =
-          initMessage.config.externalAppOwner ??
-          createAppOwner(createOwnerSecret(platformDeps));
+const createDbWorkerDeps = async (
+  platformDeps: DbWorkerPlatformDeps,
+  initMessage: Extract<DbWorkerInput, { type: "init" }>,
+  postMessage: (msg: DbWorkerOutput) => void,
+) => {
+  const sqlite = await createSqlite(platformDeps)(initMessage.config.name, {
+    memory: initMessage.config.inMemory ?? false,
+    encryptionKey: initMessage.config.encryptionKey ?? undefined,
+  });
+  if (!sqlite.ok) return sqlite;
+
+  const deps = { ...platformDeps, sqlite: sqlite.value };
+
+  return deps.sqlite.transaction(() => {
+    const dbSchema = getDbSchema(deps)();
+    if (!dbSchema.ok) return dbSchema;
+
+    const dbIsInitialized = "evolu_version" in dbSchema.value.tables;
+
+    let appOwner: AppOwner;
+    let clock: Clock;
+
+    if (dbIsInitialized) {
+      const currentVersion = deps.sqlite.exec<{
+        protocolVersion: number;
+      }>(sql`select protocolVersion from evolu_version limit 1;`);
+      if (!currentVersion.ok) return currentVersion;
+
+      const configResult = deps.sqlite.exec<{
+        clock: TimestampBytes;
+        appOwnerId: OwnerId;
+        appOwnerEncryptionKey: OwnerEncryptionKey;
+        appOwnerWriteKey: OwnerWriteKey;
+        appOwnerMnemonic: Mnemonic | null;
+      }>(sql`
+        select
+          clock,
+          appOwnerId,
+          appOwnerEncryptionKey,
+          appOwnerWriteKey,
+          appOwnerMnemonic
+        from evolu_config
+        limit 1;
+      `);
+      if (!configResult.ok) return configResult;
+
+      assertNonEmptyReadonlyArray(configResult.value.rows);
+      const config = firstInArray(configResult.value.rows);
+
+      appOwner = {
+        type: "AppOwner",
+        id: config.appOwnerId,
+        encryptionKey: config.appOwnerEncryptionKey,
+        writeKey: config.appOwnerWriteKey,
+        mnemonic: config.appOwnerMnemonic,
+      };
 
-        clock = createClock({ ...platformDeps, sqlite })();
+      clock = createClock(deps)(timestampBytesToTimestamp(config.clock));
+    } else {
+      appOwner =
+        initMessage.config.externalAppOwner ??
+        createAppOwner(createOwnerSecret(platformDeps));
 
-        const result = initializeDb({ sqlite })(appOwner, clock.get());
-        if (!result.ok) return result;
-      }
+      clock = createClock(deps)();
 
-      const result = ensureDbSchema({ sqlite })(
-        initMessage.dbSchema,
-        currentDbSchema.value,
-      );
+      const result = initializeDb(deps)(appOwner, clock.get());
       if (!result.ok) return result;
+    }
 
-      const sync = createSync({
-        ...platformDeps,
-        clock,
-        sqlite,
-        symmetricCrypto: createSymmetricCrypto(platformDeps),
-        timestampConfig: initMessage.config,
-        postMessage,
-      })({
-        appOwner,
-        transports: initMessage.config.transports,
-        onError: (error) => {
-          postMessage({ type: "onError", error });
-        },
-        onReceive: () => {
-          postMessage({ type: "refreshQueries" });
-        },
-      });
-      if (!sync.ok) return sync;
-
-      sync.value.useOwner(true, appOwner);
-
-      const tabQueryRowsCacheMap = new Map<Id, QueryRowsCache>();
-      const getQueryRowsCache = (tabId: Id) => {
-        let cache = tabQueryRowsCacheMap.get(tabId);
-        if (!cache) {
-          cache = createQueryRowsCache();
-          tabQueryRowsCacheMap.set(tabId, cache);
-        }
-        return cache;
-      };
+    {
+      const result = ensureDbSchema(deps)(initMessage.dbSchema, dbSchema.value);
+      if (!result.ok) return result;
+    }
 
-      const deps: DbWorkerDeps = {
-        ...platformDeps,
-        getQueryRowsCache,
-        postMessage,
-        sqlite,
-        sync: sync.value,
-        appOwner,
-      };
+    {
+      const result = ensureMessageQuarantineTable(deps);
+      if (!result.ok) return result;
+    }
 
-      return ok(deps);
+    const sync = createSync({
+      ...deps,
+      clock,
+      symmetricCrypto: createSymmetricCrypto(platformDeps),
+      timestampConfig: initMessage.config,
+      dbSchema: initMessage.dbSchema,
+    })({
+      appOwner,
+      transports: initMessage.config.transports,
+      onError: (error) => {
+        postMessage({ type: "onError", error });
+      },
+      onReceive: () => {
+        postMessage({ type: "refreshQueries" });
+      },
     });
+    if (!sync.ok) return sync;
 
-    if (!deps.ok) {
-      postMessage({ type: "onError", error: deps.error });
-      return null;
+    {
+      const result = tryApplyQuarantinedMessages({
+        ...deps,
+        dbSchema: initMessage.dbSchema,
+      })();
+      if (!result.ok) return result;
     }
 
-    return deps.value;
-  };
+    sync.value.useOwner(true, appOwner);
+
+    return ok({
+      ...deps,
+      getQueryRowsCache: createGetQueryRowsCache(),
+      postMessage,
+      sync: sync.value,
+      appOwner,
+    });
+  });
+};
 
 const initializeDb =
   (deps: SqliteDep) =>
@@ -548,7 +540,6 @@ const initializeDb =
       `,
 
       // Index for reading database changes by owner and timestamp.
-      // Timestamp always corresponds to a DbChange.
       sql`
         create index evolu_history_ownerId_timestamp on evolu_history (
           "ownerId",
@@ -576,6 +567,41 @@ const initializeDb =
     return ok();
   };
 
+/**
+ * Ensures the quarantine table exists for storing messages with unknown schema.
+ *
+ * When a device receives sync messages containing tables or columns that don't
+ * exist in its current schema (e.g., from a newer app version), those messages
+ * are stored here instead of being discarded. This enables forward
+ * compatibility:
+ *
+ * 1. Unknown data is preserved and can be applied when the app is updated
+ * 2. Messages are still propagated to other devices that may understand them
+ * 3. Partial messages work - known columns go to app tables, unknown to quarantine
+ *
+ * The `union all` query in `readDbChange` combines `evolu_history` and this
+ * table, ensuring all data (known and unknown) is included when syncing to
+ * other devices.
+ */
+const ensureMessageQuarantineTable = (
+  deps: SqliteDep,
+): Result<void, SqliteError> => {
+  const result = deps.sqlite.exec(sql`
+    create table if not exists evolu_message_quarantine (
+      "ownerId" blob not null,
+      "timestamp" blob not null,
+      "table" text not null,
+      "id" blob not null,
+      "column" text not null,
+      "value" any,
+      primary key ("ownerId", "timestamp", "table", "id", "column")
+    )
+    strict;
+  `);
+  if (!result.ok) return result;
+  return ok();
+};
+
 const handlers: Omit<MessageHandlers<DbWorkerInput, DbWorkerDeps>, "init"> = {
   getAppOwner: (deps) => () => {
     deps.postMessage({
@@ -647,11 +673,11 @@ const handlers: Omit<MessageHandlers<DbWorkerInput, DbWorkerDeps>, "init"> = {
   },
 
   reset: (deps) => (message) => {
-    const resetResult = deps.sqlite.transaction(() => {
+    const result = deps.sqlite.transaction(() => {
       const dbSchema = getDbSchema(deps)();
       if (!dbSchema.ok) return dbSchema;
 
-      for (const table of dbSchema.value.tables) {
+      for (const tableName in dbSchema.value.tables) {
         /**
          * The dropped table is completely removed from the database schema and
          * the disk file. The table can not be recovered. All indices and
@@ -659,34 +685,27 @@ const handlers: Omit<MessageHandlers<DbWorkerInput, DbWorkerDeps>, "init"> = {
          * https://sqlite.org/lang_droptable.html
          */
         const result = deps.sqlite.exec(sql`
-          drop table ${sql.identifier(table.name)};
+          drop table ${sql.identifier(tableName)};
         `);
         if (!result.ok) return result;
       }
 
       if (message.restore) {
-        const dbSchema = getDbSchema(deps)();
-        if (!dbSchema.ok) return dbSchema;
-
-        const ensureDbSchemaResult = ensureDbSchema(deps)(
-          message.restore.dbSchema,
-          dbSchema.value,
-        );
-        if (!ensureDbSchemaResult.ok) return ensureDbSchemaResult;
+        const result = ensureDbSchema(deps)(message.restore.dbSchema);
+        if (!result.ok) return result;
 
         const secret = mnemonicToOwnerSecret(message.restore.mnemonic);
         const appOwner = createAppOwner(secret);
         const clock = createClock(deps)();
 
-        const initializeDbResult = initializeDb(deps)(appOwner, clock.get());
-        if (!initializeDbResult.ok) return initializeDbResult;
+        return initializeDb(deps)(appOwner, clock.get());
       }
 
       return ok();
     });
 
-    if (!resetResult.ok) {
-      deps.postMessage({ type: "onError", error: resetResult.error });
+    if (!result.ok) {
+      deps.postMessage({ type: "onError", error: result.error });
       return;
     }
 
@@ -698,21 +717,12 @@ const handlers: Omit<MessageHandlers<DbWorkerInput, DbWorkerDeps>, "init"> = {
   },
 
   ensureDbSchema: (deps) => (message) => {
-    const ensureSchema = deps.sqlite.transaction(() => {
-      const dbSchema = getDbSchema(deps)();
-      if (!dbSchema.ok) return dbSchema;
-
-      const ensureDbSchemaResult = ensureDbSchema(deps)(
-        message.dbSchema,
-        dbSchema.value,
-      );
-      if (!ensureDbSchemaResult.ok) return ensureDbSchemaResult;
-
-      return ok();
-    });
+    const result = deps.sqlite.transaction(() =>
+      ensureDbSchema(deps)(message.dbSchema),
+    );
 
-    if (!ensureSchema.ok) {
-      deps.postMessage({ type: "onError", error: ensureSchema.error });
+    if (!result.ok) {
+      deps.postMessage({ type: "onError", error: result.error });
       return;
     }
   },
@@ -736,41 +746,3 @@ const handlers: Omit<MessageHandlers<DbWorkerInput, DbWorkerDeps>, "init"> = {
     deps.sync.useOwner(message.use, message.owner);
   },
 };
-
-const loadQueries =
-  (deps: GetQueryRowsCacheDep & SqliteDep) =>
-  (
-    tabId: Id,
-    queries: ReadonlyArray<Query>,
-  ): Result<ReadonlyArray<QueryPatches>, SqliteError> => {
-    const queriesRows = [];
-
-    for (const query of queries) {
-      const sqlQuery = deserializeQuery(query);
-      const result = deps.sqlite.exec(sqlQuery);
-      if (!result.ok) return result;
-
-      queriesRows.push([query, result.value.rows] as const);
-      if (sqlQuery.options?.logExplainQueryPlan) {
-        explainSqliteQueryPlan(deps)(sqlQuery);
-      }
-    }
-
-    const queryRowsCache = deps.getQueryRowsCache(tabId);
-
-    const previousState = queryRowsCache.get();
-    queryRowsCache.set(queriesRows);
-
-    const currentState = queryRowsCache.get();
-
-    const queryPatchesArray = queries.map(
-      (query): QueryPatches => ({
-        query,
-        patches: makePatches(
-          previousState.get(query),
-          currentState.get(query) ?? emptyRows,
-        ),
-      }),
-    );
-    return ok(queryPatchesArray);
-  };

package/src/local-first/Evolu.ts

@@ -37,11 +37,11 @@ import {
 } from "../Type.js";
 import { IntentionalNever } from "../Types.js";
 import { CreateDbWorkerDep, DbConfig, defaultDbConfig } from "./Db.js";
-import { applyPatches } from "./Diff.js";
 import { AppOwner } from "./Owner.js";
 import { FlushSyncDep, ReloadAppDep } from "./Platform.js";
 import { ProtocolError } from "./Protocol.js";
 import {
+  applyPatches,
   createSubscribedQueries,
   emptyRows,
   Queries,
@@ -101,6 +101,9 @@ export interface EvoluConfig extends Partial<DbConfig> {
    * URL to reload browser tabs after reset or restore.
    *
    * The default value is `/`.
+   *
+   * Note: This option will be moved to web platform deps in the next major
+   * version.
    */
   readonly reloadUrl?: string;
 }
@@ -836,15 +839,15 @@ const createEvoluInstance =
 
       subscribeQuery: (query) => (listener) => {
         // Call the listener only if the result has been changed.
-        let previousResult: unknown = null;
+        let previousRows: unknown = null;
         const unsubscribe = subscribedQueries.subscribe(query)(() => {
-          const result = evolu.getQueryRows(query);
-          if (previousResult === result) return;
-          previousResult = result;
+          const rows = evolu.getQueryRows(query);
+          if (previousRows === rows) return;
+          previousRows = rows;
           listener();
         });
         return () => {
-          previousResult = null;
+          previousRows = null;
           unsubscribe();
         };
       },

package/src/local-first/LocalAuth.ts

@@ -47,9 +47,9 @@ export interface LocalAuthDep {
 }
 
 /**
- * - **⚠️
- *
  * Secure storage interface that must be implemented by each platform.
+ *
+ * @experimental
  */
 export interface SecureStorage {
   setItem: (

package/src/local-first/Owner.ts

@@ -28,6 +28,7 @@ import { TimestampBytes } from "./Timestamp.js";
  */
 export interface ReadonlyOwner {
   readonly id: OwnerId;
+  /** TODO: Wrap with `Redacted` in the next major version. */
   readonly encryptionKey: OwnerEncryptionKey;
 }
 
@@ -63,6 +64,7 @@ export interface ReadonlyOwner {
  * @see {@link createSharedReadonlyOwner}
  */
 export interface Owner extends ReadonlyOwner {
+  /** TODO: Wrap with `Redacted` in the next major version. */
   readonly writeKey: OwnerWriteKey;
 }
 
@@ -184,10 +186,16 @@ export interface AppOwner extends Owner {
    * The mnemonic that was used to derive the AppOwner keys. Optional when the
    * AppOwner is created from external keys to avoid sharing the mnemonic with
    * the Evolu app.
+   *
+   * TODO: Wrap with `Redacted` in the next major version.
    */
   readonly mnemonic?: Mnemonic | null;
 }
 
+export interface AppOwnerDep {
+  readonly appOwner: AppOwner;
+}
+
 /** Creates an {@link AppOwner} from an {@link OwnerSecret}. */
 export const createAppOwner = (secret: OwnerSecret): AppOwner => ({
   ...createOwner(secret),

package/src/local-first/Protocol.ts

@@ -972,12 +972,9 @@ export const applyProtocolMessageAsClient =
       const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
 
       if (isNonEmptyReadonlyArray(messages)) {
-        const writeResult = await deps.storage.writeMessages(
-          ownerIdBytes,
-          messages,
-        );
+        const result = await deps.storage.writeMessages(ownerIdBytes, messages);
         // Errors are handled by the Storage. Here we just stop syncing.
-        if (!writeResult.ok) return ok({ type: "no-response" });
+        if (!result.ok) return ok({ type: "no-response" });
       }
 
       // Now: No writeKey, no sync.
@@ -1006,10 +1003,10 @@ export const applyProtocolMessageAsClient =
         rangesMaxSize: options.rangesMaxSize,
       });
 
-      const syncResult = sync(deps)(ranges, output, ownerIdBytes);
+      const result = sync(deps)(ranges, output, ownerIdBytes);
 
       // Client sync error (handled via Storage) or no changes.
-      if (!syncResult.ok || !syncResult.value) {
+      if (!result.ok || !result.value) {
         return ok({ type: "no-response" });
       }
 
@@ -1126,14 +1123,11 @@ export const applyProtocolMessageAsRelay =
           });
         }
 
-        const writeResult = await deps.storage.writeMessages(
-          ownerIdBytes,
-          messages,
-        );
+        const result = await deps.storage.writeMessages(ownerIdBytes, messages);
 
-        if (!writeResult.ok) {
+        if (!result.ok) {
           const errorCode =
-            writeResult.error.type === "StorageWriteError"
+            result.error.type === "StorageWriteError"
               ? ProtocolErrorCode.WriteError
               : ProtocolErrorCode.QuotaError;
           const message = createProtocolMessageBuffer(ownerId, {
@@ -1189,13 +1183,13 @@ export const applyProtocolMessageAsRelay =
         return ok({ type: "response", message: output.unwrap() });
       }
 
-      const syncResult = sync(deps)(ranges, output, ownerIdBytes);
+      const result = sync(deps)(ranges, output, ownerIdBytes);
 
-      const message = syncResult.ok
+      const message = result.ok
         ? output.unwrap()
         : createProtocolMessageBuffer(ownerId, {
             messageType: MessageType.Response,
-            errorCode: syncResult.error,
+            errorCode: result.error,
           }).unwrap();
 
       // Non-initiators always respond to provide sync completion feedback,

package/src/local-first/Public.ts

@@ -1,5 +1,10 @@
 /**
- * Local-first code to be imported from "@evolu/common"
+ * Public local-first API exported from "@evolu/common"
+ *
+ * This module provides the supported consumer-facing API for Evolu's
+ * local-first system: prefer importing from `@evolu/common` (this module) when
+ * using Evolu in applications. If you require access to lower level internals,
+ * consider importing from `@evolu/common/local-first`.
  *
  * @module
  */

package/src/local-first/PublicKysely.ts

@@ -13,7 +13,7 @@ import {
   TableNode,
   ValueNode,
 } from "kysely";
-import { kyselyJsonIdentifier } from "./Diff.js";
+import { kyselyJsonIdentifier } from "./Query.js";
 
 export { sql } from "kysely";
 export type { NotNull } from "kysely";