@evolu/common 7.2.3 → 7.3.0

package/src/local-first/Query.ts

@@ -1,14 +1,26 @@
 import { Brand } from "../Brand.js";
 import { bytesToHex, hexToBytes } from "../Buffer.js";
-import { objectToEntries } from "../Object.js";
+import { createRandomBytes } from "../Crypto.js";
 import {
+  createRecord,
+  isPlainObject,
+  objectToEntries,
+  ReadonlyRecord,
+} from "../Object.js";
+import { ok, Result } from "../Result.js";
+import {
+  eqSqliteValue,
+  explainSqliteQueryPlan,
   SafeSql,
+  SqliteDep,
+  SqliteError,
   SqliteQuery,
   SqliteQueryOptions,
   SqliteRow,
   SqliteValue,
 } from "../Sqlite.js";
 import { Store, StoreSubscribe } from "../Store.js";
+import { createId, Id, String } from "../Type.js";
 import { Simplify } from "../Types.js";
 
 /**
@@ -111,26 +123,6 @@ export type QueriesToQueryRowsPromises<Q extends Queries> = {
 
 export type QueryRowsMap = ReadonlyMap<Query, ReadonlyArray<Row>>;
 
-export interface QueryRowsCache {
-  readonly set: (
-    queriesRows: ReadonlyArray<readonly [Query, ReadonlyArray<SqliteRow>]>,
-  ) => void;
-  readonly get: () => QueryRowsMap;
-}
-
-export const createQueryRowsCache = (): QueryRowsCache => {
-  let queryRowsCache: QueryRowsMap = new Map();
-
-  const cache: QueryRowsCache = {
-    set: (queriesRows) => {
-      queryRowsCache = new Map([...queryRowsCache, ...queriesRows]);
-    },
-    get: () => queryRowsCache,
-  };
-
-  return cache;
-};
-
 export interface SubscribedQueries {
   subscribe: (query: Query) => StoreSubscribe;
 
@@ -166,3 +158,206 @@ export const createSubscribedQueries = (
 
   return subscribedQueries;
 };
+
+export interface GetQueryRowsCacheDep {
+  readonly getQueryRowsCache: GetQueryRowsCache;
+}
+
+export type GetQueryRowsCache = (tabId: Id) => QueryRowsCache;
+
+export interface QueryRowsCache {
+  readonly set: (
+    queriesRows: ReadonlyArray<readonly [Query, ReadonlyArray<SqliteRow>]>,
+  ) => void;
+  readonly get: () => QueryRowsMap;
+}
+
+export const createGetQueryRowsCache = (): GetQueryRowsCache => {
+  const tabQueryRowsCacheMap = new Map<Id, QueryRowsCache>();
+
+  return (tabId: Id) => {
+    let cache = tabQueryRowsCacheMap.get(tabId);
+    if (!cache) {
+      let queryRowsCache: QueryRowsMap = new Map();
+      cache = {
+        set: (queriesRows) => {
+          queryRowsCache = new Map([...queryRowsCache, ...queriesRows]);
+        },
+        get: () => queryRowsCache,
+      };
+      tabQueryRowsCacheMap.set(tabId, cache);
+    }
+    return cache;
+  };
+};
+
+export const loadQueries =
+  (deps: GetQueryRowsCacheDep & SqliteDep) =>
+  (
+    tabId: Id,
+    queries: ReadonlyArray<Query>,
+  ): Result<ReadonlyArray<QueryPatches>, SqliteError> => {
+    const queriesRows = [];
+
+    for (const query of queries) {
+      const sqlQuery = deserializeQuery(query);
+      const result = deps.sqlite.exec(sqlQuery);
+      if (!result.ok) return result;
+
+      queriesRows.push([query, result.value.rows] as const);
+      if (sqlQuery.options?.logExplainQueryPlan) {
+        explainSqliteQueryPlan(deps)(sqlQuery);
+      }
+    }
+
+    const queryRowsCache = deps.getQueryRowsCache(tabId);
+
+    const previousState = queryRowsCache.get();
+    queryRowsCache.set(queriesRows);
+
+    const currentState = queryRowsCache.get();
+
+    const queryPatchesArray = queries.map(
+      (query): QueryPatches => ({
+        query,
+        patches: makePatches(
+          previousState.get(query),
+          currentState.get(query) ?? emptyRows,
+        ),
+      }),
+    );
+    return ok(queryPatchesArray);
+  };
+
+export interface QueryPatches {
+  readonly query: Query;
+  readonly patches: ReadonlyArray<Patch>;
+}
+
+export type Patch = ReplaceAllPatch | ReplaceAtPatch;
+
+export interface ReplaceAllPatch {
+  readonly op: "replaceAll";
+  readonly value: ReadonlyArray<Row>;
+}
+
+export interface ReplaceAtPatch {
+  readonly op: "replaceAt";
+  readonly index: number;
+  readonly value: Row;
+}
+
+/**
+ * We detect only changes in the whole result and in-place edits. In the future,
+ * we will add more heuristics. We will probably not implement the Myers diff
+ * algorithm because it's faster to rerender all than to compute many detailed
+ * patches. We will only implement logic a developer would implement manually,
+ * if necessary.
+ */
+export const makePatches = (
+  previousRows: ReadonlyArray<Row> | undefined,
+  nextRows: ReadonlyArray<Row>,
+): ReadonlyArray<Patch> => {
+  if (previousRows === undefined)
+    return [{ op: "replaceAll", value: nextRows }];
+  // TODO: Detect prepend and append, it's cheap.
+  if (previousRows.length !== nextRows.length) {
+    return [{ op: "replaceAll", value: nextRows }];
+  }
+
+  const length = previousRows.length;
+  const replaceAtPatches: Array<ReplaceAtPatch> = [];
+
+  for (let i = 0; i < length; i++) {
+    const previousRow = previousRows[i];
+    const nextRow = nextRows[i];
+
+    // We expect the same shape for both rows.
+    for (const key in previousRow)
+      if (
+        !eqSqliteValue(
+          previousRow[key] as SqliteValue,
+          nextRow[key] as SqliteValue,
+        )
+      ) {
+        replaceAtPatches.push({ op: "replaceAt", value: nextRow, index: i });
+        break;
+      }
+  }
+
+  if (length > 0 && replaceAtPatches.length === length) {
+    return [{ op: "replaceAll", value: nextRows }];
+  }
+  return replaceAtPatches;
+};
+
+export const applyPatches = (
+  patches: ReadonlyArray<Patch>,
+  current: ReadonlyArray<Row>,
+): ReadonlyArray<Row> =>
+  patches.reduce((next, patch) => {
+    switch (patch.op) {
+      case "replaceAll":
+        return parseSqliteJsonArray(patch.value);
+      case "replaceAt": {
+        const parsedRow = parseSqliteJsonArray([patch.value])[0];
+        return next.toSpliced(patch.index, 1, parsedRow);
+      }
+    }
+  }, current);
+
+/**
+ * A unique identifier prepended to JSON-encoded strings. This allows safe
+ * detection and parsing of only those columns that require JSON.parse.
+ *
+ * The identifier is a cryptographically random Evolu Id, ensuring uniqueness
+ * and preventing malicious actors from inserting fake data that could be
+ * misinterpreted as JSON by the application.
+ *
+ * Note: The same queries created by different browser tabs will have different
+ * identifiers and thus be considered different and cached separately. This is
+ * usually not a big deal, but if needed, the DB cache can be optimized by
+ * passing the kyselyJsonIdentifier into the DB worker during initialization,
+ * allowing queries to be grouped and recognized across tabs or sessions.
+ *
+ * See: https://github.com/kysely-org/kysely/issues/1372#issuecomment-2702773948
+ */
+export const kyselyJsonIdentifier = createId({
+  randomBytes: createRandomBytes(),
+});
+
+export const parseSqliteJsonArray = <T>(
+  arr: ReadonlyArray<T>,
+): ReadonlyArray<T> => {
+  const result = new Array<T>(arr.length);
+  for (let i = 0; i < arr.length; ++i) {
+    result[i] = parse(arr[i]) as T;
+  }
+  return result;
+};
+
+const parse = (obj: unknown): unknown => {
+  if (String.is(obj) && obj.startsWith(kyselyJsonIdentifier)) {
+    return JSON.parse(obj.slice(kyselyJsonIdentifier.length));
+  }
+
+  if (Array.isArray(obj)) {
+    return parseSqliteJsonArray(obj);
+  }
+
+  if (isPlainObject(obj)) {
+    return parseObject(obj);
+  }
+
+  return obj;
+};
+
+const parseObject = (
+  obj: ReadonlyRecord<string, unknown>,
+): ReadonlyRecord<string, unknown> => {
+  const result = createRecord();
+  for (const key in obj) {
+    result[key] = parse(obj[key]);
+  }
+  return result as ReadonlyRecord<string, unknown>;
+};

package/src/local-first/Relay.ts

@@ -217,13 +217,8 @@ export const createRelaySqliteStorage =
               (storedBytes ?? 0) + incomingBytes,
             );
 
-            const withinQuotaResult = config.isOwnerWithinQuota(
-              ownerId,
-              newStoredBytes,
-            );
-            const isWithinQuota = isAsync(withinQuotaResult)
-              ? await withinQuotaResult
-              : withinQuotaResult;
+            const result = config.isOwnerWithinQuota(ownerId, newStoredBytes);
+            const isWithinQuota = isAsync(result) ? await result : result;
             if (!isWithinQuota) {
               return err({ type: "StorageQuotaError", ownerId });
             }
@@ -240,30 +235,31 @@ export const createRelaySqliteStorage =
                     lastTimestamp,
                   );
 
-                const insertTimestampResult = sqliteStorageBase.insertTimestamp(
-                  ownerIdBytes,
-                  timestamp,
-                  strategy,
-                );
-                if (!insertTimestampResult.ok) return insertTimestampResult;
-
-                const insertMessage = deps.sqlite.exec(sql`
-                  insert into evolu_message ("ownerId", "timestamp", "change")
-                  values (${ownerIdBytes}, ${timestamp}, ${change})
-                  on conflict do nothing;
-                `);
-                if (!insertMessage.ok) return insertMessage;
+                {
+                  const result = sqliteStorageBase.insertTimestamp(
+                    ownerIdBytes,
+                    timestamp,
+                    strategy,
+                  );
+                  if (!result.ok) return result;
+                }
+
+                {
+                  const result = deps.sqlite.exec(sql`
+                    insert into evolu_message ("ownerId", "timestamp", "change")
+                    values (${ownerIdBytes}, ${timestamp}, ${change})
+                    on conflict do nothing;
+                  `);
+                  if (!result.ok) return result;
+                }
               }
 
-              const updateUsage = updateOwnerUsage(deps)(
+              return updateOwnerUsage(deps)(
                 ownerIdBytes,
                 newStoredBytes,
                 firstTimestamp,
                 lastTimestamp,
               );
-              if (!updateUsage.ok) return updateUsage;
-
-              return ok();
             });
           })();
 

package/src/local-first/Schema.ts

@@ -1,5 +1,10 @@
 import * as Kysely from "kysely";
-import { mapObject, objectToEntries, ReadonlyRecord } from "../Object.js";
+import {
+  createRecord,
+  getProperty,
+  mapObject,
+  ReadonlyRecord,
+} from "../Object.js";
 import { ok, Result } from "../Result.js";
 import {
   SafeSql,
@@ -30,6 +35,8 @@ import {
   omit,
   optional,
   OptionalType,
+  record,
+  set,
   String,
   TableId,
   Type,
@@ -40,8 +47,9 @@ import {
 import { Simplify } from "../Types.js";
 import { AppOwner, OwnerId } from "./Owner.js";
 import { Query, Row } from "./Query.js";
-import { CrdtMessage, DbChange } from "./Storage.js";
+import type { CrdtMessage, DbChange } from "./Storage.js";
 import { Timestamp, TimestampBytes } from "./Timestamp.js";
+import { readonly } from "../Function.js";
 
 /**
  * Defines the schema of an Evolu database.
@@ -169,12 +177,10 @@ export const evoluSchemaToDbSchema = (
   schema: EvoluSchema,
   indexesConfig?: IndexesConfig,
 ): DbSchema => {
-  const tables = objectToEntries(schema).map(([tableName, table]) => ({
-    name: tableName,
-    columns: objectToEntries(table)
-      .filter(([k]) => k !== "id")
-      .map(([k]) => k),
-  }));
+  const tables = mapObject(
+    schema,
+    (table) => new Set(Object.keys(table).filter((k) => k !== "id")),
+  );
 
   const indexes = indexesConfig
     ? indexesConfig(createIndex).map(
@@ -206,6 +212,13 @@ export type CreateQuery<S extends EvoluSchema> = <R extends Row>(
             readonly column: string;
             readonly value: SqliteValue;
           };
+          readonly evolu_message_quarantine: {
+            readonly timestamp: TimestampBytes;
+            readonly table: string;
+            readonly id: IdBytes;
+            readonly column: string;
+            readonly value: SqliteValue;
+          };
         }
       >,
       "selectFrom" | "fn" | "with" | "withRecursive"
@@ -231,7 +244,11 @@ export const SystemColumns = object({
 });
 export type SystemColumns = typeof SystemColumns.Type;
 
-export const systemColumns = Object.keys(SystemColumns.props);
+export const systemColumns = readonly(
+  new Set(Object.keys(SystemColumns.props)),
+);
+
+export const systemColumnsWithId = readonly([...systemColumns, "id"]);
 
 export type MutationKind = "insert" | "update" | "upsert";
 
@@ -443,21 +460,21 @@ export type InferColumnErrors<
   >;
 }[keyof MutationMapping<T, M>];
 
-export const DbTable = object({
-  name: String,
-  columns: array(String),
-});
-export type DbTable = typeof DbTable.Type;
-
 export const DbIndex = object({ name: String, sql: String });
 export type DbIndex = typeof DbIndex.Type;
 
 export const DbSchema = object({
-  tables: array(DbTable),
+  tables: record(String, set(String)),
   indexes: array(DbIndex),
 });
 export type DbSchema = typeof DbSchema.Type;
 
+// TODO: Use a ref and update dbSchema on hot reloading to support
+// development workflows where schema changes without full app restart.
+export interface DbSchemaDep {
+  readonly dbSchema: DbSchema;
+}
+
 /** Get the current database schema by reading SQLite metadata. */
 export const getDbSchema =
   (deps: SqliteDep) =>
@@ -465,7 +482,7 @@ export const getDbSchema =
     DbSchema,
     SqliteError
   > => {
-    const map = new Map<string, Array<string>>();
+    const tables = createRecord<string, Set<string>>();
 
     const tableAndColumnInfoRows = deps.sqlite.exec(sql`
       select
@@ -483,12 +500,9 @@ export const getDbSchema =
         tableName: string;
         columnName: string;
       };
-      if (!map.has(tableName)) map.set(tableName, []);
-      map.get(tableName)?.push(columnName);
+      (tables[tableName] ??= new Set()).add(columnName);
     });
 
-    const tables = Array.from(map, ([name, columns]) => ({ name, columns }));
-
     const indexesRows = deps.sqlite.exec(
       allIndexes
         ? sql`
@@ -532,55 +546,54 @@ export const ensureDbSchema =
   (deps: SqliteDep) =>
   (
     newSchema: DbSchema,
-    currentSchema: DbSchema,
-    options?: { ignoreIndexes: boolean },
+    currentSchema?: DbSchema,
   ): Result<void, SqliteError> => {
     const queries: Array<SqliteQuery> = [];
 
-    newSchema.tables.forEach((newTable) => {
-      const currentTable = currentSchema.tables.find(
-        (t) => t.name === newTable.name,
-      );
-      if (!currentTable) {
-        queries.push(createAppTable(newTable));
+    if (!currentSchema) {
+      const dbSchema = getDbSchema(deps)();
+      if (!dbSchema.ok) return dbSchema;
+      currentSchema = dbSchema.value;
+    }
+
+    for (const [tableName, newColumns] of Object.entries(newSchema.tables)) {
+      const currentColumns = getProperty(currentSchema.tables, tableName);
+      if (!currentColumns) {
+        queries.push(createAppTable(tableName, newColumns));
       } else {
-        newTable.columns
-          .filter((newColumn) => !currentTable.columns.includes(newColumn))
-          .forEach((newColumn) => {
-            queries.push(sql`
-              alter table ${sql.identifier(newTable.name)}
-              add column ${sql.identifier(newColumn)} blob;
-            `);
-          });
+        for (const newColumn of newColumns.difference(currentColumns)) {
+          queries.push(sql`
+            alter table ${sql.identifier(tableName)}
+            add column ${sql.identifier(newColumn)} any;
+          `);
+        }
       }
-    });
-
-    if (options?.ignoreIndexes !== true) {
-      // Remove current indexes that are not in the newSchema.
-      currentSchema.indexes
-        .filter(
-          (currentIndex) =>
-            !newSchema.indexes.some((newIndex) =>
-              indexesAreEqual(newIndex, currentIndex),
-            ),
-        )
-        .forEach((index) => {
-          queries.push(sql`drop index ${sql.identifier(index.name)};`);
-        });
-
-      // Add new indexes that are not in the currentSchema.
-      newSchema.indexes
-        .filter(
-          (newIndex) =>
-            !currentSchema.indexes.some((currentIndex) =>
-              indexesAreEqual(newIndex, currentIndex),
-            ),
-        )
-        .forEach((newIndex) => {
-          queries.push({ sql: `${newIndex.sql};` as SafeSql, parameters: [] });
-        });
     }
 
+    // Remove current indexes that are not in the newSchema.
+    currentSchema.indexes
+      .filter(
+        (currentIndex) =>
+          !newSchema.indexes.some((newIndex) =>
+            indexesAreEqual(newIndex, currentIndex),
+          ),
+      )
+      .forEach((index) => {
+        queries.push(sql`drop index ${sql.identifier(index.name)};`);
+      });
+
+    // Add new indexes that are not in the currentSchema.
+    newSchema.indexes
+      .filter(
+        (newIndex) =>
+          !currentSchema.indexes.some((currentIndex) =>
+            indexesAreEqual(newIndex, currentIndex),
+          ),
+      )
+      .forEach((newIndex) => {
+        queries.push({ sql: `${newIndex.sql};` as SafeSql, parameters: [] });
+      });
+
     for (const query of queries) {
       const result = deps.sqlite.exec(query);
       if (!result.ok) return result;
@@ -588,19 +601,19 @@ export const ensureDbSchema =
     return ok();
   };
 
-const createAppTable = (table: DbTable) => sql`
-  create table ${sql.identifier(table.name)} (
+const createAppTable = (tableName: string, columns: ReadonlySet<string>) => sql`
+  create table ${sql.identifier(tableName)} (
     "id" text,
     ${sql.raw(
-      `${systemColumns
-        .concat(table.columns)
-        .filter((c) => c !== "id")
+      `${[...systemColumns, ...columns]
         // With strict tables and any type, data is preserved exactly as received
         // without any type affinity coercion. This allows storing any data type
         // while maintaining strict null enforcement for primary key columns.
+        // TODO: Use proper SQLite types for system columns (text for createdAt,
+        // updatedAt, ownerId, integer for isDeleted) instead of "any".
         .map((name) => `${sql.identifier(name).sql} any`)
-        .join(", ")}`,
-    )},
+        .join(", ")}, `,
+    )}
     primary key ("ownerId", "id")
   )
   without rowid, strict;

package/src/local-first/Storage.ts

@@ -26,13 +26,13 @@ import {
   TypeError,
 } from "../Type.js";
 import {
-  OwnerError,
   Owner,
+  OwnerError,
   OwnerId,
   OwnerIdBytes,
   OwnerWriteKey,
 } from "./Owner.js";
-import { systemColumns } from "./Schema.js";
+import { systemColumnsWithId } from "./Schema.js";
 import { orderTimestampBytes, Timestamp, TimestampBytes } from "./Timestamp.js";
 
 export interface StorageConfig {
@@ -256,16 +256,11 @@ export interface CrdtMessage {
 export const DbChangeValues = record(String, SqliteValue);
 export type DbChangeValues = typeof DbChangeValues.Type;
 
-// "createdAt" and "updatedAt" are derived from Timestamp.
-// "id" and "isDeleted" are encoded separately to save bytes
-// "ownerId" is part of the protocol message
-const forbiddenSystemColumns = systemColumns.concat("id");
-
 export const ValidDbChangeValues = brand(
   "ValidDbChangeValues",
   DbChangeValues,
   (value) => {
-    const invalidColumns = forbiddenSystemColumns.filter((key) => key in value);
+    const invalidColumns = systemColumnsWithId.filter((key) => key in value);
     if (invalidColumns.length > 0)
       return err<ValidDbChangeValuesError>({
         type: "ValidDbChangeValues",