@evolu/common 6.0.1-preview.1 → 6.0.1-preview.11

package/src/Evolu/Db.ts

@@ -34,7 +34,7 @@ import {
   SqliteValue,
 } from "../Sqlite.js";
 import { TimeDep } from "../Time.js";
-import { Id, Mnemonic, object, SimpleName, String } from "../Type.js";
+import { array, Id, Mnemonic, object, SimpleName, String } from "../Type.js";
 import {
   createInitializedWorker,
   Worker,
@@ -54,6 +54,7 @@ import {
   Base64Url256,
   BinaryId,
   binaryIdToId,
+  BinaryOwnerId,
   CrdtMessage,
   createProtocolMessageForSync,
   createProtocolMessageFromCrdtMessages,
@@ -96,19 +97,21 @@ import {
   timestampToTimestampString,
 } from "./Timestamp.js";
 
-export interface DbSchema {
-  readonly tables: ReadonlyArray<DbTable>;
-  readonly indexes: ReadonlyArray<DbIndex>;
-}
-
-export interface DbTable {
-  readonly name: Base64Url256;
-  readonly columns: ReadonlyArray<Base64Url256>;
-}
+export const DbTable = object({
+  name: Base64Url256,
+  columns: array(Base64Url256),
+});
+export type DbTable = typeof DbTable.Type;
 
 export const DbIndex = object({ name: String, sql: String });
 export type DbIndex = typeof DbIndex.Type;
 
+export const DbSchema = object({
+  tables: array(DbTable),
+  indexes: array(DbIndex),
+});
+export type DbSchema = typeof DbSchema.Type;
+
 export type DbWorker = Worker<DbWorkerInput, DbWorkerOutput>;
 
 export type CreateDbWorker = (name: SimpleName) => DbWorker;
@@ -241,6 +244,7 @@ export const createDbWorkerForPlatform = (
 
       const sqliteResult = await createSqlite(platformDeps)(
         initMessage.config.name,
+        { memory: initMessage.config.inMemory ?? false },
       );
 
       if (!sqliteResult.ok) {
@@ -496,7 +500,6 @@ export const createDbWorkerForPlatform = (
             onCompleteId: message.onCompleteId,
             reload: message.reload,
           });
-          deps.sqlite[Symbol.dispose]();
 
           break;
         }
@@ -621,34 +624,32 @@ const indexesAreEqual = (self: DbIndex, that: DbIndex): boolean =>
 
 export interface DbSnapshot {
   readonly schema: DbSchema;
-  readonly rows: Array<{
-    rows: ReadonlyArray<SqliteRow>;
+  readonly tables: Array<{
     name: string;
+    rows: ReadonlyArray<SqliteRow>;
   }>;
 }
 
-// TODO: Move it to Sqlite.
-export const getDbSnapshot = (
-  deps: SqliteDep,
-): Result<DbSnapshot, SqliteError> => {
+// TODO: Move to test helpers.
+export const getDbSnapshot = (deps: SqliteDep): DbSnapshot => {
   const schema = getDbSchema(deps)({ allIndexes: true });
-  if (!schema.ok) return schema;
+  assert(schema.ok, "bug");
 
-  const rows = [];
+  const tables = [];
 
   for (const table of schema.value.tables) {
     const result = deps.sqlite.exec(sql`
       select * from ${sql.identifier(table.name)};
     `);
-    if (!result.ok) return result;
+    assert(result.ok, "bug");
 
-    rows.push({
-      rows: result.value.rows,
+    tables.push({
       name: table.name,
+      rows: result.value.rows,
     });
   }
 
-  return ok({ schema: schema.value, rows });
+  return { schema: schema.value, tables };
 };
 
 const ensureDbSchema =
@@ -787,41 +788,39 @@ const initializeDb =
       `,
 
       /**
-       * The History table stores all values per timestamp, table, row, and
-       * column. It's required for merging without conflicts. Evolu uses
-       * last-write-win CRDT. In case of last-write-win value isn't what we
-       * want, we can use time travel. The current implementation prefers
-       * performance over storage size. The History table denormalizes Timestamp
-       * and DbChange to leverage a covering index. Hence, every value change
-       * has its timestamp, table, row, and column. In the future, we will
-       * rethink that and store history more efficiently.
-       *
-       * There is no need to use `OwnerId` on the client, because timestamps
-       * (which include a `NodeId`) are globally unique. The relay needs
-       * `OwnerId` because it hosts multiple apps/owners, but client storage
-       * represents one DB, and even when it hosts multiple owners, their
-       * timestamps remain unique.
+       * The History table stores all values per ownerId, timestamp, table, id,
+       * and column for conflict-free merging using last-write-win CRDT.
+       * Denormalizes Timestamp and DbChange for covering index performance.
+       * Time travel is available when last-write-win isn't desired. Future
+       * optimization will store history more efficiently.
        */
       sql`
         create table evolu_history (
-          "timestamp" blob not null,
+          "ownerId" blob not null,
           "table" text not null,
-          "row" blob not null,
+          "id" blob not null,
           "column" text not null,
+          "timestamp" blob not null,
           "value" any
         )
         strict;
       `,
 
+      // Index for reading database changes by owner and timestamp.
+      // Timestamp always corresponds to a DbChange.
       sql`
-        create index evolu_history_timestamp on evolu_history ("timestamp");
+        create index evolu_history_ownerId_timestamp on evolu_history (
+          "ownerId",
+          "timestamp"
+        );
       `,
 
       sql`
-        create unique index evolu_history_row_column_table_timestampDesc on evolu_history (
-          "row",
-          "column",
+        create unique index evolu_history_ownerId_table_id_column_timestampDesc on evolu_history (
+          "ownerId",
           "table",
+          "id",
+          "column",
           "timestamp" desc
         );
       `,
@@ -847,7 +846,14 @@ const initializeDb =
 
     const result = deps.sqlite.exec(sql`
       insert into evolu_owner
-        (mnemonic, id, encryptionKey, createdAt, writeKey, timestamp)
+        (
+          "mnemonic",
+          "id",
+          "encryptionKey",
+          "createdAt",
+          "writeKey",
+          "timestamp"
+        )
       values
         (
           ${ownerRow.mnemonic},
@@ -910,16 +916,18 @@ const applyMessages =
   (
     messages: ReadonlyArray<CrdtMessage>,
     lastTimestamp: Timestamp,
-    options: { isMigrationFromVersion0?: boolean } = {},
   ): Result<void, SqliteError> => {
+    const ownerId = ownerIdToBinaryOwnerId(deps.ownerRowRef.get().id);
+
     for (const message of messages) {
-      if (!options.isMigrationFromVersion0) {
-        const apply1 = applyMessageToAppTable(deps)(message);
-        if (!apply1.ok) return apply1;
-      }
+      const result1 = applyMessageToAppTable(deps)(ownerId, message);
+      if (!result1.ok) return result1;
 
-      const apply2 = applyMessageToTimestampAndHistoryTables(deps)(message);
-      if (!apply2.ok) return apply2;
+      const result2 = applyMessageToTimestampAndHistoryTables(deps)(
+        ownerId,
+        message,
+      );
+      if (!result2.ok) return result2;
     }
 
     const timestamp = timestampToTimestampString(lastTimestamp);
@@ -933,8 +941,8 @@ const applyMessages =
   };
 
 const applyMessageToAppTable =
-  (deps: SqliteDep) =>
-  (message: CrdtMessage): Result<void, SqliteError> => {
+  (deps: SqliteDep & OwnerRowRefDep) =>
+  (ownerId: BinaryOwnerId, message: CrdtMessage): Result<void, SqliteError> => {
     const date = new Date(message.timestamp.millis).toISOString();
     const timestamp = timestampToBinaryTimestamp(message.timestamp);
 
@@ -942,28 +950,29 @@ const applyMessageToAppTable =
       const result = deps.sqlite.exec(sql.prepared`
         with
           lastTimestamp as (
-            select timestamp
+            select "timestamp"
             from evolu_history
             where
-              "row" = ${message.change.id}
-              and "column" = ${column}
+              "ownerId" = ${ownerId}
               and "table" = ${message.change.table}
-            order by timestamp desc
+              and "id" = ${message.change.id}
+              and "column" = ${column}
+            order by "timestamp" desc
             limit 1
           )
         insert into ${sql.identifier(message.change.table)}
           ("id", ${sql.identifier(column)}, createdAt, updatedAt)
         select ${message.change.id}, ${value}, ${date}, ${date}
         where
-          (select timestamp from lastTimestamp) is null
-          or (select timestamp from lastTimestamp) < ${timestamp}
+          (select "timestamp" from lastTimestamp) is null
+          or (select "timestamp" from lastTimestamp) < ${timestamp}
         on conflict ("id") do update
           set
             ${sql.identifier(column)} = ${value},
             updatedAt = ${date}
           where
-            (select timestamp from lastTimestamp) is null
-            or (select timestamp from lastTimestamp) < ${timestamp};
+            (select "timestamp" from lastTimestamp) is null
+            or (select "timestamp" from lastTimestamp) < ${timestamp};
       `);
 
       if (!result.ok) return result;
@@ -973,10 +982,9 @@ const applyMessageToAppTable =
   };
 
 export const applyMessageToTimestampAndHistoryTables =
-  (deps: SqliteDep & RandomDep & OwnerRowRefDep & ClientStorageDep) =>
-  (message: CrdtMessage): Result<void, SqliteError> => {
+  (deps: SqliteDep & ClientStorageDep) =>
+  (ownerId: BinaryOwnerId, message: CrdtMessage): Result<void, SqliteError> => {
     const timestamp = timestampToBinaryTimestamp(message.timestamp);
-    const ownerId = ownerIdToBinaryOwnerId(deps.ownerRowRef.get().id);
     const id = idToBinaryId(message.change.id);
 
     const result = deps.storage.insertTimestamp(ownerId, timestamp);
@@ -985,9 +993,16 @@ export const applyMessageToTimestampAndHistoryTables =
     for (const [column, value] of Object.entries(message.change.values)) {
       const result = deps.sqlite.exec(sql.prepared`
         insert into evolu_history
-          ("timestamp", "table", "row", "column", "value")
+          ("ownerId", "table", "id", "column", "value", "timestamp")
         values
-          (${timestamp}, ${message.change.table}, ${id}, ${column}, ${value})
+          (
+            ${ownerId},
+            ${message.change.table},
+            ${id},
+            ${column},
+            ${value},
+            ${timestamp}
+          )
         on conflict do nothing;
       `);
       if (!result.ok) return result;
@@ -1063,11 +1078,11 @@ export const maybeMigrateToVersion0 =
     const messagesRows = deps.sqlite.exec<{
       timestamp: TimestampString;
       table: Base64Url256;
-      row: Id;
+      id: Id;
       column: Base64Url256;
       value: SqliteValue;
     }>(sql`
-      select "timestamp", "table", "row", "column", "value" from evolu_message;
+      select "timestamp", "table", "id", "column", "value" from evolu_message;
     `);
 
     if (!messagesRows.ok) return messagesRows;
@@ -1083,7 +1098,7 @@ export const maybeMigrateToVersion0 =
     const messages = messagesRows.value.rows.map((message) => ({
       timestamp: timestampStringToTimestamp(message.timestamp),
       change: {
-        id: message.row,
+        id: message.id,
         table: message.table,
         values: { [message.column]: message.value },
       },
@@ -1176,6 +1191,7 @@ const createClientStorage =
       ...sqliteStorageBase.value,
 
       validateWriteKey: constFalse,
+      setWriteKey: constFalse,
 
       writeMessages: (_ownerId, messages) => {
         // TODO: Get owner by _ownerId when we support more.
@@ -1231,16 +1247,16 @@ const createClientStorage =
         return true;
       },
 
-      readDbChange: (_ownerId, timestamp) => {
+      readDbChange: (ownerId, timestamp) => {
         const result = deps.sqlite.exec<{
           table: Base64Url256;
-          row: BinaryId;
+          id: BinaryId;
           column: Base64Url256;
           value: SqliteValue;
         }>(sql`
-          select "table", "row", "column", "value"
+          select "table", "id", "column", "value"
           from evolu_history
-          where "timestamp" = ${timestamp};
+          where "ownerId" = ${ownerId} and "timestamp" = ${timestamp};
         `);
         if (!result.ok) {
           deps.postMessage({ type: "onError", error: result.error });
@@ -1250,18 +1266,18 @@ const createClientStorage =
         const { rows } = result.value;
         assert(rows.length > 0, "Rows must not be empty");
 
-        const { table, row } = rows[0];
+        const { table, id } = rows[0];
         const values: Record<string, SqliteValue> = {};
 
         for (const r of rows) {
           assert(r.table === table, "All rows must have the same table");
-          assert(eqArrayNumber(r.row, row), "All rows must have the same Id");
+          assert(eqArrayNumber(r.id, id), "All rows must have the same Id");
           values[r.column] = r.value;
         }
 
         const change: DbChange = {
           table: rows[0].table,
-          id: binaryIdToId(rows[0].row),
+          id: binaryIdToId(rows[0].id),
           values,
         };
 

package/src/Evolu/Evolu.ts

@@ -1,5 +1,5 @@
 import { isNonEmptyArray, isNonEmptyReadonlyArray } from "../Array.js";
-import { assertNonEmptyArray } from "../Assert.js";
+import { assert, assertNonEmptyArray } from "../Assert.js";
 import { createCallbacks } from "../Callbacks.js";
 import { ConsoleDep } from "../Console.js";
 import { SymmetricCryptoDecryptError } from "../Crypto.js";
@@ -44,9 +44,9 @@ import {
   SubscribedQueries,
 } from "./Query.js";
 import {
-  assertValidEvoluSchema,
   CreateQuery,
   EvoluSchema,
+  evoluSchemaToDbSchema,
   insertable,
   Mutation,
   MutationKind,
@@ -54,7 +54,7 @@ import {
   MutationOptions,
   updateable,
   upsertable,
-  validEvoluSchemaToDbSchema,
+  ValidateSchema,
   ValidMutationSize,
   ValidMutationSizeError,
 } from "./Schema.js";
@@ -88,7 +88,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> {
    * All this function does is compile the Kysely query and serialize it into a
    * unique string. Both operations are fast and cheap.
    *
-   * For mutations, use {@link Evolu.insert} and {@link Evolu.update}.
+   * For mutations, use {@link Evolu#insert} and {@link Evolu#update}.
    *
    * ### Example
    *
@@ -111,7 +111,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> {
    * A returned promise always resolves successfully because there is no reason
    * why loading should fail. All data are local, and the query is typed. A
    * serious unexpected Evolu error shall be handled with
-   * {@link Evolu.subscribeError}.
+   * {@link Evolu#subscribeError}.
    *
    * Loading is batched, and returned promises are cached, so there is no need
    * for an additional cache. Evolu's internal cache is invalidated on
@@ -144,7 +144,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> {
    * If you are curious why Evolu does not do that for all queries by default,
    * the answer is simple: performance. Tracking changes is costly and
    * meaningful only for visible (hence subscribed) queries anyway. To subscribe
-   * to a query, use {@link Evolu.subscribeQuery}.
+   * to a query, use {@link Evolu#subscribeQuery}.
    *
    * ### Example
    *
@@ -300,7 +300,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> {
 
   /**
    * Restore {@link AppOwner} with all their synced data. It uses
-   * {@link Evolu.resetAppOwner}, so be careful.
+   * {@link Evolu#resetAppOwner}, so be careful.
    */
   readonly restoreAppOwner: (
     mnemonic: Mnemonic,
@@ -431,9 +431,7 @@ let tabId: Id | null = null;
 export const createEvolu =
   (deps: EvoluDeps) =>
   <S extends EvoluSchema>(
-    // TODO: Validate missing Id, unsupported types, used default types via TS types
-    // with type errors messages as we had it in the old Evolu.
-    schema: S,
+    schema: ValidateSchema<S> extends never ? S : ValidateSchema<S>,
     partialConfig: Partial<EvoluConfigWithInitialData<S>> = {},
   ): Evolu<S> => {
     const config = { ...defaultConfig, ...partialConfig };
@@ -441,11 +439,14 @@ export const createEvolu =
     let evolu = evoluInstances.get(config.name);
 
     if (evolu == null) {
-      evolu = createEvoluInstance(deps)(schema, config as IntentionalNever);
+      evolu = createEvoluInstance(deps)(
+        schema as EvoluSchema,
+        config as IntentionalNever,
+      );
       evoluInstances.set(config.name, evolu);
     } else {
       // Hot reloading. Note that indexes are intentionally omitted.
-      evolu.ensureSchema(schema);
+      evolu.ensureSchema(schema as EvoluSchema);
     }
 
     return evolu as IntentionalNever;
@@ -554,10 +555,7 @@ const createEvoluInstance =
       }
     });
 
-    const dbSchema = validEvoluSchemaToDbSchema(
-      assertValidEvoluSchema(schema),
-      indexes,
-    );
+    const dbSchema = evoluSchemaToDbSchema(schema, indexes);
 
     const mutationTypesCache = new Map<
       MutationKind,
@@ -592,21 +590,17 @@ const createEvoluInstance =
     if (initialData)
       initialData({
         insert: (table, props) => {
-          const Type = getMutationType(table, "insert");
           const id = createId(deps);
+          const values = getMutationType(table, "insert").fromUnknown(props);
 
-          const result = Type.fromUnknown(props);
-
-          if (result.ok) {
-            initialDataDbChanges.push({
-              id,
-              table,
-              values: result.value,
-            } as unknown as DbChange);
+          if (values.ok) {
+            const dbChange = { table, id, values: values.value };
+            assertValidDbChange(dbChange);
+            initialDataDbChanges.push(dbChange);
             return ok({ id });
           }
 
-          return result;
+          return values;
         },
       });
 
@@ -654,14 +648,14 @@ const createEvoluInstance =
           } else {
             // Remove `id` from values.
             const { id: _id, ...values } = result.value;
-            // EvoluSchema Types ensure valid types.
-            const change = { table, id, values } as unknown as DbChange;
-            mutateMicrotaskQueue.push([change, options?.onComplete]);
+            const dbChange = { table, id, values };
+            assertValidDbChange(dbChange);
+            mutateMicrotaskQueue.push([dbChange, options?.onComplete]);
           }
 
           if (mutateMicrotaskQueue.length === 1)
             queueMicrotask(() => {
-              const changes = [];
+              const changes: Array<DbChange> = [];
               const onCompletes = [];
 
               for (const [change, onComplete] of mutateMicrotaskQueue) {
@@ -807,6 +801,7 @@ const createEvoluInstance =
         const onCompleteId = callbacks.register(() => {
           resolve();
         });
+
         dbWorker.postMessage({
           type: "reset",
           onCompleteId,
@@ -822,11 +817,8 @@ const createEvoluInstance =
 
       ensureSchema: (schema) => {
         mutationTypesCache.clear();
-        const validSchema = assertValidEvoluSchema(schema);
-        dbWorker.postMessage({
-          type: "ensureDbSchema",
-          dbSchema: validEvoluSchemaToDbSchema(validSchema),
-        });
+        const dbSchema = evoluSchemaToDbSchema(schema);
+        dbWorker.postMessage({ type: "ensureDbSchema", dbSchema });
       },
 
       exportDatabase: () => {
@@ -945,3 +937,14 @@ const createLoadingPromises = (
 
   return loadingPromises;
 };
+
+const assertValidDbChange: (dbChange: {
+  table: string;
+  id: Id;
+  values: unknown;
+}) => asserts dbChange is DbChange = (dbChange) => {
+  assert(
+    DbChange.is(dbChange),
+    `Failed to create DbChange for table "${dbChange.table}". If you see this message, you either disabled EvoluSchema validation or Evolu has a bug - please report it.`,
+  );
+};