@evolu/common 4.0.4 → 4.1.0

package/src/Config.ts

@@ -1,5 +1,6 @@
 import * as Context from "effect/Context";
 import * as Layer from "effect/Layer";
+import { CreateIndexBuilder } from "kysely";
 
 export interface Config {
   /** Alternate URL to Evolu sync and backup server. */
@@ -23,6 +24,24 @@ export interface Config {
    * default value is: "Evolu".
    */
   readonly name: string;
+
+  /**
+   * Use the `indexes` property to define SQLite indexes.
+   *
+   * Table and column names are not typed because Kysely doesn't support it.
+   *
+   * https://medium.com/@JasonWyatt/squeezing-performance-from-sqlite-indexes-indexes-c4e175f3c346
+   *
+   * @example
+   *   const indexes = [
+   *     createIndex("indexTodoCreatedAt").on("todo").column("createdAt"),
+   *
+   *     createIndex("indexTodoCategoryCreatedAt")
+   *       .on("todoCategory")
+   *       .column("createdAt"),
+   *   ];
+   */
+  readonly indexes: ReadonlyArray<CreateIndexBuilder<any>>;
 }
 
 export const Config = Context.GenericTag<Config>("@services/Config");
@@ -35,6 +54,7 @@ export const ConfigLive = (config?: Partial<Config>): Layer.Layer<Config> =>
       maxDrift: 5 * 60 * 1000,
       reloadUrl: "/",
       name: "Evolu",
+      indexes: [],
       ...config,
     }),
   );

package/src/Crdt.ts

@@ -42,14 +42,14 @@ export const Millis = S.number.pipe(
   S.brand("Millis"),
 );
 
-export type Millis = S.Schema.To<typeof Millis>;
+export type Millis = S.Schema.Type<typeof Millis>;
 
 export const initialMillis = S.decodeSync(Millis)(
   AllowedTimeRange.greaterThan + 1,
 );
 
 export const Counter = S.number.pipe(S.between(0, 65535), S.brand("Counter"));
-export type Counter = S.Schema.To<typeof Counter>;
+export type Counter = S.Schema.Type<typeof Counter>;
 
 const initialCounter = S.decodeSync(Counter)(0);
 

package/src/Crypto.ts

@@ -51,7 +51,7 @@ export const NodeId = S.string.pipe(
   S.pattern(/^[\w-]{16}$/),
   S.brand("NodeId"),
 );
-export type NodeId = S.Schema.To<typeof NodeId>;
+export type NodeId = S.Schema.Type<typeof NodeId>;
 
 const nanoidForNodeId = customAlphabet("0123456789abcdef", 16);
 

package/src/Db.ts

@@ -31,9 +31,14 @@ import {
   insertOwner,
 } from "./Sql.js";
 import {
+  Index,
+  indexEquivalence,
   JsonObjectOrArray,
   Sqlite,
   SqliteQuery,
+  SqliteQueryOptions,
+  SqliteSchema,
+  Table,
   Value,
   isJsonObjectOrArray,
 } from "./Sqlite.js";
@@ -56,14 +61,14 @@ type TableSchema = ReadonlyRecord.ReadonlyRecord<string, Value> & {
  *
  * @example
  *   const TodoId = id("Todo");
- *   type TodoId = S.Schema.To<typeof TodoId>;
+ *   type TodoId = S.Schema.Type<typeof TodoId>;
  *
  *   const TodoTable = table({
  *     id: TodoId,
  *     title: NonEmptyString1000,
  *     isCompleted: S.nullable(SqliteBoolean),
  *   });
- *   type TodoTable = S.Schema.To<typeof TodoTable>;
+ *   type TodoTable = S.Schema.Type<typeof TodoTable>;
  */
 export const table = <Fields extends TableFields>(
   fields: Fields,
@@ -73,10 +78,10 @@ export const table = <Fields extends TableFields>(
     ? ValidateFieldsHasId<Fields> extends true
       ? S.Schema<
           Types.Simplify<
-            S.ToStruct<Fields> & S.Schema.To<typeof ReservedColumns>
+            S.Struct.Type<Fields> & S.Schema.Type<typeof ReservedColumns>
           >,
           Types.Simplify<
-            S.FromStruct<Fields> & S.Schema.From<typeof ReservedColumns>
+            S.Struct.Encoded<Fields> & S.Schema.Encoded<typeof ReservedColumns>
           >
         >
       : EvoluTypeError<"table() called without id column.">
@@ -132,7 +137,7 @@ type ValidateFieldsHasId<Fields extends TableFields> = "id" extends keyof Fields
  *     todo: TodoTable,
  *     todoCategory: TodoCategoryTable,
  *   });
- *   type Database = S.Schema.To<typeof Database>;
+ *   type Database = S.Schema.Type<typeof Database>;
  */
 export const database = S.struct;
 
@@ -157,22 +162,25 @@ interface SerializedSqliteQuery {
     | Array<number>
     | { json: JsonObjectOrArray }
   )[];
+  readonly options?: SqliteQueryOptions;
 }
 
 // We use queries as keys, hence JSON.stringify.
 export const serializeQuery = <R extends Row>({
   sql,
-  parameters,
+  parameters = [],
+  options,
 }: SqliteQuery): Query<R> => {
   const query: SerializedSqliteQuery = {
     sql,
-    parameters: parameters.map((p) => {
-      return Predicate.isUint8Array(p)
+    parameters: parameters.map((p) =>
+      Predicate.isUint8Array(p)
         ? Array.from(p)
         : isJsonObjectOrArray(p)
           ? { json: p }
-          : p;
-    }),
+          : p,
+    ),
+    ...(options && { options }),
   };
   return JSON.stringify(query) as Query<R>;
 };
@@ -249,13 +257,7 @@ export const queryResultFromRows = <R extends Row>(
   return queryResult as QueryResult<R>;
 };
 
-export type Tables = ReadonlyArray<Table>;
-
-export interface Table {
-  readonly name: string;
-  readonly columns: ReadonlyArray<string>;
-}
-
+// TODO: https://discord.com/channels/795981131316985866/1218626687546294386/1218796529725476935
 // https://github.com/Effect-TS/schema/releases/tag/v0.18.0
 const getPropertySignatures = <I extends { [K in keyof A]: any }, A>(
   schema: S.Schema<A, I>,
@@ -270,7 +272,7 @@ const getPropertySignatures = <I extends { [K in keyof A]: any }, A>(
   return out as any;
 };
 
-export const schemaToTables = (schema: S.Schema<any>): Tables =>
+export const schemaToTables = (schema: S.Schema<any>): ReadonlyArray<Table> =>
   pipe(
     getPropertySignatures(schema),
     ReadonlyRecord.toEntries,
@@ -287,10 +289,12 @@ export const transaction = <R, E, A>(
 ): Effect.Effect<A, E, Sqlite | R> =>
   Effect.flatMap(Sqlite, (sqlite) =>
     Effect.acquireUseRelease(
-      sqlite.exec("begin"),
+      sqlite.exec({ sql: "begin" }),
       () => effect,
       (_, exit) =>
-        Exit.isFailure(exit) ? sqlite.exec("rollback") : sqlite.exec("end"),
+        Exit.isFailure(exit)
+          ? sqlite.exec({ sql: "rollback" })
+          : sqlite.exec({ sql: "end" }),
     ),
   );
 
@@ -305,7 +309,7 @@ export const SqliteNoSuchTableOrColumnError = S.struct({
     S.string.pipe(S.includes("has no column")),
   ),
 });
-export type SqliteNoSuchTableOrColumnError = S.Schema.To<
+export type SqliteNoSuchTableOrColumnError = S.Schema.Type<
   typeof SqliteNoSuchTableOrColumnError
 >;
 
@@ -336,7 +340,7 @@ export const lazyInit = (
         sqlite.exec(createMessageTableIndex),
         sqlite.exec(createOwnerTable),
         sqlite.exec({
-          sql: insertOwner,
+          ...insertOwner,
           parameters: [
             owner.id,
             owner.mnemonic,
@@ -351,78 +355,150 @@ export const lazyInit = (
     return owner;
   });
 
-const getTables: Effect.Effect<
-  ReadonlyArray<string>,
-  never,
-  Sqlite
-> = Sqlite.pipe(
-  Effect.flatMap((sqlite) =>
-    sqlite.exec(`select "name" from "sqlite_schema" where "type" = 'table'`),
-  ),
-  Effect.map((result) => result.rows),
-  Effect.map(ReadonlyArray.map((row) => (row.name as string) + "")),
-  Effect.map(ReadonlyArray.filter(Predicate.not(String.startsWith("__")))),
-  Effect.map(ReadonlyArray.dedupeWith(String.Equivalence)),
-);
-
-const updateTable = ({
-  name,
-  columns,
-}: Table): Effect.Effect<void, never, Sqlite> =>
-  Effect.gen(function* (_) {
+const getSchema: Effect.Effect<SqliteSchema, never, Sqlite> = Effect.gen(
+  function* (_) {
     const sqlite = yield* _(Sqlite);
-    const sql = yield* _(
-      sqlite.exec(`pragma table_info (${name})`),
-      Effect.map((result) => result.rows),
-      Effect.map(ReadonlyArray.map((row) => row.name as string)),
-      Effect.map((existingColumns) =>
-        ReadonlyArray.differenceWith(String.Equivalence)(existingColumns)(
+
+    const tables = yield* _(
+      sqlite.exec({
+        // https://til.simonwillison.net/sqlite/list-all-columns-in-a-database
+        sql: `
+          select
+            sqlite_master.name as tableName,
+            table_info.name as columnName
+          from
+            sqlite_master
+            join pragma_table_info(sqlite_master.name) as table_info
+          `.trim(),
+      }),
+      Effect.map(({ rows }) => {
+        const map = new Map<string, string[]>();
+        rows.forEach((row) => {
+          const { tableName, columnName } = row as {
+            tableName: string;
+            columnName: string;
+          };
+          if (!map.has(tableName)) map.set(tableName, []);
+          map.get(tableName)?.push(columnName);
+        });
+        return Array.from(map, ([name, columns]) => ({
+          name,
           columns,
-        ),
-      ),
-      Effect.map(
+        }));
+      }),
+    );
+
+    const indexes = yield* _(
+      sqlite.exec({
+        sql: `
+          select
+            name, sql
+          from
+            sqlite_master
+          where
+            type='index' and
+            name not like 'sqlite_%' and
+            name not like 'index_evolu_%'`,
+      }),
+      Effect.map((result) =>
         ReadonlyArray.map(
-          (newColumn) =>
-            `alter table "${name}" add column "${newColumn}" blob;`,
+          result.rows,
+          (row): Index => ({
+            name: row.name as string,
+            /**
+             * SQLite returns "CREATE INDEX" for "create index" for some reason.
+             * Other keywords remain unchanged. We have to normalize the casing
+             * for `indexEquivalence` manually.
+             */
+            sql: (row.sql as string).replace("CREATE INDEX", "create index"),
+          }),
         ),
       ),
-      Effect.map(ReadonlyArray.join("")),
     );
-    if (sql) yield* _(sqlite.exec(sql));
-  });
 
-const createTable = ({
-  name,
-  columns,
-}: Table): Effect.Effect<void, never, Sqlite> =>
-  Effect.flatMap(Sqlite, (sqlite) =>
-    sqlite.exec(`
-      create table ${name} (
-        "id" text primary key,
-        ${columns
-          .filter((c) => c !== "id")
-          // "A column with affinity BLOB does not prefer one storage class over another
-          // and no attempt is made to coerce data from one storage class into another."
-          // https://www.sqlite.org/datatype3.html
-          .map((name) => `"${name}" blob`)
-          .join(", ")}
-      );
-    `),
-  );
+    return { tables, indexes };
+  },
+);
 
 export const ensureSchema = (
-  tables: Tables,
+  schema: SqliteSchema,
 ): Effect.Effect<void, never, Sqlite> =>
-  Effect.flatMap(getTables, (existingTables) =>
-    Effect.forEach(
-      tables,
-      (tableDefinition) =>
-        existingTables.includes(tableDefinition.name)
-          ? updateTable(tableDefinition)
-          : createTable(tableDefinition),
-      { discard: true },
-    ),
-  );
+  Effect.gen(function* (_) {
+    const sqlite = yield* _(Sqlite);
+    const currentSchema = yield* _(getSchema);
+
+    const sql: string[] = [];
+
+    schema.tables.forEach((table) => {
+      const currentTable = currentSchema.tables.find(
+        (t) => t.name === table.name,
+      );
+      if (!currentTable) {
+        sql.push(`
+          create table ${table.name} (
+            "id" text primary key,
+            ${table.columns
+              .filter((c) => c !== "id")
+              // "A column with affinity BLOB does not prefer one storage class over another
+              // and no attempt is made to coerce data from one storage class into another."
+              // https://www.sqlite.org/datatype3.html
+              .map((name) => `"${name}" blob`)
+              .join(", ")}
+            );
+        `);
+      } else {
+        ReadonlyArray.differenceWith(String.Equivalence)(
+          table.columns,
+          currentTable.columns,
+        ).forEach((newColumn) => {
+          sql.push(
+            `alter table "${table.name}" add column "${newColumn}" blob;`,
+          );
+        });
+      }
+    });
+
+    // Remove old indexes.
+    ReadonlyArray.differenceWith(indexEquivalence)(
+      currentSchema.indexes,
+      ReadonlyArray.intersectionWith(indexEquivalence)(
+        currentSchema.indexes,
+        schema.indexes,
+      ),
+    ).forEach((indexToDrop) => {
+      sql.push(`drop index "${indexToDrop.name}";`);
+    });
+
+    // Add new indexes.
+    ReadonlyArray.differenceWith(indexEquivalence)(
+      schema.indexes,
+      currentSchema.indexes,
+    ).forEach((newIndex) => {
+      sql.push(`${newIndex.sql};`);
+    });
+
+    if (sql.length > 0)
+      yield* _(
+        sqlite.exec({
+          sql: sql.join(""),
+        }),
+      );
+  });
+
+export const dropAllTables: Effect.Effect<void, never, Sqlite> = Effect.gen(
+  function* (_) {
+    const sqlite = yield* _(Sqlite);
+    const schema = yield* _(getSchema);
+    const sql = schema.tables
+      // The dropped table is completely removed from the database schema and
+      // the disk file. The table can not be recovered.
+      // All indices and triggers associated with the table are also deleted.
+      // https://sqlite.org/lang_droptable.html
+      .map((table) => `drop table "${table.name}";`)
+      .join("");
+    yield* _(sqlite.exec({ sql }));
+  },
+);
 
 export type RowsStore = Store<RowsStoreValue>;
 export const RowsStore = Context.GenericTag<RowsStore>("@services/RowsStore");

package/src/DbWorker.ts

@@ -34,9 +34,8 @@ import {
   Query,
   RowsStore,
   RowsStoreLive,
-  Table,
-  Tables,
   deserializeQuery,
+  dropAllTables,
   ensureSchema,
   lazyInit,
   someDefectToNoSuchTableOrColumnError,
@@ -49,7 +48,14 @@ import { OnCompleteId } from "./OnCompletes.js";
 import { Owner, OwnerId } from "./Owner.js";
 import { DbWorkerLock } from "./Platform.js";
 import * as Sql from "./Sql.js";
-import { Sqlite, Value } from "./Sqlite.js";
+import {
+  Sqlite,
+  SqliteQueryPlanRow,
+  SqliteSchema,
+  Table,
+  Value,
+  drawSqliteQueryPlan as drawExplainQueryPlan,
+} from "./Sqlite.js";
 import {
   Message,
   NewMessage,
@@ -99,9 +105,8 @@ interface DbWorkerInputReset {
   readonly mnemonic?: Mnemonic;
 }
 
-interface DbWorkerInputEnsureSchema {
+interface DbWorkerInputEnsureSchema extends SqliteSchema {
   readonly _tag: "ensureSchema";
-  readonly tables: Tables;
 }
 
 type DbWorkerOnMessage = DbWorker["onMessage"];
@@ -189,11 +194,30 @@ const query = ({
 
     const queriesRows = yield* _(
       ReadonlyArray.dedupe(queries),
-      Effect.forEach((query) =>
-        sqlite
-          .exec(deserializeQuery(query))
-          .pipe(Effect.map((result) => [query, result.rows] as const)),
-      ),
+      Effect.forEach((query) => {
+        const sqliteQuery = deserializeQuery(query);
+        return sqlite.exec(sqliteQuery).pipe(
+          Effect.map((result) => [query, result.rows] as const),
+          Effect.tap(() => {
+            if (!sqliteQuery.options?.logExplainQueryPlan) return;
+            return sqlite
+              .exec({
+                ...sqliteQuery,
+                sql: `EXPLAIN QUERY PLAN ${sqliteQuery.sql}`,
+              })
+              .pipe(
+                Effect.tap(() => Effect.log("ExplainQueryPlan")),
+                Effect.tap(({ rows }) => {
+                  // Not using Effect.log because of formating
+                  // eslint-disable-next-line no-console
+                  console.log(
+                    drawExplainQueryPlan(rows as SqliteQueryPlanRow[]),
+                  );
+                }),
+              );
+          }),
+        );
+      }),
     );
 
     const previous = rowsStore.getState();
@@ -288,7 +312,8 @@ const ensureSchemaByNewMessages = (
         columns: table.columns.concat(message.column),
       });
     });
-    yield* _(ensureSchema(Array.from(tablesMap.values())));
+    const tables = Array.from(tablesMap.values());
+    yield* _(ensureSchema({ tables, indexes: [] }));
   });
 
 export const upsertValueIntoTableRowColumn = (
@@ -300,7 +325,15 @@ export const upsertValueIntoTableRowColumn = (
     const sqlite = yield* _(Sqlite);
     const createdAtOrUpdatedAt = cast(new Date(millis));
     const insert = sqlite.exec({
-      sql: Sql.upsertValueIntoTableRowColumn(message.table, message.column),
+      sql: `
+        insert into
+          "${message.table}" ("id", "${message.column}", "createdAt", "updatedAt")
+        values
+          (?, ?, ?, ?)
+        on conflict do update set
+          "${message.column}" = ?,
+          "updatedAt" = ?
+        `.trim(),
       parameters: [
         message.row,
         message.value,
@@ -334,8 +367,8 @@ const applyMessages = ({
     for (const message of messages) {
       const timestamp: TimestampString | null = yield* _(
         sqlite.exec({
-          sql: Sql.selectLastTimestampForTableRowColumn,
-          parameters: [message.table, message.row, message.column],
+          ...Sql.selectLastTimestampForTableRowColumn,
+          parameters: [message.table, message.row, message.column, 1],
         }),
         Effect.map((result) => result.rows),
         Effect.flatMap(ReadonlyArray.head),
@@ -351,7 +384,7 @@ const applyMessages = ({
       if (timestamp == null || timestamp !== message.timestamp) {
         const { changes } = yield* _(
           sqlite.exec({
-            sql: Sql.insertIntoMessagesIfNew,
+            ...Sql.insertIntoMessagesIfNew,
             parameters: [
               message.timestamp,
               message.table,
@@ -377,10 +410,10 @@ const writeTimestampAndMerkleTree = ({
 }: TimestampAndMerkleTree): Effect.Effect<void, never, Sqlite> =>
   Effect.flatMap(Sqlite, (sqlite) =>
     sqlite.exec({
-      sql: Sql.updateOwnerTimestampAndMerkleTree,
+      ...Sql.updateOwnerTimestampAndMerkleTree,
       parameters: [
-        timestampToString(timestamp),
         merkleTreeToString(merkleTree),
+        timestampToString(timestamp),
       ],
     }),
   );
@@ -420,7 +453,10 @@ const mutate = ({
     const { exec } = yield* _(Sqlite);
     yield* _(
       Effect.forEach(toDelete, ({ table, row }) =>
-        exec({ sql: Sql.deleteTableRow(table), parameters: [row] }),
+        exec({
+          sql: `delete from "${table}" where "id" = ?;`,
+          parameters: [row],
+        }),
       ),
     );
 
@@ -504,7 +540,7 @@ const handleSyncResponse = ({
 
     const messagesToSync = yield* _(
       sqlite.exec({
-        sql: Sql.selectMessagesToSync,
+        ...Sql.selectMessagesToSync,
         parameters: [timestampToString(makeSyncTimestamp(diff.value))],
       }),
       Effect.map(({ rows }) => rows as unknown as ReadonlyArray<Message>),
@@ -561,25 +597,8 @@ const reset = (
   Sqlite | Bip39 | NanoIdGenerator | DbWorkerOnMessage
 > =>
   Effect.gen(function* (_) {
-    const sqlite = yield* _(Sqlite);
-
-    yield* _(
-      sqlite.exec(`SELECT "name" FROM "sqlite_master" WHERE "type" = 'table'`),
-      Effect.map((result) => result.rows),
-      Effect.flatMap(
-        // The dropped table is completely removed from the database schema and
-        // the disk file. The table can not be recovered.
-        // All indices and triggers associated with the table are also deleted.
-        // https://sqlite.org/lang_droptable.html
-        Effect.forEach(
-          ({ name }) => sqlite.exec(`DROP TABLE "${name as string}"`),
-          { discard: true },
-        ),
-      ),
-    );
-
+    yield* _(dropAllTables);
     if (input.mnemonic) yield* _(lazyInit(input.mnemonic));
-
     const onMessage = yield* _(DbWorkerOnMessage);
     onMessage({ _tag: "onResetOrRestore" });
   });
@@ -666,7 +685,7 @@ export const DbWorkerCommonLive = Layer.effect(
               skipAllBecauseOfReset = true;
               return reset(input);
             },
-            ensureSchema: ({ tables }) => ensureSchema(tables),
+            ensureSchema,
             SyncWorkerOutputSyncResponse: handleSyncResponse,
           }),
           Effect.provide(layer),