@evolu/common 4.0.5 → 4.1.0

package/src/Db.ts

@@ -31,9 +31,14 @@ import {
   insertOwner,
 } from "./Sql.js";
 import {
+  Index,
+  indexEquivalence,
   JsonObjectOrArray,
   Sqlite,
   SqliteQuery,
+  SqliteQueryOptions,
+  SqliteSchema,
+  Table,
   Value,
   isJsonObjectOrArray,
 } from "./Sqlite.js";
@@ -157,22 +162,25 @@ interface SerializedSqliteQuery {
     | Array<number>
     | { json: JsonObjectOrArray }
   )[];
+  readonly options?: SqliteQueryOptions;
 }
 
 // We use queries as keys, hence JSON.stringify.
 export const serializeQuery = <R extends Row>({
   sql,
-  parameters,
+  parameters = [],
+  options,
 }: SqliteQuery): Query<R> => {
   const query: SerializedSqliteQuery = {
     sql,
-    parameters: parameters.map((p) => {
-      return Predicate.isUint8Array(p)
+    parameters: parameters.map((p) =>
+      Predicate.isUint8Array(p)
         ? Array.from(p)
         : isJsonObjectOrArray(p)
           ? { json: p }
-          : p;
-    }),
+          : p,
+    ),
+    ...(options && { options }),
   };
   return JSON.stringify(query) as Query<R>;
 };
@@ -249,13 +257,7 @@ export const queryResultFromRows = <R extends Row>(
   return queryResult as QueryResult<R>;
 };
 
-export type Tables = ReadonlyArray<Table>;
-
-export interface Table {
-  readonly name: string;
-  readonly columns: ReadonlyArray<string>;
-}
-
+// TODO: https://discord.com/channels/795981131316985866/1218626687546294386/1218796529725476935
 // https://github.com/Effect-TS/schema/releases/tag/v0.18.0
 const getPropertySignatures = <I extends { [K in keyof A]: any }, A>(
   schema: S.Schema<A, I>,
@@ -270,7 +272,7 @@ const getPropertySignatures = <I extends { [K in keyof A]: any }, A>(
   return out as any;
 };
 
-export const schemaToTables = (schema: S.Schema<any>): Tables =>
+export const schemaToTables = (schema: S.Schema<any>): ReadonlyArray<Table> =>
   pipe(
     getPropertySignatures(schema),
     ReadonlyRecord.toEntries,
@@ -287,10 +289,12 @@ export const transaction = <R, E, A>(
 ): Effect.Effect<A, E, Sqlite | R> =>
   Effect.flatMap(Sqlite, (sqlite) =>
     Effect.acquireUseRelease(
-      sqlite.exec("begin"),
+      sqlite.exec({ sql: "begin" }),
       () => effect,
       (_, exit) =>
-        Exit.isFailure(exit) ? sqlite.exec("rollback") : sqlite.exec("end"),
+        Exit.isFailure(exit)
+          ? sqlite.exec({ sql: "rollback" })
+          : sqlite.exec({ sql: "end" }),
     ),
   );
 
@@ -336,7 +340,7 @@ export const lazyInit = (
         sqlite.exec(createMessageTableIndex),
         sqlite.exec(createOwnerTable),
         sqlite.exec({
-          sql: insertOwner,
+          ...insertOwner,
           parameters: [
             owner.id,
             owner.mnemonic,
@@ -351,78 +355,150 @@ export const lazyInit = (
     return owner;
   });
 
-const getTables: Effect.Effect<
-  ReadonlyArray<string>,
-  never,
-  Sqlite
-> = Sqlite.pipe(
-  Effect.flatMap((sqlite) =>
-    sqlite.exec(`select "name" from "sqlite_schema" where "type" = 'table'`),
-  ),
-  Effect.map((result) => result.rows),
-  Effect.map(ReadonlyArray.map((row) => (row.name as string) + "")),
-  Effect.map(ReadonlyArray.filter(Predicate.not(String.startsWith("__")))),
-  Effect.map(ReadonlyArray.dedupeWith(String.Equivalence)),
-);
-
-const updateTable = ({
-  name,
-  columns,
-}: Table): Effect.Effect<void, never, Sqlite> =>
-  Effect.gen(function* (_) {
+const getSchema: Effect.Effect<SqliteSchema, never, Sqlite> = Effect.gen(
+  function* (_) {
     const sqlite = yield* _(Sqlite);
-    const sql = yield* _(
-      sqlite.exec(`pragma table_info (${name})`),
-      Effect.map((result) => result.rows),
-      Effect.map(ReadonlyArray.map((row) => row.name as string)),
-      Effect.map((existingColumns) =>
-        ReadonlyArray.differenceWith(String.Equivalence)(existingColumns)(
+
+    const tables = yield* _(
+      sqlite.exec({
+        // https://til.simonwillison.net/sqlite/list-all-columns-in-a-database
+        sql: `
+          select
+            sqlite_master.name as tableName,
+            table_info.name as columnName
+          from
+            sqlite_master
+            join pragma_table_info(sqlite_master.name) as table_info
+          `.trim(),
+      }),
+      Effect.map(({ rows }) => {
+        const map = new Map<string, string[]>();
+        rows.forEach((row) => {
+          const { tableName, columnName } = row as {
+            tableName: string;
+            columnName: string;
+          };
+          if (!map.has(tableName)) map.set(tableName, []);
+          map.get(tableName)?.push(columnName);
+        });
+        return Array.from(map, ([name, columns]) => ({
+          name,
           columns,
-        ),
-      ),
-      Effect.map(
+        }));
+      }),
+    );
+
+    const indexes = yield* _(
+      sqlite.exec({
+        sql: `
+          select
+            name, sql
+          from
+            sqlite_master
+          where
+            type='index' and
+            name not like 'sqlite_%' and
+            name not like 'index_evolu_%'`,
+      }),
+      Effect.map((result) =>
         ReadonlyArray.map(
-          (newColumn) =>
-            `alter table "${name}" add column "${newColumn}" blob;`,
+          result.rows,
+          (row): Index => ({
+            name: row.name as string,
+            /**
+             * SQLite returns "CREATE INDEX" for "create index" for some reason.
+             * Other keywords remain unchanged. We have to normalize the casing
+             * for `indexEquivalence` manually.
+             */
+            sql: (row.sql as string).replace("CREATE INDEX", "create index"),
+          }),
         ),
       ),
-      Effect.map(ReadonlyArray.join("")),
     );
-    if (sql) yield* _(sqlite.exec(sql));
-  });
 
-const createTable = ({
-  name,
-  columns,
-}: Table): Effect.Effect<void, never, Sqlite> =>
-  Effect.flatMap(Sqlite, (sqlite) =>
-    sqlite.exec(`
-      create table ${name} (
-        "id" text primary key,
-        ${columns
-          .filter((c) => c !== "id")
-          // "A column with affinity BLOB does not prefer one storage class over another
-          // and no attempt is made to coerce data from one storage class into another."
-          // https://www.sqlite.org/datatype3.html
-          .map((name) => `"${name}" blob`)
-          .join(", ")}
-      );
-    `),
-  );
+    return { tables, indexes };
+  },
+);
 
 export const ensureSchema = (
-  tables: Tables,
+  schema: SqliteSchema,
 ): Effect.Effect<void, never, Sqlite> =>
-  Effect.flatMap(getTables, (existingTables) =>
-    Effect.forEach(
-      tables,
-      (tableDefinition) =>
-        existingTables.includes(tableDefinition.name)
-          ? updateTable(tableDefinition)
-          : createTable(tableDefinition),
-      { discard: true },
-    ),
-  );
+  Effect.gen(function* (_) {
+    const sqlite = yield* _(Sqlite);
+    const currentSchema = yield* _(getSchema);
+
+    const sql: string[] = [];
+
+    schema.tables.forEach((table) => {
+      const currentTable = currentSchema.tables.find(
+        (t) => t.name === table.name,
+      );
+      if (!currentTable) {
+        sql.push(`
+          create table ${table.name} (
+            "id" text primary key,
+            ${table.columns
+              .filter((c) => c !== "id")
+              // "A column with affinity BLOB does not prefer one storage class over another
+              // and no attempt is made to coerce data from one storage class into another."
+              // https://www.sqlite.org/datatype3.html
+              .map((name) => `"${name}" blob`)
+              .join(", ")}
+            );
+        `);
+      } else {
+        ReadonlyArray.differenceWith(String.Equivalence)(
+          table.columns,
+          currentTable.columns,
+        ).forEach((newColumn) => {
+          sql.push(
+            `alter table "${table.name}" add column "${newColumn}" blob;`,
+          );
+        });
+      }
+    });
+
+    // Remove old indexes.
+    ReadonlyArray.differenceWith(indexEquivalence)(
+      currentSchema.indexes,
+      ReadonlyArray.intersectionWith(indexEquivalence)(
+        currentSchema.indexes,
+        schema.indexes,
+      ),
+    ).forEach((indexToDrop) => {
+      sql.push(`drop index "${indexToDrop.name}";`);
+    });
+
+    // Add new indexes.
+    ReadonlyArray.differenceWith(indexEquivalence)(
+      schema.indexes,
+      currentSchema.indexes,
+    ).forEach((newIndex) => {
+      sql.push(`${newIndex.sql};`);
+    });
+
+    if (sql.length > 0)
+      yield* _(
+        sqlite.exec({
+          sql: sql.join(""),
+        }),
+      );
+  });
+
+export const dropAllTables: Effect.Effect<void, never, Sqlite> = Effect.gen(
+  function* (_) {
+    const sqlite = yield* _(Sqlite);
+    const schema = yield* _(getSchema);
+    const sql = schema.tables
+      // The dropped table is completely removed from the database schema and
+      // the disk file. The table can not be recovered.
+      // All indices and triggers associated with the table are also deleted.
+      // https://sqlite.org/lang_droptable.html
+      .map((table) => `drop table "${table.name}";`)
+      .join("");
+    yield* _(sqlite.exec({ sql }));
+  },
+);
 
 export type RowsStore = Store<RowsStoreValue>;
 export const RowsStore = Context.GenericTag<RowsStore>("@services/RowsStore");

package/src/DbWorker.ts

@@ -34,9 +34,8 @@ import {
   Query,
   RowsStore,
   RowsStoreLive,
-  Table,
-  Tables,
   deserializeQuery,
+  dropAllTables,
   ensureSchema,
   lazyInit,
   someDefectToNoSuchTableOrColumnError,
@@ -49,7 +48,14 @@ import { OnCompleteId } from "./OnCompletes.js";
 import { Owner, OwnerId } from "./Owner.js";
 import { DbWorkerLock } from "./Platform.js";
 import * as Sql from "./Sql.js";
-import { Sqlite, Value } from "./Sqlite.js";
+import {
+  Sqlite,
+  SqliteQueryPlanRow,
+  SqliteSchema,
+  Table,
+  Value,
+  drawSqliteQueryPlan as drawExplainQueryPlan,
+} from "./Sqlite.js";
 import {
   Message,
   NewMessage,
@@ -99,9 +105,8 @@ interface DbWorkerInputReset {
   readonly mnemonic?: Mnemonic;
 }
 
-interface DbWorkerInputEnsureSchema {
+interface DbWorkerInputEnsureSchema extends SqliteSchema {
   readonly _tag: "ensureSchema";
-  readonly tables: Tables;
 }
 
 type DbWorkerOnMessage = DbWorker["onMessage"];
@@ -189,11 +194,30 @@ const query = ({
 
     const queriesRows = yield* _(
       ReadonlyArray.dedupe(queries),
-      Effect.forEach((query) =>
-        sqlite
-          .exec(deserializeQuery(query))
-          .pipe(Effect.map((result) => [query, result.rows] as const)),
-      ),
+      Effect.forEach((query) => {
+        const sqliteQuery = deserializeQuery(query);
+        return sqlite.exec(sqliteQuery).pipe(
+          Effect.map((result) => [query, result.rows] as const),
+          Effect.tap(() => {
+            if (!sqliteQuery.options?.logExplainQueryPlan) return;
+            return sqlite
+              .exec({
+                ...sqliteQuery,
+                sql: `EXPLAIN QUERY PLAN ${sqliteQuery.sql}`,
+              })
+              .pipe(
+                Effect.tap(() => Effect.log("ExplainQueryPlan")),
+                Effect.tap(({ rows }) => {
+                  // Not using Effect.log because of formating
+                  // eslint-disable-next-line no-console
+                  console.log(
+                    drawExplainQueryPlan(rows as SqliteQueryPlanRow[]),
+                  );
+                }),
+              );
+          }),
+        );
+      }),
     );
 
     const previous = rowsStore.getState();
@@ -288,7 +312,8 @@ const ensureSchemaByNewMessages = (
         columns: table.columns.concat(message.column),
       });
     });
-    yield* _(ensureSchema(Array.from(tablesMap.values())));
+    const tables = Array.from(tablesMap.values());
+    yield* _(ensureSchema({ tables, indexes: [] }));
   });
 
 export const upsertValueIntoTableRowColumn = (
@@ -300,7 +325,15 @@ export const upsertValueIntoTableRowColumn = (
     const sqlite = yield* _(Sqlite);
     const createdAtOrUpdatedAt = cast(new Date(millis));
     const insert = sqlite.exec({
-      sql: Sql.upsertValueIntoTableRowColumn(message.table, message.column),
+      sql: `
+        insert into
+          "${message.table}" ("id", "${message.column}", "createdAt", "updatedAt")
+        values
+          (?, ?, ?, ?)
+        on conflict do update set
+          "${message.column}" = ?,
+          "updatedAt" = ?
+        `.trim(),
       parameters: [
         message.row,
         message.value,
@@ -334,8 +367,8 @@ const applyMessages = ({
     for (const message of messages) {
       const timestamp: TimestampString | null = yield* _(
         sqlite.exec({
-          sql: Sql.selectLastTimestampForTableRowColumn,
-          parameters: [message.table, message.row, message.column],
+          ...Sql.selectLastTimestampForTableRowColumn,
+          parameters: [message.table, message.row, message.column, 1],
         }),
         Effect.map((result) => result.rows),
         Effect.flatMap(ReadonlyArray.head),
@@ -351,7 +384,7 @@ const applyMessages = ({
       if (timestamp == null || timestamp !== message.timestamp) {
         const { changes } = yield* _(
           sqlite.exec({
-            sql: Sql.insertIntoMessagesIfNew,
+            ...Sql.insertIntoMessagesIfNew,
             parameters: [
               message.timestamp,
               message.table,
@@ -377,10 +410,10 @@ const writeTimestampAndMerkleTree = ({
 }: TimestampAndMerkleTree): Effect.Effect<void, never, Sqlite> =>
   Effect.flatMap(Sqlite, (sqlite) =>
     sqlite.exec({
-      sql: Sql.updateOwnerTimestampAndMerkleTree,
+      ...Sql.updateOwnerTimestampAndMerkleTree,
       parameters: [
-        timestampToString(timestamp),
         merkleTreeToString(merkleTree),
+        timestampToString(timestamp),
       ],
     }),
   );
@@ -420,7 +453,10 @@ const mutate = ({
     const { exec } = yield* _(Sqlite);
     yield* _(
       Effect.forEach(toDelete, ({ table, row }) =>
-        exec({ sql: Sql.deleteTableRow(table), parameters: [row] }),
+        exec({
+          sql: `delete from "${table}" where "id" = ?;`,
+          parameters: [row],
+        }),
       ),
     );
 
@@ -504,7 +540,7 @@ const handleSyncResponse = ({
 
     const messagesToSync = yield* _(
       sqlite.exec({
-        sql: Sql.selectMessagesToSync,
+        ...Sql.selectMessagesToSync,
         parameters: [timestampToString(makeSyncTimestamp(diff.value))],
       }),
       Effect.map(({ rows }) => rows as unknown as ReadonlyArray<Message>),
@@ -561,25 +597,8 @@ const reset = (
   Sqlite | Bip39 | NanoIdGenerator | DbWorkerOnMessage
 > =>
   Effect.gen(function* (_) {
-    const sqlite = yield* _(Sqlite);
-
-    yield* _(
-      sqlite.exec(`SELECT "name" FROM "sqlite_master" WHERE "type" = 'table'`),
-      Effect.map((result) => result.rows),
-      Effect.flatMap(
-        // The dropped table is completely removed from the database schema and
-        // the disk file. The table can not be recovered.
-        // All indices and triggers associated with the table are also deleted.
-        // https://sqlite.org/lang_droptable.html
-        Effect.forEach(
-          ({ name }) => sqlite.exec(`DROP TABLE "${name as string}"`),
-          { discard: true },
-        ),
-      ),
-    );
-
+    yield* _(dropAllTables);
     if (input.mnemonic) yield* _(lazyInit(input.mnemonic));
-
     const onMessage = yield* _(DbWorkerOnMessage);
     onMessage({ _tag: "onResetOrRestore" });
   });
@@ -666,7 +685,7 @@ export const DbWorkerCommonLive = Layer.effect(
               skipAllBecauseOfReset = true;
               return reset(input);
             },
-            ensureSchema: ({ tables }) => ensureSchema(tables),
+            ensureSchema,
             SyncWorkerOutputSyncResponse: handleSyncResponse,
           }),
           Effect.provide(layer),

package/src/Evolu.ts

@@ -32,7 +32,12 @@ import { SqliteBoolean, SqliteDate } from "./Model.js";
 import { OnCompletes, OnCompletesLive } from "./OnCompletes.js";
 import { Owner } from "./Owner.js";
 import { AppState, FlushSync } from "./Platform.js";
-import { SqliteQuery, isSqlMutation } from "./Sqlite.js";
+import {
+  Index,
+  SqliteQuery,
+  SqliteQueryOptions,
+  isSqlMutation,
+} from "./Sqlite.js";
 import { Store, Unsubscribe, makeStore } from "./Store.js";
 import { SyncState } from "./SyncWorker.js";
 
@@ -297,6 +302,7 @@ export interface Evolu<S extends DatabaseSchema = DatabaseSchema> {
    */
   readonly ensureSchema: <From, To extends S>(
     schema: S.Schema<To, From>,
+    indexes?: ReadonlyArray<Index>,
   ) => void;
 
   /**
@@ -313,6 +319,7 @@ export const Evolu = Context.GenericTag<Evolu>("@services/Evolu");
 
 type CreateQuery<S extends DatabaseSchema> = <R extends Row>(
   queryCallback: QueryCallback<S, R>,
+  options?: SqliteQueryOptions,
 ) => Query<R>;
 
 type QueryCallback<S extends DatabaseSchema, R extends Row> = (
@@ -334,6 +341,7 @@ type NullableExceptForIdAndAutomaticColumns<T> = {
     : T[K] | null;
 };
 
+// https://kysely.dev/docs/recipes/splitting-query-building-and-execution
 const kysely = new Kysely.Kysely<QuerySchema<DatabaseSchema>>({
   dialect: {
     createAdapter: (): Kysely.DialectAdapter => new Kysely.SqliteAdapter(),
@@ -346,18 +354,29 @@ const kysely = new Kysely.Kysely<QuerySchema<DatabaseSchema>>({
   },
 });
 
+export const createIndex = kysely.schema.createIndex.bind(kysely.schema);
+
 export const makeCreateQuery =
   <S extends DatabaseSchema = DatabaseSchema>(): CreateQuery<S> =>
-  <R extends Row>(queryCallback: QueryCallback<S, R>) =>
+  <R extends Row>(
+    queryCallback: QueryCallback<S, R>,
+    options?: SqliteQueryOptions,
+  ) =>
     pipe(
       queryCallback(kysely as Kysely.Kysely<QuerySchema<S>>).compile(),
-      ({ sql, parameters }): SqliteQuery => {
-        if (isSqlMutation(sql))
+      (compiledQuery): SqliteQuery => {
+        if (isSqlMutation(compiledQuery.sql))
           throw new Error(
             "SQL mutation (INSERT, UPDATE, DELETE, etc.) isn't allowed in the Evolu `createQuery` function. Kysely suggests it because there is no read-only Kysely yet, and removing such an API is not possible. For mutations, use Evolu mutation API.",
           );
-
-        return { sql, parameters: parameters as SqliteQuery["parameters"] };
+        const parameters = compiledQuery.parameters as NonNullable<
+          SqliteQuery["parameters"]
+        >;
+        return {
+          sql: compiledQuery.sql,
+          parameters,
+          ...(options && { options }),
+        };
       },
       (query) => serializeQuery<R>(query),
     );
@@ -739,7 +758,6 @@ const EvoluCommon = Layer.effect(
     };
 
     appState.init({ onRequestSync: sync });
-
     sync();
 
     return Evolu.of({
@@ -770,10 +788,11 @@ const EvoluCommon = Layer.effect(
       restoreOwner: (mnemonic) =>
         dbWorker.postMessage({ _tag: "reset", mnemonic }),
 
-      ensureSchema: (schema) =>
+      ensureSchema: (schema, indexes = []) =>
         dbWorker.postMessage({
           _tag: "ensureSchema",
           tables: schemaToTables(schema),
+          indexes,
         }),
 
       sync,
@@ -815,6 +834,14 @@ export const makeCreateEvolu =
         Effect.runSync,
       ),
     );
-    evolu.ensureSchema(schema);
+
+    const indexes = config?.indexes?.map(
+      (index): Index => ({
+        name: index.toOperationNode().name.name,
+        sql: index.compile().sql,
+      }),
+    );
+
+    evolu.ensureSchema(schema, indexes);
     return evolu as Evolu<To>;
   };

package/src/Public.ts

@@ -1,3 +1,4 @@
+export { sql } from "kysely";
 export type { NotNull } from "kysely";
 export { jsonArrayFrom, jsonObjectFrom } from "kysely/helpers/sqlite";
 export type { Timestamp, TimestampError } from "./Crdt.js";
@@ -5,6 +6,7 @@ export type { InvalidMnemonicError, Mnemonic } from "./Crypto.js";
 export { database, table } from "./Db.js";
 export type { ExtractRow, QueryResult } from "./Db.js";
 export type { EvoluError, UnexpectedError } from "./ErrorStore.js";
+export { createIndex } from "./Evolu.js";
 export * from "./Model.js";
 export type { Owner, OwnerId } from "./Owner.js";
 export { canUseDom } from "./Platform.js";