@evolu/common 6.0.1-preview.8 → 7.0.0

package/src/Evolu/Schema.ts

@@ -1,19 +1,26 @@
-import { Kysely, SelectQueryBuilder } from "kysely";
-import { pack } from "msgpackr";
-import { assert } from "../Assert.js";
+import * as Kysely from "kysely";
 import { mapObject, objectToEntries, ReadonlyRecord } from "../Object.js";
-import { err, ok, Result } from "../Result.js";
-import { SqliteBoolean, SqliteQueryOptions, SqliteValue } from "../Sqlite.js";
+import { ok, Result } from "../Result.js";
+import {
+  SafeSql,
+  sql,
+  SqliteBoolean,
+  SqliteDep,
+  SqliteError,
+  SqliteQuery,
+  SqliteQueryOptions,
+  SqliteValue,
+} from "../Sqlite.js";
 import {
   AnyType,
-  brand,
-  BrandType,
-  createTypeErrorFormatter,
-  DateIsoString,
-  IdType,
+  array,
+  createIdFromString,
+  DateIso,
+  IdBytes,
   InferErrors,
   InferInput,
   InferType,
+  maxMutationSize,
   MergeObjectTypeErrors,
   nullableToOptional,
   NullableToOptionalProps,
@@ -23,16 +30,18 @@ import {
   omit,
   optional,
   OptionalType,
+  String,
+  TableId,
   Type,
-  TypeError,
+  ValidMutationSize,
+  validMutationSize,
+  ValidMutationSizeError,
 } from "../Type.js";
 import { Simplify } from "../Types.js";
-import { DbSchema } from "./Db.js";
-import { createIndexes, DbIndexesBuilder } from "./Kysely.js";
-import { AppOwner, ShardOwner, SharedOwner } from "./Owner.js";
-import { BinaryId, maxProtocolMessageRangesSize } from "./Protocol.js";
+import { AppOwner, OwnerId } from "./Owner.js";
 import { Query, Row } from "./Query.js";
-import { BinaryTimestamp } from "./Timestamp.js";
+import { CrdtMessage, DbChange } from "./Storage.js";
+import { Timestamp, TimestampBytes } from "./Timestamp.js";
 
 /**
  * Defines the schema of an Evolu database.
@@ -71,7 +80,7 @@ import { BinaryTimestamp } from "./Timestamp.js";
 export type EvoluSchema = ReadonlyRecord<
   string,
   // TypeScript errors are cryptic so we use ValidateSchema.
-  ReadonlyRecord<string, Type<any, any, any, any, any>>
+  ReadonlyRecord<string, Type<any, any, any, any, any, any>>
 >;
 
 /**
@@ -84,17 +93,17 @@ export type EvoluSchema = ReadonlyRecord<
  *
  * 1. All tables must have an 'id' column
  * 2. The 'id' column must be a branded ID type (created with id() function)
- * 3. Tables cannot use default column names (createdAt, updatedAt, isDeleted)
+ * 3. Tables cannot use system column names (createdAt, updatedAt, isDeleted)
  * 4. All column types must be compatible with SQLite (extend SqliteValue)
  */
 export type ValidateSchema<S extends EvoluSchema> =
   ValidateSchemaHasId<S> extends never
     ? ValidateIdColumnType<S> extends never
-      ? ValidateNoDefaultColumns<S> extends never
+      ? ValidateNoSystemColumns<S> extends never
         ? ValidateColumnTypes<S> extends never
           ? S
           : ValidateColumnTypes<S>
-        : ValidateNoDefaultColumns<S>
+        : ValidateNoSystemColumns<S>
       : ValidateIdColumnType<S>
     : ValidateSchemaHasId<S>;
 
@@ -111,20 +120,24 @@ export type ValidateIdColumnType<S extends EvoluSchema> =
   keyof S extends infer TableName
     ? TableName extends keyof S
       ? "id" extends keyof S[TableName]
-        ? S[TableName]["id"] extends IdType<any>
+        ? S[TableName]["id"] extends TableId<any>
           ? never
           : SchemaValidationError<`Table "${TableName & string}" id column must be a branded ID type (created with id("${TableName & string}")).`>
         : never
       : never
     : never;
 
-export type ValidateNoDefaultColumns<S extends EvoluSchema> =
+export type ValidateNoSystemColumns<S extends EvoluSchema> =
   keyof S extends infer TableName
     ? TableName extends keyof S
       ? keyof S[TableName] extends infer ColumnName
         ? ColumnName extends keyof S[TableName]
-          ? ColumnName extends "createdAt" | "updatedAt" | "isDeleted"
-            ? SchemaValidationError<`Table "${TableName & string}" uses default column name "${ColumnName & string}". Default columns (createdAt, updatedAt, isDeleted) are added automatically.`>
+          ? ColumnName extends
+              | "createdAt"
+              | "updatedAt"
+              | "isDeleted"
+              | "ownerId"
+            ? SchemaValidationError<`Table "${TableName & string}" uses system column name "${ColumnName & string}". System columns (createdAt, updatedAt, isDeleted, ownerId) are added automatically.`>
             : never
           : never
         : never
@@ -148,9 +161,13 @@ export type ValidateColumnTypes<S extends EvoluSchema> =
 export type SchemaValidationError<Message extends string> =
   `❌ Schema Error: ${Message}`;
 
+export type IndexesConfig = (
+  create: (indexName: string) => Kysely.CreateIndexBuilder,
+) => ReadonlyArray<Kysely.CreateIndexBuilder<any>>;
+
 export const evoluSchemaToDbSchema = (
   schema: EvoluSchema,
-  indexes?: DbIndexesBuilder,
+  indexesConfig?: IndexesConfig,
 ): DbSchema => {
   const tables = objectToEntries(schema).map(([tableName, table]) => ({
     name: tableName,
@@ -159,34 +176,33 @@ export const evoluSchemaToDbSchema = (
       .map(([k]) => k),
   }));
 
-  const dbSchema = { tables, indexes: createIndexes(indexes) };
-
-  assert(
-    DbSchema.is(dbSchema),
-    "Invalid EvoluSchema: Table and column names must use only characters A-Za-z0-9_- and be at most 256 characters long.",
-  );
+  const indexes = indexesConfig
+    ? indexesConfig(createIndex).map(
+        (index): DbIndex => ({
+          name: index.toOperationNode().name.name,
+          sql: index.compile().sql,
+        }),
+      )
+    : [];
 
-  return dbSchema;
+  return { tables, indexes };
 };
 
 export type CreateQuery<S extends EvoluSchema> = <R extends Row>(
   queryCallback: (
     db: Pick<
-      Kysely<
+      Kysely.Kysely<
         {
           [Table in keyof S]: {
-            readonly [Column in keyof S[Table]]: Column extends
-              | "id"
-              | "createdAt"
-              | "updatedAt"
+            readonly [Column in keyof S[Table]]: Column extends "id"
               ? InferType<S[Table][Column]>
               : InferType<S[Table][Column]> | null;
-          } & DefaultColumns;
+          } & SystemColumns;
         } & {
           readonly evolu_history: {
-            readonly timestamp: BinaryTimestamp;
+            readonly timestamp: TimestampBytes;
             readonly table: keyof S;
-            readonly id: BinaryId;
+            readonly id: IdBytes;
             readonly column: string;
             readonly value: SqliteValue;
           };
@@ -194,16 +210,28 @@ export type CreateQuery<S extends EvoluSchema> = <R extends Row>(
       >,
       "selectFrom" | "fn" | "with" | "withRecursive"
     >,
-  ) => SelectQueryBuilder<any, any, R>,
+  ) => Kysely.SelectQueryBuilder<any, any, R>,
   options?: SqliteQueryOptions,
 ) => Query<Simplify<R>>;
 
-export const DefaultColumns = object({
-  createdAt: DateIsoString,
-  updatedAt: DateIsoString,
+/**
+ * System columns that are implicitly defined by Evolu.
+ *
+ * - `createdAt`: Set by Evolu on row creation, derived from {@link Timestamp}.
+ * - `updatedAt`: Set by Evolu on every row change, derived from {@link Timestamp}.
+ * - `isDeleted`: Soft delete flag created by Evolu and used by the developer to
+ *   mark rows as deleted.
+ * - `ownerId`: Represents ownership and logically partitions the database.
+ */
+export const SystemColumns = object({
+  createdAt: DateIso,
+  updatedAt: DateIso,
   isDeleted: nullOr(SqliteBoolean),
+  ownerId: OwnerId,
 });
-export type DefaultColumns = typeof DefaultColumns.Type;
+export type SystemColumns = typeof SystemColumns.Type;
+
+export const systemColumns = Object.keys(SystemColumns.props);
 
 export type MutationKind = "insert" | "update" | "upsert";
 
@@ -229,7 +257,48 @@ export type MutationMapping<
     : UpsertableProps<P>;
 
 export interface MutationOptions {
+  /**
+   * Called after the mutation is completed and the local state is updated.
+   * Useful for triggering side effects (e.g., notifications, UI updates) after
+   * insert, update, or upsert.
+   */
   readonly onComplete?: () => void;
+
+  /**
+   * Specifies the owner ID for this mutation. If omitted, the default
+   * {@link AppOwner} is used.
+   *
+   * The owner must be used with `evolu.useOwner()` to enable sync. Mutations
+   * with unused owners are stored locally but not synced until the owner is
+   * used.
+   *
+   * ### Example
+   *
+   * ```ts
+   * // Partition your own data by project (derived from your AppOwner)
+   * const projectOwner = deriveShardOwner(appOwner, [
+   *   "project",
+   *   projectId,
+   * ]);
+   * evolu.insert(
+   *   "task",
+   *   { title: "Task 1" },
+   *   { ownerId: projectOwner.id },
+   * );
+   *
+   * // Collaborative data (independent owner shared with others)
+   * const sharedOwner = createSharedOwner(sharedSecret);
+   * evolu.insert(
+   *   "comment",
+   *   { text: "Hello" },
+   *   { ownerId: sharedOwner.id },
+   * );
+   * ```
+   *
+   * @experimental
+   */
+  readonly ownerId?: OwnerId;
+
   /**
    * Only validate, don't mutate.
    *
@@ -237,50 +306,17 @@ export interface MutationOptions {
    * `onlyValidate: true`.
    */
   readonly onlyValidate?: boolean;
-
-  /**
-   * The owner to use for this mutation. Can be a {@link ShardOwner} for sharding
-   * app data or a {@link SharedOwner} for collaborative write access. If
-   * omitted, defaults to the app's {@link AppOwner}.
-   */
-  readonly owner?: ShardOwner | SharedOwner;
 }
 
-/**
- * Evolu has to limit the maximum mutation size. Otherwise, sync couldn't use
- * the {@link maxProtocolMessageRangesSize}. The max size is 640KB in bytes,
- * measured via MessagePack. Evolu Protocol DbChange will be smaller thanks to
- * various optimizations.
- */
-export const maxMutationSize = 655360;
-
-const validMutationSize = <T extends AnyType>(type: T) =>
-  brand("ValidMutationSize", type, (value) =>
-    pack(value).byteLength <= maxMutationSize
-      ? ok(value)
-      : err<ValidMutationSizeError>({ type: "ValidMutationSize", value }),
-  );
-
-export interface ValidMutationSizeError
-  extends TypeError<"ValidMutationSize"> {}
-
-export const formatValidMutationSizeError =
-  createTypeErrorFormatter<ValidMutationSizeError>(
-    (error) =>
-      `The mutation size exceeds the maximum limit of ${maxMutationSize} bytes. The provided mutation has a size of ${pack(error.value).byteLength} bytes.`,
-  );
-
-export type ValidMutationSize<Props extends Record<string, AnyType>> =
-  BrandType<
-    ObjectType<Props>,
-    "ValidMutationSize",
-    ValidMutationSizeError,
-    InferErrors<ObjectType<Props>>
-  >;
+export interface MutationChange extends DbChange {
+  /** Owner of the change. If undefined, the change belongs to the AppOwner. */
+  readonly ownerId?: OwnerId | undefined;
+}
 
 /**
  * Type Factory to create insertable {@link Type}. It makes nullable Types
- * optional, omits Id, and ensures the {@link maxMutationSize}.
+ * optional (so they are not required), omits Id, and ensures the
+ * {@link maxMutationSize}.
  *
  * ### Example
  *
@@ -310,7 +346,7 @@ export type Insertable<Props extends Record<string, AnyType>> = InferInput<
 
 /**
  * Type Factory to create updateable {@link Type}. It makes everything except for
- * the `id` column partial (i.e. optional) and ensures the
+ * the `id` column optional (so they are not required) and ensures the
  * {@link maxMutationSize}.
  *
  * ### Example
@@ -346,25 +382,44 @@ export type Updateable<Props extends Record<string, AnyType>> = InferInput<
 >;
 
 /**
- * Type Factory to create upsertable Type. It makes nullable Types optional and
- * ensures the {@link maxMutationSize}.
+ * Type Factory to create an upsertable Type. It makes nullable Types optional
+ * (so they are not required) and ensures the {@link maxMutationSize}.
+ *
+ * Upsert is like insert, except it requires an ID. It's useful for inserting
+ * rows with external ID via {@link createIdFromString}.
+ *
+ * Note that it's not possible to upsert a row with `createdAt` nor `updatedAt`,
+ * because they are derived from {@link CrdtMessage} timestamp. For external
+ * createdAt, use a different column.
  *
  * ### Example
  *
  * ```ts
  * const UpsertableTodo = upsertable(Schema.todo);
  * type UpsertableTodo = typeof UpsertableTodo.Type;
- * const todo = UpsertableTodo.from({ id, title });
+ * const todo = UpsertableTodo.from({
+ *   id,
+ *   title,
+ * });
  * if (!todo.ok) return; // handle errors
  * ```
  */
 export const upsertable = <Props extends Record<string, AnyType>>(
   props: Props,
-): ValidMutationSize<UpsertableProps<Props>> =>
-  validMutationSize(nullableToOptional(props));
+): ValidMutationSize<UpsertableProps<Props>> => {
+  const propsWithDefaults = {
+    ...props,
+    isDeleted: optional(SqliteBoolean),
+  };
+  return validMutationSize(nullableToOptional(propsWithDefaults));
+};
 
 export type UpsertableProps<Props extends Record<string, AnyType>> =
-  NullableToOptionalProps<Props>;
+  NullableToOptionalProps<
+    Props & {
+      isDeleted: OptionalType<typeof SqliteBoolean>;
+    }
+  >;
 
 export type Upsertable<Props extends Record<string, AnyType>> = InferInput<
   ObjectType<UpsertableProps<Props>>
@@ -387,3 +442,180 @@ export type InferColumnErrors<
     MutationMapping<T, M>[Column]
   >;
 }[keyof MutationMapping<T, M>];
+
+export const DbTable = object({
+  name: String,
+  columns: array(String),
+});
+export type DbTable = typeof DbTable.Type;
+
+export const DbIndex = object({ name: String, sql: String });
+export type DbIndex = typeof DbIndex.Type;
+
+export const DbSchema = object({
+  tables: array(DbTable),
+  indexes: array(DbIndex),
+});
+export type DbSchema = typeof DbSchema.Type;
+
+/** Get the current database schema by reading SQLite metadata. */
+export const getDbSchema =
+  (deps: SqliteDep) =>
+  ({ allIndexes = false }: { allIndexes?: boolean } = {}): Result<
+    DbSchema,
+    SqliteError
+  > => {
+    const map = new Map<string, Array<string>>();
+
+    const tableAndColumnInfoRows = deps.sqlite.exec(sql`
+      select
+        sqlite_master.name as tableName,
+        table_info.name as columnName
+      from
+        sqlite_master
+        join pragma_table_info(sqlite_master.name) as table_info;
+    `);
+
+    if (!tableAndColumnInfoRows.ok) return tableAndColumnInfoRows;
+
+    tableAndColumnInfoRows.value.rows.forEach((row) => {
+      const { tableName, columnName } = row as unknown as {
+        tableName: string;
+        columnName: string;
+      };
+      if (!map.has(tableName)) map.set(tableName, []);
+      map.get(tableName)?.push(columnName);
+    });
+
+    const tables = Array.from(map, ([name, columns]) => ({ name, columns }));
+
+    const indexesRows = deps.sqlite.exec(
+      allIndexes
+        ? sql`
+            select name, sql
+            from sqlite_master
+            where type = 'index' and name not like 'sqlite_%';
+          `
+        : sql`
+            select name, sql
+            from sqlite_master
+            where
+              type = 'index'
+              and name not like 'sqlite_%'
+              and name not like 'evolu_%';
+          `,
+    );
+
+    if (!indexesRows.ok) return indexesRows;
+
+    const indexes = indexesRows.value.rows.map(
+      (row): DbIndex => ({
+        name: row.name as string,
+        /**
+         * SQLite returns "CREATE INDEX" for "create index" for some reason.
+         * Other keywords remain unchanged. We have to normalize the casing for
+         * {@link indexesAreEqual} manually.
+         */
+        sql: (row.sql as string)
+          .replace("CREATE INDEX", "create index")
+          .replace("CREATE UNIQUE INDEX", "create unique index"),
+      }),
+    );
+
+    return ok({ tables, indexes });
+  };
+
+const indexesAreEqual = (self: DbIndex, that: DbIndex): boolean =>
+  self.name === that.name && self.sql === that.sql;
+
+export const ensureDbSchema =
+  (deps: SqliteDep) =>
+  (
+    newSchema: DbSchema,
+    currentSchema: DbSchema,
+    options?: { ignoreIndexes: boolean },
+  ): Result<void, SqliteError> => {
+    const queries: Array<SqliteQuery> = [];
+
+    newSchema.tables.forEach((newTable) => {
+      const currentTable = currentSchema.tables.find(
+        (t) => t.name === newTable.name,
+      );
+      if (!currentTable) {
+        queries.push(createAppTable(newTable));
+      } else {
+        newTable.columns
+          .filter((newColumn) => !currentTable.columns.includes(newColumn))
+          .forEach((newColumn) => {
+            queries.push(sql`
+              alter table ${sql.identifier(newTable.name)}
+              add column ${sql.identifier(newColumn)} blob;
+            `);
+          });
+      }
+    });
+
+    if (options?.ignoreIndexes !== true) {
+      // Remove current indexes that are not in the newSchema.
+      currentSchema.indexes
+        .filter(
+          (currentIndex) =>
+            !newSchema.indexes.some((newIndex) =>
+              indexesAreEqual(newIndex, currentIndex),
+            ),
+        )
+        .forEach((index) => {
+          queries.push(sql`drop index ${sql.identifier(index.name)};`);
+        });
+
+      // Add new indexes that are not in the currentSchema.
+      newSchema.indexes
+        .filter(
+          (newIndex) =>
+            !currentSchema.indexes.some((currentIndex) =>
+              indexesAreEqual(newIndex, currentIndex),
+            ),
+        )
+        .forEach((newIndex) => {
+          queries.push({ sql: `${newIndex.sql};` as SafeSql, parameters: [] });
+        });
+    }
+
+    for (const query of queries) {
+      const result = deps.sqlite.exec(query);
+      if (!result.ok) return result;
+    }
+    return ok();
+  };
+
+const createAppTable = (table: DbTable) => sql`
+  create table ${sql.identifier(table.name)} (
+    "id" text,
+    ${sql.raw(
+      `${systemColumns
+        .concat(table.columns)
+        .filter((c) => c !== "id")
+        // With strict tables and any type, data is preserved exactly as received
+        // without any type affinity coercion. This allows storing any data type
+        // while maintaining strict null enforcement for primary key columns.
+        .map((name) => `${sql.identifier(name).sql} any`)
+        .join(", ")}`,
+    )},
+    primary key ("ownerId", "id")
+  )
+  without rowid, strict;
+`;
+
+// https://kysely.dev/docs/recipes/splitting-query-building-and-execution
+export const kysely = new Kysely.Kysely({
+  dialect: {
+    createAdapter: () => new Kysely.SqliteAdapter(),
+    createDriver: () => new Kysely.DummyDriver(),
+    createIntrospector() {
+      throw new Error("Not implemeneted");
+    },
+    createQueryCompiler: () => new Kysely.SqliteQueryCompiler(),
+  },
+});
+
+const createIndex = kysely.schema.createIndex.bind(kysely.schema);