@evolu/common 8.11.0 → 8.13.0

package/src/local-first/Evolu.ts

@@ -15,6 +15,7 @@ import {
   assertNonEmptyReadonlyArray,
   assertNotUndefined,
 } from "../Assert.ts";
+import { createBuffer } from "../Bytes.ts";
 import { createCallbacks } from "../Callbacks.ts";
 import type { ConsoleDep } from "../Console.ts";
 import { createConsole } from "../Console.ts";
@@ -28,11 +29,22 @@ import {
   type LockManagerDep,
 } from "../LockManager.ts";
 import { createMicrotaskBatch } from "../Microtask.ts";
+import {
+  createMutableRecord,
+  isPlainObject,
+  objectToEntries,
+  shareStructure,
+  type ReadonlyRecord,
+} from "../Object.ts";
 import type { FlushSyncDep, ReloadAppDep } from "../Platform.ts";
 import { createRefCountByKey } from "../RefCount.ts";
 import { err, ok } from "../Result.ts";
 import { isNonEmptySet } from "../Set.ts";
-import { SqliteBoolean, sqliteBooleanToBoolean } from "../Sqlite.ts";
+import {
+  SqliteBoolean,
+  sqliteBooleanToBoolean,
+  type SqliteValue,
+} from "../Sqlite.ts";
 import type { Listener, ReadonlyStore, Unsubscribe } from "../Store.ts";
 import { createStore } from "../Store.ts";
 import type { Task } from "../Task.ts";
@@ -41,8 +53,9 @@ import {
   createId,
   createIdFromString,
   type ExtractTyped,
-  type Id,
+  Id,
   Name,
+  PositiveInt,
   type TypeError,
   UrlSafeString,
 } from "../Type.ts";
@@ -61,7 +74,14 @@ import type {
   SyncOwner,
 } from "./Owner.ts";
 import { createOwnerWebSocketTransport } from "./Owner.ts";
-import type { ProtocolError, ProtocolQuotaError } from "./Protocol.ts";
+import {
+  encodeDbChange,
+  type ProtocolError,
+  type ProtocolInvalidDataError,
+  type ProtocolQuotaError,
+  type ProtocolTimestampMismatchError,
+  type ProtocolVersionError,
+} from "./Protocol.ts";
 import type {
   Queries,
   QueriesToQueryRowsPromises,
@@ -77,21 +97,25 @@ import type {
   Mutation,
   MutationChange,
   MutationOptions,
+  MutationValues,
   ValidateSchema,
 } from "./Schema.ts";
-import { evoluSchemaToSqliteSchema } from "./Schema.ts";
+import { evoluSchemaToSqliteSchema, isLocalOnlyTable } from "./Schema.ts";
 import type {
   ConsoleEntryOrError,
   EvoluInput,
   EvoluOutput,
   OtherBuildRunningError,
+  OwnerSyncStatus,
+  PendingSyncRoute,
   SharedWorkerDep,
   SyncState,
-  syncStateToOwnerSyncStates,
+  syncStateToOwnerSyncStatus,
+  syncStateToRelaySyncStates,
 } from "./Shared.ts";
 import { consoleEntryOrErrorBroadcastChannelName } from "./Shared.ts";
-import { DbChange, type StorageQuotaError } from "./Storage.ts";
-import type { Timestamp } from "./Timestamp.ts";
+import { DbChange } from "./Storage.ts";
+import { createTimestamp, type Timestamp } from "./Timestamp.ts";
 
 /**
  * Configuration for {@link createEvolu}.
@@ -371,8 +395,10 @@ export interface Evolu<
    *
    * Pass `onComplete` when follow-up work must wait until the mutation is
    * stored and subscribed queries reflect it. It never runs when the database
-   * is unavailable; see {@link Evolu.loadQuery}. A stored change is not always
-   * visible; see {@link MutationOptions.onComplete}.
+   * is unavailable; see {@link Evolu.loadQuery}. It also never runs when the
+   * mutation could not be stored, which {@link EvoluErrorDep.evoluError}
+   * reports. A stored change is not always visible; see
+   * {@link MutationOptions.onComplete}.
    *
    * ### Example
    *
@@ -492,6 +518,41 @@ export interface Evolu<
    */
   readonly upsert: Mutation<S, "upsert">;
 
+  /**
+   * Returns the size of a mutation of `table` with `values`, measured as
+   * {@link maxMutationSize} describes.
+   *
+   * Use it to check values that column Types do not bound before mutating, or
+   * to show how much of the limit a mutation uses. It accepts any of the
+   * table's columns, so the values of an insert, update, or upsert fit, and it
+   * ignores `id` and `isDeleted`, which are not columns. Mutations of
+   * local-only tables are measured too, although they are exempt from the
+   * limit.
+   *
+   * ### Example
+   *
+   * ```ts
+   * import {
+   *   assertType,
+   *   type Evolu,
+   *   maxMutationSize,
+   *   type NonEmptyTrimmedString100,
+   *   type TestEvoluSchema,
+   * } from "@evolu/common";
+   *
+   * const fitsTodo = (
+   *   evolu: Evolu<TestEvoluSchema>,
+   *   title: NonEmptyTrimmedString100,
+   * ) => evolu.getMutationSize("todo", { title }) <= maxMutationSize;
+   *
+   * assertType<ReturnType<typeof fitsTodo>, boolean>();
+   * ```
+   */
+  readonly getMutationSize: <TableName extends keyof S>(
+    table: TableName,
+    values: Partial<MutationValues<S[TableName], "update">>,
+  ) => PositiveInt;
+
   /**
    * Load {@link Query} and return a promise with {@link QueryRows}.
    *
@@ -655,6 +716,9 @@ export interface Evolu<
   /**
    * Exports the SQLite database file.
    *
+   * The exported file is not encrypted, even when the local database is. Any
+   * SQLite tool can read all local data from it, so treat it as plaintext.
+   *
    * Exports are sequential: concurrent calls share one pending export instead
    * of starting parallel exports.
    *
@@ -766,9 +830,12 @@ export interface Evolu<
    *
    * Reconciles locally stored changes, including writes previously rejected
    * with {@link ProtocolQuotaError}, through the owner's active transports. Call
-   * this after the relay provider confirms additional quota is available.
-   * Existing connections and {@link Evolu.useOwner} registrations are retained,
-   * including registrations shared by multiple instances or tabs.
+   * this after the relay provider confirms additional quota is available. It
+   * also checks again the changes this database skipped from the owner's
+   * relays, and sends such a relay the changes received from other relays,
+   * which otherwise wait for its next sync, such as after a reconnect. Existing
+   * connections and {@link Evolu.useOwner} registrations are retained, including
+   * registrations shared by multiple instances or tabs.
    *
    * The owner must have an active writable registration in this database.
    * Unregistered or read-only owners are ignored. Requests are skipped while
@@ -776,8 +843,9 @@ export interface Evolu<
    * reopen. Disposal drops locally buffered requests. Calling a disposed
    * instance throws, like other Evolu operations.
    *
-   * Returns immediately, without waiting for synchronization to complete.
-   * Errors are reported through {@link EvoluErrorDep.evoluError}.
+   * Returns immediately, without waiting for synchronization to complete. The
+   * owner's routes in sync state show a failure, such as a quota rejection, as
+   * `failure`, and a skipped change as `skippedError`.
    *
    * ### Example
    *
@@ -806,17 +874,112 @@ export interface Evolu<
 export type UnuseOwner = () => void;
 
 /**
- * Represents errors that can occur in {@link Evolu}.
+ * Represents app-level errors that can occur in {@link Evolu}.
+ *
+ * Apps show them from {@link EvoluErrorDep.evoluError}. Problems a relay causes
+ * while syncing an owner are not EvoluErrors; see below. An unexpected failure
+ * while syncing is an {@link UnknownError}.
+ *
+ * An error that leaves the app unusable deserves a modal dialog, which moves
+ * focus into itself and restores it when closed. Any other error is a status
+ * message: show it in a region with `role="alert"`, which screen readers
+ * announce without moving focus, so the user keeps working and a background tab
+ * shows it when the user returns. Avoid `alert()`, which blocks the page, once
+ * in every open tab.
+ *
+ * - {@link UnsupportedDbVersionError} blocks the app: the local data needs a newer
+ *   version of it. Ask the user to update the app or close all its tabs.
+ * - {@link OtherBuildRunningError} blocks the app while it lasts: another version
+ *   of the app holds the local data. Ask the user to close the app's other
+ *   tabs. It clears when the wait ends.
+ * - {@link UnknownError} does not block the app: Evolu logged an unexpected
+ *   failure, such as a mutation that could not be stored. Show a generic
+ *   message.
+ *
+ * A problem syncing an owner through a relay belongs to that relay and often
+ * repeats in every round, so sync state shows it on the relay's route, as its
+ * {@link PendingSyncRoute.failure} with its details, and apps show it as the
+ * owner's `Error` {@link OwnerSyncStatus}. When the relay rejected or failed a
+ * request, or sent a frame that could not be decoded, the route shows a
+ * {@link ProtocolError}. A {@link ProtocolQuotaError} needs more relay quota,
+ * then {@link Evolu.requestSync}, and a {@link ProtocolVersionError} needs an app
+ * or relay update.
+ *
+ * A received change that was not created with the owner's encryption key, was
+ * altered afterwards, or cannot be decoded by this app version is skipped,
+ * while everything else still syncs. The relay offers it again in every round,
+ * so the route shows it as its `skippedError`: the
+ * {@link DecryptWithXChaCha20Poly1305Error},
+ * {@link ProtocolTimestampMismatchError}, or {@link ProtocolInvalidDataError} of
+ * the first change skipped in a reply, with its details. The owner's status is
+ * `Error` with it too, unless a relay has a failure. Once this database stores
+ * a valid change with that timestamp, for example from another relay, the relay
+ * no longer offers its copy, and the route completes with its next sync, such
+ * as after a reconnect or {@link Evolu.requestSync}, during which no changes
+ * arrive from other relays. If you don't trust that relay, stop using it for
+ * the owner. For owners that use the default transports, such as
+ * {@link Evolu.appOwner} when {@link EvoluConfig.transports} is not empty,
+ * replace the relay there; an empty list stops syncing the app owner and makes
+ * {@link Evolu.useOwner} require explicit transports. For an owner you passed
+ * explicit transports to {@link Evolu.useOwner}, call its {@link UnuseOwner} and
+ * use it again without that relay. Do this wherever the owner is used, such as
+ * in every tab, because the owner syncs through every transport any of its uses
+ * claims. If every route of the owner shows
+ * {@link DecryptWithXChaCha20Poly1305Error} and your code creates or shares the
+ * owner, check the owner's keys.
  *
  * @group Core
  */
 export type EvoluError =
-  | DecryptWithXChaCha20Poly1305Error
-  | OtherBuildRunningError
-  | ProtocolError
-  | StorageQuotaError
-  | UnknownError
-  | UnsupportedDbVersionError;
+  OtherBuildRunningError | UnknownError | UnsupportedDbVersionError;
+
+/**
+ * The largest {@link Mutation}, in bytes.
+ *
+ * A mutation's size is its change as encoded for sync: the table name, the ID,
+ * and every column name and value, with a few bytes of overhead. A string takes
+ * at most three bytes per UTF-16 code unit. Every mutation within the limit
+ * fits one protocol message, so it can always sync. A larger mutation throws
+ * and is not saved. Mutations of local-only tables are exempt, because they
+ * never sync. Measure a mutation in advance with {@link Evolu.getMutationSize}.
+ *
+ * @group Core
+ */
+export const maxMutationSize: PositiveInt =
+  /*#__PURE__*/ PositiveInt.orThrow(640_000);
+
+/** Measures a mutation as {@link maxMutationSize} describes. */
+const measureMutation = (
+  table: string,
+  values: ReadonlyRecord<string, SqliteValue | undefined>,
+): PositiveInt => {
+  const columns = createMutableRecord<string, SqliteValue>();
+  for (const [column, value] of objectToEntries(values)) {
+    if (value !== undefined && column !== "id" && column !== "isDeleted") {
+      columns[column] = value;
+    }
+  }
+
+  // Measuring with the protocol's encoding keeps the limit and the message size
+  // from disagreeing. A separate formula would save the tab about 3 KB
+  // compressed, but it would be a second encoding to keep in sync, and it
+  // would have to overcount.
+  const buffer = createBuffer();
+  encodeDbChange(buffer, {
+    // Every timestamp and ID takes 16 bytes, and the flags always take one.
+    timestamp: createTimestamp(),
+    change: DbChange.orThrow({
+      table,
+      id: mutationSizeId,
+      values: columns,
+      isInsert: true,
+      isDelete: null,
+    }),
+  });
+  return buffer.getLength() as PositiveInt;
+};
+
+const mutationSizeId = /*#__PURE__*/ Id.orThrow("A".repeat(22));
 
 /**
  * Dependency wrapper for the shared {@link EvoluError} store.
@@ -845,6 +1008,13 @@ export interface EvoluErrorDep {
    * instead; see {@link UnsupportedDbVersionError}. Show that blocking message
    * outside any query-loading boundary, so pending queries do not hide it.
    *
+   * An {@link UnknownError} leaves Evolu in an unknown state, so the app can
+   * only ask the user to close the tab. A failed shared worker closes itself,
+   * so the app opened again starts a new one. Some errors reach every tab, such
+   * as an unexpected failure of the shared worker. An app that forwards this
+   * store to an error tracker from each tab then reports such an error once per
+   * tab.
+   *
    * ### Example
    *
    * ```ts
@@ -858,14 +1028,13 @@ export interface EvoluErrorDep {
    * // The message for the current error, or null for none.
    * const errorMessage = (error: EvoluError | null): string | null => {
    *   if (!error) return null;
-   *   // oxlint-disable-next-line typescript/switch-exhaustiveness-check -- The default handles every other EvoluError.
    *   switch (error.type) {
    *     case "UnsupportedDbVersionError":
    *       return "Your data requires a newer version of this app. Please update it.";
    *     case "OtherBuildRunningError":
    *       return "This app is open in another tab with a different version. Close that tab to continue.";
-   *     default:
-   *       return "Something went wrong. Please try again.";
+   *     case "UnknownError":
+   *       return "Something went wrong. Please close this tab and open the app again.";
    *   }
    * };
    *
@@ -901,8 +1070,15 @@ export interface SyncStateDep {
   /**
    * {@link ReadonlyStore} of the latest {@link SyncState} shared by all
    * {@link Evolu} instances, or null before the shared worker sends its first
-   * snapshot. Derive what to show from it, such as one indicator per relay, or
-   * use {@link syncStateToOwnerSyncStates} for one state per owner.
+   * snapshot. It lists every database the shared worker serves, including other
+   * tabs', so an app finds its own by {@link Evolu.name}. Apps show users an
+   * owner's {@link OwnerSyncStatus}, from {@link syncStateToOwnerSyncStatus} or a
+   * framework binding's `useOwnerSyncStatus`; views of each relay use
+   * {@link syncStateToRelaySyncStates}.
+   *
+   * Each snapshot keeps the previous snapshot's object for every part that did
+   * not change, and a snapshot equal to the previous one leaves the store
+   * unchanged, so parts and derived statuses can be compared with `===`.
    *
    * ### Example
    *
@@ -911,6 +1087,7 @@ export interface SyncStateDep {
    *   assertEqual,
    *   createId,
    *   createStore,
+   *   Millis,
    *   testCreateDeps,
    * } from "@evolu/common";
    * import type {
@@ -920,7 +1097,7 @@ export interface SyncStateDep {
    *
    * const openRelayLabels = (deps: SyncStateDep): ReadonlyArray<string> =>
    *   (deps.syncState.get()?.transports ?? [])
-   *     .filter(({ readyState }) => readyState === "open")
+   *     .filter(({ connection }) => connection.type === "Open")
    *     .map(({ label }) => label);
    *
    * using syncState = createStore<SyncState | null>(null);
@@ -930,12 +1107,14 @@ export interface SyncStateDep {
    * syncState.set({
    *   transports: [
    *     {
+   *       type: "WebSocket",
    *       id: createId<"SyncTransport">(deps),
    *       label: "wss://relay.example",
-   *       readyState: "open",
-   *       openedAt: null,
-   *       closedAt: null,
-   *       error: null,
+   *       connection: {
+   *         type: "Open",
+   *         openedAt: Millis.orThrow(1000),
+   *         error: null,
+   *       },
    *     },
    *   ],
    *   tenants: [],
@@ -1025,11 +1204,15 @@ export const createEvoluDeps = (deps: EvoluPlatformDeps): EvoluDeps => {
         }
         break;
 
-      case "Error":
-        setEvoluError(message.error);
+      case "Error": {
+        // Every build shares this channel, so another build's worker can post
+        // an error type this build does not post, such as an older build's
+        // sync error. That build's own tabs show it, so this tab only logs it.
+        if (message.error.type === "UnknownError") setEvoluError(message.error);
         // Keep typed errors visible in logs as operational failures.
         console.error(message.error);
         break;
+      }
 
       default:
         exhaustiveCheck(message);
@@ -1043,8 +1226,9 @@ export const createEvoluDeps = (deps: EvoluPlatformDeps): EvoluDeps => {
         break;
 
       case "Error":
-        // Sent to this tab only: its database refused startup, or its worker
-        // still waits for another build.
+        // Sent to this tab only: its database refused startup, a mutation it
+        // made could not be stored, or its worker still waits for another
+        // build.
         setEvoluError(message.error);
         console.error(message.error);
         break;
@@ -1067,7 +1251,15 @@ export const createEvoluDeps = (deps: EvoluPlatformDeps): EvoluDeps => {
           message.syncStateChannelName,
         );
         syncStateBroadcastChannel.onMessage = (state) => {
-          syncState.set(state);
+          // Every snapshot arrives as a new structured clone, so its unchanged
+          // parts get the previous snapshot's objects back, and an unchanged
+          // snapshot leaves the store as it is.
+          const previous = syncState.get();
+          syncState.set(
+            previous
+              ? shareStructure(previous, state, syncStateItemToKey)
+              : state,
+          );
         };
         // Asking only after listening misses no snapshot.
         sharedWorker.port.postMessage({ type: "RequestSyncState" });
@@ -1105,6 +1297,14 @@ export const createEvoluDeps = (deps: EvoluPlatformDeps): EvoluDeps => {
   );
 };
 
+// Transports, tenants, owners, and routes come and go, so each is compared
+// with its previous self by its ID: a transport's id, a tenant's name, an
+// owner's ownerId, or a route's transportId. A wrong key only loses sharing.
+const syncStateItemToKey = (item: unknown): unknown =>
+  isPlainObject(item)
+    ? (item.id ?? item.name ?? item.ownerId ?? item.transportId)
+    : undefined;
+
 /**
  * Creates an {@link Evolu} instance from {@link EvoluSchema} and
  * {@link EvoluConfig}.
@@ -1304,6 +1504,14 @@ export const createEvolu =
             break;
           }
 
+          case "OnMutateFailed": {
+            // The tab reports the failure, and these callbacks never run.
+            for (const onCompleteId of message.onCompleteIds) {
+              onMutateCompleteCallbacks.unregister(onCompleteId);
+            }
+            break;
+          }
+
           default:
             exhaustiveCheck(message);
         }
@@ -1396,6 +1604,41 @@ export const createEvolu =
           `Invalid DbChange for table '${String(table)}'.`,
         );
 
+        // Copy binary values, so what is saved is what was passed: the caller
+        // can change a Uint8Array before the batch is sent, a view of a
+        // resizable buffer can grow, and a view of a shared buffer cannot be
+        // sent to a SharedWorker at all. Node's Buffer passes validation and
+        // its slice shares memory, so only the constructor copies reliably.
+        for (const [column, value] of objectToEntries(dbChange.values)) {
+          if (typeof value === "object" && value !== null) {
+            changeValues[column] = new Uint8Array(value);
+          }
+        }
+
+        // A change over maxMutationSize could never sync, and a shared worker
+        // hosting two databases would stop all work of that database. Correct
+        // apps bound column Types, so it is a programmer error and throws like
+        // the DbChange check above, before batching: it gets no timestamp, the
+        // database worker never sees it, and the code after the call does not
+        // run, so a form keeps its input and no row refers to the missing one.
+        //
+        // Rejected: reporting it through evoluError, which lets that code run
+        // as if the mutation was saved; checking in the database worker, which
+        // gets it already batched and cannot reach the call site; quarantine,
+        // which holds changes stored for sync; and skipping it in sync, which
+        // keeps range fingerprints disagreeing. Received changes are not
+        // checked, and oversized changes stored before this limit are not
+        // recovered, which needs a history-aware design.
+        //
+        // Local-only tables never sync, so they have no limit.
+        if (!isLocalOnlyTable(dbChange.table)) {
+          const size = measureMutation(dbChange.table, dbChange.values);
+          assert(
+            size <= maxMutationSize,
+            `The mutation of table '${dbChange.table}' is ${size} bytes, over maxMutationSize (${maxMutationSize}). Bound the Types of its columns or check it with evolu.getMutationSize.`,
+          );
+        }
+
         mutateBatch.push({
           change: { ...dbChange, ownerId: options?.ownerId ?? appOwner.id },
           onComplete: options?.onComplete,
@@ -1484,6 +1727,12 @@ export const createEvolu =
           update: createMutation("update"),
           upsert: createMutation("upsert"),
 
+          getMutationSize: (table, values) =>
+            measureMutation(
+              String(table),
+              values as ReadonlyRecord<string, SqliteValue | undefined>,
+            ),
+
           loadQuery,
           loadQueries: <Q extends Queries<S>>(
             queries: [...Q],

package/src/local-first/Owner.ts

@@ -28,6 +28,15 @@
  * - {@link OwnerEncryptionKey}: the symmetric key that protects the data.
  * - {@link OwnerWriteKey}: the rotatable token that authorizes writes.
  *
+ * Only holders of the encryption key can create an owner's changes. Each change
+ * is encrypted and authenticated together with its timestamp, so a relay, or
+ * anyone who can write to a relay for the owner, can store bytes for the owner
+ * but cannot forge or move a change. A client stores only the changes it can
+ * decrypt, verify, and decode, and syncs only what it stored. It skips any
+ * other change, which sync state shows on the route of the relay that holds it,
+ * and the relay offers it again on each sync until the client stores a valid
+ * change with that timestamp.
+ *
  * @module
  */