@evolu/common 8.10.0 → 8.12.0

package/src/local-first/Storage.ts

@@ -9,7 +9,8 @@ import type { NonEmptyReadonlyArray } from "../Array.ts";
 import { firstInArray, isNonEmptyArray } from "../Array.ts";
 import { assert, assertNonNullable } from "../Assert.ts";
 import type { Brand } from "../Brand.ts";
-import { concatBytes } from "../Bytes.ts";
+import { concatByteArrays } from "../Bytes.ts";
+import type { DecryptWithXChaCha20Poly1305Error } from "../Crypto.ts";
 import { decrement } from "../Number.ts";
 import type { RandomDep } from "../Random.ts";
 import { err, ok } from "../Result.ts";
@@ -38,6 +39,10 @@ import {
 import type { Awaitable } from "../Types.ts";
 import type { Owner, OwnerError, OwnerIdBytes } from "./Owner.ts";
 import { OwnerId, OwnerWriteKey } from "./Owner.ts";
+import type {
+  ProtocolInvalidDataError,
+  ProtocolTimestampMismatchError,
+} from "./Protocol.ts";
 import { systemColumnsWithId } from "./Schema.ts";
 import {
   createTimestamp,
@@ -46,6 +51,11 @@ import {
   TimestampBytes,
 } from "./Timestamp.ts";
 
+/**
+ * Configuration for {@link Storage}, such as quota checks.
+ *
+ * @group Core
+ */
 export interface StorageConfig {
   /**
    * Callback called before an attempt to write, to check if an {@link OwnerId}
@@ -99,17 +109,23 @@ export interface StorageConfig {
    */
   readonly isOwnerWithinQuota: (
     ownerId: OwnerId,
-    requiredBytes: PositiveInt,
+    requiredBytes: NonNegativeInt,
   ) => Awaitable<boolean>;
 }
 
 /**
- * Evolu Storage.
+ * Replica storage used by Evolu's synchronization protocol.
+ *
+ * Protocol owns message framing, set reconciliation, and sync continuation.
+ * Storage owns batch acceptance, persistence, and the timestamp and fingerprint
+ * queries used by reconciliation. Protocol functions accept any implementation
+ * that satisfies this contract.
  *
- * Evolu Protocol is agnostic to storage implementation—any storage can be
- * plugged in, as long as it implements this interface. Implementations must
- * handle their own errors; return values only indicate overall success or
- * failure.
+ * {@link Storage.writeMessages} returns the {@link StorageWriteMessagesError}
+ * that made it store none of a batch. Implementations return expected write
+ * rejections without reporting them; the caller owns reporting. The client
+ * protocol forwards these errors unchanged, while the relay protocol maps them
+ * to wire error codes.
  *
  * The Storage API is synchronous because SQLite's synchronous API is the
  * fastest way to use SQLite. Synchronous bindings (like better-sqlite3) call
@@ -119,6 +135,8 @@ export interface StorageConfig {
  * The only exception is {@link Storage.writeMessages}, which is async to allow
  * for async validation logic before writing to storage. The write operation
  * itself remains synchronous.
+ *
+ * @group Core
  */
 export interface Storage {
   readonly getSize: (ownerId: OwnerIdBytes) => NonNegativeInt;
@@ -175,13 +193,16 @@ export interface Storage {
   /**
    * Write encrypted {@link CrdtMessage}s to storage.
    *
+   * Stores none of the messages and returns the cause when the batch cannot be
+   * accepted.
+   *
    * Must use a mutex per ownerId to ensure sequential processing and proper
    * protocol logic handling during sync operations.
    */
   readonly writeMessages: (
     ownerIdBytes: OwnerIdBytes,
     messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>,
-  ) => Task<void, StorageQuotaError>;
+  ) => Task<void, StorageWriteMessagesError>;
 
   /** Read encrypted {@link DbChange}s from storage. */
   readonly readDbChange: (
@@ -193,30 +214,72 @@ export interface Storage {
   readonly deleteOwner: (ownerId: OwnerIdBytes) => void;
 }
 
+/**
+ * Dependency wrapper for {@link Storage}.
+ *
+ * @group Core
+ */
 export interface StorageDep {
   readonly storage: Storage;
 }
 
-/** Error when storage or billing quota is exceeded. */
+/**
+ * Error when storage or billing quota is exceeded.
+ *
+ * @group Core
+ */
 export interface StorageQuotaError
   extends OwnerError, Typed<"StorageQuotaError"> {}
 
+/**
+ * Expected reasons why {@link Storage.writeMessages} stored none of a batch.
+ *
+ * Each implementation returns the members that apply to it. The built-in relay
+ * storage currently stores opaque encrypted messages and rejects batches over
+ * quota. The built-in client storage decrypts and validates incoming messages
+ * before updating its clock and database tables. The contract permits quota
+ * checks on either side.
+ *
+ * @group Core
+ */
+export type StorageWriteMessagesError =
+  | DecryptWithXChaCha20Poly1305Error
+  | ProtocolInvalidDataError
+  | ProtocolTimestampMismatchError
+  | StorageQuotaError;
+
 /**
  * A cryptographic hash used for efficiently comparing collections of
  * {@link TimestampBytes}es.
  *
  * It consists of the first {@link fingerprintSize} bytes of the SHA-256 hash of
  * one or more timestamps.
+ *
+ * @group Ranges
  */
 export type Fingerprint = Uint8Array & Brand<"Fingerprint">;
 
+/**
+ * Number of leading SHA-256 bytes that form a {@link Fingerprint}.
+ *
+ * @group Ranges
+ */
 export const fingerprintSize = /*#__PURE__*/ NonNegativeInt.orThrow(12);
 
-/** A fingerprint of an empty range. */
+/**
+ * A fingerprint of an empty range.
+ *
+ * @group Ranges
+ */
 export const zeroFingerprint = /*#__PURE__*/ new Uint8Array(
   fingerprintSize,
 ) as Fingerprint;
 
+/**
+ * Common shape of every {@link Range}.
+ *
+ * @group Ranges
+ */
 export interface BaseRange {
   readonly upperBound: RangeUpperBound;
 }
@@ -224,45 +287,96 @@ export interface BaseRange {
 /**
  * Union type for Range's upperBound: either a {@link TimestampBytes} or
  * {@link InfiniteUpperBound}.
+ *
+ * @group Ranges
  */
 export type RangeUpperBound = TimestampBytes | InfiniteUpperBound;
 
+/**
+ * Sentinel {@link RangeUpperBound} for a range without an upper limit.
+ *
+ * @group Ranges
+ */
 export const InfiniteUpperBound = /*#__PURE__*/ Symbol(
   "evolu.local-first.Storage.InfiniteUpperBound",
 );
+/**
+ * Type of the {@link InfiniteUpperBound} sentinel.
+ *
+ * @group Ranges
+ */
 export type InfiniteUpperBound = typeof InfiniteUpperBound;
 
+/**
+ * Numeric tags discriminating {@link Range} variants.
+ *
+ * @group Ranges
+ */
 export const RangeType = {
   Fingerprint: 1,
   Skip: 0,
   Timestamps: 2,
 } as const;
 
+/**
+ * Numeric tag of one {@link Range} variant.
+ *
+ * @group Ranges
+ */
 export type RangeType = (typeof RangeType)[keyof typeof RangeType];
 
+/**
+ * Range with nothing to reconcile.
+ *
+ * @group Ranges
+ */
 export interface SkipRange extends BaseRange {
   readonly type: typeof RangeType.Skip;
 }
 
+/**
+ * Range summarized by a {@link Fingerprint} for comparison.
+ *
+ * @group Ranges
+ */
 export interface FingerprintRange extends BaseRange {
   readonly type: typeof RangeType.Fingerprint;
   readonly fingerprint: Fingerprint;
 }
 
+/**
+ * Range listing its {@link TimestampBytes} explicitly.
+ *
+ * @group Ranges
+ */
 export interface TimestampsRange extends BaseRange {
   readonly type: typeof RangeType.Timestamps;
   readonly timestamps: ReadonlyArray<TimestampBytes>;
 }
 
+/**
+ * Range exchanged during sync: {@link SkipRange}, {@link FingerprintRange}, or
+ * {@link TimestampsRange}.
+ *
+ * @group Ranges
+ */
 export type Range = SkipRange | FingerprintRange | TimestampsRange;
 
-/** An encrypted {@link CrdtMessage}. */
+/**
+ * An encrypted {@link CrdtMessage}.
+ *
+ * @group Messages
+ */
 export interface EncryptedCrdtMessage {
   readonly timestamp: Timestamp;
   readonly change: EncryptedDbChange;
 }
 
-/** Encrypted DbChange */
+/**
+ * Encrypted DbChange
+ *
+ * @group Messages
+ */
 export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
 
 /**
@@ -271,13 +385,19 @@ export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
  * Used in Evolu's sync protocol to replicate data changes across devices. Evolu
  * operates as a durable queue, providing exactly-once delivery guarantees for
  * reliable synchronization across application restarts and network failures.
+ *
+ * @group Messages
  */
 export interface CrdtMessage {
   readonly timestamp: Timestamp;
   readonly change: DbChange;
 }
 
-/** Test helper for creating a simple {@link CrdtMessage}. */
+/**
+ * Test helper for creating a simple {@link CrdtMessage}.
+ *
+ * @group Testing
+ */
 export const testCreateCrdtMessage = (
   id: Id,
   millis: number,
@@ -296,9 +416,19 @@ export const testCreateCrdtMessage = (
   }),
 });
 
+/**
+ * Column values of a {@link DbChange}, keyed by column name.
+ *
+ * @group Messages
+ */
 export const DbChangeValues = /*#__PURE__*/ record(String, SqliteValue);
 export type DbChangeValues = typeof DbChangeValues.Output;
 
+/**
+ * Column values that contain no reserved system columns.
+ *
+ * @group Messages
+ */
 export const ValidDbChangeValues = /*#__PURE__*/ brand(
   "ValidDbChangeValues",
   DbChangeValues,
@@ -318,6 +448,11 @@ export const ValidDbChangeValues = /*#__PURE__*/ brand(
 );
 export type ValidDbChangeValues = typeof ValidDbChangeValues.Output;
 
+/**
+ * Error produced when {@link DbChangeValues} contain reserved system columns.
+ *
+ * @group Messages
+ */
 export interface ValidDbChangeValuesError extends TypeError<"ValidDbChangeValues"> {
   readonly value: DbChangeValues;
   readonly invalidColumns: ReadonlyArray<string>;
@@ -326,6 +461,8 @@ export interface ValidDbChangeValuesError extends TypeError<"ValidDbChangeValues
 /**
  * A DbChange is a change to a table row. Together with a unique
  * {@link Timestamp}, it forms a {@link CrdtMessage}.
+ *
+ * @group Messages
  */
 export const DbChange: ObjectType<{
   readonly table: typeof String;
@@ -371,17 +508,24 @@ export interface DbChange extends InferType<typeof DbChange> {}
  * each other, if necessary. One relay should handle hundreds of thousands of
  * users, and when it goes down, nothing happens, because it will be
  * synchronized later.
+ *
+ * @group SQLite
  */
 export interface BaseSqliteStorage extends Omit<
   Storage,
   "validateWriteKey" | "setWriteKey" | "writeMessages" | "readDbChange"
 > {
-  /** Inserts a timestamp for an owner into the skiplist-based storage. */
+  /**
+   * Inserts a timestamp for an owner into the skiplist-based storage.
+   *
+   * Returns whether the timestamp was new. An existing timestamp is left
+   * unchanged.
+   */
   readonly insertTimestamp: (
     ownerId: OwnerIdBytes,
     timestamp: TimestampBytes,
     strategy: StorageInsertTimestampStrategy,
-  ) => void;
+  ) => boolean;
 
   /**
    * Efficiently checks which timestamps already exist in the database using a
@@ -393,10 +537,20 @@ export interface BaseSqliteStorage extends Omit<
   ) => ReadonlyArray<TimestampBytes>;
 }
 
+/**
+ * Dependency wrapper for {@link BaseSqliteStorage}.
+ *
+ * @group SQLite
+ */
 export interface BaseSqliteStorageDep {
   readonly baseSqliteStorage: BaseSqliteStorage;
 }
 
+/**
+ * Dependencies required by {@link createBaseSqliteStorage}.
+ *
+ * @group SQLite
+ */
 export type SqliteStorageDeps = RandomDep & SqliteDep;
 
 /**
@@ -411,6 +565,8 @@ export type SqliteStorageDeps = RandomDep & SqliteDep;
  * Cloudflare Workers with Durable Objects, and other platforms where memory
  * doesn't persist between requests. While not extensively tested in all these
  * environments yet, the stateless design should work well across them.
+ *
+ * @group SQLite
  */
 export const createBaseSqliteStorage = (
   deps: SqliteStorageDeps,
@@ -421,11 +577,13 @@ export const createBaseSqliteStorage = (
     strategy: StorageInsertTimestampStrategy,
   ) => {
     const level = randomSkiplistLevel(deps);
-    insertTimestamp(deps)(ownerId, timestamp, level, strategy);
+    return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
   },
 
   getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
-    const concatenatedTimestamps = concatBytes(...timestampsBytes);
+    // A batch can hold hundreds of thousands of timestamps, too many to spread
+    // into concatBytes.
+    const concatenatedTimestamps = concatByteArrays(timestampsBytes);
 
     const result = deps.sqlite.exec<{
       timestampBytes: TimestampBytes;
@@ -509,6 +667,11 @@ const assertBeginEnd = (begin: NonNegativeInt, end: NonNegativeInt) => {
   assert(begin <= end, "invalid begin or end");
 };
 
+/**
+ * Creates the SQLite tables used by {@link BaseSqliteStorage}.
+ *
+ * @group SQLite
+ */
 export const createBaseSqliteStorageTables = (deps: SqliteDep): void => {
   for (const query of [
     /**
@@ -575,6 +738,11 @@ export const createBaseSqliteStorageTables = (deps: SqliteDep): void => {
   }
 };
 
+/**
+ * Position at which a timestamp is inserted relative to existing timestamps.
+ *
+ * @group SQLite
+ */
 export type StorageInsertTimestampStrategy = "append" | "prepend" | "insert";
 
 /**
@@ -582,6 +750,8 @@ export type StorageInsertTimestampStrategy = "append" | "prepend" | "insert";
  * relative to the current first and last timestamps.
  *
  * Returns a tuple with the strategy and updated timestamp bounds.
+ *
+ * @group SQLite
  */
 export const getTimestampInsertStrategy = (
   timestamp: TimestampBytes,
@@ -624,9 +794,9 @@ export const getTimestampInsertStrategy = (
  * key instead of repeating the same correlated range lookup for every column.
  *
  * Inserts are idempotent to support direct calls and message replay. `on
- * conflict do nothing` makes a duplicate insertion a no-op, and `changes() > 0`
- * ensures ancestor metadata is updated only when the preceding insertion added
- * a timestamp.
+ * conflict do nothing` makes a duplicate insertion a no-op that reports the
+ * timestamp as not new, so the follow-up statements update metadata only for a
+ * new timestamp.
  */
 const insertTimestamp =
   (deps: SqliteDep) =>
@@ -635,12 +805,15 @@ const insertTimestamp =
     timestamp: TimestampBytes,
     level: PositiveInt,
     strategy: StorageInsertTimestampStrategy,
-  ): void => {
+  ): boolean => {
     const [h1, h2] = fingerprintToSqliteFingerprint(
       timestampBytesToFingerprint(timestamp),
     );
 
-    let queries: Array<ReturnType<typeof sql.prepared>> = [];
+    let queries: [
+      insert: ReturnType<typeof sql.prepared>,
+      ...updates: Array<ReturnType<typeof sql.prepared>>,
+    ];
 
     switch (strategy) {
       case "append":
@@ -816,10 +989,7 @@ const insertTimestamp =
               h2 = u.h2,
               c = c + 1
             from u
-            where
-              changes() > 0
-              and ownerId = ${ownerId}
-              and evolu_timestamp.t = u.t;
+            where ownerId = ${ownerId} and evolu_timestamp.t = u.t;
           `,
         ];
         break;
@@ -886,10 +1056,7 @@ const insertTimestamp =
                     h2 = u.h2,
                     c = c + 1
                   from u
-                  where
-                    changes() > 0
-                    and ownerId = ${ownerId}
-                    and evolu_timestamp.t = u.t;
+                  where ownerId = ${ownerId} and evolu_timestamp.t = u.t;
                 `,
               ]
             : [
@@ -1072,17 +1239,25 @@ const insertTimestamp =
                     h2 = uh2,
                     c = uc
                   from u
-                  where changes() > 0 and ownerId = ${ownerId} and t = ut;
+                  where ownerId = ${ownerId} and t = ut;
                 `,
               ];
         break;
     }
 
-    for (const query of queries) {
-      deps.sqlite.exec(query);
-    }
+    // The insert uses "on conflict do nothing". An existing timestamp changes
+    // nothing, and the metadata updates are skipped.
+    const [insert, ...updates] = queries;
+    if (deps.sqlite.exec(insert).changes === 0) return false;
+    for (const update of updates) deps.sqlite.exec(update);
+    return true;
   };
 
+/**
+ * Computes the {@link Fingerprint} of a single timestamp.
+ *
+ * @group Ranges
+ */
 export const timestampBytesToFingerprint = (
   timestamp: TimestampBytes,
 ): Fingerprint => {
@@ -1093,6 +1268,8 @@ export const timestampBytesToFingerprint = (
 /**
  * Computes a brute-force {@link Fingerprint} from {@link TimestampBytes} values
  * for tests and benchmarks.
+ *
+ * @group Testing
  */
 export const testFingerprintTimestamps = (
   timestamps: ReadonlyArray<TimestampBytes>,
@@ -1492,6 +1669,11 @@ const fingerprintRanges =
 // XOR in SQLite
 const x = (a: string, b: string) => sql.raw(`(${a} | ${b}) - (${a} & ${b})`);
 
+/**
+ * Reads the timestamp at a position within an owner's ordered timestamps.
+ *
+ * @group SQLite
+ */
 export const getTimestampByIndex =
   (deps: SqliteDep) =>
   (ownerId: OwnerIdBytes, index: NonNegativeInt): TimestampBytes => {
@@ -1572,7 +1754,11 @@ export const getTimestampByIndex =
     return result.rows[0].pt;
   };
 
-/** Reads owner usage from SQLite and returns default bounds when absent. */
+/**
+ * Reads owner usage from SQLite and returns default bounds when absent.
+ *
+ * @group SQLite
+ */
 export const readOwnerUsageOrDefault =
   (deps: SqliteDep) =>
   (
@@ -1617,12 +1803,14 @@ export const readOwnerUsageOrDefault =
  *
  * Used by both relay and client to maintain firstTimestamp/lastTimestamp after
  * processing messages.
+ *
+ * @group SQLite
  */
 export const updateOwnerUsage =
   (deps: SqliteDep) =>
   (
     ownerIdBytes: OwnerIdBytes,
-    storedBytes: PositiveInt,
+    storedBytes: NonNegativeInt,
     firstTimestamp: TimestampBytes,
     lastTimestamp: TimestampBytes,
   ): void => {