@evolu/common 6.0.1-preview.19 → 6.0.1-preview.20

package/src/Evolu/Storage.ts

@@ -1,10 +1,230 @@
+import { sha256 } from "@noble/hashes/sha2.js";
+import { NonEmptyReadonlyArray } from "../Array.js";
+import { assert } from "../Assert.js";
+import { Brand } from "../Brand.js";
+import { decrement } from "../Number.js";
+import { RandomDep } from "../Random.js";
+import { ok, Result } from "../Result.js";
+import { sql, SqliteDep, SqliteError, SqliteValue } from "../Sqlite.js";
+import {
+  Id,
+  Int64String,
+  NonNegativeInt,
+  object,
+  PositiveInt,
+  record,
+  String,
+} from "../Type.js";
+import {
+  Owner,
+  OwnerId,
+  OwnerIdBytes,
+  ownerIdBytesToOwnerId,
+  OwnerWriteKey,
+} from "./Owner.js";
+import { orderTimestampBytes, Timestamp, TimestampBytes } from "./Timestamp.js";
+
 /**
- * Evolu Storage for SQLite
+ * Evolu Storage
+ *
+ * The protocol using Storage is agnostic to storage implementation details—any
+ * storage can be plugged in, as long as it implements this interface.
+ * Implementations must handle their own errors; return values only indicate
+ * overall success or failure.
+ *
+ * The Storage API is synchronous because SQLite's synchronous API is the
+ * fastest way to use SQLite. Synchronous bindings (like better-sqlite3) call
+ * SQLite's C API directly with no context switching between the event loop and
+ * native code, and no promise microtasks or await overhead.
+ *
+ * The only exception is {@link Storage#writeMessages}, which is async to allow
+ * for async validation logic before writing to storage. The write operation
+ * itself remains synchronous.
+ */
+export interface Storage {
+  readonly getSize: (ownerId: OwnerIdBytes) => NonNegativeInt | null;
+
+  readonly fingerprint: (
+    ownerId: OwnerIdBytes,
+    begin: NonNegativeInt,
+    end: NonNegativeInt,
+  ) => Fingerprint | null;
+
+  /**
+   * Computes fingerprints with their upper bounds in one call.
+   *
+   * This function can be replaced with many fingerprint/findLowerBound calls,
+   * but implementations can leverage it for batching and more efficient
+   * fingerprint computation.
+   */
+  readonly fingerprintRanges: (
+    ownerId: OwnerIdBytes,
+    buckets: ReadonlyArray<NonNegativeInt>,
+    upperBound?: RangeUpperBound,
+  ) => ReadonlyArray<FingerprintRange> | null;
+
+  readonly findLowerBound: (
+    ownerId: OwnerIdBytes,
+    begin: NonNegativeInt,
+    end: NonNegativeInt,
+    upperBound: RangeUpperBound,
+  ) => NonNegativeInt | null;
+
+  readonly iterate: (
+    ownerId: OwnerIdBytes,
+    begin: NonNegativeInt,
+    end: NonNegativeInt,
+    callback: (timestamp: TimestampBytes, index: NonNegativeInt) => boolean,
+  ) => void;
+
+  /**
+   * Validates the {@link OwnerWriteKey} for the given {@link Owner}.
+   *
+   * Returns `true` if the write key is valid, `false` otherwise.
+   */
+  readonly validateWriteKey: (
+    ownerId: OwnerIdBytes,
+    writeKey: OwnerWriteKey,
+  ) => boolean;
+
+  /** Sets the {@link OwnerWriteKey} for the given {@link Owner}. */
+  readonly setWriteKey: (
+    ownerId: OwnerIdBytes,
+    writeKey: OwnerWriteKey,
+  ) => boolean;
+
+  /**
+   * Write encrypted {@link CrdtMessage}s to storage.
+   *
+   * Must use a mutex (per ownerId on Relay) to ensure sequential processing and
+   * proper protocol logic handling during sync operations.
+   *
+   * Returns `true` on success, `false` on failure.
+   */
+  readonly writeMessages: (
+    ownerId: OwnerIdBytes,
+    messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>,
+  ) => Promise<boolean>;
+
+  /** Read encrypted {@link DbChange}s from storage. */
+  readonly readDbChange: (
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
+  ) => EncryptedDbChange | null;
+
+  /**
+   * Delete all data for the given {@link Owner}.
+   *
+   * Returns `true` on success, `false` on failure.
+   */
+  readonly deleteOwner: (ownerId: OwnerIdBytes) => boolean;
+}
+
+export interface StorageDep {
+  readonly storage: Storage;
+}
+
+/**
+ * A cryptographic hash used for efficiently comparing collections of
+ * {@link TimestampBytes}s.
+ *
+ * It consists of the first {@link fingerprintSize} bytes of the SHA-256 hash of
+ * one or more timestamps.
+ */
+export type Fingerprint = Uint8Array & Brand<"Fingerprint">;
+
+export const fingerprintSize = 12 as NonNegativeInt;
+
+/** A fingerprint of an empty range. */
+export const zeroFingerprint = new Uint8Array(fingerprintSize) as Fingerprint;
+
+export interface BaseRange {
+  readonly upperBound: RangeUpperBound;
+}
+
+/**
+ * Union type for Range's upperBound: either a {@link TimestampBytes} or
+ * {@link InfiniteUpperBound}.
+ */
+export type RangeUpperBound = TimestampBytes | InfiniteUpperBound;
+
+export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
+export type InfiniteUpperBound = typeof InfiniteUpperBound;
+
+export const RangeType = {
+  Fingerprint: 1,
+  Skip: 0,
+  Timestamps: 2,
+} as const;
+
+export type RangeType = (typeof RangeType)[keyof typeof RangeType];
+
+export interface SkipRange extends BaseRange {
+  readonly type: typeof RangeType.Skip;
+}
+
+export interface FingerprintRange extends BaseRange {
+  readonly type: typeof RangeType.Fingerprint;
+  readonly fingerprint: Fingerprint;
+}
+
+export interface TimestampsRange extends BaseRange {
+  readonly type: typeof RangeType.Timestamps;
+  readonly timestamps: ReadonlyArray<TimestampBytes>;
+}
+
+export type Range = SkipRange | FingerprintRange | TimestampsRange;
+
+/** An encrypted {@link CrdtMessage}. */
+export interface EncryptedCrdtMessage {
+  readonly timestamp: Timestamp;
+  readonly change: EncryptedDbChange;
+}
+
+/** Encrypted DbChange */
+export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
+
+/**
+ * A CRDT message combining a unique {@link Timestamp} with a {@link DbChange}.
+ *
+ * Used in Evolu's sync protocol to replicate data changes across devices. Evolu
+ * operates as a durable queue, providing exactly-once delivery guarantees for
+ * reliable synchronization across application restarts and network failures.
+ */
+export interface CrdtMessage {
+  readonly timestamp: Timestamp;
+  readonly change: DbChange;
+}
+
+/**
+ * A DbChange is a change to a table row. Together with a unique
+ * {@link Timestamp}, it forms a {@link CrdtMessage}.
+ */
+export const DbChange = object({
+  table: String,
+  id: Id,
+  values: record(String, SqliteValue),
+});
+export type DbChange = typeof DbChange.Type;
+
+/**
+ * Common interface for both client and relay SQLite storages.
  *
  * Evolu uses a Skiplist, which leverages SQLite indexes. The core logic is
  * implemented in SQL, so it doesn't have to make roundtrips to the DB.
  *
- * The ideal storage for a Relay should use a similar architecture to
+ * While the SQL implementation may look sophisticated, it's conceptually simple
+ * and LLMs can explain how it works. The Skiplist data structure is well
+ * explained in [this Stack Overflow
+ * answer](https://stackoverflow.com/questions/61944198/what-is-a-zip-tree-and-how-does-it-work).
+ * The logic resembles [Negentropy's C++
+ * storage](https://github.com/hoytech/negentropy), except we use a Skiplist to
+ * leverage SQLite indexes, which makes the code simpler.
+ *
+ * Note: A paid review by the SQLite team is planned, as they use the same
+ * algorithm for their rsync tool.
+ *
+ * The ideal storage for a Relay should use an architecture like
  * [strfry](https://github.com/hoytech/strfry) (a KV storage), but with Skiplist
  * to ensure that insertion order doesn't matter (local-first apps can often
  * write in the past.)
@@ -19,37 +239,18 @@
  * each other, if necessary. One relay should handle hundreds of thousands of
  * users, and when it goes down, nothing happens, because it will be
  * synchronized later.
- *
- * @module
  */
-
-import { assert } from "../Assert.js";
-import { decrement } from "../Number.js";
-import { RandomDep } from "../Random.js";
-import { ok, Result } from "../Result.js";
-import { sql, SqliteDep, SqliteError } from "../Sqlite.js";
-import { Int64String, NonNegativeInt, PositiveInt } from "../Type.js";
-import { Brand } from "../Brand.js";
-import { OwnerId } from "./Owner.js";
-import {
-  BinaryOwnerId,
-  binaryOwnerIdToOwnerId,
-  binaryTimestampToFingerprint,
-  Fingerprint,
-  FingerprintRange,
-  InfiniteUpperBound,
-  RangeType,
-  RangeUpperBound,
-  Storage,
-  zeroFingerprint,
-} from "./Protocol.js";
-import { BinaryTimestamp, orderBinaryTimestamp } from "./Timestamp.js";
-
-/** Common interface for both client and relay SQLite storages. */
 export interface SqliteStorageBase {
+  /**
+   * Inserts a timestamp for an owner into the skiplist-based storage.
+   *
+   * Must be idempotent - inserting the same timestamp multiple times has no
+   * effect after the first insertion. This is crucial for sync reliability as
+   * messages may be received and processed multiple times.
+   */
   readonly insertTimestamp: (
-    ownerId: BinaryOwnerId,
-    timestamp: BinaryTimestamp,
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
   ) => Result<void, SqliteError>;
 
   readonly getSize: Storage["getSize"];
@@ -64,7 +265,7 @@ export interface SqliteStorageBaseDep {
   readonly storage: SqliteStorageBase;
 }
 
-export type SqliteStorageDeps = SqliteDep & RandomDep;
+export type SqliteStorageDeps = RandomDep & SqliteDep;
 
 export interface CreateSqliteStorageBaseOptions {
   onStorageError: (error: SqliteError) => void;
@@ -81,22 +282,22 @@ export const createSqliteStorageBase =
     const ownerStats = new Map<
       OwnerId,
       {
-        minT: BinaryTimestamp;
-        maxT: BinaryTimestamp;
+        minT: TimestampBytes;
+        maxT: TimestampBytes;
       }
     >();
 
     return ok({
-      insertTimestamp: (ownerId: BinaryOwnerId, timestamp: BinaryTimestamp) => {
-        const ownerIdString = binaryOwnerIdToOwnerId(ownerId);
+      insertTimestamp: (ownerId: OwnerIdBytes, timestamp: TimestampBytes) => {
+        const ownerIdString = ownerIdBytesToOwnerId(ownerId);
         const level = randomSkiplistLevel(deps);
 
         let stats = ownerStats.get(ownerIdString);
 
         if (!stats) {
           const result = deps.sqlite.exec<{
-            maxT: BinaryTimestamp | null;
-            minT: BinaryTimestamp | null;
+            maxT: TimestampBytes | null;
+            minT: TimestampBytes | null;
           }>(sql.prepared`
             select min(t) as minT, max(t) as maxT
             from evolu_timestamp
@@ -113,10 +314,10 @@ export const createSqliteStorageBase =
 
         let strategy: InsertTimestampStrategy;
 
-        if (orderBinaryTimestamp(timestamp, stats.maxT) === 1) {
+        if (orderTimestampBytes(timestamp, stats.maxT) === 1) {
           strategy = "append";
           stats.maxT = timestamp;
-        } else if (orderBinaryTimestamp(timestamp, stats.minT) === -1) {
+        } else if (orderTimestampBytes(timestamp, stats.minT) === -1) {
           strategy = "prepend";
           stats.minT = timestamp;
         } else {
@@ -193,7 +394,7 @@ export const createSqliteStorageBase =
          * implementing chunking, be sure to run performance tests (including
          * fetching one by one).
          */
-        const result = deps.sqlite.exec<{ t: BinaryTimestamp }>(sql`
+        const result = deps.sqlite.exec<{ t: TimestampBytes }>(sql`
           select t
           from evolu_timestamp
           where ownerId = ${ownerId} and t > ${first.value}
@@ -241,7 +442,7 @@ const createTables = (deps: SqliteDep): Result<void, SqliteError> => {
      *
      * Columns:
      *
-     * - `t` – globally unique binary timestamp
+     * - `t` – TimestampBytes
      * - `h1`/`h2` – 12-byte fingerprint split into two integers for fast XOR
      * - `c` – incremental count
      * - `l` – Skiplist level (1 to 32)
@@ -295,13 +496,13 @@ type InsertTimestampStrategy = "append" | "prepend" | "insert";
 const insertTimestamp =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
-    timestamp: BinaryTimestamp,
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
     level: PositiveInt,
     strategy: InsertTimestampStrategy,
   ): Result<void, SqliteError> => {
     const [h1, h2] = fingerprintToSqliteFingerprint(
-      binaryTimestampToFingerprint(timestamp),
+      timestampBytesToFingerprint(timestamp),
     );
 
     let queries: Array<ReturnType<typeof sql.prepared>> = [];
@@ -819,6 +1020,13 @@ const insertTimestamp =
     return ok();
   };
 
+export const timestampBytesToFingerprint = (
+  timestamp: TimestampBytes,
+): Fingerprint => {
+  const hash = sha256(timestamp).slice(0, fingerprintSize);
+  return hash as Fingerprint;
+};
+
 /**
  * Generates a random skiplist level in the range [1, skiplistMaxLevel].
  * Probabilistic approach avoids the need for explicit tree balancing.
@@ -899,7 +1107,7 @@ const sqliteFingerprintToFingerprint = ([
 
 const getSize =
   (deps: SqliteDep) =>
-  (ownerId: BinaryOwnerId): Result<NonNegativeInt, SqliteError> => {
+  (ownerId: OwnerIdBytes): Result<NonNegativeInt, SqliteError> => {
     const result = deps.sqlite.exec<{ size: NonNegativeInt }>(sql.prepared`
       with
         ml(ml) as (
@@ -942,7 +1150,7 @@ const getSize =
 const findLowerBound =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     begin: NonNegativeInt,
     end: NonNegativeInt,
     upperBound: RangeUpperBound,
@@ -954,7 +1162,7 @@ const findLowerBound =
     }
 
     const result = deps.sqlite.exec<{
-      t: BinaryTimestamp;
+      t: TimestampBytes;
     }>(sql.prepared`
       select t
       from evolu_timestamp
@@ -978,8 +1186,8 @@ const findLowerBound =
 const getTimestampCount =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
-    timestamp: BinaryTimestamp,
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
   ): Result<PositiveInt, SqliteError> => {
     const result = deps.sqlite.exec<{
       count: PositiveInt;
@@ -1036,7 +1244,7 @@ const getTimestampCount =
 const fingerprint =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     begin: NonNegativeInt,
     end: NonNegativeInt,
   ): Result<Fingerprint, SqliteError> => {
@@ -1068,14 +1276,14 @@ const fingerprint =
 const fingerprintRanges =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     buckets: ReadonlyArray<NonNegativeInt>,
     upperBound: RangeUpperBound = InfiniteUpperBound,
   ): Result<ReadonlyArray<FingerprintRange>, SqliteError> => {
     const bucketsJson = JSON.stringify(buckets);
 
     const result = deps.sqlite.exec<{
-      b: BinaryTimestamp | null;
+      b: TimestampBytes | null;
       h1: Int64String;
       h2: Int64String;
     }>(sql.prepared`
@@ -1211,11 +1419,11 @@ const x = (a: string, b: string) => sql.raw(`(${a} | ${b}) - (${a} & ${b})`);
 export const getTimestampByIndex =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     index: NonNegativeInt,
-  ): Result<BinaryTimestamp, SqliteError> => {
+  ): Result<TimestampBytes, SqliteError> => {
     const result = deps.sqlite.exec<{
-      readonly pt: BinaryTimestamp;
+      readonly pt: TimestampBytes;
     }>(sql.prepared`
       with
         fi(b, cl, ic, pt, mt, nt, nc) as (