@evolu/common 6.0.1-preview.28 → 6.0.1-preview.29

package/src/Evolu/Storage.ts

@@ -2,10 +2,12 @@ import { sha256 } from "@noble/hashes/sha2.js";
 import { NonEmptyReadonlyArray } from "../Array.js";
 import { assert } from "../Assert.js";
 import { Brand } from "../Brand.js";
+import { concatBytes } from "../Buffer.js";
 import { decrement } from "../Number.js";
 import { RandomDep } from "../Random.js";
 import { ok, Result } from "../Result.js";
 import { sql, SqliteDep, SqliteError, SqliteValue } from "../Sqlite.js";
+import { MaybeAsync } from "../Task.js";
 import {
   Id,
   Int64String,
@@ -16,6 +18,7 @@ import {
   String,
 } from "../Type.js";
 import {
+  BaseOwnerError,
   Owner,
   OwnerId,
   OwnerIdBytes,
@@ -24,13 +27,50 @@ import {
 } from "./Owner.js";
 import { orderTimestampBytes, Timestamp, TimestampBytes } from "./Timestamp.js";
 
+export interface StorageConfig {
+  /**
+   * Optional callback to check if an {@link OwnerId} is within their quota for
+   * the requested write. If this callback is not provided, all writes are
+   * allowed regardless of size.
+   *
+   * If provided, the callback receives the OwnerId and the number of bytes
+   * required for the write, and should return a {@link MaybeAsync} boolean:
+   * `true` to allow the write, or `false` to deny it due to quota limits.
+   *
+   * The callback can be synchronous (for SQLite or in-memory checks) or
+   * asynchronous (for calling remote APIs).
+   *
+   * The callback returns a boolean rather than an error type because error
+   * handling and logging are the responsibility of the callback
+   * implementation.
+   *
+   * ### Example
+   *
+   * ```ts
+   * // Client
+   * // evolu.subscribeError
+   *
+   * // Relay
+   * isOwnerWithinQuota: (ownerId, requiredBytes) => {
+   *   console.log(ownerId, requiredBytes);
+   *   // Check error via evolu.subscribeError
+   *   return true;
+   * };
+   * ```
+   */
+  readonly isOwnerWithinQuota?: (
+    ownerId: OwnerId,
+    requiredBytes: PositiveInt,
+  ) => MaybeAsync<boolean>;
+}
+
 /**
  * Evolu Storage
  *
- * The protocol using Storage is agnostic to storage implementation details—any
- * storage can be plugged in, as long as it implements this interface.
- * Implementations must handle their own errors; return values only indicate
- * overall success or failure.
+ * Evolu protocol using Storage is agnostic to storage implementation
+ * details—any storage can be plugged in, as long as it implements this
+ * interface. Implementations must handle their own errors; return values only
+ * indicate overall success or failure.
  *
  * The Storage API is synchronous because SQLite's synchronous API is the
  * fastest way to use SQLite. Synchronous bindings (like better-sqlite3) call
@@ -96,15 +136,15 @@ export interface Storage {
   /**
    * Write encrypted {@link CrdtMessage}s to storage.
    *
-   * Must use a mutex (per ownerId on Relay) to ensure sequential processing and
-   * proper protocol logic handling during sync operations.
+   * Must use a mutex per ownerId to ensure sequential processing and proper
+   * protocol logic handling during sync operations.
    *
-   * Returns `true` on success, `false` on failure.
+   * TODO: Use MaybeAsync
    */
   readonly writeMessages: (
-    ownerId: OwnerIdBytes,
+    ownerIdBytes: OwnerIdBytes,
     messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>,
-  ) => Promise<boolean>;
+  ) => MaybeAsync<Result<void, StorageWriteError | StorageQuotaError>>;
 
   /** Read encrypted {@link DbChange}s from storage. */
   readonly readDbChange: (
@@ -124,6 +164,16 @@ export interface StorageDep {
   readonly storage: Storage;
 }
 
+/** Error indicating a serious write failure. */
+export interface StorageWriteError extends BaseOwnerError {
+  readonly type: "StorageWriteError";
+}
+
+/** Error when storage or billing quota is exceeded. */
+export interface StorageQuotaError extends BaseOwnerError {
+  readonly type: "StorageQuotaError";
+}
+
 /**
  * A cryptographic hash used for efficiently comparing collections of
  * {@link TimestampBytes}s.
@@ -261,6 +311,15 @@ export interface BaseSqliteStorage
     ownerId: OwnerIdBytes,
     timestamp: TimestampBytes,
   ) => Result<void, SqliteError>;
+
+  /**
+   * Efficiently checks which timestamps already exist in the database using a
+   * single CTE query instead of N individual queries.
+   */
+  readonly getExistingTimestamps: (
+    ownerIdBytes: OwnerIdBytes,
+    timestampsBytes: NonEmptyReadonlyArray<TimestampBytes>,
+  ) => Result<ReadonlyArray<TimestampBytes>, SqliteError>;
 }
 
 export interface BaseSqliteStorageDep {
@@ -269,14 +328,14 @@ export interface BaseSqliteStorageDep {
 
 export type SqliteStorageDeps = RandomDep & SqliteDep;
 
-export interface CreateBaseSqliteStorageOptions {
+export interface CreateBaseSqliteStorageConfig extends StorageConfig {
   onStorageError: (error: SqliteError) => void;
 }
 
 export const createBaseSqliteStorage =
   (deps: SqliteStorageDeps) =>
-  (options: CreateBaseSqliteStorageOptions): BaseSqliteStorage => {
-    // TODO: Use OwnerUsage table.
+  (config: CreateBaseSqliteStorageConfig): BaseSqliteStorage => {
+    // TODO: Use evolu_usage table.
     const ownerStats = new Map<
       OwnerId,
       {
@@ -286,49 +345,10 @@ export const createBaseSqliteStorage =
     >();
 
     return {
-      insertTimestamp: (ownerId: OwnerIdBytes, timestamp: TimestampBytes) => {
-        const ownerIdString = ownerIdBytesToOwnerId(ownerId);
-        const level = randomSkiplistLevel(deps);
-
-        let stats = ownerStats.get(ownerIdString);
-
-        if (!stats) {
-          const result = deps.sqlite.exec<{
-            maxT: TimestampBytes | null;
-            minT: TimestampBytes | null;
-          }>(sql.prepared`
-            select min(t) as minT, max(t) as maxT
-            from evolu_timestamp
-            where ownerId = ${ownerId};
-          `);
-          if (!result.ok) return result;
-
-          stats = {
-            minT: result.value.rows[0].minT ?? timestamp,
-            maxT: result.value.rows[0].maxT ?? timestamp,
-          };
-          ownerStats.set(ownerIdString, stats);
-        }
-
-        let strategy: InsertTimestampStrategy;
-
-        if (orderTimestampBytes(timestamp, stats.maxT) === 1) {
-          strategy = "append";
-          stats.maxT = timestamp;
-        } else if (orderTimestampBytes(timestamp, stats.minT) === -1) {
-          strategy = "prepend";
-          stats.minT = timestamp;
-        } else {
-          strategy = "insert";
-        }
-
-        return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
-      },
-
       getSize: (ownerId) => {
         const size = getSize(deps)(ownerId);
         if (!size.ok) {
-          options.onStorageError(size.error);
+          config.onStorageError(size.error);
           return null;
         }
         return size.value;
@@ -338,7 +358,7 @@ export const createBaseSqliteStorage =
         assertBeginEnd(begin, end);
         const result = fingerprint(deps)(ownerId, begin, end);
         if (!result.ok) {
-          options.onStorageError(result.error);
+          config.onStorageError(result.error);
           return null;
         }
         return result.value;
@@ -347,7 +367,7 @@ export const createBaseSqliteStorage =
       fingerprintRanges: (ownerId, buckets, upperBound) => {
         const ranges = fingerprintRanges(deps)(ownerId, buckets, upperBound);
         if (!ranges.ok) {
-          options.onStorageError(ranges.error);
+          config.onStorageError(ranges.error);
           return null;
         }
         return ranges.value;
@@ -361,7 +381,7 @@ export const createBaseSqliteStorage =
           upperBound,
         );
         if (!lowerBound.ok) {
-          options.onStorageError(lowerBound.error);
+          config.onStorageError(lowerBound.error);
           return null;
         }
         return lowerBound.value;
@@ -375,7 +395,7 @@ export const createBaseSqliteStorage =
         // This is much faster than SQL limit with offset.
         const first = getTimestampByIndex(deps)(ownerId, begin);
         if (!first.ok) {
-          options.onStorageError(first.error);
+          config.onStorageError(first.error);
           return;
         }
 
@@ -400,7 +420,7 @@ export const createBaseSqliteStorage =
           limit ${length - 1};
         `);
         if (!result.ok) {
-          options.onStorageError(result.error);
+          config.onStorageError(result.error);
           return;
         }
 
@@ -415,11 +435,80 @@ export const createBaseSqliteStorage =
           delete from evolu_timestamp where ownerId = ${ownerId};
         `);
         if (!result.ok) {
-          options.onStorageError(result.error);
+          config.onStorageError(result.error);
           return false;
         }
         return true;
       },
+
+      insertTimestamp: (ownerId: OwnerIdBytes, timestamp: TimestampBytes) => {
+        const ownerIdString = ownerIdBytesToOwnerId(ownerId);
+        const level = randomSkiplistLevel(deps);
+
+        let stats = ownerStats.get(ownerIdString);
+
+        if (!stats) {
+          const result = deps.sqlite.exec<{
+            maxT: TimestampBytes | null;
+            minT: TimestampBytes | null;
+          }>(sql.prepared`
+            select min(t) as minT, max(t) as maxT
+            from evolu_timestamp
+            where ownerId = ${ownerId};
+          `);
+          if (!result.ok) return result;
+
+          stats = {
+            minT: result.value.rows[0].minT ?? timestamp,
+            maxT: result.value.rows[0].maxT ?? timestamp,
+          };
+          ownerStats.set(ownerIdString, stats);
+        }
+
+        let strategy: InsertTimestampStrategy;
+
+        if (orderTimestampBytes(timestamp, stats.maxT) === 1) {
+          strategy = "append";
+          stats.maxT = timestamp;
+        } else if (orderTimestampBytes(timestamp, stats.minT) === -1) {
+          strategy = "prepend";
+          stats.minT = timestamp;
+        } else {
+          strategy = "insert";
+        }
+
+        return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
+      },
+
+      getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
+        const concatenatedTimestamps = concatBytes(...timestampsBytes);
+
+        const result = deps.sqlite.exec<{
+          timestampBytes: TimestampBytes;
+        }>(sql`
+          with recursive
+            split_timestamps(timestampBytes, pos) as (
+              select
+                substr(${concatenatedTimestamps}, 1, 16),
+                17 as pos
+              union all
+              select
+                substr(${concatenatedTimestamps}, pos, 16),
+                pos + 16
+              from split_timestamps
+              where pos <= length(${concatenatedTimestamps})
+            )
+          select s.timestampBytes
+          from
+            split_timestamps s
+            join evolu_timestamp t
+              on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
+        `);
+
+        if (!result.ok) return result;
+
+        return ok(result.value.rows.map((row) => row.timestampBytes));
+      },
     };
   };
 
@@ -484,19 +573,20 @@ export const createBaseSqliteStorageTables = (
      *
      * - `ownerId` – OwnerIdBytes (primary key)
      * - `storedBytes` – total bytes stored in database
-     * - `receivedBytes` – total bytes received from clients
-     * - `sentBytes` – total bytes sent to clients
-     * - `firstTimestamp` – minimum timestamp (nullable)
-     * - `lastTimestamp` – maximum timestamp (nullable)
+     * - `receivedBytes` – TODO: Decide how to use
+     * - `sentBytes` – TODO: Decide how to use
+     * - `firstTimestamp` – TODO: Decide how to use (nullable)
+     * - `lastTimestamp` – TODO: Decide how to use (nullable)
      */
     sql`
       create table evolu_usage (
         "ownerId" blob primary key,
-        "storedBytes" integer not null,
-        "receivedBytes" integer not null,
-        "sentBytes" integer not null,
-        "firstTimestamp" blob,
-        "lastTimestamp" blob
+        "storedBytes" integer not null
+        -- TODO: Decide how to use receivedBytes, sentBytes, firstTimestamp, lastTimestamp
+        -- "receivedBytes" integer not null,
+        -- "sentBytes" integer not null,
+        -- "firstTimestamp" blob,
+        -- "lastTimestamp" blob
       )
       strict;
     `,

package/src/Evolu/Sync.ts

@@ -13,7 +13,7 @@ import { constFalse } from "../Function.js";
 import { objectToEntries } from "../Object.js";
 import { RandomDep } from "../Random.js";
 import { createRefCountedResourceManager } from "../RefCountedResourceManager.js";
-import { ok, Result } from "../Result.js";
+import { err, ok, Result } from "../Result.js";
 import { sql, SqliteDep, SqliteError, SqliteValue } from "../Sqlite.js";
 import { AbortError, createMutex } from "../Task.js";
 import { TimeDep } from "../Time.js";
@@ -48,10 +48,11 @@ import {
 import { MutationChange } from "./Schema.js";
 import {
   BaseSqliteStorage,
-  createBaseSqliteStorage,
   CrdtMessage,
+  createBaseSqliteStorage,
   DbChange,
   Storage,
+  StorageWriteError,
 } from "./Storage.js";
 import {
   createInitialTimestamp,
@@ -452,15 +453,20 @@ const createClientStorage =
       onStorageError: config.onError,
     });
 
+    // TODO: Mutex per OwnerId
     const mutex = createMutex();
 
     const storage: ClientStorage = {
       ...sqliteStorageBase,
 
+      // Not implemented yet.
       validateWriteKey: constFalse,
       setWriteKey: constFalse,
 
       writeMessages: async (ownerIdBytes, encryptedMessages) => {
+        const ownerId = ownerIdBytesToOwnerId(ownerIdBytes);
+
+        // Everything is sync now, but we will need async crypto in the future.
         const writeResult = await mutex.withLock<
           boolean,
           | AbortError
@@ -473,10 +479,16 @@ const createClientStorage =
           | TimestampTimeOutOfRangeError
           // eslint-disable-next-line @typescript-eslint/require-await
         >(async () => {
-          const ownerId = ownerIdBytesToOwnerId(ownerIdBytes);
           const owner = deps.getSyncOwner(ownerId);
           // Owner can be removed during syncing.
-          if (!owner) return ok(false);
+          // `ok(true)` means success, we just skipped the write.
+          if (!owner) return ok(true);
+
+          // TODO: Add quota checking for collaborative scenarios.
+          // When receiving messages from other owners via relay broadcast,
+          // check if this owner is within quota before accepting the data.
+          // This prevents an owner from exceeding storage limits when receiving
+          // data shared by other collaborators.
 
           const messages: Array<CrdtMessage> = [];
 
@@ -530,12 +542,12 @@ const createClientStorage =
           if (writeResult.error.type !== "AbortError") {
             config.onError(writeResult.error);
           }
-          return false;
+          return err<StorageWriteError>({ type: "StorageWriteError", ownerId });
         }
 
         config.onReceive();
 
-        return true;
+        return ok();
       },
 
       readDbChange: (ownerId, timestamp) => {

package/src/Evolu/Timestamp.ts

@@ -117,6 +117,31 @@ export const maxNodeId = "ffffffffffffffff" as NodeId;
  * Timestamps serve as globally unique, causally ordered identifiers for CRDT
  * messages in Evolu's sync protocol.
  *
+ * ### Why Hybrid Logical Clocks
+ *
+ * Evolu uses Hybrid Logical Clocks (HLC), which combine physical time (millis)
+ * with a logical counter. This hybrid approach preserves causality like logical
+ * clocks while staying close to physical time for better human
+ * interpretability.
+ *
+ * The counter component ensures causality is maintained even when physical
+ * clocks are imperfect. When clocks drift or operations occur concurrently, the
+ * counter increments to establish a total order. This means Evolu achieves
+ * well-defined, eventually-consistent behavior regardless of physical clock
+ * accuracy.
+ *
+ * Vector clocks can accurately track causality and detect concurrent
+ * operations, but they require unbounded space in peer-to-peer systems and
+ * crucially, still don't solve our fundamental problem: when they detect
+ * operations as concurrent, we still need a deterministic way to choose a
+ * winner. Additionally, any deterministic conflict resolution can be gamed by
+ * malicious actors.
+ *
+ * HLC timestamps work well in practice because modern device clocks accurately
+ * reflect the order of sequential edits in the common case. Evolu's `maxDrift`
+ * configuration protects against buggy clocks and prevents problematic
+ * future-dated entries from propagating through the network.
+ *
  * ### References
  *
  * - https://muratbuffalo.blogspot.com/2014/07/hybrid-logical-clocks.html

package/src/Instances.ts

@@ -0,0 +1,90 @@
+/**
+ * Manages disposable instances by key, ensuring exactly one instance per key.
+ *
+ * Use cases:
+ *
+ * - One mutex per key to prevent concurrent writes
+ * - Preserving state during hot module reloading
+ *
+ * **Important:** Do not use this as global shared state. Use it locally or pass
+ * it as a dependency instead. The only exception is for hot reloading support,
+ * where Evolu uses it to ensure only one instance exists across module reloads
+ * (having two Evolu instances with the same name would mean two SQLite
+ * connections to the same file, which could corrupt data).
+ */
+export interface Instances<K extends string, T extends Disposable>
+  extends Disposable {
+  /**
+   * Ensures an instance exists for the given key, creating it if necessary. If
+   * the instance already exists, the optional `onCacheHit` callback is invoked
+   * to update the existing instance.
+   */
+  readonly ensure: (
+    key: K,
+    create: () => T,
+    onCacheHit?: (instance: T) => void,
+  ) => T;
+
+  /** Gets an instance by key, or returns `null` if it doesn't exist. */
+  readonly get: (key: K) => T | null;
+
+  /** Checks if an instance exists for the given key. */
+  readonly has: (key: K) => boolean;
+
+  /**
+   * Deletes and disposes an instance by key. Returns `true` if the instance
+   * existed and was deleted, `false` otherwise.
+   */
+  readonly delete: (key: K) => boolean;
+}
+
+/** Creates an {@link Instances} manager. */
+export const createInstances = <
+  K extends string,
+  T extends Disposable,
+>(): Instances<K, T> => {
+  const instances = new Map<K, T>();
+
+  return {
+    ensure: (key, create, onCacheHit) => {
+      let instance = instances.get(key);
+
+      if (instance == null) {
+        instance = create();
+        instances.set(key, instance);
+      } else if (onCacheHit) {
+        onCacheHit(instance);
+      }
+
+      return instance;
+    },
+
+    get: (key) => instances.get(key) ?? null,
+
+    has: (key) => instances.has(key),
+
+    delete: (key) => {
+      const instance = instances.get(key);
+      if (instance == null) return false;
+      instances.delete(key);
+      instance[Symbol.dispose]();
+      return true;
+    },
+
+    [Symbol.dispose]: () => {
+      const errors: Array<unknown> = [];
+      for (const instance of instances.values()) {
+        try {
+          instance[Symbol.dispose]();
+        } catch (error) {
+          errors.push(error);
+        }
+      }
+      instances.clear();
+      if (errors.length === 1) throw errors[0];
+      if (errors.length > 1) {
+        throw new AggregateError(errors, "Multiple disposal errors occurred");
+      }
+    },
+  };
+};

package/src/Sqlite.ts

@@ -334,6 +334,12 @@ export type SqlTemplateParam = SqliteValue | SqlIdentifier | RawSql;
  * const orderBy = "created_at desc";
  * sqlite.exec(sql`select * from users order by ${sql.raw(orderBy)};`);
  * ```
+ *
+ * ### TIP
+ *
+ * Use `prettier-plugin-sql-cst` for SQL formatting. Like Prettier for
+ * JavaScript, this plugin formats SQL expressions differently depending on
+ * their length.
  */
 export const sql = (
   strings: TemplateStringsArray,

package/src/Task.ts

@@ -778,13 +778,6 @@ export const createMutex = (): Mutex => {
   };
 };
 
-// TODO: Add tracing support
-// - Extend TaskContext with optional tracing field
-// - Add traced(name, task) helper that wraps Task execution
-// - Collect span data (name, timing, parent-child relationships, status)
-// - Support OpenTelemetry export format with proper traceId/spanId generation
-// - Automatic parent-child span relationships through context propagation
-
 /**
  * Schedule a task to run after all interactions (animations, gestures,
  * navigation) have completed.
@@ -818,3 +811,91 @@ const idleCallback: (callback: () => void) => void =
   typeof globalThis.requestIdleCallback === "function"
     ? globalThis.requestIdleCallback
     : (callback) => setTimeout(callback, 0);
+
+/**
+ * Represents a value that can be either synchronous or asynchronous.
+ *
+ * This type is useful for functions that may complete synchronously or
+ * asynchronously depending on runtime conditions (e.g., cache hit vs network
+ * fetch).
+ *
+ * ### Why MaybeAsync?
+ *
+ * When a function can be sync or async, the typical approaches are:
+ *
+ * 1. **Always return Promise** - Simple but forces microtask overhead even for
+ *    sync values (see "await always adds microtask" test in Task.test.ts)
+ * 2. **Use callbacks** - Can avoid microtask, but calling code must still `await`
+ *    for sane composition, which adds microtask anyway
+ * 3. **Return `T | PromiseLike<T>`** - Calling code can check the value and only
+ *    `await` when needed, avoiding microtask overhead for sync cases
+ *
+ * The third approach (MaybeAsync) provides:
+ *
+ * - **Performance**: No microtask overhead for synchronous operations
+ * - **Reliability**: No interleaving via microtask queue when operations are
+ *   _synchronous_, reducing need for mutexes to protect shared state
+ *
+ * ### Example
+ *
+ * ```ts
+ * // Function that may be sync or async
+ * const getData = (id: string): MaybeAsync<Data> => {
+ *   const cached = cache.get(id);
+ *   if (cached) return cached; // Sync path
+ *   return fetchData(id); // Async path
+ * };
+ *
+ * // Caller can optimize based on actual behavior
+ * const result = getData(id);
+ * const data = isAsync(result) ? await result : result;
+ * ```
+ *
+ * ### Alternative Approaches
+ *
+ * It's possible to eliminate the sync/async distinction using complex
+ * frameworks with custom schedulers. However, such frameworks require depending
+ * on other people's code that controls how your code executes, resulting in
+ * more complex stack traces and debugging experiences. With MaybeAsync, we
+ * don't need that machinery - it works directly with JavaScript's native
+ * primitives and TypeScript's type system.
+ *
+ * ### TODO: Consider
+ *
+ * Use MaybeAsync in Task and Task helpers to preserve synchronous execution
+ * when possible (e.g., mutex with available permit, retry on first success).
+ */
+export type MaybeAsync<T> = T | PromiseLike<T>;
+
+/**
+ * Type guard to check if a {@link MaybeAsync} value is async (a promise).
+ *
+ * This function narrows the type of a {@link MaybeAsync} value, allowing you to
+ * conditionally `await` only when necessary.
+ *
+ * ### Example
+ *
+ * ```ts
+ * const getData = (id: string): MaybeAsync<Data> => {
+ *   const cached = cache.get(id);
+ *   if (cached) return cached; // Sync path
+ *   return fetchData(id); // Async path
+ * };
+ *
+ * const result = getData(id);
+ * const data = isAsync(result) ? await result : result;
+ * // No microtask overhead when cached!
+ * ```
+ */
+export const isAsync = <T>(
+  value: MaybeAsync<T>,
+): value is T extends PromiseLike<unknown> ? never : PromiseLike<T> =>
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+  typeof (value as any)?.then === "function";
+
+// TODO: Add tracing support
+// - Extend TaskContext with optional tracing field
+// - Add traced(name, task) helper that wraps Task execution
+// - Collect span data (name, timing, parent-child relationships, status)
+// - Support OpenTelemetry export format with proper traceId/spanId generation
+// - Automatic parent-child span relationships through context propagation