@evolu/common 6.0.1-preview.9 → 7.1.0

package/src/Evolu/Storage.ts

@@ -1,10 +1,319 @@
+import { sha256 } from "@noble/hashes/sha2.js";
+import {
+  firstInArray,
+  isNonEmptyReadonlyArray,
+  NonEmptyReadonlyArray,
+} from "../Array.js";
+import { assert } from "../Assert.js";
+import { Brand } from "../Brand.js";
+import { concatBytes } from "../Buffer.js";
+import { decrement } from "../Number.js";
+import { RandomDep } from "../Random.js";
+import { err, ok, Result } from "../Result.js";
+import { sql, SqliteDep, SqliteError, SqliteValue } from "../Sqlite.js";
+import { MaybeAsync } from "../Task.js";
+import {
+  Boolean,
+  brand,
+  Id,
+  Int64String,
+  NonNegativeInt,
+  nullOr,
+  object,
+  PositiveInt,
+  record,
+  String,
+  TypeError,
+} from "../Type.js";
+import {
+  BaseOwnerError,
+  Owner,
+  OwnerId,
+  OwnerIdBytes,
+  OwnerWriteKey,
+} from "./Owner.js";
+import { systemColumns } from "./Schema.js";
+import { orderTimestampBytes, Timestamp, TimestampBytes } from "./Timestamp.js";
+
+export interface StorageConfig {
+  /**
+   * Callback called before an attempt to write, to check if an {@link OwnerId}
+   * has sufficient quota for the write.
+   *
+   * The callback receives the {@link OwnerId} and the total bytes that would be
+   * stored after the write (current stored bytes plus incoming bytes), and
+   * returns a {@link MaybeAsync} boolean: `true` to allow the write, or `false`
+   * to deny it due to quota limits.
+   *
+   * The callback can be synchronous (for SQLite or in-memory checks) or
+   * asynchronous (for calling remote APIs).
+   *
+   * The callback returns a boolean rather than an error because error handling
+   * and logging are the responsibility of the callback implementation.
+   *
+   * ### Example
+   *
+   * ```ts
+   * // Client
+   * // evolu.subscribeError
+   *
+   * // Relay
+   * isOwnerWithinQuota: (ownerId, requiredBytes) => {
+   *   console.log(ownerId, requiredBytes);
+   *   // Check error via evolu.subscribeError
+   *   return true;
+   * };
+   * ```
+   */
+  readonly isOwnerWithinQuota: (
+    ownerId: OwnerId,
+    requiredBytes: PositiveInt,
+  ) => MaybeAsync<boolean>;
+}
+
+/**
+ * Evolu Storage
+ *
+ * Evolu protocol using Storage is agnostic to storage implementation
+ * details—any storage can be plugged in, as long as it implements this
+ * interface. Implementations must handle their own errors; return values only
+ * indicate overall success or failure.
+ *
+ * The Storage API is synchronous because SQLite's synchronous API is the
+ * fastest way to use SQLite. Synchronous bindings (like better-sqlite3) call
+ * SQLite's C API directly with no context switching between the event loop and
+ * native code, and no promise microtasks or await overhead.
+ *
+ * The only exception is {@link Storage#writeMessages}, which is async to allow
+ * for async validation logic before writing to storage. The write operation
+ * itself remains synchronous.
+ */
+export interface Storage {
+  readonly getSize: (ownerId: OwnerIdBytes) => NonNegativeInt | null;
+
+  readonly fingerprint: (
+    ownerId: OwnerIdBytes,
+    begin: NonNegativeInt,
+    end: NonNegativeInt,
+  ) => Fingerprint | null;
+
+  /**
+   * Computes fingerprints with their upper bounds in one call.
+   *
+   * This function can be replaced with many fingerprint/findLowerBound calls,
+   * but implementations can leverage it for batching and more efficient
+   * fingerprint computation.
+   */
+  readonly fingerprintRanges: (
+    ownerId: OwnerIdBytes,
+    buckets: ReadonlyArray<NonNegativeInt>,
+    upperBound?: RangeUpperBound,
+  ) => ReadonlyArray<FingerprintRange> | null;
+
+  readonly findLowerBound: (
+    ownerId: OwnerIdBytes,
+    begin: NonNegativeInt,
+    end: NonNegativeInt,
+    upperBound: RangeUpperBound,
+  ) => NonNegativeInt | null;
+
+  readonly iterate: (
+    ownerId: OwnerIdBytes,
+    begin: NonNegativeInt,
+    end: NonNegativeInt,
+    callback: (timestamp: TimestampBytes, index: NonNegativeInt) => boolean,
+  ) => void;
+
+  /**
+   * Validates the {@link OwnerWriteKey} for the given {@link Owner}.
+   *
+   * Returns `true` if the write key is valid, `false` otherwise.
+   */
+  readonly validateWriteKey: (
+    ownerId: OwnerIdBytes,
+    writeKey: OwnerWriteKey,
+  ) => boolean;
+
+  /** Sets the {@link OwnerWriteKey} for the given {@link Owner}. */
+  readonly setWriteKey: (
+    ownerId: OwnerIdBytes,
+    writeKey: OwnerWriteKey,
+  ) => boolean;
+
+  /**
+   * Write encrypted {@link CrdtMessage}s to storage.
+   *
+   * Must use a mutex per ownerId to ensure sequential processing and proper
+   * protocol logic handling during sync operations.
+   *
+   * TODO: Use MaybeAsync
+   */
+  readonly writeMessages: (
+    ownerIdBytes: OwnerIdBytes,
+    messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>,
+  ) => MaybeAsync<Result<void, StorageWriteError | StorageQuotaError>>;
+
+  /** Read encrypted {@link DbChange}s from storage. */
+  readonly readDbChange: (
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
+  ) => EncryptedDbChange | null;
+
+  /**
+   * Delete all data for the given {@link Owner}.
+   *
+   * Returns `true` on success, `false` on failure.
+   */
+  readonly deleteOwner: (ownerId: OwnerIdBytes) => boolean;
+}
+
+export interface StorageDep {
+  readonly storage: Storage;
+}
+
+/** Error indicating a serious write failure. */
+export interface StorageWriteError extends BaseOwnerError {
+  readonly type: "StorageWriteError";
+}
+
+/** Error when storage or billing quota is exceeded. */
+export interface StorageQuotaError extends BaseOwnerError {
+  readonly type: "StorageQuotaError";
+}
+
 /**
- * Evolu Storage for SQLite
+ * A cryptographic hash used for efficiently comparing collections of
+ * {@link TimestampBytes}s.
+ *
+ * It consists of the first {@link fingerprintSize} bytes of the SHA-256 hash of
+ * one or more timestamps.
+ */
+export type Fingerprint = Uint8Array & Brand<"Fingerprint">;
+
+export const fingerprintSize = NonNegativeInt.orThrow(12);
+
+/** A fingerprint of an empty range. */
+export const zeroFingerprint = new Uint8Array(fingerprintSize) as Fingerprint;
+
+export interface BaseRange {
+  readonly upperBound: RangeUpperBound;
+}
+
+/**
+ * Union type for Range's upperBound: either a {@link TimestampBytes} or
+ * {@link InfiniteUpperBound}.
+ */
+export type RangeUpperBound = TimestampBytes | InfiniteUpperBound;
+
+export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
+export type InfiniteUpperBound = typeof InfiniteUpperBound;
+
+export const RangeType = {
+  Fingerprint: 1,
+  Skip: 0,
+  Timestamps: 2,
+} as const;
+
+export type RangeType = (typeof RangeType)[keyof typeof RangeType];
+
+export interface SkipRange extends BaseRange {
+  readonly type: typeof RangeType.Skip;
+}
+
+export interface FingerprintRange extends BaseRange {
+  readonly type: typeof RangeType.Fingerprint;
+  readonly fingerprint: Fingerprint;
+}
+
+export interface TimestampsRange extends BaseRange {
+  readonly type: typeof RangeType.Timestamps;
+  readonly timestamps: ReadonlyArray<TimestampBytes>;
+}
+
+export type Range = SkipRange | FingerprintRange | TimestampsRange;
+
+/** An encrypted {@link CrdtMessage}. */
+export interface EncryptedCrdtMessage {
+  readonly timestamp: Timestamp;
+  readonly change: EncryptedDbChange;
+}
+
+/** Encrypted DbChange */
+export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
+
+/**
+ * A CRDT message combining a unique {@link Timestamp} with a {@link DbChange}.
+ *
+ * Used in Evolu's sync protocol to replicate data changes across devices. Evolu
+ * operates as a durable queue, providing exactly-once delivery guarantees for
+ * reliable synchronization across application restarts and network failures.
+ */
+export interface CrdtMessage {
+  readonly timestamp: Timestamp;
+  readonly change: DbChange;
+}
+
+export const DbChangeValues = record(String, SqliteValue);
+export type DbChangeValues = typeof DbChangeValues.Type;
+
+// "createdAt" and "updatedAt" are derived from Timestamp.
+// "id" and "isDeleted" are encoded separately to save bytes
+// "ownerId" is part of the protocol message
+const forbiddenSystemColumns = systemColumns.concat("id");
+
+export const ValidDbChangeValues = brand(
+  "ValidDbChangeValues",
+  DbChangeValues,
+  (value) => {
+    const invalidColumns = forbiddenSystemColumns.filter((key) => key in value);
+    if (invalidColumns.length > 0)
+      return err<ValidDbChangeValuesError>({
+        type: "ValidDbChangeValues",
+        value,
+        invalidColumns,
+      });
+
+    return ok(value);
+  },
+);
+export type ValidDbChangeValues = typeof ValidDbChangeValues.Type;
+
+export interface ValidDbChangeValuesError
+  extends TypeError<"ValidDbChangeValues"> {
+  readonly invalidColumns: ReadonlyArray<string>;
+}
+
+/**
+ * A DbChange is a change to a table row. Together with a unique
+ * {@link Timestamp}, it forms a {@link CrdtMessage}.
+ */
+export const DbChange = object({
+  table: String,
+  id: Id,
+  values: ValidDbChangeValues,
+  isInsert: Boolean,
+  isDelete: nullOr(Boolean),
+});
+export type DbChange = typeof DbChange.Type;
+
+/**
+ * Common interface for both client and relay SQLite storages.
  *
  * Evolu uses a Skiplist, which leverages SQLite indexes. The core logic is
  * implemented in SQL, so it doesn't have to make roundtrips to the DB.
  *
- * The ideal storage for a Relay should use a similar architecture to
+ * While the SQL implementation may look sophisticated, it's conceptually simple
+ * and LLMs can explain how it works. The Skiplist data structure is well
+ * explained in [this Stack Overflow
+ * answer](https://stackoverflow.com/questions/61944198/what-is-a-zip-tree-and-how-does-it-work).
+ * The logic resembles [Negentropy's C++
+ * storage](https://github.com/hoytech/negentropy), except we use a Skiplist to
+ * leverage SQLite indexes, which makes the code simpler.
+ *
+ * Note: A paid review by the SQLite team is planned, as they use the same
+ * algorithm for their rsync tool.
+ *
+ * The ideal storage for a Relay should use an architecture like
  * [strfry](https://github.com/hoytech/strfry) (a KV storage), but with Skiplist
  * to ensure that insertion order doesn't matter (local-first apps can often
  * write in the past.)
@@ -19,117 +328,104 @@
  * each other, if necessary. One relay should handle hundreds of thousands of
  * users, and when it goes down, nothing happens, because it will be
  * synchronized later.
- *
- * @module
  */
-
-import { assert } from "../Assert.js";
-import { decrement } from "../Number.js";
-import { RandomDep } from "../Random.js";
-import { ok, Result } from "../Result.js";
-import { sql, SqliteDep, SqliteError } from "../Sqlite.js";
-import { Int64String, NonNegativeInt, PositiveInt } from "../Type.js";
-import { Brand } from "../Types.js";
-import { OwnerId } from "./Owner.js";
-import {
-  BinaryOwnerId,
-  binaryOwnerIdToOwnerId,
-  binaryTimestampToFingerprint,
-  Fingerprint,
-  FingerprintRange,
-  InfiniteUpperBound,
-  RangeType,
-  RangeUpperBound,
-  Storage,
-  zeroFingerprint,
-} from "./Protocol.js";
-import { BinaryTimestamp, orderBinaryTimestamp } from "./Timestamp.js";
-
-/** Common interface for both client and relay SQLite storages. */
-export interface SqliteStorageBase {
+export interface BaseSqliteStorage
+  extends Pick<
+    Storage,
+    | "getSize"
+    | "fingerprint"
+    | "fingerprintRanges"
+    | "findLowerBound"
+    | "iterate"
+    | "deleteOwner"
+  > {
+  /** Inserts a timestamp for an owner into the skiplist-based storage. */
   readonly insertTimestamp: (
-    ownerId: BinaryOwnerId,
-    timestamp: BinaryTimestamp,
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
+    strategy: StorageInsertTimestampStrategy,
   ) => Result<void, SqliteError>;
 
-  readonly getSize: Storage["getSize"];
-  readonly fingerprint: Storage["fingerprint"];
-  readonly fingerprintRanges: Storage["fingerprintRanges"];
-  readonly findLowerBound: Storage["findLowerBound"];
-  readonly iterate: Storage["iterate"];
-  readonly deleteOwner: Storage["deleteOwner"];
+  /**
+   * Efficiently checks which timestamps already exist in the database using a
+   * single CTE query instead of N individual queries.
+   */
+  readonly getExistingTimestamps: (
+    ownerIdBytes: OwnerIdBytes,
+    timestampsBytes: NonEmptyReadonlyArray<TimestampBytes>,
+  ) => Result<ReadonlyArray<TimestampBytes>, SqliteError>;
 }
 
-export interface SqliteStorageBaseDep {
-  readonly storage: SqliteStorageBase;
+export interface BaseSqliteStorageDep {
+  readonly storage: BaseSqliteStorage;
 }
 
-export type SqliteStorageDeps = SqliteDep & RandomDep;
+export type SqliteStorageDeps = RandomDep & SqliteDep;
 
-export interface CreateSqliteStorageBaseOptions {
+export interface CreateBaseSqliteStorageConfig extends StorageConfig {
   onStorageError: (error: SqliteError) => void;
 }
 
-export const createSqliteStorageBase =
+/**
+ * Creates a {@link BaseSqliteStorage} implementation.
+ *
+ * # Stateless Design
+ *
+ * This implementation is fully stateless - it requires no in-memory state
+ * between invocations. All necessary metadata (timestamp bounds for insertion
+ * strategy optimization) is persisted in the evolu_usage table. This makes
+ * Evolu Relay suitable for stateless serverless environments like AWS Lambda,
+ * Cloudflare Workers with Durable Objects, and other platforms where memory
+ * doesn't persist between requests. While not extensively tested in all these
+ * environments yet, the stateless design should work well across them.
+ */
+export const createBaseSqliteStorage =
   (deps: SqliteStorageDeps) =>
-  (
-    options: CreateSqliteStorageBaseOptions,
-  ): Result<SqliteStorageBase, SqliteError> => {
-    const createTablesResult = createTables(deps);
-    if (!createTablesResult.ok) return createTablesResult;
-
-    const ownerStats = new Map<
-      OwnerId,
-      {
-        minT: BinaryTimestamp;
-        maxT: BinaryTimestamp;
-      }
-    >();
-
-    return ok({
-      insertTimestamp: (ownerId: BinaryOwnerId, timestamp: BinaryTimestamp) => {
-        const ownerIdString = binaryOwnerIdToOwnerId(ownerId);
+  (config: CreateBaseSqliteStorageConfig): BaseSqliteStorage => {
+    return {
+      insertTimestamp: (
+        ownerId: OwnerIdBytes,
+        timestamp: TimestampBytes,
+        strategy: StorageInsertTimestampStrategy,
+      ) => {
         const level = randomSkiplistLevel(deps);
+        return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
+      },
 
-        let stats = ownerStats.get(ownerIdString);
-
-        if (!stats) {
-          const result = deps.sqlite.exec<{
-            maxT: BinaryTimestamp | null;
-            minT: BinaryTimestamp | null;
-          }>(sql.prepared`
-            select min(t) as minT, max(t) as maxT
-            from evolu_timestamp
-            where ownerId = ${ownerId};
-          `);
-          if (!result.ok) return result;
-
-          stats = {
-            minT: result.value.rows[0].minT ?? timestamp,
-            maxT: result.value.rows[0].maxT ?? timestamp,
-          };
-          ownerStats.set(ownerIdString, stats);
-        }
-
-        let strategy: InsertTimestampStrategy;
+      getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
+        const concatenatedTimestamps = concatBytes(...timestampsBytes);
+
+        const result = deps.sqlite.exec<{
+          timestampBytes: TimestampBytes;
+        }>(sql`
+          with recursive
+            split_timestamps(timestampBytes, pos) as (
+              select
+                substr(${concatenatedTimestamps}, 1, 16),
+                17 as pos
+              union all
+              select
+                substr(${concatenatedTimestamps}, pos, 16),
+                pos + 16
+              from split_timestamps
+              where pos <= length(${concatenatedTimestamps})
+            )
+          select s.timestampBytes
+          from
+            split_timestamps s
+            join evolu_timestamp t
+              on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
+        `);
 
-        if (orderBinaryTimestamp(timestamp, stats.maxT) === 1) {
-          strategy = "append";
-          stats.maxT = timestamp;
-        } else if (orderBinaryTimestamp(timestamp, stats.minT) === -1) {
-          strategy = "prepend";
-          stats.minT = timestamp;
-        } else {
-          strategy = "insert";
-        }
+        if (!result.ok) return result;
 
-        return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
+        return ok(result.value.rows.map((row) => row.timestampBytes));
       },
 
       getSize: (ownerId) => {
         const size = getSize(deps)(ownerId);
         if (!size.ok) {
-          options.onStorageError(size.error);
+          config.onStorageError(size.error);
           return null;
         }
         return size.value;
@@ -139,7 +435,7 @@ export const createSqliteStorageBase =
         assertBeginEnd(begin, end);
         const result = fingerprint(deps)(ownerId, begin, end);
         if (!result.ok) {
-          options.onStorageError(result.error);
+          config.onStorageError(result.error);
           return null;
         }
         return result.value;
@@ -148,7 +444,7 @@ export const createSqliteStorageBase =
       fingerprintRanges: (ownerId, buckets, upperBound) => {
         const ranges = fingerprintRanges(deps)(ownerId, buckets, upperBound);
         if (!ranges.ok) {
-          options.onStorageError(ranges.error);
+          config.onStorageError(ranges.error);
           return null;
         }
         return ranges.value;
@@ -162,7 +458,7 @@ export const createSqliteStorageBase =
           upperBound,
         );
         if (!lowerBound.ok) {
-          options.onStorageError(lowerBound.error);
+          config.onStorageError(lowerBound.error);
           return null;
         }
         return lowerBound.value;
@@ -176,7 +472,7 @@ export const createSqliteStorageBase =
         // This is much faster than SQL limit with offset.
         const first = getTimestampByIndex(deps)(ownerId, begin);
         if (!first.ok) {
-          options.onStorageError(first.error);
+          config.onStorageError(first.error);
           return;
         }
 
@@ -193,7 +489,7 @@ export const createSqliteStorageBase =
          * implementing chunking, be sure to run performance tests (including
          * fetching one by one).
          */
-        const result = deps.sqlite.exec<{ t: BinaryTimestamp }>(sql`
+        const result = deps.sqlite.exec<{ t: TimestampBytes }>(sql`
           select t
           from evolu_timestamp
           where ownerId = ${ownerId} and t > ${first.value}
@@ -201,12 +497,12 @@ export const createSqliteStorageBase =
           limit ${length - 1};
         `);
         if (!result.ok) {
-          options.onStorageError(result.error);
+          config.onStorageError(result.error);
           return;
         }
 
         for (let i = 0; i < result.value.rows.length; i++) {
-          const index = (begin + 1 + i) as NonNegativeInt;
+          const index = NonNegativeInt.orThrow(begin + 1 + i);
           if (!callback(result.value.rows[i].t, index)) return;
         }
       },
@@ -216,19 +512,21 @@ export const createSqliteStorageBase =
           delete from evolu_timestamp where ownerId = ${ownerId};
         `);
         if (!result.ok) {
-          options.onStorageError(result.error);
+          config.onStorageError(result.error);
           return false;
         }
         return true;
       },
-    });
+    };
   };
 
 const assertBeginEnd = (begin: NonNegativeInt, end: NonNegativeInt) => {
   assert(begin <= end, "invalid begin or end");
 };
 
-const createTables = (deps: SqliteDep): Result<void, SqliteError> => {
+export const createBaseSqliteStorageTables = (
+  deps: SqliteDep,
+): Result<void, SqliteError> => {
   for (const query of [
     /**
      * Creates the `evolu_timestamp` table for storing timestamps of multiple
@@ -241,19 +539,13 @@ const createTables = (deps: SqliteDep): Result<void, SqliteError> => {
      *
      * Columns:
      *
-     * - `t` – globally unique binary timestamp
+     * - `t` – TimestampBytes
      * - `h1`/`h2` – 12-byte fingerprint split into two integers for fast XOR
      * - `c` – incremental count
-     * - `l` – Skiplist level (1 to 32)
-     *
-     * For scaling or isolation, sharding is possible—each owner can have a
-     * separate SQLite database.
-     *
-     * Maybe we could use an integer surrogate key for ownerId, but it's fast
-     * enough even without it.
+     * - `l` – Skiplist level (1 to 10)
      */
     sql`
-      create table if not exists evolu_timestamp (
+      create table evolu_timestamp (
         "ownerId" blob not null,
         "t" blob not null,
         "h1" integer,
@@ -266,7 +558,7 @@ const createTables = (deps: SqliteDep): Result<void, SqliteError> => {
     `,
 
     sql`
-      create index if not exists evolu_timestamp_index on evolu_timestamp (
+      create index evolu_timestamp_index on evolu_timestamp (
         "ownerId",
         "l",
         "t",
@@ -275,6 +567,26 @@ const createTables = (deps: SqliteDep): Result<void, SqliteError> => {
         "c"
       );
     `,
+
+    /**
+     * Creates the `evolu_usage` table for tracking data consumption per owner.
+     *
+     * Columns:
+     *
+     * - `ownerId` – OwnerIdBytes (primary key)
+     * - `storedBytes` – total bytes stored in database
+     * - `firstTimestamp` – for timestamp insertion strategies
+     * - `lastTimestamp` – for timestamp insertion strategies
+     */
+    sql`
+      create table evolu_usage (
+        "ownerId" blob primary key,
+        "storedBytes" integer not null,
+        "firstTimestamp" blob,
+        "lastTimestamp" blob
+      )
+      strict;
+    `,
   ]) {
     const result = deps.sqlite.exec(query);
     if (!result.ok) return result;
@@ -282,26 +594,56 @@ const createTables = (deps: SqliteDep): Result<void, SqliteError> => {
   return ok();
 };
 
-type InsertTimestampStrategy = "append" | "prepend" | "insert";
+export type StorageInsertTimestampStrategy = "append" | "prepend" | "insert";
+
+/**
+ * Determines the insertion strategy for a timestamp based on its position
+ * relative to the current first and last timestamps.
+ *
+ * Returns a tuple with the strategy and updated timestamp bounds.
+ */
+export const getTimestampInsertStrategy = (
+  timestamp: TimestampBytes,
+  firstTimestamp: TimestampBytes,
+  lastTimestamp: TimestampBytes,
+): [
+  strategy: StorageInsertTimestampStrategy,
+  firstTimestamp: TimestampBytes,
+  lastTimestamp: TimestampBytes,
+] => {
+  if (orderTimestampBytes(timestamp, lastTimestamp) === 1) {
+    return ["append", firstTimestamp, timestamp];
+  }
+  if (orderTimestampBytes(timestamp, firstTimestamp) === -1) {
+    return ["prepend", timestamp, lastTimestamp];
+  }
+  return ["insert", firstTimestamp, lastTimestamp];
+};
 
-// AFAIK, we can't do both insert and update in one query, and that's probably
-// why append is 2x faster than insert. Prepend also has to update parents, but
-// it's constantly fast. Insert degrades for reversed (yet LIMIT X magically
-// makes it much faster) but it's OK for append. It's probably because it's the
-// most complicated SQL, but I believe it can be simplified. If not, we can
-// optimize prepending by reversing the incoming timestamps if we detect that
-// they will prepend. They are always sorted in ascending order by the
-// Protocol.
+/**
+ * AFAIK, we can't do both insert and update in one query, and that's probably
+ * why append is 2x faster than insert. Prepend also has to update parents, but
+ * it's constantly fast. Insert degrades for reversed (yet LIMIT X magically
+ * fixes that) but it's OK for append.
+ *
+ * Note: SQL operations are idempotent (using `on conflict do nothing` and
+ * `changes() > 0`), but this is no longer required here since we use
+ * {@link BaseSqliteStorage.getExistingTimestamps} to filter out duplicates
+ * before insertion, which we need for quota checks anyway.
+ *
+ * TODO: Remove idempotency (`on conflict do nothing` and `changes() > 0`) since
+ * duplicates are now filtered before insertion.
+ */
 const insertTimestamp =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
-    timestamp: BinaryTimestamp,
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
     level: PositiveInt,
-    strategy: InsertTimestampStrategy,
+    strategy: StorageInsertTimestampStrategy,
   ): Result<void, SqliteError> => {
     const [h1, h2] = fingerprintToSqliteFingerprint(
-      binaryTimestampToFingerprint(timestamp),
+      timestampBytesToFingerprint(timestamp),
     );
 
     let queries: Array<ReturnType<typeof sql.prepared>> = [];
@@ -819,6 +1161,13 @@ const insertTimestamp =
     return ok();
   };
 
+export const timestampBytesToFingerprint = (
+  timestamp: TimestampBytes,
+): Fingerprint => {
+  const hash = sha256(timestamp).slice(0, fingerprintSize);
+  return hash as Fingerprint;
+};
+
 /**
  * Generates a random skiplist level in the range [1, skiplistMaxLevel].
  * Probabilistic approach avoids the need for explicit tree balancing.
@@ -831,7 +1180,7 @@ const randomSkiplistLevel = (deps: RandomDep): PositiveInt => {
   ) {
     level += 1;
   }
-  return level as PositiveInt;
+  return PositiveInt.orThrow(level);
 };
 
 /**
@@ -899,7 +1248,7 @@ const sqliteFingerprintToFingerprint = ([
 
 const getSize =
   (deps: SqliteDep) =>
-  (ownerId: BinaryOwnerId): Result<NonNegativeInt, SqliteError> => {
+  (ownerId: OwnerIdBytes): Result<NonNegativeInt, SqliteError> => {
     const result = deps.sqlite.exec<{ size: NonNegativeInt }>(sql.prepared`
       with
         ml(ml) as (
@@ -942,7 +1291,7 @@ const getSize =
 const findLowerBound =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     begin: NonNegativeInt,
     end: NonNegativeInt,
     upperBound: RangeUpperBound,
@@ -954,7 +1303,7 @@ const findLowerBound =
     }
 
     const result = deps.sqlite.exec<{
-      t: BinaryTimestamp;
+      t: TimestampBytes;
     }>(sql.prepared`
       select t
       from evolu_timestamp
@@ -972,14 +1321,14 @@ const findLowerBound =
     if (!count.ok) return count;
 
     // `decrement` converts a count to an index.
-    return ok(decrement(count.value) as NonNegativeInt);
+    return ok(NonNegativeInt.orThrow(decrement(count.value)));
   };
 
 const getTimestampCount =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
-    timestamp: BinaryTimestamp,
+    ownerId: OwnerIdBytes,
+    timestamp: TimestampBytes,
   ): Result<PositiveInt, SqliteError> => {
     const result = deps.sqlite.exec<{
       count: PositiveInt;
@@ -1036,7 +1385,7 @@ const getTimestampCount =
 const fingerprint =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     begin: NonNegativeInt,
     end: NonNegativeInt,
   ): Result<Fingerprint, SqliteError> => {
@@ -1068,14 +1417,14 @@ const fingerprint =
 const fingerprintRanges =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     buckets: ReadonlyArray<NonNegativeInt>,
     upperBound: RangeUpperBound = InfiniteUpperBound,
   ): Result<ReadonlyArray<FingerprintRange>, SqliteError> => {
     const bucketsJson = JSON.stringify(buckets);
 
     const result = deps.sqlite.exec<{
-      b: BinaryTimestamp | null;
+      b: TimestampBytes | null;
       h1: Int64String;
       h2: Int64String;
     }>(sql.prepared`
@@ -1211,11 +1560,11 @@ const x = (a: string, b: string) => sql.raw(`(${a} | ${b}) - (${a} & ${b})`);
 export const getTimestampByIndex =
   (deps: SqliteDep) =>
   (
-    ownerId: BinaryOwnerId,
+    ownerId: OwnerIdBytes,
     index: NonNegativeInt,
-  ): Result<BinaryTimestamp, SqliteError> => {
+  ): Result<TimestampBytes, SqliteError> => {
     const result = deps.sqlite.exec<{
-      readonly pt: BinaryTimestamp;
+      readonly pt: TimestampBytes;
     }>(sql.prepared`
       with
         fi(b, cl, ic, pt, mt, nt, nc) as (
@@ -1291,3 +1640,76 @@ export const getTimestampByIndex =
     if (!result.ok) return result;
     return ok(result.value.rows[0].pt);
   };
+
+/** Retrieves usage information for an owner from the evolu_usage table. */
+export const getOwnerUsage =
+  (deps: SqliteDep) =>
+  (
+    ownerIdBytes: OwnerIdBytes,
+    initialTimestamp: TimestampBytes,
+  ): Result<
+    {
+      storedBytes: NonNegativeInt | null;
+      firstTimestamp: TimestampBytes;
+      lastTimestamp: TimestampBytes;
+    },
+    SqliteError
+  > => {
+    const result = deps.sqlite.exec<{
+      storedBytes: NonNegativeInt;
+      firstTimestamp: TimestampBytes | null;
+      lastTimestamp: TimestampBytes | null;
+    }>(sql`
+      select storedBytes, firstTimestamp, lastTimestamp
+      from evolu_usage
+      where ownerId = ${ownerIdBytes};
+    `);
+    if (!result.ok) return result;
+
+    if (!isNonEmptyReadonlyArray(result.value.rows)) {
+      return ok({
+        storedBytes: null,
+        firstTimestamp: initialTimestamp,
+        lastTimestamp: initialTimestamp,
+      });
+    }
+
+    const row = firstInArray(result.value.rows);
+    assert(row.firstTimestamp, "not null");
+    assert(row.lastTimestamp, "not null");
+
+    return ok({
+      storedBytes: row.storedBytes,
+      firstTimestamp: row.firstTimestamp,
+      lastTimestamp: row.lastTimestamp,
+    });
+  };
+
+/**
+ * Updates timestamp bounds in evolu_usage table.
+ *
+ * Used by both relay and client to maintain firstTimestamp/lastTimestamp after
+ * processing messages.
+ */
+export const updateOwnerUsage =
+  (deps: SqliteDep) =>
+  (
+    ownerIdBytes: OwnerIdBytes,
+    storedBytes: PositiveInt,
+    firstTimestamp: TimestampBytes,
+    lastTimestamp: TimestampBytes,
+  ): Result<void, SqliteError> => {
+    const result = deps.sqlite.exec(sql`
+      insert into evolu_usage
+        ("ownerId", "storedBytes", "firstTimestamp", "lastTimestamp")
+      values
+        (${ownerIdBytes}, ${storedBytes}, ${firstTimestamp}, ${lastTimestamp})
+      on conflict (ownerId) do update
+        set
+          storedBytes = ${storedBytes},
+          firstTimestamp = ${firstTimestamp},
+          lastTimestamp = ${lastTimestamp};
+    `);
+    if (!result.ok) return result;
+    return ok();
+  };