@evolu/common 6.0.1-preview.3 → 6.0.1-preview.30

package/dist/src/Evolu/Storage.d.ts

@@ -1,10 +1,196 @@
+import { NonEmptyReadonlyArray } from "../Array.js";
+import { Brand } from "../Brand.js";
+import { RandomDep } from "../Random.js";
+import { Result } from "../Result.js";
+import { SqliteDep, SqliteError } from "../Sqlite.js";
+import { MaybeAsync } from "../Task.js";
+import { NonNegativeInt, PositiveInt } from "../Type.js";
+import { BaseOwnerError, OwnerId, OwnerIdBytes, OwnerWriteKey } from "./Owner.js";
+import { Timestamp, TimestampBytes } from "./Timestamp.js";
+export interface StorageConfig {
+    /**
+     * Callback called before an attempt to write, to check if an {@link OwnerId}
+     * has sufficient quota for the write.
+     *
+     * The callback receives the {@link OwnerId} and the number of bytes required
+     * for the write, and returns a {@link MaybeAsync} boolean: `true` to allow the
+     * write, or `false` to deny it due to quota limits.
+     *
+     * The callback can be synchronous (for SQLite or in-memory checks) or
+     * asynchronous (for calling remote APIs).
+     *
+     * The callback returns a boolean rather than an error type because error
+     * handling and logging are the responsibility of the callback
+     * implementation.
+     *
+     * ### Example
+     *
+     * ```ts
+     * // Client
+     * // evolu.subscribeError
+     *
+     * // Relay
+     * isOwnerWithinQuota: (ownerId, requiredBytes) => {
+     *   console.log(ownerId, requiredBytes);
+     *   // Check error via evolu.subscribeError
+     *   return true;
+     * };
+     * ```
+     */
+    readonly isOwnerWithinQuota: (ownerId: OwnerId, requiredBytes: PositiveInt) => MaybeAsync<boolean>;
+}
 /**
- * Evolu Storage for SQLite
+ * Evolu Storage
+ *
+ * Evolu protocol using Storage is agnostic to storage implementation
+ * details—any storage can be plugged in, as long as it implements this
+ * interface. Implementations must handle their own errors; return values only
+ * indicate overall success or failure.
+ *
+ * The Storage API is synchronous because SQLite's synchronous API is the
+ * fastest way to use SQLite. Synchronous bindings (like better-sqlite3) call
+ * SQLite's C API directly with no context switching between the event loop and
+ * native code, and no promise microtasks or await overhead.
+ *
+ * The only exception is {@link Storage#writeMessages}, which is async to allow
+ * for async validation logic before writing to storage. The write operation
+ * itself remains synchronous.
+ */
+export interface Storage {
+    readonly getSize: (ownerId: OwnerIdBytes) => NonNegativeInt | null;
+    readonly fingerprint: (ownerId: OwnerIdBytes, begin: NonNegativeInt, end: NonNegativeInt) => Fingerprint | null;
+    /**
+     * Computes fingerprints with their upper bounds in one call.
+     *
+     * This function can be replaced with many fingerprint/findLowerBound calls,
+     * but implementations can leverage it for batching and more efficient
+     * fingerprint computation.
+     */
+    readonly fingerprintRanges: (ownerId: OwnerIdBytes, buckets: ReadonlyArray<NonNegativeInt>, upperBound?: RangeUpperBound) => ReadonlyArray<FingerprintRange> | null;
+    readonly findLowerBound: (ownerId: OwnerIdBytes, begin: NonNegativeInt, end: NonNegativeInt, upperBound: RangeUpperBound) => NonNegativeInt | null;
+    readonly iterate: (ownerId: OwnerIdBytes, begin: NonNegativeInt, end: NonNegativeInt, callback: (timestamp: TimestampBytes, index: NonNegativeInt) => boolean) => void;
+    /**
+     * Validates the {@link OwnerWriteKey} for the given {@link Owner}.
+     *
+     * Returns `true` if the write key is valid, `false` otherwise.
+     */
+    readonly validateWriteKey: (ownerId: OwnerIdBytes, writeKey: OwnerWriteKey) => boolean;
+    /** Sets the {@link OwnerWriteKey} for the given {@link Owner}. */
+    readonly setWriteKey: (ownerId: OwnerIdBytes, writeKey: OwnerWriteKey) => boolean;
+    /**
+     * Write encrypted {@link CrdtMessage}s to storage.
+     *
+     * Must use a mutex per ownerId to ensure sequential processing and proper
+     * protocol logic handling during sync operations.
+     *
+     * TODO: Use MaybeAsync
+     */
+    readonly writeMessages: (ownerIdBytes: OwnerIdBytes, messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>) => MaybeAsync<Result<void, StorageWriteError | StorageQuotaError>>;
+    /** Read encrypted {@link DbChange}s from storage. */
+    readonly readDbChange: (ownerId: OwnerIdBytes, timestamp: TimestampBytes) => EncryptedDbChange | null;
+    /**
+     * Delete all data for the given {@link Owner}.
+     *
+     * Returns `true` on success, `false` on failure.
+     */
+    readonly deleteOwner: (ownerId: OwnerIdBytes) => boolean;
+}
+export interface StorageDep {
+    readonly storage: Storage;
+}
+/** Error indicating a serious write failure. */
+export interface StorageWriteError extends BaseOwnerError {
+    readonly type: "StorageWriteError";
+}
+/** Error when storage or billing quota is exceeded. */
+export interface StorageQuotaError extends BaseOwnerError {
+    readonly type: "StorageQuotaError";
+}
+/**
+ * A cryptographic hash used for efficiently comparing collections of
+ * {@link TimestampBytes}s.
+ *
+ * It consists of the first {@link fingerprintSize} bytes of the SHA-256 hash of
+ * one or more timestamps.
+ */
+export type Fingerprint = Uint8Array & Brand<"Fingerprint">;
+export declare const fingerprintSize: number & Brand<"Int"> & Brand<"NonNegative">;
+/** A fingerprint of an empty range. */
+export declare const zeroFingerprint: Fingerprint;
+export interface BaseRange {
+    readonly upperBound: RangeUpperBound;
+}
+/**
+ * Union type for Range's upperBound: either a {@link TimestampBytes} or
+ * {@link InfiniteUpperBound}.
+ */
+export type RangeUpperBound = TimestampBytes | InfiniteUpperBound;
+export declare const InfiniteUpperBound: unique symbol;
+export type InfiniteUpperBound = typeof InfiniteUpperBound;
+export declare const RangeType: {
+    readonly Fingerprint: 1;
+    readonly Skip: 0;
+    readonly Timestamps: 2;
+};
+export type RangeType = (typeof RangeType)[keyof typeof RangeType];
+export interface SkipRange extends BaseRange {
+    readonly type: typeof RangeType.Skip;
+}
+export interface FingerprintRange extends BaseRange {
+    readonly type: typeof RangeType.Fingerprint;
+    readonly fingerprint: Fingerprint;
+}
+export interface TimestampsRange extends BaseRange {
+    readonly type: typeof RangeType.Timestamps;
+    readonly timestamps: ReadonlyArray<TimestampBytes>;
+}
+export type Range = SkipRange | FingerprintRange | TimestampsRange;
+/** An encrypted {@link CrdtMessage}. */
+export interface EncryptedCrdtMessage {
+    readonly timestamp: Timestamp;
+    readonly change: EncryptedDbChange;
+}
+/** Encrypted DbChange */
+export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
+/**
+ * A CRDT message combining a unique {@link Timestamp} with a {@link DbChange}.
+ *
+ * Used in Evolu's sync protocol to replicate data changes across devices. Evolu
+ * operates as a durable queue, providing exactly-once delivery guarantees for
+ * reliable synchronization across application restarts and network failures.
+ */
+export interface CrdtMessage {
+    readonly timestamp: Timestamp;
+    readonly change: DbChange;
+}
+/**
+ * A DbChange is a change to a table row. Together with a unique
+ * {@link Timestamp}, it forms a {@link CrdtMessage}.
+ */
+export declare const DbChange: import("../Type.js").ObjectType<{
+    table: import("../Type.js").Type<"String", string, string, import("../Type.js").StringError, string, import("../Type.js").StringError>;
+    id: import("../Type.js").BrandType<import("../Type.js").Type<"String", string, string, import("../Type.js").StringError, string, import("../Type.js").StringError>, "Id", import("../Type.js").IdError, import("../Type.js").StringError>;
+    values: import("../Type.js").RecordType<"String", string, string, import("../Type.js").StringError, string, import("../Type.js").StringError, import("../Type.js").UnionType<[import("../Type.js").Type<"Null", null, null, import("../Type.js").NullError, null, import("../Type.js").NullError>, import("../Type.js").Type<"String", string, string, import("../Type.js").StringError, string, import("../Type.js").StringError>, import("../Type.js").Type<"Number", number, number, import("../Type.js").NumberError, number, import("../Type.js").NumberError>, import("../Type.js").Type<"Uint8Array", Uint8Array<ArrayBufferLike>, Uint8Array<ArrayBufferLike>, import("../Type.js").Uint8ArrayError, Uint8Array<ArrayBufferLike>, import("../Type.js").Uint8ArrayError>]>>;
+}>;
+export type DbChange = typeof DbChange.Type;
+/**
+ * Common interface for both client and relay SQLite storages.
  *
  * Evolu uses a Skiplist, which leverages SQLite indexes. The core logic is
  * implemented in SQL, so it doesn't have to make roundtrips to the DB.
  *
- * The ideal storage for a Relay should use a similar architecture to
+ * While the SQL implementation may look sophisticated, it's conceptually simple
+ * and LLMs can explain how it works. The Skiplist data structure is well
+ * explained in [this Stack Overflow
+ * answer](https://stackoverflow.com/questions/61944198/what-is-a-zip-tree-and-how-does-it-work).
+ * The logic resembles [Negentropy's C++
+ * storage](https://github.com/hoytech/negentropy), except we use a Skiplist to
+ * leverage SQLite indexes, which makes the code simpler.
+ *
+ * Note: A paid review by the SQLite team is planned, as they use the same
+ * algorithm for their rsync tool.
+ *
+ * The ideal storage for a Relay should use an architecture like
  * [strfry](https://github.com/hoytech/strfry) (a KV storage), but with Skiplist
  * to ensure that insertion order doesn't matter (local-first apps can often
  * write in the past.)
@@ -19,31 +205,31 @@
  * each other, if necessary. One relay should handle hundreds of thousands of
  * users, and when it goes down, nothing happens, because it will be
  * synchronized later.
- *
- * @module
  */
-import { RandomDep } from "../Random.js";
-import { Result } from "../Result.js";
-import { SqliteDep, SqliteError } from "../Sqlite.js";
-import { NonNegativeInt } from "../Type.js";
-import { BinaryOwnerId, Storage } from "./Protocol.js";
-import { BinaryTimestamp } from "./Timestamp.js";
-/** Common interface for both client and relay SQLite storages. */
-export interface SqliteStorageBase {
-    readonly insertTimestamp: (ownerId: BinaryOwnerId, timestamp: BinaryTimestamp) => Result<void, SqliteError>;
-    readonly getSize: Storage["getSize"];
-    readonly fingerprint: Storage["fingerprint"];
-    readonly fingerprintRanges: Storage["fingerprintRanges"];
-    readonly findLowerBound: Storage["findLowerBound"];
-    readonly iterate: Storage["iterate"];
-}
-export interface SqliteStorageBaseDep {
-    readonly storage: SqliteStorageBase;
-}
-export type SqliteStorageDeps = SqliteDep & RandomDep;
-export interface CreateSqliteStorageBaseOptions {
+export interface BaseSqliteStorage extends Pick<Storage, "getSize" | "fingerprint" | "fingerprintRanges" | "findLowerBound" | "iterate" | "deleteOwner"> {
+    /**
+     * Inserts a timestamp for an owner into the skiplist-based storage.
+     *
+     * Must be idempotent - inserting the same timestamp multiple times has no
+     * effect after the first insertion. This is crucial for sync reliability as
+     * messages may be received and processed multiple times.
+     */
+    readonly insertTimestamp: (ownerId: OwnerIdBytes, timestamp: TimestampBytes) => Result<void, SqliteError>;
+    /**
+     * Efficiently checks which timestamps already exist in the database using a
+     * single CTE query instead of N individual queries.
+     */
+    readonly getExistingTimestamps: (ownerIdBytes: OwnerIdBytes, timestampsBytes: NonEmptyReadonlyArray<TimestampBytes>) => Result<ReadonlyArray<TimestampBytes>, SqliteError>;
+}
+export interface BaseSqliteStorageDep {
+    readonly storage: BaseSqliteStorage;
+}
+export type SqliteStorageDeps = RandomDep & SqliteDep;
+export interface CreateBaseSqliteStorageConfig extends StorageConfig {
     onStorageError: (error: SqliteError) => void;
 }
-export declare const createSqliteStorageBase: (deps: SqliteStorageDeps) => (options: CreateSqliteStorageBaseOptions) => Result<SqliteStorageBase, SqliteError>;
-export declare const getTimestampByIndex: (deps: SqliteDep) => (ownerId: BinaryOwnerId, index: NonNegativeInt) => Result<BinaryTimestamp, SqliteError>;
+export declare const createBaseSqliteStorage: (deps: SqliteStorageDeps) => (config: CreateBaseSqliteStorageConfig) => BaseSqliteStorage;
+export declare const createBaseSqliteStorageTables: (deps: SqliteDep) => Result<void, SqliteError>;
+export declare const timestampBytesToFingerprint: (timestamp: TimestampBytes) => Fingerprint;
+export declare const getTimestampByIndex: (deps: SqliteDep) => (ownerId: OwnerIdBytes, index: NonNegativeInt) => Result<TimestampBytes, SqliteError>;
 //# sourceMappingURL=Storage.d.ts.map

package/dist/src/Evolu/Storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Storage.d.ts","sourceRoot":"","sources":["../../../src/Evolu/Storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAM,MAAM,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAO,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAe,cAAc,EAAe,MAAM,YAAY,CAAC;AAGtE,OAAO,EACL,aAAa,EAQb,OAAO,EAER,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,eAAe,EAAwB,MAAM,gBAAgB,CAAC;AAEvE,kEAAkE;AAClE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,eAAe,EAAE,CACxB,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,eAAe,KACvB,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAE/B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7C,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACnD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;CACrC;AAED,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,CAAC;AAEtD,MAAM,WAAW,8BAA8B;IAC7C,cAAc,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;CAC9C;AAED,eAAO,MAAM,uBAAuB,GACjC,MAAM,iBAAiB,MAEtB,SAAS,8BAA8B,KACtC,MAAM,CAAC,iBAAiB,EAAE,WAAW,CAyIvC,CAAC;AA09BJ,eAAO,MAAM,mBAAmB,GAC7B,MAAM,SAAS,MAEd,SAAS,aAAa,EACtB,OAAO,cAAc,KACpB,MAAM,CAAC,eAAe,EAAE,WAAW,CA6ErC,CAAC"}
+{"version":3,"file":"Storage.d.ts","sourceRoot":"","sources":["../../../src/Evolu/Storage.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEpD,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAGpC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAM,MAAM,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAO,SAAS,EAAE,WAAW,EAAe,MAAM,cAAc,CAAC;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAGL,cAAc,EAEd,WAAW,EAGZ,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,cAAc,EAEd,OAAO,EACP,YAAY,EAEZ,aAAa,EACd,MAAM,YAAY,CAAC;AACpB,OAAO,EAAuB,SAAS,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhF,MAAM,WAAW,aAAa;IAC5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,WAAW,KACvB,UAAU,CAAC,OAAO,CAAC,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,cAAc,GAAG,IAAI,CAAC;IAEnE,QAAQ,CAAC,WAAW,EAAE,CACpB,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,cAAc,EACrB,GAAG,EAAE,cAAc,KAChB,WAAW,GAAG,IAAI,CAAC;IAExB;;;;;;OAMG;IACH,QAAQ,CAAC,iBAAiB,EAAE,CAC1B,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,EACtC,UAAU,CAAC,EAAE,eAAe,KACzB,aAAa,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IAE5C,QAAQ,CAAC,cAAc,EAAE,CACvB,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,cAAc,EACrB,GAAG,EAAE,cAAc,EACnB,UAAU,EAAE,eAAe,KACxB,cAAc,GAAG,IAAI,CAAC;IAE3B,QAAQ,CAAC,OAAO,EAAE,CAChB,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,cAAc,EACrB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,CAAC,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,KAAK,OAAO,KACpE,IAAI,CAAC;IAEV;;;;OAIG;IACH,QAAQ,CAAC,gBAAgB,EAAE,CACzB,OAAO,EAAE,YAAY,EACrB,QAAQ,EAAE,aAAa,KACpB,OAAO,CAAC;IAEb,kEAAkE;IAClE,QAAQ,CAAC,WAAW,EAAE,CACpB,OAAO,EAAE,YAAY,EACrB,QAAQ,EAAE,aAAa,KACpB,OAAO,CAAC;IAEb;;;;;;;OAOG;IACH,QAAQ,CAAC,aAAa,EAAE,CACtB,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,CAAC,oBAAoB,CAAC,KAClD,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,iBAAiB,GAAG,iBAAiB,CAAC,CAAC,CAAC;IAErE,qDAAqD;IACrD,QAAQ,CAAC,YAAY,EAAE,CACrB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,cAAc,KACtB,iBAAiB,GAAG,IAAI,CAAC;IAE9B;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC;CAC1D;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,gDAAgD;AAChD,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;CACpC;AAED,uDAAuD;AACvD,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;CACpC;AAED;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC;AAE5D,eAAO,MAAM,eAAe,8CAA6B,CAAC;AAE1D,uCAAuC;AACvC,eAAO,MAAM,eAAe,EAAsC,WAAW,CAAC;AAE9E,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;CACtC;AAED;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,kBAAkB,CAAC;AAElE,eAAO,MAAM,kBAAkB,eAA+B,CAAC;AAC/D,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC;AAE3D,eAAO,MAAM,SAAS;;;;CAIZ,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,CAAC,MAAM,OAAO,SAAS,CAAC,CAAC;AAEnE,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,QAAQ,CAAC,IAAI,EAAE,OAAO,SAAS,CAAC,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,gBAAiB,SAAQ,SAAS;IACjD,QAAQ,CAAC,IAAI,EAAE,OAAO,SAAS,CAAC,WAAW,CAAC;IAC5C,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;CACnC;AAED,MAAM,WAAW,eAAgB,SAAQ,SAAS;IAChD,QAAQ,CAAC,IAAI,EAAE,OAAO,SAAS,CAAC,UAAU,CAAC;IAC3C,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACpD;AAED,MAAM,MAAM,KAAK,GAAG,SAAS,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAEnE,wCAAwC;AACxC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;CACpC;AAED,yBAAyB;AACzB,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,KAAK,CAAC,mBAAmB,CAAC,CAAC;AAExE;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;CAC3B;AAED;;;GAGG;AACH,eAAO,MAAM,QAAQ;;;;EAInB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,OAAO,QAAQ,CAAC,IAAI,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,WAAW,iBACf,SAAQ,IAAI,CACV,OAAO,EACL,SAAS,GACT,aAAa,GACb,mBAAmB,GACnB,gBAAgB,GAChB,SAAS,GACT,aAAa,CAChB;IACD;;;;;;OAMG;IACH,QAAQ,CAAC,eAAe,EAAE,CACxB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,cAAc,KACtB,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAE/B;;;OAGG;IACH,QAAQ,CAAC,qBAAqB,EAAE,CAC9B,YAAY,EAAE,YAAY,EAC1B,eAAe,EAAE,qBAAqB,CAAC,cAAc,CAAC,KACnD,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE,WAAW,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;CACrC;AAED,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,CAAC;AAEtD,MAAM,WAAW,6BAA8B,SAAQ,aAAa;IAClE,cAAc,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;CAC9C;AAED,eAAO,MAAM,uBAAuB,GACjC,MAAM,iBAAiB,MACvB,QAAQ,6BAA6B,KAAG,iBAgLxC,CAAC;AAMJ,eAAO,MAAM,6BAA6B,GACxC,MAAM,SAAS,KACd,MAAM,CAAC,IAAI,EAAE,WAAW,CA6E1B,CAAC;AA2hBF,eAAO,MAAM,2BAA2B,GACtC,WAAW,cAAc,KACxB,WAGF,CAAC;AAuYF,eAAO,MAAM,mBAAmB,GAC7B,MAAM,SAAS,MAEd,SAAS,YAAY,EACrB,OAAO,cAAc,KACpB,MAAM,CAAC,cAAc,EAAE,WAAW,CA6EpC,CAAC"}

package/dist/src/Evolu/Storage.js

@@ -1,75 +1,38 @@
-/**
- * Evolu Storage for SQLite
- *
- * Evolu uses a Skiplist, which leverages SQLite indexes. The core logic is
- * implemented in SQL, so it doesn't have to make roundtrips to the DB.
- *
- * The ideal storage for a Relay should use a similar architecture to
- * [strfry](https://github.com/hoytech/strfry) (a KV storage), but with Skiplist
- * to ensure that insertion order doesn't matter (local-first apps can often
- * write in the past.)
- *
- * The ideal client implementation should probably use the SQLite extension
- * instead of SQL or even a KV storage, when such a thing for browsers/native
- * will exist and will be faster than SQLite.
- *
- * # Scaling
- *
- * The load can be distributed by deploying multiple relays, synchronized with
- * each other, if necessary. One relay should handle hundreds of thousands of
- * users, and when it goes down, nothing happens, because it will be
- * synchronized later.
- *
- * @module
- */
+import { sha256 } from "@noble/hashes/sha2.js";
 import { assert } from "../Assert.js";
+import { concatBytes } from "../Buffer.js";
 import { decrement } from "../Number.js";
 import { ok } from "../Result.js";
-import { sql } from "../Sqlite.js";
-import { binaryOwnerIdToOwnerId, binaryTimestampToFingerprint, InfiniteUpperBound, RangeType, zeroFingerprint, } from "./Protocol.js";
-import { orderBinaryTimestamp } from "./Timestamp.js";
-export const createSqliteStorageBase = (deps) => (options) => {
-    const createTablesResult = createTables(deps);
-    if (!createTablesResult.ok)
-        return createTablesResult;
+import { sql, SqliteValue } from "../Sqlite.js";
+import { Id, NonNegativeInt, object, PositiveInt, record, String, } from "../Type.js";
+import { ownerIdBytesToOwnerId, } from "./Owner.js";
+import { orderTimestampBytes } from "./Timestamp.js";
+export const fingerprintSize = NonNegativeInt.orThrow(12);
+/** A fingerprint of an empty range. */
+export const zeroFingerprint = new Uint8Array(fingerprintSize);
+export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
+export const RangeType = {
+    Fingerprint: 1,
+    Skip: 0,
+    Timestamps: 2,
+};
+/**
+ * A DbChange is a change to a table row. Together with a unique
+ * {@link Timestamp}, it forms a {@link CrdtMessage}.
+ */
+export const DbChange = object({
+    table: String,
+    id: Id,
+    values: record(String, SqliteValue),
+});
+export const createBaseSqliteStorage = (deps) => (config) => {
+    // TODO: Use evolu_usage table.
     const ownerStats = new Map();
-    return ok({
-        insertTimestamp: (ownerId, timestamp) => {
-            const ownerIdString = binaryOwnerIdToOwnerId(ownerId);
-            const level = randomSkiplistLevel(deps);
-            let stats = ownerStats.get(ownerIdString);
-            if (!stats) {
-                const result = deps.sqlite.exec(sql.prepared `
-            select min(t) as minT, max(t) as maxT
-            from evolu_timestamp
-            where ownerId = ${ownerId};
-          `);
-                if (!result.ok)
-                    return result;
-                stats = {
-                    minT: result.value.rows[0].minT ?? timestamp,
-                    maxT: result.value.rows[0].maxT ?? timestamp,
-                };
-                ownerStats.set(ownerIdString, stats);
-            }
-            let strategy;
-            if (orderBinaryTimestamp(timestamp, stats.maxT) === 1) {
-                strategy = "append";
-                stats.maxT = timestamp;
-            }
-            else if (orderBinaryTimestamp(timestamp, stats.minT) === -1) {
-                strategy = "prepend";
-                stats.minT = timestamp;
-            }
-            else {
-                strategy = "insert";
-            }
-            return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
-        },
+    return {
         getSize: (ownerId) => {
             const size = getSize(deps)(ownerId);
             if (!size.ok) {
-                options.onStorageError(size.error);
+                config.onStorageError(size.error);
                 return null;
             }
             return size.value;
@@ -78,7 +41,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
             assertBeginEnd(begin, end);
             const result = fingerprint(deps)(ownerId, begin, end);
             if (!result.ok) {
-                options.onStorageError(result.error);
+                config.onStorageError(result.error);
                 return null;
             }
             return result.value;
@@ -86,7 +49,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
         fingerprintRanges: (ownerId, buckets, upperBound) => {
             const ranges = fingerprintRanges(deps)(ownerId, buckets, upperBound);
             if (!ranges.ok) {
-                options.onStorageError(ranges.error);
+                config.onStorageError(ranges.error);
                 return null;
             }
             return ranges.value;
@@ -94,7 +57,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
         findLowerBound: (ownerId, begin, end, upperBound) => {
             const lowerBound = findLowerBound(deps)(ownerId, begin, end, upperBound);
             if (!lowerBound.ok) {
-                options.onStorageError(lowerBound.error);
+                config.onStorageError(lowerBound.error);
                 return null;
             }
             return lowerBound.value;
@@ -107,7 +70,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
             // This is much faster than SQL limit with offset.
             const first = getTimestampByIndex(deps)(ownerId, begin);
             if (!first.ok) {
-                options.onStorageError(first.error);
+                config.onStorageError(first.error);
                 return;
             }
             if (!callback(first.value, begin))
@@ -132,21 +95,88 @@ export const createSqliteStorageBase = (deps) => (options) => {
           limit ${length - 1};
         `);
             if (!result.ok) {
-                options.onStorageError(result.error);
+                config.onStorageError(result.error);
                 return;
             }
             for (let i = 0; i < result.value.rows.length; i++) {
-                const index = (begin + 1 + i);
+                const index = NonNegativeInt.orThrow(begin + 1 + i);
                 if (!callback(result.value.rows[i].t, index))
                     return;
             }
         },
-    });
+        deleteOwner: (ownerId) => {
+            const result = deps.sqlite.exec(sql `
+          delete from evolu_timestamp where ownerId = ${ownerId};
+        `);
+            if (!result.ok) {
+                config.onStorageError(result.error);
+                return false;
+            }
+            return true;
+        },
+        insertTimestamp: (ownerId, timestamp) => {
+            const ownerIdString = ownerIdBytesToOwnerId(ownerId);
+            const level = randomSkiplistLevel(deps);
+            let stats = ownerStats.get(ownerIdString);
+            if (!stats) {
+                const result = deps.sqlite.exec(sql.prepared `
+            select min(t) as minT, max(t) as maxT
+            from evolu_timestamp
+            where ownerId = ${ownerId};
+          `);
+                if (!result.ok)
+                    return result;
+                stats = {
+                    minT: result.value.rows[0].minT ?? timestamp,
+                    maxT: result.value.rows[0].maxT ?? timestamp,
+                };
+                ownerStats.set(ownerIdString, stats);
+            }
+            let strategy;
+            if (orderTimestampBytes(timestamp, stats.maxT) === 1) {
+                strategy = "append";
+                stats.maxT = timestamp;
+            }
+            else if (orderTimestampBytes(timestamp, stats.minT) === -1) {
+                strategy = "prepend";
+                stats.minT = timestamp;
+            }
+            else {
+                strategy = "insert";
+            }
+            return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
+        },
+        getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
+            const concatenatedTimestamps = concatBytes(...timestampsBytes);
+            const result = deps.sqlite.exec(sql `
+          with recursive
+            split_timestamps(timestampBytes, pos) as (
+              select
+                substr(${concatenatedTimestamps}, 1, 16),
+                17 as pos
+              union all
+              select
+                substr(${concatenatedTimestamps}, pos, 16),
+                pos + 16
+              from split_timestamps
+              where pos <= length(${concatenatedTimestamps})
+            )
+          select s.timestampBytes
+          from
+            split_timestamps s
+            join evolu_timestamp t
+              on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
+        `);
+            if (!result.ok)
+                return result;
+            return ok(result.value.rows.map((row) => row.timestampBytes));
+        },
+    };
 };
 const assertBeginEnd = (begin, end) => {
     assert(begin <= end, "invalid begin or end");
 };
-const createTables = (deps) => {
+export const createBaseSqliteStorageTables = (deps) => {
     for (const query of [
         /**
          * Creates the `evolu_timestamp` table for storing timestamps of multiple
@@ -159,7 +189,7 @@ const createTables = (deps) => {
          *
          * Columns:
          *
-         * - `t` – globally unique binary timestamp
+         * - `t` – TimestampBytes
          * - `h1`/`h2` – 12-byte fingerprint split into two integers for fast XOR
          * - `c` – incremental count
          * - `l` – Skiplist level (1 to 32)
@@ -171,7 +201,7 @@ const createTables = (deps) => {
          * enough even without it.
          */
         sql `
-      create table if not exists evolu_timestamp (
+      create table evolu_timestamp (
         "ownerId" blob not null,
         "t" blob not null,
         "h1" integer,
@@ -183,7 +213,7 @@ const createTables = (deps) => {
       strict;
     `,
         sql `
-      create index if not exists evolu_timestamp_index on evolu_timestamp (
+      create index evolu_timestamp_index on evolu_timestamp (
         "ownerId",
         "l",
         "t",
@@ -191,6 +221,30 @@ const createTables = (deps) => {
         "h2",
         "c"
       );
+    `,
+        /**
+         * Creates the `evolu_usage` table for tracking data consumption per owner.
+         *
+         * Columns:
+         *
+         * - `ownerId` – OwnerIdBytes (primary key)
+         * - `storedBytes` – total bytes stored in database
+         * - `receivedBytes` – TODO: Decide how to use
+         * - `sentBytes` – TODO: Decide how to use
+         * - `firstTimestamp` – TODO: Decide how to use (nullable)
+         * - `lastTimestamp` – TODO: Decide how to use (nullable)
+         */
+        sql `
+      create table evolu_usage (
+        "ownerId" blob primary key,
+        "storedBytes" integer not null
+        -- TODO: Decide how to use receivedBytes, sentBytes, firstTimestamp, lastTimestamp
+        -- "receivedBytes" integer not null,
+        -- "sentBytes" integer not null,
+        -- "firstTimestamp" blob,
+        -- "lastTimestamp" blob
+      )
+      strict;
     `,
     ]) {
         const result = deps.sqlite.exec(query);
@@ -208,7 +262,7 @@ const createTables = (deps) => {
 // they will prepend. They are always sorted in ascending order by the
 // Protocol.
 const insertTimestamp = (deps) => (ownerId, timestamp, level, strategy) => {
-    const [h1, h2] = fingerprintToSqliteFingerprint(binaryTimestampToFingerprint(timestamp));
+    const [h1, h2] = fingerprintToSqliteFingerprint(timestampBytesToFingerprint(timestamp));
     let queries = [];
     switch (strategy) {
         case "append":
@@ -717,6 +771,10 @@ const insertTimestamp = (deps) => (ownerId, timestamp, level, strategy) => {
     }
     return ok();
 };
+export const timestampBytesToFingerprint = (timestamp) => {
+    const hash = sha256(timestamp).slice(0, fingerprintSize);
+    return hash;
+};
 /**
  * Generates a random skiplist level in the range [1, skiplistMaxLevel].
  * Probabilistic approach avoids the need for explicit tree balancing.
@@ -727,7 +785,7 @@ const randomSkiplistLevel = (deps) => {
         level < skiplistMaxLevel) {
         level += 1;
     }
-    return level;
+    return PositiveInt.orThrow(level);
 };
 /**
  * Probability used for generating Skiplist levels.
@@ -830,7 +888,7 @@ const findLowerBound = (deps) => (ownerId, begin, end, upperBound) => {
     if (!count.ok)
         return count;
     // `decrement` converts a count to an index.
-    return ok(decrement(count.value));
+    return ok(NonNegativeInt.orThrow(decrement(count.value)));
 };
 const getTimestampCount = (deps) => (ownerId, timestamp) => {
     const result = deps.sqlite.exec(sql.prepared `