@evolu/common 8.10.0 → 8.12.0

package/src/local-first/Protocol.ts

@@ -103,9 +103,9 @@
  * fit within the limit, the protocol automatically continues synchronization in
  * subsequent rounds using range-based reconciliation.
  *
- * Database mutations are limited to 640KB, which is smaller than the protocol
- * message limit to ensure efficient sync with
- * {@link defaultProtocolMessageRangesMaxSize}.
+ * Each mutation is limited to {@link maxMutationSize}, so every change fits one
+ * message of {@link defaultProtocolMessageMaxSize} next to the largest ranges
+ * section.
  *
  * ## Why Binary?
  *
@@ -182,6 +182,7 @@ import {
 } from "../Array.ts";
 import {
   assert,
+  assertNonEmptyArray,
   assertNonNullable,
   assertNotUndefined,
   assertSame,
@@ -189,6 +190,7 @@ import {
 import type { Brand } from "../Brand.ts";
 import {
   type Buffer,
+  BufferError,
   createBuffer,
   createRunLengthEncoder,
   decodeFlags,
@@ -247,6 +249,7 @@ import {
   zeroNonNegativeInt,
 } from "../Type.ts";
 import type { Predicate } from "../Types.ts";
+import type { Evolu, maxMutationSize } from "./Evolu.ts";
 import {
   type Owner,
   type OwnerError,
@@ -271,6 +274,7 @@ import {
   type RangeUpperBound,
   type SkipRange,
   type StorageDep,
+  type StorageWriteMessagesError,
   type TimestampsRange,
 } from "./Storage.ts";
 import {
@@ -367,18 +371,27 @@ export const MessageType = {
 
 export type MessageType = (typeof MessageType)[keyof typeof MessageType];
 
-/** Parsed protocol header for supported protocol messages. */
+/**
+ * The routing prefix of a {@link ProtocolMessage}.
+ *
+ * The version and the {@link OwnerId} begin every message of every protocol
+ * version. The message type follows them only in messages of this peer's
+ * version; the layout after the prefix is unknown for any other version.
+ */
 export interface ProtocolHeader extends Typed<"ProtocolHeader"> {
-  readonly version: 1;
+  readonly version: NonNegativeInt;
   readonly ownerId: OwnerId;
-  readonly messageType: MessageType;
+  /** Absent when `version` differs from {@link protocolVersion}. */
+  readonly messageType?: MessageType;
 }
 
 /**
- * Parses the protocol header needed for routing.
+ * Parses the {@link ProtocolHeader} a transport needs to route a message.
  *
- * Every protocol message begins with `protocolVersion`, `ownerId`, and
- * `messageType`.
+ * Parsing succeeds for versions this peer cannot process, because a
+ * non-initiator answers a version mismatch with only its version and the owner
+ * ID. Rejecting that reply here would drop it before
+ * {@link applyProtocolMessageAsClient} can report {@link ProtocolVersionError}.
  */
 export const parseProtocolHeader = (
   inputMessage: Uint8Array,
@@ -468,8 +481,9 @@ export interface ProtocolWriteError
  * still sync normally.
  *
  * Clients should prompt the user to contact the relay provider or upgrade their
- * plan. Quota monitoring and management is the relay provider's
- * responsibility.
+ * plan. Quota monitoring and management is the relay provider's responsibility.
+ * After additional quota is available, call {@link Evolu.requestSync} with the
+ * affected owner's ID to retry locally stored changes.
  */
 export interface ProtocolQuotaError
   extends OwnerError, Typed<"ProtocolQuotaError"> {}
@@ -558,9 +572,87 @@ export const createProtocolMessageFromCrdtMessages =
     return buffer.unwrap();
   };
 
+/**
+ * Creates size-limited broadcast {@link ProtocolMessage}s containing every
+ * supplied {@link CrdtMessage}.
+ *
+ * Broadcasts contain no synchronization ranges or write key. Unlike
+ * {@link createProtocolMessageFromCrdtMessages}, this function splits all
+ * messages into complete frames rather than relying on later synchronization
+ * rounds to deliver messages that do not fit. Each individual message must fit
+ * the configured size limit.
+ *
+ * ### Example
+ *
+ * ```ts
+ * import {
+ *   assertSame,
+ *   createId,
+ *   getOrThrow,
+ *   testCreateDeps,
+ * } from "@evolu/common";
+ * import {
+ *   createProtocolBroadcastMessagesFromCrdtMessages,
+ *   MessageType,
+ *   parseProtocolHeader,
+ *   testAppOwner,
+ *   testCreateCrdtMessage,
+ * } from "@evolu/common/local-first";
+ *
+ * const deps = testCreateDeps();
+ * const broadcasts = createProtocolBroadcastMessagesFromCrdtMessages(deps)(
+ *   testAppOwner,
+ *   [testCreateCrdtMessage(createId(deps), 1, "Ada")],
+ * );
+ * assertSame(broadcasts.length, 1);
+ * assertSame(
+ *   getOrThrow(parseProtocolHeader(broadcasts[0])).messageType,
+ *   MessageType.Broadcast,
+ * );
+ * ```
+ */
+export const createProtocolBroadcastMessagesFromCrdtMessages =
+  (deps: RandomBytesDep) =>
+  (
+    owner: Owner,
+    messages: NonEmptyReadonlyArray<CrdtMessage>,
+    maxSize: ProtocolMessageMaxSize = defaultProtocolMessageMaxSize,
+  ): NonEmptyReadonlyArray<ProtocolMessage> => {
+    const broadcasts: Array<ProtocolMessage> = [];
+    let buffer = createProtocolMessageBuffer(owner.id, {
+      messageType: MessageType.Broadcast,
+      totalMaxSize: maxSize,
+    });
+    for (const message of messages) {
+      const encryptedMessage = {
+        timestamp: message.timestamp,
+        change: encodeAndEncryptDbChange(deps)(message, owner.encryptionKey),
+      };
+
+      if (!buffer.canAddMessage(encryptedMessage)) {
+        const nextBuffer = createProtocolMessageBuffer(owner.id, {
+          messageType: MessageType.Broadcast,
+          totalMaxSize: maxSize,
+        });
+        assert(
+          nextBuffer.canAddMessage(encryptedMessage),
+          "the message is too big",
+        );
+        broadcasts.push(buffer.unwrap());
+        buffer = nextBuffer;
+      }
+
+      buffer.addMessage(encryptedMessage);
+    }
+
+    broadcasts.push(buffer.unwrap());
+    assertNonEmptyArray(broadcasts);
+    return broadcasts;
+  };
+
 /** Creates a {@link ProtocolMessage} for sync. */
 export const createProtocolMessageForSync =
-  (deps: StorageDep & ConsoleDep) =>
+  (deps: StorageDep) =>
   (ownerId: OwnerId, subscriptionFlag?: SubscriptionFlag): ProtocolMessage => {
     const buffer = createProtocolMessageBuffer(ownerId, {
       messageType: MessageType.Request,
@@ -902,21 +994,53 @@ export interface ApplyProtocolMessageAsClientOptions {
 }
 
 /**
- * Result type for {@link applyProtocolMessageAsClient} that distinguishes
- * between responses to client requests and broadcast messages.
+ * Result of {@link applyProtocolMessageAsClient}: a continuation for the
+ * non-initiator.
  */
 export interface ApplyProtocolMessageAsClientResponse extends Typed<"Response"> {
   readonly message: ProtocolMessage;
+  /** The uploaded messages, if any, for delivery to other local databases. */
+  readonly broadcast?: ProtocolMessage;
 }
 
-export interface ApplyProtocolMessageAsClientNoResponse extends Typed<"NoResponse"> {}
+/**
+ * Result of {@link applyProtocolMessageAsClient}: the response needed nothing
+ * more, because its ranges all matched or it carried none.
+ */
+export interface ApplyProtocolMessageAsClientConverged extends Typed<"Converged"> {}
 
+/**
+ * Result of {@link applyProtocolMessageAsClient}: a Broadcast message whose
+ * messages were written.
+ */
 export interface ApplyProtocolMessageAsClientBroadcast extends Typed<"Broadcast"> {}
 
+/**
+ * Result of {@link applyProtocolMessageAsClient}: the messages were written, but
+ * without a write key no ranges are reconciled.
+ */
+export interface ApplyProtocolMessageAsClientReadonly extends Typed<"Readonly"> {}
+
+/**
+ * Result of {@link applyProtocolMessageAsClient}: the protocol logged an
+ * exception thrown by calling the storage's `writeMessages` (`Write`) or a
+ * failed range reconciliation (`Sync`) to the console. An exception while the
+ * returned Task runs, as the built-in storages throw, is a defect that aborts
+ * the Run instead. Expected write rejections return the original
+ * {@link StorageWriteMessagesError} through {@link Result}. Reconciliation can
+ * fail after messages have been committed; that failure does not roll back the
+ * write.
+ */
+export interface ApplyProtocolMessageAsClientFailed extends Typed<"Failed"> {
+  readonly cause: "Write" | "Sync";
+}
+
 export type ApplyProtocolMessageAsClientResult =
   | ApplyProtocolMessageAsClientResponse
-  | ApplyProtocolMessageAsClientNoResponse
-  | ApplyProtocolMessageAsClientBroadcast;
+  | ApplyProtocolMessageAsClientConverged
+  | ApplyProtocolMessageAsClientBroadcast
+  | ApplyProtocolMessageAsClientReadonly
+  | ApplyProtocolMessageAsClientFailed;
 
 export const applyProtocolMessageAsClient =
   (
@@ -924,12 +1048,7 @@ export const applyProtocolMessageAsClient =
     options: ApplyProtocolMessageAsClientOptions = {},
   ): Task<
     ApplyProtocolMessageAsClientResult,
-    | ProtocolInvalidDataError
-    | ProtocolSyncError
-    | ProtocolVersionError
-    | ProtocolWriteError
-    | ProtocolWriteKeyError
-    | ProtocolQuotaError,
+    ProtocolError | StorageWriteMessagesError,
     StorageDep
   > =>
   async (run) => {
@@ -995,15 +1114,18 @@ export const applyProtocolMessageAsClient =
           const result = await run(
             storage.writeMessages(ownerIdBytes, messages),
           );
-          // Quota errors are handled by Storage; protocol just stops syncing.
-          if (!result.ok) return ok({ type: "NoResponse" });
+          if (!result.ok) return result;
         } catch (error) {
           if (AbortError.is(error)) throw error;
           run.deps.console.error(error);
-          return ok({ type: "NoResponse" });
+          return ok({ type: "Failed", cause: "Write" });
         }
       }
 
+      if (messageType === MessageType.Broadcast) {
+        return ok({ type: "Broadcast" });
+      }
+
       // Now: No writeKey, no sync.
       // TODO: Allow to sync SharedReadonlyOwner
       // Without local changes, writeKey will not be required.
@@ -1011,17 +1133,13 @@ export const applyProtocolMessageAsClient =
       // the sync will stop.
       const writeKey = options.writeKey;
       if (writeKey == null) {
-        return ok({ type: "NoResponse" });
-      }
-
-      if (messageType === MessageType.Broadcast) {
-        return ok({ type: "Broadcast" });
+        return ok({ type: "Readonly" });
       }
 
       const ranges = decodeRanges(input);
 
       if (!isNonEmptyArray(ranges)) {
-        return ok({ type: "NoResponse" });
+        return ok({ type: "Converged" });
       }
 
       const output = createProtocolMessageBuffer(ownerId, {
@@ -1030,14 +1148,23 @@ export const applyProtocolMessageAsClient =
         rangesMaxSize: options.rangesMaxSize,
       });
 
-      const result = sync(run.deps)(ranges, output, ownerIdBytes);
+      let broadcast: ProtocolMessageBuffer | undefined;
+      const result = sync(run.deps)(ranges, output, ownerIdBytes, (message) => {
+        broadcast ??= createProtocolMessageBuffer(ownerId, {
+          messageType: MessageType.Broadcast,
+        });
+        broadcast.addMessage(message);
+      });
 
-      // Client sync error (handled via Storage) or no changes.
-      if (!result.ok || !result.value) {
-        return ok({ type: "NoResponse" });
-      }
+      // A failure was logged by sync.
+      if (!result.ok) return ok({ type: "Failed", cause: "Sync" });
+      if (!result.value) return ok({ type: "Converged" });
 
-      return ok({ type: "Response", message: output.unwrap() });
+      return ok({
+        type: "Response",
+        message: output.unwrap(),
+        ...(broadcast && { broadcast: broadcast.unwrap() }),
+      });
     } catch (error) {
       if (AbortError.is(error)) throw error;
       return err<ProtocolInvalidDataError>({
@@ -1159,9 +1286,13 @@ export const applyProtocolMessageAsRelay =
           );
 
           if (!result.ok) {
+            const isQuotaError = result.error.type === "StorageQuotaError";
+            if (!isQuotaError) run.deps.console.error(result.error);
             const message = createProtocolMessageBuffer(ownerId, {
               messageType: MessageType.Response,
-              errorCode: ProtocolErrorCode.QuotaError,
+              errorCode: isQuotaError
+                ? ProtocolErrorCode.QuotaError
+                : ProtocolErrorCode.WriteError,
             }).unwrap();
             return ok({ type: "Response", message });
           }
@@ -1256,7 +1387,7 @@ const parseProtocolHeaderFromBuffer = (input: Buffer): ProtocolHeader => {
   const [version, ownerId] = decodeVersionAndOwner(input);
 
   if (version !== protocolVersion) {
-    throw new ProtocolDecodeError(`Unsupported protocol version: ${version}`);
+    return { type: "ProtocolHeader", version, ownerId };
   }
 
   const messageTypeValue = input.shift();
@@ -1276,12 +1407,7 @@ const parseProtocolHeaderFromBuffer = (input: Buffer): ProtocolHeader => {
       throw new ProtocolDecodeError("Invalid MessageType");
   }
 
-  return {
-    type: "ProtocolHeader",
-    version: 1,
-    ownerId,
-    messageType,
-  };
+  return { type: "ProtocolHeader", version, ownerId, messageType };
 };
 
 /**
@@ -1319,6 +1445,7 @@ const sync =
     ranges: NonEmptyReadonlyArray<Range>,
     output: ProtocolMessageBuffer,
     ownerIdBytes: OwnerIdBytes,
+    onMessage?: (message: EncryptedCrdtMessage) => void,
   ): Result<boolean, typeof ProtocolErrorCode.SyncError> => {
     const outputInitialSize = output.getSize();
     let storageSize: NonNegativeInt;
@@ -1427,13 +1554,19 @@ const sync =
             skipRange(range);
           } else if (output.canSplitRange()) {
             coalesceSkipsBeforeAdd();
-            splitRange(deps)(
-              ownerIdBytes,
-              lower,
-              upper,
-              currentUpperBound,
-              output,
-            );
+            try {
+              splitRange(deps)(
+                ownerIdBytes,
+                lower,
+                upper,
+                currentUpperBound,
+                output,
+              );
+            } catch (error) {
+              if (AbortError.is(error)) throw error;
+              deps.console.error(error);
+              return err(ProtocolErrorCode.SyncError);
+            }
           } else {
             return addFingerprintForRemainingRange(upper)
               ? ok(true)
@@ -1495,7 +1628,10 @@ const sync =
                 }
 
                 ourTimestamps.add(timestampBinary);
-                if (message) output.addMessage(message);
+                if (message) {
+                  output.addMessage(message);
+                  onMessage?.(message);
+                }
                 return true;
               },
             );
@@ -1547,7 +1683,7 @@ const sync =
   };
 
 const splitRange =
-  (deps: StorageDep & ConsoleDep) =>
+  (deps: StorageDep) =>
   (
     ownerId: OwnerIdBytes,
     lower: NonNegativeInt,
@@ -1565,15 +1701,10 @@ const splitRange =
         timestamps: createTimestampsBuffer(),
       };
 
-      deps.storage.iterate(
-        ownerId,
-        zeroNonNegativeInt,
-        itemCount,
-        (timestamp) => {
-          range.timestamps.add(timestampBytesToTimestamp(timestamp));
-          return true;
-        },
-      );
+      deps.storage.iterate(ownerId, lower, upper, (timestamp) => {
+        range.timestamps.add(timestampBytesToTimestamp(timestamp));
+        return true;
+      });
 
       buffer.addRange(range);
       return;
@@ -1588,17 +1719,11 @@ const splitRange =
             ...buckets.value.map((b) => NonNegativeInt.orThrow(b + lower)),
           ];
 
-    let fingerprintRanges: ReadonlyArray<FingerprintRange>;
-    try {
-      fingerprintRanges = deps.storage.fingerprintRanges(
-        ownerId,
-        fingerprintRangesBuckets,
-        upperBound,
-      );
-    } catch (error) {
-      deps.console.error(error);
-      return;
-    }
+    const fingerprintRanges = deps.storage.fingerprintRanges(
+      ownerId,
+      fingerprintRangesBuckets,
+      upperBound,
+    );
 
     const rangesToUse =
       lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;
@@ -1731,30 +1856,7 @@ export const encodeAndEncryptDbChange =
   (message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
     const buffer = createBuffer();
 
-    encodeNonNegativeInt(buffer, protocolVersion);
-
-    // Encode the timestamp to prevent tampering (e.g., a malicious relay
-    // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
-    buffer.extend(timestampToTimestampBytes(message.timestamp));
-
-    encodeFlags(buffer, [
-      message.change.isInsert,
-      // Encode nullable boolean as two flags: presence + value.
-      message.change.isDelete != null,
-      message.change.isDelete ?? false,
-    ]);
-
-    encodeString(buffer, message.change.table);
-    buffer.extend(idToIdBytes(message.change.id));
-
-    const entries = objectToEntries(message.change.values);
-
-    encodeLength(buffer, entries);
-    for (const [column, value] of entries) {
-      assertNotUndefined(value);
-      encodeString(buffer, column);
-      encodeSqliteValue(buffer, value);
-    }
+    encodeDbChange(buffer, message);
 
     // Add PADMÉ padding (ignored during decoding)
     buffer.extend(createPadmePadding(buffer.getLength()));
@@ -1772,6 +1874,41 @@ export const encodeAndEncryptDbChange =
     return buffer.unwrap() as EncryptedDbChange;
   };
 
+/**
+ * Encodes a {@link CrdtMessage} as {@link encodeAndEncryptDbChange} does before
+ * padding and encryption.
+ *
+ * {@link Evolu.getMutationSize} measures mutations with this encoding, so
+ * {@link maxMutationSize} limits exactly what is encoded, and every change
+ * within it fits one protocol message.
+ */
+export const encodeDbChange = (buffer: Buffer, message: CrdtMessage): void => {
+  encodeNonNegativeInt(buffer, protocolVersion);
+
+  // Encode the timestamp to prevent tampering (e.g., a malicious relay
+  // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
+  buffer.extend(timestampToTimestampBytes(message.timestamp));
+
+  encodeFlags(buffer, [
+    message.change.isInsert,
+    // Encode nullable boolean as two flags: presence + value.
+    message.change.isDelete != null,
+    message.change.isDelete ?? false,
+  ]);
+
+  encodeString(buffer, message.change.table);
+  buffer.extend(idToIdBytes(message.change.id));
+
+  const entries = objectToEntries(message.change.values);
+
+  encodeLength(buffer, entries);
+  for (const [column, value] of entries) {
+    assertNotUndefined(value);
+    encodeString(buffer, column);
+    encodeSqliteValue(buffer, value);
+  }
+};
+
 /**
  * Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
  * owner's encryption key. Verifies that the embedded timestamp matches the
@@ -1945,21 +2082,30 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
       }
 
       const json = Json.from.parent(value);
-      // Only encode as Json if it survives JSON.parse/JSON.stringify round-trip.
-      // Some valid JSON strings like "-0E0" get normalized to "0" during parsing,
-      // which would cause data corruption if we don't verify round-trip safety.
-      if (json.ok && JSON.stringify(jsonToJsonValue(json.value)) === value) {
+      if (json.ok) {
+        const jsonValue = jsonToJsonValue(json.value);
         jsonBuffer.reset();
         try {
-          encodeJsonValue(jsonBuffer, jsonToJsonValue(json.value));
-          const jsonBytes = jsonBuffer.unwrap();
-          encodeNonNegativeInt(buffer, ProtocolValueType.Json);
-          encodeLength(buffer, jsonBytes);
-          buffer.extend(jsonBytes);
+          // Encoding first rejects nesting deeper than decoding allows, before
+          // the recursive JSON.stringify below could overflow the stack. Such
+          // a value is encoded as a plain string.
+          encodeJsonValue(jsonBuffer, jsonValue);
+          // Only encode as Json if it survives JSON.parse/JSON.stringify
+          // round-trip. Some valid JSON strings like "-0E0" get normalized to
+          // "0" during parsing, which would cause data corruption if we don't
+          // verify round-trip safety.
+          if (JSON.stringify(jsonValue) === value) {
+            const jsonBytes = jsonBuffer.unwrap();
+            encodeNonNegativeInt(buffer, ProtocolValueType.Json);
+            encodeLength(buffer, jsonBytes);
+            buffer.extend(jsonBytes);
+            return;
+          }
+        } catch (error) {
+          if (!(error instanceof BufferError)) throw error;
         } finally {
           jsonBuffer.reset();
         }
-        return;
       }
 
       const base64Url = Base64Url.from.parent(value);

package/src/local-first/Query.ts

@@ -43,25 +43,22 @@ export type { NotNull as KyselyNotNull } from "kysely";
  * import {
  *   assertType,
  *   createQueryBuilder,
- *   id,
+ *   testEvoluSchema,
+ *   type TestEvoluSchema,
  *   NonEmptyTrimmedString100,
  *   type Query,
+ *   type TestTodoId,
  * } from "@evolu/common";
  *
- * const TodoId = id("Todo");
- * type TodoId = typeof TodoId.Output;
- * const Schema = {
- *   todo: { id: TodoId, title: NonEmptyTrimmedString100 },
- * };
- * const createQuery = createQueryBuilder(Schema);
+ * const createQuery = createQueryBuilder(testEvoluSchema);
  * const allTodos = createQuery((db) => db.selectFrom("todo").selectAll());
  *
  * type AllTodosRow = typeof allTodos.Row;
  * assertType<
- *   typeof allTodos extends Query<typeof Schema> ? true : false,
+ *   typeof allTodos extends Query<TestEvoluSchema> ? true : false,
  *   true
  * >();
- * assertType<AllTodosRow["id"], TodoId>();
+ * assertType<AllTodosRow["id"], TestTodoId>();
  * assertType<AllTodosRow["title"], NonEmptyTrimmedString100 | null>();
  * ```
  */
@@ -82,15 +79,11 @@ export type Query<
      * import {
      *   assertType,
      *   createQueryBuilder,
-     *   id,
+     *   testEvoluSchema,
      *   type InferRow,
-     *   NonEmptyTrimmedString100,
      * } from "@evolu/common";
      *
-     * const Schema = {
-     *   todo: { id: id("Todo"), title: NonEmptyTrimmedString100 },
-     * };
-     * const createQuery = createQueryBuilder(Schema);
+     * const createQuery = createQueryBuilder(testEvoluSchema);
      * const allTodos = createQuery((db) =>
      *   db.selectFrom("todo").selectAll(),
      * );

package/src/local-first/Relay.ts

@@ -17,7 +17,7 @@ import { err, ok } from "../Result.ts";
 import type { SqliteDep } from "../Sqlite.ts";
 import { sql } from "../Sqlite.ts";
 import { createMutexByKey } from "../Task.ts";
-import { Name, PositiveInt, uint8ArrayToBase64Url } from "../Type.ts";
+import { Name, NonNegativeInt, uint8ArrayToBase64Url } from "../Type.ts";
 import { isPromiseLike, type Awaitable } from "../Types.ts";
 import {
   OwnerId,
@@ -246,7 +246,9 @@ export const createRelaySqliteStorage =
               (sum, m) => sum + m.change.length,
               0,
             );
-            const newStoredBytes = PositiveInt.orThrow(
+            // A sum of lengths can be zero. Throwing here would panic the
+            // relay's shared Run.
+            const newStoredBytes = NonNegativeInt.orThrow(
               (usage.storedBytes ?? 0) + incomingBytes,
             );