@evolu/common 8.17.0 → 8.18.0

package/src/Bytes.ts

@@ -103,16 +103,11 @@ export const concatByteArrays = (
   return result;
 };
 
-/**
- * Custom error for {@link Buffer}-related failures like premature end of data.
- * Provides better stack traces for debugging binary protocol issues.
- */
+/** Custom error for {@link Buffer}-related failures like premature end of data. */
 export class BufferError extends Error {
   constructor(message: string) {
     super(message);
     this.name = this.constructor.name;
-
-    Error.captureStackTrace(this, this.constructor);
   }
 }
 
@@ -173,34 +168,34 @@ export class BufferError extends Error {
  */
 export interface Buffer {
   /** Returns the current capacity of the buffer. */
-  getCapacity: () => NonNegativeInt;
+  readonly getCapacity: () => NonNegativeInt;
 
   /** Returns the current number of bytes stored in the buffer. */
-  getLength: () => NonNegativeInt;
+  readonly getLength: () => NonNegativeInt;
 
   /**
    * Appends binary data to the buffer, resizing if necessary. Throws if
    * `arg.length` is not a non-negative safe integer.
    */
-  extend: (arg: Uint8Array | ArrayLike<number>) => void;
+  readonly extend: (arg: Uint8Array | ArrayLike<number>) => void;
 
   /**
    * Removes and returns the first byte. Throws an `Error` with message "Buffer
    * parse ended prematurely" if the buffer is empty.
    */
-  shift: () => NonNegativeInt;
+  readonly shift: () => NonNegativeInt;
 
   /**
    * Removes and returns the first `n` bytes. Throws an `Error` with message
    * "Buffer parse ended prematurely" if fewer than `n` bytes remain.
    */
-  shiftN: (n: NonNegativeInt) => Uint8Array;
+  readonly shiftN: (n: NonNegativeInt) => Uint8Array;
 
   /**
    * Truncates the buffer to the specified length, discarding data from the end.
    * Throws if the new length is greater than the current length.
    */
-  truncate: (length: NonNegativeInt) => void;
+  readonly truncate: (length: NonNegativeInt) => void;
 
   /**
    * Resets the buffer to its initial empty state, preserving its capacity.
@@ -209,14 +204,14 @@ export interface Buffer {
    * when you want to clear the buffer and write new data, avoiding unnecessary
    * allocations.
    */
-  reset: () => void;
+  readonly reset: () => void;
 
   /**
    * Returns a view of the buffer’s current data. Do not modify this array, as
    * it directly alters the buffer’s internal state, potentially breaking
    * subsequent operations.
    */
-  unwrap: () => Uint8Array;
+  readonly unwrap: () => Uint8Array;
 }
 
 /** Creates a {@link Buffer} for efficient byte operations. */
@@ -438,18 +433,37 @@ export const encodeLength = (
 /** Decodes an array-like value length. */
 export const decodeLength = decodeNonNegativeInt;
 
-/** Encodes a length-prefixed UTF-8 string. */
+/**
+ * Encodes a length-prefixed UTF-8 string.
+ *
+ * A lone surrogate becomes U+FFFD, so the bytes are always valid UTF-8. Unlike
+ * {@link encodeJsonValue}, which keeps lone surrogates, this does not round-trip
+ * every JavaScript string.
+ */
 export const encodeString = (buffer: Buffer, value: string): void => {
   const bytes = utf8ToBytes(value);
   encodeLength(buffer, bytes);
   buffer.extend(bytes);
 };
 
-/** Decodes a length-prefixed UTF-8 string. */
+/**
+ * Decodes a length-prefixed UTF-8 string.
+ *
+ * Invalid UTF-8 decodes as U+FFFD instead of throwing. A leading U+FEFF is
+ * kept. Decoders in `@evolu/common` 8.17 and earlier drop it, so such peers
+ * still store a string that starts with it without the mark.
+ */
 export const decodeString = (buffer: Buffer): string => {
   const length = decodeLength(buffer);
   const bytes = buffer.shiftN(length);
-  return bytesToUtf8(bytes);
+  const string = bytesToUtf8(bytes);
+  // bytesToUtf8 uses the default TextDecoder, which drops a leading U+FEFF
+  // (EF BB BF in UTF-8), so it is restored here. Passing { ignoreBOM: true }
+  // to a TextDecoder would also keep it, but costs more compiler types in the
+  // type benchmark.
+  return bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
+    ? `\uFEFF${string}`
+    : string;
 };
 
 /** Incrementally encodes consecutive equal values using run-length encoding. */
@@ -457,6 +471,15 @@ export interface RunLengthEncoder<T> {
   readonly add: (value: T) => void;
   readonly getLength: () => NonNegativeInt;
   readonly unwrap: () => Uint8Array;
+
+  /**
+   * Returns a function that restores the encoder, in constant time, to the
+   * state it had when this was called, discarding every value added since.
+   *
+   * A restore function can be called repeatedly. It is valid until the encoder
+   * is restored to an earlier checkpoint.
+   */
+  readonly checkpoint: () => () => void;
 }
 
 /** Creates an incremental run-length encoder. */
@@ -485,6 +508,26 @@ export const createRunLengthEncoder = <T>(
     getLength: () => buffer.getLength(),
 
     unwrap: () => buffer.unwrap(),
+
+    checkpoint: () => {
+      const checkpointLength = previousLength;
+      const checkpointValue = previousValue;
+      const checkpointRunLength = runLength;
+
+      return () => {
+        // `add` rewrites the last run in place, so the bytes after its start
+        // may differ from the checkpoint's even at the same length. Bytes
+        // before it never change, so the last run is encoded again.
+        buffer.truncate(checkpointLength);
+        previousLength = checkpointLength;
+        previousValue = checkpointValue;
+        runLength = checkpointRunLength;
+        if (runLength > 0) {
+          encodeValue(buffer, previousValue as T);
+          encodeNonNegativeInt(buffer, runLength);
+        }
+      };
+    },
   };
 };
 
@@ -557,6 +600,9 @@ interface JsonKeyCacheEntry {
 const jsonKeyCacheSize = 4096;
 const maxCachedJsonKeyByteLength = 32;
 const maxJsonNestingDepth = 1_000;
+// A string reserves 3 bytes per UTF-16 code unit, so one large value would
+// otherwise keep several times its size allocated after encoding.
+const maxRetainedJsonEncoderLength = 1024 * 1024;
 let jsonEncoderTarget = new Uint8Array(8192);
 let jsonEncoderTargetView = new DataView(jsonEncoderTarget.buffer);
 let jsonEncoderPosition = 0;
@@ -622,6 +668,10 @@ export const encodeJsonValue = (buffer: Buffer, value: JsonValue): void => {
     jsonEncoderPosition = 0;
     jsonEncoderDepth = 0;
     jsonEncoderIsActive = false;
+    if (jsonEncoderTarget.length > maxRetainedJsonEncoderLength) {
+      jsonEncoderTarget = new Uint8Array(8192);
+      jsonEncoderTargetView = new DataView(jsonEncoderTarget.buffer);
+    }
   }
 };
 

package/src/Error.ts

@@ -172,7 +172,11 @@ export const defectToError = (reported: unknown): Error => {
   // instanceof fails.
   const tag = Object.prototype.toString.call(defect);
   // Chromium reports a DOMException from a worker without its name or message,
-  // so it is described in an Error.
+  // so it is described in an Error. Its fix covered only exceptions thrown in
+  // classic workers (https://issues.chromium.org/issues/41241359); Chrome 154
+  // still drops both for reportError
+  // (https://issues.chromium.org/issues/568850673) and for a module worker's
+  // evaluation.
   if (tag === "[object DOMException]") {
     const { name, message } = defect as Error;
     return new Error(`${name}: ${message}`, { cause: defect });

package/src/Polyfills.ts

@@ -37,8 +37,9 @@ export const installPolyfills = (): void => {
    * This module intentionally owns `DisposableStack` and `AsyncDisposableStack`
    * polyfills instead of depending on `es-shims/DisposableStack` at runtime.
    *
-   * Evolu originally used the upstream package, but WebKit hit a known async
-   * disposal completion bug (`completion["?"]` crash, see issue #9). The local
+   * Evolu originally used the upstream package, but WebKit, which needs the
+   * polyfill, hit its async disposal completion bug (`completion["?"]` crash,
+   * https://github.com/es-shims/DisposableStack/issues/9). The local
    * implementation applies the fix and keeps behavior deterministic across
    * runtimes used by Evolu.
    *

package/src/Task.ts

@@ -1798,9 +1798,10 @@ export interface AbortReason extends InferType<typeof AbortReason> {}
  * Helpers that abort their own child Tasks should catch or normalize AbortError
  * before it escapes the helper boundary. The reason carries typed domain data.
  *
- * WebKit fetch rejects with its own abort error instead of `signal.reason`.
- * Native wrappers should treat `signal.reason` as the source of truth and
- * normalize aborts to AbortError.
+ * WebKit fetch rejects with its own abort error instead of `signal.reason`
+ * ([WebKit bug 246069](https://bugs.webkit.org/show_bug.cgi?id=246069)). Native
+ * wrappers should treat `signal.reason` as the source of truth and normalize
+ * aborts to AbortError.
  *
  * @group Core
  */

package/src/local-first/Db.ts

@@ -57,7 +57,7 @@ import {
   type RandomBytesDep,
 } from "../Crypto.ts";
 import { createUnknownError } from "../Error.ts";
-import { constFalse, constVoid } from "../Function.ts";
+import { constFalse } from "../Function.ts";
 import type { LockManagerDep } from "../LockManager.ts";
 import {
   acquireLeaderLock,
@@ -116,6 +116,7 @@ import {
   decryptAndDecodeDbChange,
   encodeAndEncryptDbChange,
   SubscriptionFlags,
+  type ProtocolChangeTooLargeError,
   type ProtocolInvalidDataError,
   type ProtocolMessage,
   type ProtocolTimestampMismatchError,
@@ -374,6 +375,7 @@ export const startDbWorker =
                 const result = await run.abortable(
                   applyProtocolMessageAsClient(inputMessage, {
                     writeKey: owner.writeKey,
+                    onChangeTooLarge: storage.onChangeTooLarge,
                   }),
                 );
                 postQueuedResponse({
@@ -475,7 +477,8 @@ export const startDbWorker =
 
         // Stops once the SharedWorker no longer leads for its ID, which the
         // platform releases when it closes, because Firefox can lose a Dispose
-        // posted right before that.
+        // posted right before that
+        // (https://bugzilla.mozilla.org/show_bug.cgi?id=2077609).
         const sharedWorkerEnded = acquireLeaderLockCallback(deps)(
           initMessage.sharedWorkerId,
           () => resolve(ok()),
@@ -917,14 +920,18 @@ interface ClientStorage extends Storage, BaseSqliteStorage {
   ) => void;
   readonly didWriteMessages: () => boolean;
   /**
-   * The first error of a message that {@link Storage.writeMessages} skipped in
-   * this request, or null.
+   * The first error of a message skipped in this request, or null: one that
+   * {@link Storage.writeMessages} could not store, or a stored change that sync
+   * could not send.
    */
   readonly skippedError: () =>
     | DecryptWithXChaCha20Poly1305Error
     | ProtocolInvalidDataError
     | ProtocolTimestampMismatchError
+    | ProtocolChangeTooLargeError
     | null;
+  /** Records a stored change that sync skipped, unless an error came first. */
+  readonly onChangeTooLarge: (error: ProtocolChangeTooLargeError) => void;
 }
 
 const createClientStorage = (
@@ -961,10 +968,12 @@ const createClientStorage = (
 
     didWriteMessages: () => didWriteMessages,
     skippedError: () => skippedError,
+    onChangeTooLarge: (error) => {
+      skippedError ??= error;
+    },
 
     // Not implemented yet.
     validateWriteKey: constFalse,
-    setWriteKey: constVoid,
 
     writeMessages: (ownerIdBytes, encryptedMessages) => () => {
       // TODO: Add quota checking for collaborative scenarios.
@@ -1020,15 +1029,21 @@ const createClientStorage = (
       }
 
       let wroteNewMessages = false;
-      deps.sqlite.transaction(() => {
-        wroteNewMessages = applyMessages(deps)(
-          ownerIdBytesToOwnerId(ownerIdBytes),
-          messages,
-          QuarantineOrigin.ReceivedMessage,
-          now,
-        );
-        saveClock(deps)(clockTimestamp);
-      });
+      // SQLite can fail the write, for example on a full disk. The transaction
+      // has rolled back, so the route reports the error instead of a throw
+      // panicking the DbWorker.
+      const written = trySync(() => {
+        deps.sqlite.transaction(() => {
+          wroteNewMessages = applyMessages(deps)(
+            ownerIdBytesToOwnerId(ownerIdBytes),
+            messages,
+            QuarantineOrigin.ReceivedMessage,
+            now,
+          );
+          saveClock(deps)(clockTimestamp);
+        });
+      }, createUnknownError);
+      if (!written.ok) return written;
       clock.set(clockTimestamp);
       // A batch of duplicates changes no table, so queries need no refresh.
       if (wroteNewMessages) didWriteMessages = true;

package/src/local-first/Evolu.test.ts

@@ -75,7 +75,7 @@ import { err, ok } from "../Result.ts";
 import { SqliteBoolean } from "../Sqlite.ts";
 import { explicitAbortReason, testCreateDeps, testCreateRun } from "../Task.ts";
 import { testCreateId } from "../Test.ts";
-import { Millis } from "../Time.ts";
+import { maxMillis, Millis } from "../Time.ts";
 import {
   assertType,
   createIdFromString,
@@ -90,8 +90,14 @@ import {
   Uint8Array as Uint8ArrayType,
 } from "../Type.ts";
 import type { ExtractTyped } from "../Type.ts";
-import { DbChange } from "./Storage.ts";
-import { createTimestamp, type NodeId } from "./Timestamp.ts";
+import { DbChange, RangeType } from "./Storage.ts";
+import {
+  createTimestamp,
+  maxCounter,
+  maxNodeId,
+  type NodeId,
+  timestampToTimestampBytes,
+} from "./Timestamp.ts";
 import {
   testCreateBroadcastChannel,
   testCreateMessageChannel,
@@ -3144,13 +3150,25 @@ describe("Evolu", () => {
         }),
       };
 
-      // A request and a broadcast assert that the change fits.
-      createProtocolMessageFromCrdtMessages(deps)(testAppOwner, [message]);
+      const change = encodeAndEncryptDbChange(deps)(
+        message,
+        testAppOwner.encryptionKey,
+      );
+
+      // A request leaves out a change that does not fit and asks for another
+      // round instead, so it must be longer than the change. A broadcast
+      // throws when the change does not fit.
+      assertTrue(
+        createProtocolMessageFromCrdtMessages(deps)(testAppOwner, [message])
+          .length > change.length,
+      );
       createProtocolBroadcastMessagesFromCrdtMessages(deps)(testAppOwner, [
         message,
       ]);
 
-      // A relay's response can pair it with the largest ranges section.
+      // A relay's response can pair it with the largest ranges section: a
+      // Skip range and a Timestamps range listing nearly 100,000 bytes, with
+      // room left to close the frame.
       const rangesMaxSize = ProtocolMessageRangesMaxSize.orThrow(100_000);
       const timestamps = createTimestampsBuffer();
       for (let index = 1; timestamps.getLength() < 99_900; index++) {
@@ -3167,12 +3185,25 @@ describe("Evolu", () => {
         rangesMaxSize,
       });
       assertTrue(
-        response.canAddTimestampsRangeAndMessage(timestamps, {
-          timestamp: message.timestamp,
-          change: encodeAndEncryptDbChange(deps)(
-            message,
-            testAppOwner.encryptionKey,
-          ),
+        response.tryWrite(() => {
+          response.addMessage({ timestamp: message.timestamp, change });
+          response.addRange({
+            type: RangeType.Skip,
+            upperBound: timestampToTimestampBytes(
+              createTimestamp({ millis: Millis.orThrow(2 ** 47) }),
+            ),
+          });
+          response.addRange({
+            type: RangeType.Timestamps,
+            upperBound: timestampToTimestampBytes(
+              createTimestamp({
+                millis: maxMillis,
+                counter: maxCounter,
+                nodeId: maxNodeId,
+              }),
+            ),
+            timestamps,
+          });
         }),
       );
     });

package/src/local-first/Evolu.ts

@@ -76,6 +76,7 @@ import type {
 import { createOwnerWebSocketTransport } from "./Owner.ts";
 import {
   encodeDbChange,
+  type ProtocolChangeTooLargeError,
   type ProtocolError,
   type ProtocolInvalidDataError,
   type ProtocolQuotaError,
@@ -960,7 +961,8 @@ export type DevicePersistence = "Persisted" | "NotPersisted" | "Unknown";
  * request, or sent a frame that could not be decoded, the route shows a
  * {@link ProtocolError}. A {@link ProtocolQuotaError} needs more relay quota,
  * then {@link Evolu.requestSync}, and a {@link ProtocolVersionError} needs an app
- * or relay update.
+ * or relay update. A route shows an {@link UnknownError} failure when this
+ * device could not store received changes, for example on a full disk.
  *
  * A received change that was not created with the owner's encryption key, was
  * altered afterwards, or cannot be decoded by this app version is skipped,
@@ -985,6 +987,14 @@ export type DevicePersistence = "Persisted" | "NotPersisted" | "Unknown";
  * {@link DecryptWithXChaCha20Poly1305Error} and your code creates or shares the
  * owner, check the owner's keys.
  *
+ * A {@link ProtocolChangeTooLargeError} skip is a change no protocol message can
+ * hold: one this device saved before {@link maxMutationSize} existed, or a
+ * crafted one received from a relay. Every route through a relay that lacks it
+ * shows it after each sync, and everything else still syncs, but changes
+ * received from one relay reach the others only after a reconnect or
+ * {@link Evolu.requestSync}. The change is not recovered, and replacing relays
+ * does not help.
+ *
  * @group Core
  */
 export type EvoluError =
@@ -1708,7 +1718,11 @@ export const createEvolu =
         // gets it already batched and cannot reach the call site; quarantine,
         // which holds changes stored for sync; and skipping it in sync, which
         // keeps range fingerprints disagreeing. Received changes are not
-        // checked, and oversized changes stored before this limit are not
+        // checked. Changes saved before this limit can be too large for any
+        // protocol message, as can crafted changes on a relay. Their
+        // fingerprints disagree either way, so sync skips them, which ends
+        // every round instead of retrying forever. A client reports such a
+        // change as its route's skippedError, and a relay logs it. They are not
         // recovered, which needs a history-aware design.
         //
         // Local-only tables never sync, so they have no limit.