@evolu/common 8.6.2 → 8.8.0

package/src/local-first/Protocol.ts

@@ -170,29 +170,41 @@
  *   initiator/non-initiator terminology instead, and consolidate into a single
  *   `applyProtocolMessage` function with conditional arguments to reduce code
  *   duplication.
- * - Replace try-catch with Result + new Error (to preserve stacktraces). Measure
- *   Result overhead, it should be super small.
  * - Allow clients to broadcast messages that are not persisted by relays. This
  *   would enable real-time ephemeral data (like cursor positions, typing
  *   indicators) to be forwarded by relays without storage overhead.
  */
 
-import { Packr } from "msgpackr";
 import {
   createMutableArray,
   isNonEmptyArray,
   type NonEmptyReadonlyArray,
 } from "../Array.ts";
-import { assert, assertNonNullable, assertNotUndefined } from "../Assert.ts";
+import {
+  assert,
+  assertNonNullable,
+  assertNotUndefined,
+  assertSame,
+} from "../Assert.ts";
 import type { Brand } from "../Brand.ts";
 import {
   type Buffer,
-  bytesToHex,
-  bytesToUtf8,
   createBuffer,
-  hexToBytes,
-  utf8ToBytes,
-} from "../Buffer.ts";
+  createRunLengthEncoder,
+  decodeFlags,
+  decodeJsonValue,
+  decodeLength,
+  decodeNonNegativeInt,
+  decodeNumber,
+  decodeRle,
+  decodeString,
+  encodeFlags,
+  encodeJsonValue,
+  encodeLength,
+  encodeNonNegativeInt,
+  encodeNumber,
+  encodeString,
+} from "../Binary.ts";
 import type { ConsoleDep } from "../Console.ts";
 import {
   createPadmePadding,
@@ -209,10 +221,11 @@ import { eqArrayNumber } from "../Eq.ts";
 import { computeBalancedBuckets } from "../Number.ts";
 import { createMutableRecord, objectToEntries } from "../Object.ts";
 import { err, ok, type Result } from "../Result.ts";
-import { SqliteValue } from "../Sqlite.ts";
+import type { SqliteValue } from "../Sqlite.ts";
 import { AbortError, type Task } from "../Task.ts";
 import { Millis } from "../Time.ts";
 import {
+  assertType,
   Base64Url,
   base64UrlToUint8Array,
   between,
@@ -264,6 +277,10 @@ import {
   Counter,
   eqTimestamp,
   NodeId,
+  type NodeIdBytes,
+  nodeIdBytesLength,
+  nodeIdBytesToNodeId,
+  nodeIdToNodeIdBytes,
   Timestamp,
   TimestampBytes,
   timestampBytesLength,
@@ -271,14 +288,6 @@ import {
   timestampToTimestampBytes,
 } from "./Timestamp.ts";
 
-/**
- * Evolu uses MessagePack for numbers and JSONs.
- *
- * - `variableMapSize: true` - More compact maps, ~5-10% slower encoding
- * - `useRecords: false` - Standard MessagePack without extensions
- */
-const packr = new Packr({ variableMapSize: true, useRecords: false });
-
 const minProtocolMessageMaxSize = 1_000_000;
 const maxProtocolMessageMaxSize = 100_000_000;
 
@@ -434,7 +443,7 @@ export interface ProtocolVersionError
 
 /** Error for invalid or corrupted protocol message data. */
 export interface ProtocolInvalidDataError extends Typed<"ProtocolInvalidDataError"> {
-  readonly data: globalThis.Uint8Array;
+  readonly data: Uint8Array;
   readonly error: unknown;
 }
 
@@ -843,13 +852,13 @@ export const createTimestampsBuffer = (): TimestampsBuffer => {
     encodeNonNegativeInt(buffer, value);
   });
   const nodeIdEncoder = createRunLengthEncoder<NodeId>((buffer, value) => {
-    encodeNodeId(buffer, value);
+    buffer.extend(nodeIdToNodeIdBytes(value));
   });
 
   return {
     add: (timestamp) => {
       const delta = timestamp.millis - previousMillis;
-      assert(NonNegativeInt.is(delta), "The delta must be NonNegativeInt");
+      assertType(NonNegativeInt, delta);
 
       count++;
       syncCount();
@@ -883,40 +892,6 @@ export const createTimestampsBuffer = (): TimestampsBuffer => {
   };
 };
 
-interface RunLengthEncoder<T> {
-  add: (value: T) => void;
-  getLength: () => NonNegativeInt;
-  unwrap: () => Uint8Array;
-}
-
-const createRunLengthEncoder = <T>(
-  encodeValue: (buffer: Buffer, value: T) => void,
-): RunLengthEncoder<T> => {
-  const buffer = createBuffer();
-  let previousLength = zeroNonNegativeInt;
-  let previousValue = null as T | null;
-  let runLength = zeroNonNegativeInt;
-
-  return {
-    add: (value) => {
-      if (value === previousValue) {
-        runLength++;
-        buffer.truncate(previousLength);
-      } else {
-        previousValue = value;
-        runLength = NonNegativeInt.orThrow(1);
-      }
-      previousLength = buffer.getLength();
-      encodeValue(buffer, value);
-      encodeNonNegativeInt(buffer, runLength);
-    },
-
-    getLength: () => buffer.getLength(),
-
-    unwrap: () => buffer.unwrap(),
-  };
-};
-
 export interface ApplyProtocolMessageAsClientOptions {
   writeKey?: OwnerWriteKey;
 
@@ -1130,7 +1105,7 @@ export const applyProtocolMessageAsRelay =
       }
 
       const messageType = input.shift() as MessageType;
-      assert(messageType === MessageType.Request, "Invalid MessageType");
+      assertSame(messageType, MessageType.Request);
 
       const hasWriteKey = input.shift();
       let writeKey: OwnerWriteKey | undefined;
@@ -1722,7 +1697,9 @@ const decodeTimestamps = (
     return counter.value;
   });
 
-  const nodeIds = decodeRle(buffer, length, (): NodeId => decodeNodeId(buffer));
+  const nodeIds = decodeRle(buffer, length, (): NodeId =>
+    nodeIdBytesToNodeId(buffer.shiftN(nodeIdBytesLength) as NodeIdBytes),
+  );
 
   const timestamps = createMutableArray<Timestamp>(length);
   for (let i = 0; i < length; i++) {
@@ -1736,135 +1713,11 @@ const decodeTimestamps = (
   return timestamps;
 };
 
-export const decodeRle = <T>(
-  buffer: Buffer,
-  length: NonNegativeInt,
-  decodeValue: () => T,
-): ReadonlyArray<T> => {
-  const values = createMutableArray<T>(length);
-  let index = 0;
-  while (index < length) {
-    const value = decodeValue();
-    const runLength = decodeNonNegativeInt(buffer);
-
-    // Prevent infinite loop on malformed input.
-    if (runLength === 0) {
-      throw new ProtocolDecodeError(
-        "Invalid RLE encoding: runLength must be positive",
-      );
-    }
-    const remaining = length - index;
-
-    // Prevent CPU/memory amplification via oversized runLength.
-    if (runLength > remaining) {
-      throw new ProtocolDecodeError(
-        `Invalid RLE encoding: runLength ${runLength} exceeds remaining ${remaining}`,
-      );
-    }
-    for (let i = 0; i < runLength; i++) {
-      values[index] = value;
-      index++;
-    }
-  }
-  return values;
-};
-
 const decodeId = (buffer: Buffer): Id => {
   const bytes = buffer.shiftN(idBytesTypeValueLength);
   return idBytesToId(bytes as IdBytes);
 };
 
-/**
- * Evolu uses MessagePack to handle finite numbers except for NonNegativeInt.
- * For NonNegativeInt, Evolu provides more efficient encoding.
- */
-export const encodeNumber = (buffer: Buffer, number: FiniteNumber): void => {
-  buffer.extend(packr.pack(number));
-};
-
-export const decodeNumber = (buffer: Buffer): FiniteNumber => {
-  let number: unknown;
-  let end: unknown;
-
-  packr.unpackMultiple(
-    buffer.unwrap(),
-    (n: unknown, _: unknown, e: unknown) => {
-      number = n;
-      end = e;
-      return false;
-    },
-  );
-
-  const endResult = NonNegativeInt.fromUnknown(end);
-  if (!endResult.ok) throw new ProtocolDecodeError(endResult.error.type);
-
-  const numberResult = FiniteNumber.fromUnknown(number);
-  if (!numberResult.ok) throw new ProtocolDecodeError(numberResult.error.type);
-
-  buffer.shiftN(endResult.value);
-  return numberResult.value;
-};
-
-/**
- * Encodes an array of boolean flags into a single byte.
- *
- * Each element in the array corresponds to a bit (0-7). Array can have 0-8
- * elements.
- *
- * ### Example
- *
- * ```ts
- * import { assertEqual, createBuffer } from "@evolu/common";
- * import { encodeFlags } from "@evolu/common/local-first";
- *
- * const buffer = createBuffer();
- * encodeFlags(buffer, [true, false, true]);
- *
- * assertEqual(buffer.unwrap(), new Uint8Array([0b101]));
- * ```
- */
-export const encodeFlags = (
-  buffer: Buffer,
-  flags: ReadonlyArray<boolean>,
-): void => {
-  let byte = 0;
-  for (let i = 0; i < flags.length && i < 8; i++) {
-    if (flags[i]) {
-      byte |= 1 << i;
-    }
-  }
-  buffer.extend([byte]);
-};
-
-/**
- * Decodes a byte into an array of boolean flags.
- *
- * ### Example
- *
- * ```ts
- * import { assertEqual, createBuffer, PositiveInt } from "@evolu/common";
- * import { decodeFlags } from "@evolu/common/local-first";
- *
- * const buffer = createBuffer([0b101]);
- * const flags = decodeFlags(buffer, PositiveInt.orThrow(3));
- *
- * assertEqual(flags, [true, false, true]);
- * assertEqual(buffer.getLength(), 0);
- * ```
- */
-export const decodeFlags = (
-  buffer: Buffer,
-  count: PositiveInt,
-): ReadonlyArray<boolean> => {
-  const byte = buffer.shift();
-  const length = globalThis.Math.min(count, 8);
-  const flags = createMutableArray<boolean>(length);
-  for (let i = 0; i < length; i++) {
-    flags[i] = (byte & (1 << i)) !== 0;
-  }
-  return flags;
-};
-
 /**
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
@@ -1996,91 +1849,30 @@ export const decryptAndDecodeDbChange = (
 };
 
 /**
- * Encodes a non-negative integer into a variable-length integer format. It's
- * more efficient than encoding via {@link encodeNumber}.
+ * Decodes a ProtocolMessage into a readable JSON object for debugging.
  *
- * https://en.wikipedia.org/wiki/Variable-length_quantity
- */
-export const encodeNonNegativeInt = (
-  buffer: Buffer,
-  int: NonNegativeInt,
-): void => {
-  if (int === 0) {
-    buffer.extend([0]);
-    return;
-  }
-
-  let remaining = BigInt(int);
-  const bytes: Array<number> = [];
-
-  while (remaining !== 0n) {
-    const byte = globalThis.Number(remaining & 127n);
-    bytes.push(byte);
-    remaining >>= 7n;
-  }
-
-  for (let i = 0; i < bytes.length - 1; i++) {
-    bytes[i] |= 128;
-  }
-
-  buffer.extend(bytes);
-};
-
-/**
- * Decodes a non-negative integer from a variable-length integer format.
+ * Note: This is a stub for future implementation. It should use:
+ *
+ * - DecodeVersionAndOwner
+ * - DecodeError or decodeWriteKeys (depending on context)
+ * - DecodeMessages
+ * - DecodeRanges
  *
- * https://en.wikipedia.org/wiki/Variable-length_quantity
+ * If you want to help, please contribute to this function.
  */
-export const decodeNonNegativeInt = (buffer: Buffer): NonNegativeInt => {
-  let result = 0n;
-  let shift = 0n;
-  let byte;
-
-  // 8 is the smallest required count
-  for (let byteCount = 0; byteCount < 8; byteCount++) {
-    byte = buffer.shift();
-    result |= BigInt(byte & 127) << shift;
-    if ((byte & 128) === 0) break;
-    shift += 7n;
-  }
-
-  const int = NonNegativeInt.fromUnknown(globalThis.Number(result));
-  if (!int.ok) throw new ProtocolDecodeError(int.error.type);
-
-  return int.value;
-};
-
-export const encodeLength = (buffer: Buffer, value: ArrayLike<any>): void => {
-  encodeNonNegativeInt(buffer, NonNegativeInt.orThrow(value.length));
-};
-
-export const decodeLength = decodeNonNegativeInt;
-
-export const encodeString = (buffer: Buffer, value: string): void => {
-  const bytes = utf8ToBytes(value);
-  encodeLength(buffer, bytes);
-  buffer.extend(bytes);
-};
-
-export const decodeString = (buffer: Buffer): string => {
-  const length = decodeLength(buffer);
-  const bytes = buffer.shiftN(length);
-  return bytesToUtf8(bytes);
-};
-
-export const encodeNodeId = (buffer: Buffer, nodeId: NodeId): void => {
-  buffer.extend(hexToBytes(nodeId));
-};
-
-export const decodeNodeId = (buffer: Buffer): NodeId => {
-  const bytes = buffer.shiftN(NonNegativeInt.orThrow(8));
-  return bytesToHex(bytes) as NodeId;
+export const decodeProtocolMessageToJson = (
+  _protocolMessage: ProtocolMessage,
+  _isInitiator: boolean,
+): unknown => {
+  // TODO: Implement using
+  // - decodeVersionAndOwner
+  // -- decodeError or decodeWriteKeys (should be refactored out),
+  // -- decodeMessages, and decodeRanges.
+  // This is a stub for PRs and community contributions.
+  throw new Error("decodeProtocolMessageToJson is not implemented yet.");
 };
 
 // Small ints are encoded into ProtocolValueType, saving one byte per int.
-const isSmallInt: Predicate<number> = (value: number) =>
-  value >= 0 && value < 20;
-
 export const ProtocolValueType = {
   // 0-19 small ints
 
@@ -2157,10 +1949,16 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
       // Some valid JSON strings like "-0E0" get normalized to "0" during parsing,
       // which would cause data corruption if we don't verify round-trip safety.
       if (json.ok && JSON.stringify(jsonToJsonValue(json.value)) === value) {
-        const jsonBytes = packr.pack(jsonToJsonValue(json.value));
-        encodeNonNegativeInt(buffer, ProtocolValueType.Json);
-        encodeLength(buffer, jsonBytes);
-        buffer.extend(jsonBytes);
+        jsonBuffer.reset();
+        try {
+          encodeJsonValue(jsonBuffer, jsonToJsonValue(json.value));
+          const jsonBytes = jsonBuffer.unwrap();
+          encodeNonNegativeInt(buffer, ProtocolValueType.Json);
+          encodeLength(buffer, jsonBytes);
+          buffer.extend(jsonBytes);
+        } finally {
+          jsonBuffer.reset();
+        }
         return;
       }
 
@@ -2179,7 +1977,9 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
     }
 
     case "number": {
-      if (NonNegativeInt.is(value)) {
+      // Negative zero is a non-negative integer in JavaScript, but integer
+      // encoding would lose its observable sign.
+      if (!Object.is(value, -0) && NonNegativeInt.is(value)) {
         if (isSmallInt(value)) {
           encodeNonNegativeInt(buffer, value);
           return;
@@ -2229,8 +2029,12 @@ export const decodeSqliteValue = (buffer: Buffer): SqliteValue => {
 
     case ProtocolValueType.Json: {
       const length = decodeLength(buffer);
-      const bytes = buffer.shiftN(length);
-      return JSON.stringify(packr.unpack(bytes));
+      const jsonValueBuffer = createBuffer(buffer.shiftN(length));
+      const value = decodeJsonValue(jsonValueBuffer);
+      if (jsonValueBuffer.getLength() !== 0) {
+        throw new ProtocolDecodeError("Invalid JSON MessagePack length");
+      }
+      return JSON.stringify(value);
     }
 
     case ProtocolValueType.DateIsoWithNonNegativeTime:
@@ -2258,26 +2062,7 @@ export const decodeSqliteValue = (buffer: Buffer): SqliteValue => {
   }
 };
 
-/**
- * Decodes a ProtocolMessage into a readable JSON object for debugging.
- *
- * Note: This is a stub for future implementation. It should use:
- *
- * - DecodeVersionAndOwner
- * - DecodeError or decodeWriteKeys (depending on context)
- * - DecodeMessages
- * - DecodeRanges
- *
- * If you want to help, please contribute to this function.
- */
-export const decodeProtocolMessageToJson = (
-  _protocolMessage: ProtocolMessage,
-  _isInitiator: boolean,
-): unknown => {
-  // TODO: Implement using
-  // - decodeVersionAndOwner
-  // -- decodeError or decodeWriteKeys (should be refactored out),
-  // -- decodeMessages, and decodeRanges.
-  // This is a stub for PRs and community contributions.
-  throw new Error("decodeProtocolMessageToJson is not implemented yet.");
-};
+const jsonBuffer = createBuffer();
+
+const isSmallInt: Predicate<number> = (value: number) =>
+  value >= 0 && value < 20;