@evolu/common 6.0.1-preview.1 → 6.0.1-preview.11

package/src/Evolu/Protocol.ts

@@ -15,21 +15,39 @@
  *
  * ### Message Structure
  *
- * | Field                          | Notes                      |
- * | :----------------------------- | :------------------------- |
- * | **Header**                     |                            |
- * | - {@link protocolVersion}      |                            |
- * | - {@link OwnerId}              |                            |
- * | - {@link ProtocolErrorCode}    | In non-initiator response. |
- * | **Messages**                   |                            |
- * | - {@link NonNegativeInt}       | A number of messages.      |
- * | - {@link EncryptedCrdtMessage} |                            |
- * | - {@link WriteKey}             | In initiator request.      |
- * | **Ranges**                     |                            |
- * | - {@link NonNegativeInt}       | Number of ranges.          |
- * | - {@link Range}                |                            |
+ * | Field                          | Notes                     |
+ * | :----------------------------- | :------------------------ |
+ * | **Header**                     |                           |
+ * | - {@link protocolVersion}      |                           |
+ * | - {@link OwnerId}              | {@link Owner}             |
+ * | **Initiator**                  |                           |
+ * | - {@link WriteKeyMode}         |                           |
+ * | - {@link WriteKey}             | If WriteKeyMode >= 1      |
+ * | - {@link WriteKey}             | If WriteKeyMode = 2 (new) |
+ * | **Non-initiator**              |                           |
+ * | - {@link ProtocolErrorCode}    |                           |
+ * | **Messages**                   |                           |
+ * | - {@link NonNegativeInt}       | A number of messages.     |
+ * | - {@link EncryptedCrdtMessage} |                           |
+ * | **Ranges**                     |                           |
+ * | - {@link NonNegativeInt}       | Number of ranges.         |
+ * | - {@link Range}                |                           |
  *
- * Every protocol message belongs to an {@link Owner}.
+ * ### WriteKey Validation
+ *
+ * The initiator sends WriteKeyMode and optionally one or two WriteKeys. One key
+ * for write operations and two for key rotation (current and new). Note that
+ * it's ok to not send any key if initiator is going to be synced with readonly
+ * owner. The non-initiator validates them immediately after parsing the
+ * initiator header, before processing any messages or ranges.
+ *
+ * ### WriteKey Rotation
+ *
+ * When initiator's {@link WriteKeyMode} is `Rotation`, two WriteKeys are
+ * present:
+ *
+ * 1. Current WriteKey (for validation)
+ * 2. New WriteKey (to be stored)
  *
  * ### Synchronization
  *
@@ -127,10 +145,8 @@ import { isNonEmptyReadonlyArray, NonEmptyReadonlyArray } from "../Array.js";
 import { assert } from "../Assert.js";
 import {
   Buffer,
-  BufferError,
   bytesToHex,
   bytesToUtf8,
-  concatBytes,
   createBuffer,
   hexToBytes,
   utf8ToBytes,
@@ -144,11 +160,12 @@ import {
 } from "../Crypto.js";
 import { eqArrayNumber } from "../Eq.js";
 import { computeBalancedBuckets } from "../Number.js";
-import { objectToEntries, ReadonlyRecord } from "../Object.js";
+import { objectToEntries } from "../Object.js";
 import { err, ok, Result } from "../Result.js";
 import { SqliteValue } from "../Sqlite.js";
 import {
   Base64Url,
+  base64UrlAlphabet,
   DateIsoString,
   Id,
   idTypeValueLength,
@@ -157,11 +174,12 @@ import {
   NanoId,
   NonNegativeInt,
   Number,
+  object,
   PositiveInt,
+  record,
 } from "../Type.js";
 import { Brand, Predicate } from "../Types.js";
 import {
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   Owner,
   OwnerId,
   OwnerWithWriteAccess,
@@ -203,6 +221,14 @@ export const ProtocolErrorCode = {
 type ProtocolErrorCode =
   (typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
 
+export const WriteKeyMode = {
+  None: 0,
+  Single: 1,
+  Rotation: 2,
+} as const;
+
+type WriteKeyMode = (typeof WriteKeyMode)[keyof typeof WriteKeyMode];
+
 /**
  * Evolu Protocol Storage
  *
@@ -247,17 +273,15 @@ export interface Storage {
     callback: (timestamp: BinaryTimestamp, index: NonNegativeInt) => boolean,
   ) => void;
 
-  /**
-   * Authorizes the initiator's {@link WriteKey} for the given
-   * {@link BinaryOwnerId}.
-   *
-   * For a client that does not expect foreign writes, return `false`.
-   */
+  /** Validates the {@link WriteKey} for the given {@link Owner}. */
   readonly validateWriteKey: (
     ownerId: BinaryOwnerId,
     writeKey: WriteKey,
   ) => boolean;
 
+  /** Sets the {@link WriteKey} for the given {@link Owner}. */
+  readonly setWriteKey: (ownerId: BinaryOwnerId, writeKey: WriteKey) => boolean;
+
   /** Write encrypted {@link CrdtMessage}s to storage. */
   readonly writeMessages: (
     ownerId: BinaryOwnerId,
@@ -269,6 +293,9 @@ export interface Storage {
     ownerId: BinaryOwnerId,
     timestamp: BinaryTimestamp,
   ) => EncryptedDbChange | null;
+
+  /** Delete all data for the given {@link Owner}. */
+  readonly deleteOwner: (ownerId: BinaryOwnerId) => boolean;
 }
 
 export interface StorageDep {
@@ -293,15 +320,23 @@ export interface CrdtMessage {
   readonly change: DbChange;
 }
 
+/**
+ * Base64Url string with maximum length of 256 characters. Encoding strings as
+ * Base64UrlString saves up to 25% in size compared to regular strings.
+ */
+export const Base64Url256 = maxLength(256)(Base64Url);
+export type Base64Url256 = typeof Base64Url256.Type;
+
 /**
  * A DbChange is a change to a table row. Together with a unique
  * {@link Timestamp}, it forms a {@link CrdtMessage}.
  */
-export interface DbChange {
-  readonly table: Base64Url256;
-  readonly id: Id;
-  readonly values: ReadonlyRecord<Base64Url256, SqliteValue>;
-}
+export const DbChange = object({
+  table: Base64Url256,
+  id: Id,
+  values: record(Base64Url256, SqliteValue),
+});
+export type DbChange = typeof DbChange.Type;
 
 export const RangeType = {
   Fingerprint: 1,
@@ -425,6 +460,7 @@ export const createProtocolMessageFromCrdtMessages =
     maxSize?: PositiveInt,
   ): ProtocolMessage => {
     const buffer = createProtocolMessageBuffer(owner.id, {
+      type: "initiator",
       totalMaxSize: maxSize ?? maxProtocolMessageSize,
       writeKey: owner.writeKey,
     });
@@ -480,7 +516,7 @@ export const createProtocolMessageFromCrdtMessages =
 export const createProtocolMessageForSync =
   (deps: StorageDep) =>
   (ownerId: OwnerId): ProtocolMessage | null => {
-    const buffer = createProtocolMessageBuffer(ownerId);
+    const buffer = createProtocolMessageBuffer(ownerId, { type: "initiator" });
     const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
 
     const size = deps.storage.getSize(binaryOwnerId);
@@ -498,6 +534,19 @@ export const createProtocolMessageForSync =
     return buffer.unwrap();
   };
 
+/** Creates a ProtocolMessage for {@link WriteKey} rotation. */
+export const createProtocolMessageForWriteKeyRotation = (
+  ownerId: OwnerId,
+  currentWriteKey: WriteKey,
+  newWriteKey: WriteKey,
+): ProtocolMessage => {
+  const buffer = createProtocolMessageBuffer(ownerId, {
+    type: "initiator",
+    writeKey: [currentWriteKey, newWriteKey],
+  });
+  return buffer.unwrap();
+};
+
 /**
  * Mutable builder for constructing {@link ProtocolMessage} respecting size
  * limits.
@@ -525,16 +574,22 @@ export interface ProtocolMessageBuffer {
 export const createProtocolMessageBuffer = (
   ownerId: OwnerId,
   options: {
-    readonly errorCode?: ProtocolErrorCode;
-    readonly writeKey?: WriteKey;
     readonly totalMaxSize?: PositiveInt | undefined;
     readonly rangesMaxSize?: PositiveInt | undefined;
     readonly version?: NonNegativeInt;
-  } = {},
+  } & (
+    | {
+        readonly type: "initiator";
+        /** Single key or [current, new] for rotation. */
+        readonly writeKey?: WriteKey | readonly [WriteKey, WriteKey];
+      }
+    | {
+        readonly type: "non-initiator";
+        readonly errorCode: ProtocolErrorCode;
+      }
+  ),
 ): ProtocolMessageBuffer => {
   const {
-    errorCode,
-    writeKey,
     totalMaxSize = maxProtocolMessageSize,
     rangesMaxSize = maxProtocolMessageRangesSize,
     version = protocolVersion,
@@ -555,7 +610,21 @@ export const createProtocolMessageBuffer = (
 
   encodeNonNegativeInt(buffers.header, version);
   buffers.header.extend(ownerIdToBinaryOwnerId(ownerId));
-  if (errorCode != null) buffers.header.extend([errorCode]);
+
+  if (options.type === "initiator") {
+    if (!options.writeKey) {
+      buffers.header.extend([WriteKeyMode.None]);
+    } else if (!Array.isArray(options.writeKey)) {
+      buffers.header.extend([WriteKeyMode.Single]);
+      buffers.header.extend(options.writeKey as WriteKey);
+    } else {
+      buffers.header.extend([WriteKeyMode.Rotation]);
+      buffers.header.extend(options.writeKey[0] as WriteKey); // current
+      buffers.header.extend(options.writeKey[1] as WriteKey); // new
+    }
+  } else {
+    buffers.header.extend([options.errorCode]);
+  }
 
   let isLastRangeInfinite = false;
 
@@ -567,10 +636,7 @@ export const createProtocolMessageBuffer = (
   const getHeaderAndMessagesSize = () =>
     buffers.header.getLength() +
     buffers.messages.timestamps.getLength() +
-    buffers.messages.dbChanges.getLength() +
-    (buffers.messages.timestamps.getCount() > 0 && writeKey
-      ? writeKeyLength
-      : 0);
+    buffers.messages.dbChanges.getLength();
 
   const getRangesSize = () =>
     buffers.ranges.timestamps.getCount() > 0
@@ -685,8 +751,6 @@ export const createProtocolMessageBuffer = (
 
       buffers.messages.timestamps.append(buffers.header);
       buffers.header.extend(buffers.messages.dbChanges.unwrap());
-      if (buffers.messages.timestamps.getCount() > 0 && writeKey)
-        buffers.header.extend(writeKey);
 
       if (buffers.ranges.timestamps.getCount() > 0) {
         buffers.ranges.timestamps.append(buffers.header);
@@ -836,8 +900,6 @@ export const applyProtocolMessageAsClient =
           });
         }
 
-        const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
-
         const errorCode = input.shift() as ProtocolErrorCode;
         if (errorCode !== ProtocolErrorCode.NoError) {
           switch (errorCode) {
@@ -864,6 +926,7 @@ export const applyProtocolMessageAsClient =
         }
 
         const messages = decodeMessages(input);
+        const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
 
         if (
           isNonEmptyReadonlyArray(messages) &&
@@ -877,6 +940,7 @@ export const applyProtocolMessageAsClient =
         if (writeKey == null) return ok(null);
 
         const output = createProtocolMessageBuffer(ownerId, {
+          type: "initiator",
           writeKey,
           totalMaxSize,
           rangesMaxSize,
@@ -911,8 +975,7 @@ export const applyProtocolMessageAsRelay =
     version = protocolVersion,
   ): Result<ProtocolMessage | null, ProtocolInvalidDataError> =>
     tryDecodeProtocolData(inputMessage, (input) => {
-      const requestedVersion = decodeNonNegativeInt(input);
-      const ownerId = decodeOwnerId(input);
+      const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
       const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
 
       if (requestedVersion !== version) {
@@ -925,44 +988,89 @@ export const applyProtocolMessageAsRelay =
 
       subscribe?.(ownerId);
 
-      const messages = decodeMessages(input);
+      const writeKeyMode = input.shift() as WriteKeyMode;
+      let writeKey: WriteKey | undefined;
+      let newWriteKey: WriteKey | undefined;
+
+      if (writeKeyMode !== WriteKeyMode.None) {
+        writeKey = input.shiftN(writeKeyLength) as WriteKey;
+        switch (writeKeyMode) {
+          case WriteKeyMode.Single:
+            break;
+          case WriteKeyMode.Rotation:
+            newWriteKey = input.shiftN(writeKeyLength) as WriteKey;
+            break;
+          default:
+            throw new ProtocolDecodeError(
+              `Invalid WriteKeyMode: ${writeKeyMode}`,
+            );
+        }
+      }
 
-      if (isNonEmptyReadonlyArray(messages)) {
-        const messagesEnd = inputMessage.length - input.getLength();
-        const writeKey = input.shiftN(writeKeyLength) as WriteKey;
+      if (writeKey) {
+        const isValid = deps.storage.validateWriteKey(binaryOwnerId, writeKey);
+        if (!isValid) {
+          return ok(
+            createProtocolMessageBuffer(ownerId, {
+              type: "non-initiator",
+              errorCode: ProtocolErrorCode.WriteKeyError,
+            }).unwrap(),
+          );
+        }
 
-        const writeKeyIsValid = deps.storage.validateWriteKey(
-          binaryOwnerId,
-          writeKey,
-        );
+        if (newWriteKey) {
+          const rotationSuccess = deps.storage.setWriteKey(
+            binaryOwnerId,
+            newWriteKey,
+          );
+          if (!rotationSuccess) {
+            return ok(
+              createProtocolMessageBuffer(ownerId, {
+                type: "non-initiator",
+                errorCode: ProtocolErrorCode.WriteError,
+              }).unwrap(),
+            );
+          }
+        }
+      }
+
+      const messages = decodeMessages(input);
 
-        if (!writeKeyIsValid)
+      if (isNonEmptyReadonlyArray(messages)) {
+        if (!writeKey)
           return ok(
             createProtocolMessageBuffer(ownerId, {
+              type: "non-initiator",
               errorCode: ProtocolErrorCode.WriteKeyError,
             }).unwrap(),
           );
 
-        if (broadcast) {
-          // Instead of encoding a new protocol message, we reuse the inputMessage.
-          const broadcastMessage = concatBytes(
-            inputMessage.slice(0, 17),
-            new Uint8Array([ProtocolErrorCode.NoError]),
-            inputMessage.slice(17, messagesEnd),
-          ) as ProtocolMessage;
-
-          broadcast(ownerId, broadcastMessage);
+        // Only broadcast if there's no ranges.
+        if (broadcast && input.getLength() === 0) {
+          const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
+            type: "non-initiator",
+            errorCode: ProtocolErrorCode.NoError,
+            totalMaxSize,
+            rangesMaxSize,
+            version,
+          });
+          for (const message of messages) {
+            broadcastBuffer.addMessage(message);
+          }
+          broadcast(ownerId, broadcastBuffer.unwrap());
         }
 
         if (!deps.storage.writeMessages(binaryOwnerId, messages))
           return ok(
             createProtocolMessageBuffer(ownerId, {
+              type: "non-initiator",
               errorCode: ProtocolErrorCode.WriteError,
             }).unwrap(),
           );
       }
 
       const output = createProtocolMessageBuffer(ownerId, {
+        type: "non-initiator",
         errorCode: ProtocolErrorCode.NoError,
         totalMaxSize,
         rangesMaxSize,
@@ -983,18 +1091,19 @@ const tryDecodeProtocolData = <T, E>(
   try {
     return callback(createBuffer(data));
   } catch (error: unknown) {
-    if (error instanceof ProtocolDecodeError || error instanceof BufferError)
-      return err<ProtocolInvalidDataError>({
-        type: "ProtocolInvalidDataError",
-        data,
-        error,
-      });
-
-    throw error;
+    return err<ProtocolInvalidDataError>({
+      type: "ProtocolInvalidDataError",
+      data,
+      error,
+    });
   }
 };
 
 const decodeVersionAndOwner = (input: Buffer): [NonNegativeInt, OwnerId] => {
+  // This structure must never change across protocol versions. The version
+  // and owner ID must always be the first two fields in every protocol message
+  // to enable version negotiation and owner identification before any other
+  // processing occurs.
   const version = decodeNonNegativeInt(input);
   const ownerId = decodeOwnerId(input);
   return [version, ownerId];
@@ -1052,6 +1161,7 @@ const sync =
         return ok(null);
       }
       const message = createProtocolMessageBuffer(binaryOwnerId, {
+        type: "non-initiator",
         errorCode: ProtocolErrorCode.SyncError,
       });
       return ok(message.unwrap());
@@ -1434,26 +1544,12 @@ export const ownerIdToBinaryOwnerId = (ownerId: OwnerId): BinaryOwnerId =>
 export const binaryOwnerIdToOwnerId = (binaryOwnerId: BinaryOwnerId): OwnerId =>
   decodeOwnerId(createBuffer(binaryOwnerId));
 
-/**
- * Base64Url string with maximum length of 256 characters. Encoding strings as
- * Base64UrlString saves up to 25% in size compared to regular strings.
- */
-export const Base64Url256 = maxLength(256)(Base64Url);
-export type Base64Url256 = typeof Base64Url256.Type;
-
 /**
  * Union type for all variants of Base64Url strings with limited length. All
  * these types use Base64Url alphabet and are < 256 characters.
  */
 export type Base64Url256Variant = Base64Url256 | Id | NanoId | OwnerId;
 
-/**
- * Alphabet used for Base64Url encoding. This is copied from the `nanoid`
- * library to avoid dependency on a specific version of `nanoid`.
- */
-const urlAlphabet =
-  "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
-
 /**
  * Converts a Base64Url string to a Uint8Array for binary storage. This encoding
  * is more space-efficient than UTF-8 for Base64Url strings.
@@ -1470,7 +1566,7 @@ export const base64Url256ToBytes = (
   let byteIndex = 0;
 
   for (const char of string) {
-    const charValue = urlAlphabet.indexOf(char);
+    const charValue = base64UrlAlphabet.indexOf(char);
     bitBuffer = (bitBuffer << 6) | charValue;
     bitsInBuffer += 6;
     while (bitsInBuffer >= 8) {
@@ -1505,10 +1601,10 @@ export const decodeBase64Url256 = (
       bitsInBuffer -= 6;
       if (string.length < stringLength) {
         const charValue = (bitBuffer >> bitsInBuffer) & 0x3f;
-        if (charValue < 0 || charValue >= urlAlphabet.length) {
+        if (charValue < 0 || charValue >= base64UrlAlphabet.length) {
           throw new ProtocolDecodeError("invalid charValue");
         }
-        string += urlAlphabet[charValue];
+        string += base64UrlAlphabet[charValue];
       }
     }
   }
@@ -1895,3 +1991,27 @@ export const decodeSqliteValue = (buffer: Buffer): SqliteValue => {
       throw new ProtocolDecodeError("invalid ProtocolValueType");
   }
 };
+
+/**
+ * Decodes a ProtocolMessage into a readable JSON object for debugging.
+ *
+ * Note: This is a stub for future implementation. It should use:
+ *
+ * - DecodeVersionAndOwner
+ * - DecodeError or decodeWriteKeys (depending on context)
+ * - DecodeMessages
+ * - DecodeRanges
+ *
+ * If you want to help, please contribute to this function.
+ */
+export const decodeProtocolMessageToJson = (
+  _protocolMessage: ProtocolMessage,
+  _isInitiator: boolean,
+): unknown => {
+  // TODO: Implement using
+  // - decodeVersionAndOwner
+  // -- decodeError or decodeWriteKeys (should be refactored out),
+  // -- decodeMessages, and decodeRanges.
+  // This is a stub for PRs and community contributions.
+  throw new Error("decodeProtocolMessageToJson is not implemented yet.");
+};

package/src/Evolu/Relay.ts

@@ -1,7 +1,7 @@
 import { isNonEmptyReadonlyArray } from "../Array.js";
 import { ConsoleConfig } from "../Console.js";
-import { eqArrayNumber } from "../Eq.js";
-import { ok, Result } from "../Result.js";
+import { TimingSafeEqualDep } from "../Crypto.js";
+import { err, ok, Result } from "../Result.js";
 import { sql, SqliteError } from "../Sqlite.js";
 import { SimpleName } from "../Type.js";
 import { OwnerId, WriteKey } from "./Owner.js";
@@ -19,8 +19,10 @@ export interface RelayConfig extends ConsoleConfig {
   readonly name?: SimpleName;
 }
 
+export type RelaySqliteStorageDeps = SqliteStorageDeps & TimingSafeEqualDep;
+
 export const createRelayStorage =
-  (deps: SqliteStorageDeps) =>
+  (deps: RelaySqliteStorageDeps) =>
   (options: CreateSqliteStorageBaseOptions): Result<Storage, SqliteError> => {
     const sqliteStorageBase = createSqliteStorageBase(deps)(options);
     if (!sqliteStorageBase.ok) return sqliteStorageBase;
@@ -87,7 +89,22 @@ export const createRelayStorage =
           return true;
         }
 
-        return eqArrayNumber(rows[0].writeKey, writeKey);
+        return deps.timingSafeEqual(rows[0].writeKey, writeKey);
+      },
+
+      setWriteKey: (ownerId, writeKey) => {
+        const upsertWriteKey = deps.sqlite.exec(sql`
+          insert into evolu_writeKey (ownerId, writeKey)
+          values (${ownerId}, ${writeKey})
+          on conflict (ownerId) do update
+            set writeKey = excluded.writeKey;
+        `);
+        if (!upsertWriteKey.ok) {
+          options.onStorageError(upsertWriteKey.error);
+          return false;
+        }
+
+        return true;
       },
 
       writeMessages: (ownerId, messages) => {
@@ -138,5 +155,29 @@ export const createRelayStorage =
 
         return result.value.rows[0]?.change;
       },
+
+      deleteOwner: (ownerId) => {
+        const result = deps.sqlite.transaction(() => {
+          const del1 = deps.sqlite.exec(sql`
+            delete from evolu_writeKey where ownerId = ${ownerId};
+          `);
+          if (!del1.ok) return del1;
+
+          const del2 = deps.sqlite.exec(sql`
+            delete from evolu_message where ownerId = ${ownerId};
+          `);
+          if (!del2.ok) return del2;
+
+          const del3 = sqliteStorageBase.value.deleteOwner(ownerId);
+          if (!del3) return err(null);
+
+          return ok();
+        });
+        if (!result.ok) {
+          if (result.error) options.onStorageError(result.error);
+          return false;
+        }
+        return true;
+      },
     });
   };