@evolu/common 5.4.7 → 6.0.0

package/dist/src/Evolu/Protocol.js

@@ -0,0 +1,1151 @@
+/**
+ * Evolu Protocol
+ *
+ * Evolu Protocol is a local-first, end-to-end encrypted binary synchronization
+ * protocol optimized for minimal size and maximum speed. It enables data sync
+ * between a client and a relay, clients in a peer-to-peer (P2P) setup, or
+ * relays with each other.
+ *
+ * Evolu Protocol is designed for SQLite but can be extended to any database. It
+ * implements [Range-Based Set Reconciliation](https://arxiv.org/abs/2212.13567)
+ * by Aljoscha Meyer.
+ *
+ * To learn how RBSR works, check
+ * [Negentropy](https://logperiodic.com/rbsr.html). Evolu Protocol is similar to
+ * Negentropy but uses different encoding and also provides data transfer and
+ * ownership.
+ *
+ * ### Message Structure
+ *
+ * | Field                          | Notes                      |
+ * | :----------------------------- | :------------------------- |
+ * | **Header**                     |                            |
+ * | - {@link protocolVersion}      |                            |
+ * | - {@link OwnerId}              |                            |
+ * | - {@link ProtocolErrorCode}    | In non-initiator response. |
+ * | **Messages**                   |                            |
+ * | - {@link NonNegativeInt}       | A number of messages.      |
+ * | - {@link EncryptedCrdtMessage} |                            |
+ * | - {@link WriteKey}             | In initiator request.      |
+ * | **Ranges**                     |                            |
+ * | - {@link NonNegativeInt}       | Number of ranges.          |
+ * | - {@link Range}                |                            |
+ *
+ * Every protocol message belongs to an owner.
+ *
+ * ### Synchronization
+ *
+ * - **Messages**: Sends {@link EncryptedCrdtMessage}s in either direction.
+ * - **Ranges**: Determines messages to sync. Usage varies by transport—e.g., sent
+ *   only on WebSocket connection open or with every fetch request.
+ *
+ * Synchronization involves an initiator and a non-initiator. The **initiator**
+ * is typically a client, and the **non-initiator** is typically a relay. Each
+ * side processes the received message and responds with a new `ProtocolMessage`
+ * if further sync is needed or possible, continuing until both sides are
+ * synchronized.
+ *
+ * Both **Messages** and **Ranges** are optional, allowing each side to send,
+ * sync, or only subscribe data as needed.
+ *
+ * When the initiator sends data, the {@link WriteKey} is **required** in
+ * **Messages** as a secure token proving the initiator can write changes. The
+ * non-initiator responds without a {@link WriteKey}, since the initiator’s
+ * request already signals it wants data. If the non-initiator detects an issue
+ * (e.g., an invalid {@link WriteKey} causing a {@link ProtocolWriteKeyError}, or
+ * a write failure causing a {@link ProtocolWriteError}), it sends an error code
+ * via the `Error` field in the header back to the initiator. In relay-to-relay
+ * or P2P sync, both sides may require the {@link WriteKey} depending on who is
+ * the initiator.
+ *
+ * ### Message Size Limit
+ *
+ * The protocol enforces a strict maximum size for all messages, defined by
+ * {@link maxProtocolMessageSize}. This ensures every `ProtocolMessage` is less
+ * than or equal to this limit, eliminating the need for applications to
+ * fragment and reconstruct messages during transmission.
+ *
+ * ### Why Binary?
+ *
+ * The protocol avoids JSON because:
+ *
+ * - Encrypted data doesn’t compress well, unlike plain JSON.
+ * - Message size must be controlled during creation.
+ * - Sequential byte reading is faster than parsing and can avoid conversions.
+ *
+ * It uses structure-aware encoding, significantly outperforming generic binary
+ * serialization formats with the following optimizations:
+ *
+ * - **NonNegativeInt:** Up to 33% smaller than MessagePack.
+ * - **Base64Url Strings:** Up to 25% size reduction.
+ * - **DateIso:** Up to 75% smaller.
+ * - **Timestamp Encoding:** Delta encoding for milliseconds and run-length
+ *   encoding (RLE) for counters and NodeIds.
+ * - **Small Integers (0 to 19):** Reduces size by 1 byte per integer.
+ *
+ * To avoid reinventing serialization where it’s unnecessary—like for JSON and
+ * certain numbers—the Evolu Protocol relies on MessagePack.
+ *
+ * ### Versioning
+ *
+ * The initiator sends a versioned `ProtocolMessage`. If the non-initiator uses
+ * a different version, it responds with a message containing only its protocol
+ * version—**without an `ownerId`**. This allows the initiator to check protocol
+ * compatibility, for example, by sending version-only messages to multiple
+ * relays before starting synchronization.
+ *
+ * @module
+ */
+import { sha256 } from "@noble/hashes/sha2";
+import { pack, unpack, unpackMultiple } from "msgpackr";
+import { isNonEmptyReadonlyArray } from "../Array.js";
+import { assert } from "../Assert.js";
+import { BufferError, bytesToHex, bytesToUtf8, concatBytes, createBuffer, hexToBytes, utf8ToBytes, } from "../Buffer.js";
+import { padmePaddingLength, } from "../Crypto.js";
+import { eqArrayNumber } from "../Eq.js";
+import { computeBalancedBuckets } from "../Number.js";
+import { objectToEntries } from "../Object.js";
+import { err, ok } from "../Result.js";
+import { Base64Url, DateIsoString, idTypeValueLength, JsonValueFromString, maxLength, NonNegativeInt, Number, } from "../Type.js";
+import { writeKeyLength, } from "./Owner.js";
+import { binaryTimestampToTimestamp, Counter, Millis, timestampToBinaryTimestamp, } from "./Timestamp.js";
+/** Maximum size of the entire protocol message in bytes. */
+export const maxProtocolMessageSize = 1_000_000;
+/** Maximum size of the ranges in bytes. */
+export const maxProtocolMessageRangesSize = 30_000;
+/** Evolu Protocol version. */
+export const protocolVersion = 0;
+export const ProtocolErrorCode = {
+    NoError: 0,
+    /** A code for {@link ProtocolWriteKeyError}. */
+    WriteKeyError: 1,
+    /** A code for {@link ProtocolWriteError}. */
+    WriteError: 2,
+    /** A code for {@link ProtocolSyncError}. */
+    SyncError: 3,
+};
+export const RangeType = {
+    Fingerprint: 1,
+    Skip: 0,
+    Timestamps: 2,
+};
+export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
+export const fingerprintSize = 12;
+/** A fingerprint of an empty range. */
+export const zeroFingerprint = new Uint8Array(fingerprintSize);
+/**
+ * Creates a {@link ProtocolMessage} from CRDT messages.
+ *
+ * If the message size would exceed {@link maxProtocolMessageSize}, the protocol
+ * ensures all messages will be sent in the next round(s) even over
+ * unidirectional and stateless transports.
+ */
+export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages, maxSize) => {
+    const buffer = createProtocolMessageBuffer(owner.id, {
+        totalMaxSize: maxSize ?? maxProtocolMessageSize,
+        writeKey: owner.writeKey,
+    });
+    let notAllMessagesSent = false;
+    for (const message of messages) {
+        const change = encodeAndEncryptDbChange(deps)(message.change, owner.encryptionKey);
+        const encryptedCrdtMessage = { timestamp: message.timestamp, change };
+        if (buffer.canAddMessage(encryptedCrdtMessage)) {
+            buffer.addMessage(encryptedCrdtMessage);
+        }
+        else {
+            notAllMessagesSent = true;
+            break;
+        }
+    }
+    if (notAllMessagesSent) {
+        /**
+         * DEV: If not all messages fit due to size limits, we trigger a sync
+         * continuation by appending a Range with a random fingerprint. This
+         * ensures the receiver always responds with ranges, prompting another
+         * sync round.
+         *
+         * The ideal approach would be to send three ranges (skip, fingerprint,
+         * skip) where the fingerprint of unsent messages would act as narrow sync
+         * probe. I think we can send {@link zeroFingerprint} which can be
+         * interpreted as an indication that the other side should reply with
+         * {@link TimestampsRange}, so no need to restart syncing.
+         *
+         * For now, using a random fingerprint avoids extra complexity and is good
+         * enough for this case.
+         */
+        const randomFingerprint = deps.createRandomBytes(fingerprintSize);
+        // There is always a space for Fingerprint with InfiniteUpperBound.
+        buffer.addRange({
+            type: RangeType.Fingerprint,
+            upperBound: InfiniteUpperBound,
+            fingerprint: randomFingerprint,
+        });
+    }
+    return buffer.unwrap();
+};
+/** Creates a {@link ProtocolMessage} for sync. */
+export const createProtocolMessageForSync = (deps) => (ownerId) => {
+    const buffer = createProtocolMessageBuffer(ownerId);
+    const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
+    const size = deps.storage.getSize(binaryOwnerId);
+    // Errors are handled by the storage.
+    if (size == null)
+        return null;
+    splitRange(deps)(binaryOwnerId, 0, size, InfiniteUpperBound, buffer);
+    return buffer.unwrap();
+};
+export const createProtocolMessageBuffer = (ownerId, options = {}) => {
+    const { errorCode, writeKey, totalMaxSize = maxProtocolMessageSize, rangesMaxSize = maxProtocolMessageRangesSize, version = protocolVersion, } = options;
+    const buffers = {
+        header: createBuffer(),
+        messages: {
+            timestamps: createTimestampsBuffer(),
+            dbChanges: createBuffer(),
+        },
+        ranges: {
+            timestamps: createTimestampsBuffer(),
+            types: createBuffer(),
+            payloads: createBuffer(),
+        },
+    };
+    encodeNonNegativeInt(buffers.header, version);
+    buffers.header.extend(ownerIdToBinaryOwnerId(ownerId));
+    if (errorCode != null)
+        buffers.header.extend([errorCode]);
+    let isLastRangeInfinite = false;
+    const isWithinSizeLimits = () => getSize() <= totalMaxSize;
+    const getSize = () => (getHeaderAndMessagesSize() + getRangesSize());
+    const getHeaderAndMessagesSize = () => buffers.header.getLength() +
+        buffers.messages.timestamps.getLength() +
+        buffers.messages.dbChanges.getLength() +
+        (buffers.messages.timestamps.getCount() > 0 && writeKey
+            ? writeKeyLength
+            : 0);
+    const getRangesSize = () => buffers.ranges.timestamps.getCount() > 0
+        ? buffers.ranges.timestamps.getLength() +
+            buffers.ranges.types.getLength() +
+            buffers.ranges.payloads.getLength() +
+            safeMargins.remainingRange
+        : 0;
+    /**
+     * We calculated worst-case sizes as closely as possible and added a small
+     * safety margin, since computing exact worst cases is difficult due to
+     * variable-length, run-length, and delta encoding.
+     *
+     * Runtime assertions (`assert`) are used to guarantee that size limits are
+     * never exceeded. If a limit is exceeded, the assertion will fail at the
+     * precise location, making it easy to identify and fix the issue.
+     *
+     * While it would be possible to avoid the safety margin by snapshotting
+     * buffer states and rolling back changes, this would likely impact
+     * performance. If someone has time and wants to experiment with this
+     * approach, contributions are welcome.
+     */
+    const safeMargins = {
+        remainingRange: fingerprintSize + 10, // bytes: range type + possible increased count varint
+        timestamp: 30, // bytes: max millis + max count + NodeId
+        dbChangeLength: 8, // bytes: maximum encoded DbChange length varint
+        splitRange: 800, // bytes: worst case is around 650 bytes
+        timestampsRange: 50, // bytes: range type + its upperBound + possible increased count varint
+    };
+    const addMessageSafeMargin = safeMargins.timestamp +
+        safeMargins.dbChangeLength +
+        safeMargins.remainingRange;
+    return {
+        canAddMessage: (message) => getSize() + addMessageSafeMargin + message.change.length <= totalMaxSize,
+        addMessage: (message) => {
+            buffers.messages.timestamps.add(message.timestamp);
+            encodeLength(buffers.messages.dbChanges, message.change);
+            buffers.messages.dbChanges.extend(message.change);
+            assert(isWithinSizeLimits(), "the message is too big");
+        },
+        canSplitRange: () => {
+            return getRangesSize() + safeMargins.splitRange <= rangesMaxSize;
+        },
+        canAddTimestampsRangeAndMessage: (timestamps, message) => {
+            const rangesNewSize = getRangesSize() + timestamps.getLength() + safeMargins.timestampsRange;
+            return (rangesNewSize <= rangesMaxSize &&
+                (message
+                    ? getHeaderAndMessagesSize() +
+                        rangesNewSize +
+                        addMessageSafeMargin +
+                        message.change.length <=
+                        totalMaxSize
+                    : true));
+        },
+        addRange: (range) => {
+            assert(!isLastRangeInfinite, "Cannot add a range after an InfiniteUpperBound range");
+            isLastRangeInfinite = range.upperBound === InfiniteUpperBound;
+            /**
+             * We don't have to encode InfiniteUpperBound timestamp since it's always
+             * the last because ranges cover the whole universe. For partial sync, we
+             * use SkipRange.
+             */
+            if (range.upperBound !== InfiniteUpperBound)
+                buffers.ranges.timestamps.add(binaryTimestampToTimestamp(range.upperBound));
+            else {
+                buffers.ranges.timestamps.addInfinite();
+            }
+            encodeNonNegativeInt(buffers.ranges.types, range.type);
+            switch (range.type) {
+                case RangeType.Skip:
+                    break;
+                case RangeType.Fingerprint:
+                    buffers.ranges.payloads.extend(range.fingerprint);
+                    break;
+                case RangeType.Timestamps: {
+                    range.timestamps.append(buffers.ranges.payloads);
+                    break;
+                }
+            }
+            assert(isWithinSizeLimits(), `the range ${range.type} is too big`);
+        },
+        unwrap: () => {
+            if (buffers.ranges.timestamps.getCount() > 0) {
+                assert(isLastRangeInfinite, "The last range's upperBound must be InfiniteUpperBound");
+            }
+            buffers.messages.timestamps.append(buffers.header);
+            buffers.header.extend(buffers.messages.dbChanges.unwrap());
+            if (buffers.messages.timestamps.getCount() > 0 && writeKey)
+                buffers.header.extend(writeKey);
+            if (buffers.ranges.timestamps.getCount() > 0) {
+                buffers.ranges.timestamps.append(buffers.header);
+                buffers.header.extend(buffers.ranges.types.unwrap());
+                buffers.header.extend(buffers.ranges.payloads.unwrap());
+            }
+            return buffers.header.unwrap();
+        },
+        getSize,
+    };
+};
+export const createTimestampsBuffer = () => {
+    let count = 0;
+    const countBuffer = createBuffer();
+    const syncCount = () => {
+        countBuffer.reset();
+        encodeNonNegativeInt(countBuffer, count);
+    };
+    syncCount();
+    const millisBuffer = createBuffer();
+    let previousMillis = 0;
+    const counterEncoder = createRunLengthEncoder((buffer, value) => {
+        encodeNonNegativeInt(buffer, value);
+    });
+    const nodeIdEncoder = createRunLengthEncoder((buffer, value) => {
+        encodeNodeId(buffer, value);
+    });
+    return {
+        add: (timestamp) => {
+            const delta = timestamp.millis - previousMillis;
+            assert(NonNegativeInt.is(delta), "The delta must be NonNegativeInt");
+            count++;
+            syncCount();
+            previousMillis = timestamp.millis;
+            encodeNonNegativeInt(millisBuffer, delta);
+            counterEncoder.add(timestamp.counter);
+            nodeIdEncoder.add(timestamp.nodeId);
+        },
+        addInfinite: () => {
+            count++;
+            syncCount();
+        },
+        getCount: () => count,
+        getLength: () => countBuffer.getLength() +
+            millisBuffer.getLength() +
+            counterEncoder.getLength() +
+            nodeIdEncoder.getLength(),
+        append: (buffer) => {
+            buffer.extend(countBuffer.unwrap());
+            buffer.extend(millisBuffer.unwrap());
+            buffer.extend(counterEncoder.unwrap());
+            buffer.extend(nodeIdEncoder.unwrap());
+        },
+    };
+};
+const createRunLengthEncoder = (encodeValue) => {
+    const buffer = createBuffer();
+    let previousLength = 0;
+    let previousValue = null;
+    let runLength = 0;
+    return {
+        add: (value) => {
+            if (value === previousValue) {
+                runLength++;
+                buffer.truncate(previousLength);
+            }
+            else {
+                previousValue = value;
+                runLength = 1;
+            }
+            previousLength = buffer.getLength();
+            encodeValue(buffer, value);
+            encodeNonNegativeInt(buffer, runLength);
+        },
+        getLength: () => buffer.getLength(),
+        unwrap: () => buffer.unwrap(),
+    };
+};
+export const applyProtocolMessageAsClient = (deps) => (inputMessage, { getWriteKey, version = protocolVersion, totalMaxSize, rangesMaxSize, } = {}) => tryDecodeProtocolData(inputMessage, (input) => {
+    const requestedVersion = decodeNonNegativeInt(input);
+    if (requestedVersion !== version) {
+        return err({
+            type: "ProtocolUnsupportedVersionError",
+            unsupportedVersion: requestedVersion,
+            isInitiator: version < requestedVersion,
+        });
+    }
+    const ownerId = decodeOwnerId(input);
+    const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
+    const errorCode = input.shift();
+    if (errorCode !== ProtocolErrorCode.NoError) {
+        switch (errorCode) {
+            case ProtocolErrorCode.WriteKeyError:
+                return err({
+                    type: "ProtocolWriteKeyError",
+                });
+            case ProtocolErrorCode.WriteError:
+                return err({
+                    type: "ProtocolWriteError",
+                });
+            case ProtocolErrorCode.SyncError:
+                return err({
+                    type: "ProtocolSyncError",
+                });
+            default:
+                throw new ProtocolDecodeError(`Invalid ProtocolErrorCode: ${errorCode}`);
+        }
+    }
+    const messages = decodeMessages(input);
+    if (isNonEmptyReadonlyArray(messages) &&
+        !deps.storage.writeMessages(binaryOwnerId, messages)) {
+        return ok(null);
+    }
+    if (!getWriteKey)
+        return ok(null);
+    const writeKey = getWriteKey(ownerId);
+    if (writeKey == null)
+        return ok(null);
+    const output = createProtocolMessageBuffer(ownerId, {
+        writeKey,
+        totalMaxSize,
+        rangesMaxSize,
+    });
+    return sync(deps)("initiator", input, output, binaryOwnerId);
+});
+export const applyProtocolMessageAsRelay = (deps) => (inputMessage, { subscribe, broadcast, totalMaxSize, rangesMaxSize, } = {}, 
+/** For testing purposes only; should not be used in production. */
+version = protocolVersion) => tryDecodeProtocolData(inputMessage, (input) => {
+    const requestedVersion = decodeNonNegativeInt(input);
+    if (requestedVersion !== version) {
+        // Non-initiator responds with its version.
+        const output = createBuffer();
+        encodeNonNegativeInt(output, version);
+        return ok(output.unwrap());
+    }
+    const ownerId = decodeOwnerId(input);
+    const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
+    subscribe?.(ownerId);
+    const messages = decodeMessages(input);
+    if (isNonEmptyReadonlyArray(messages)) {
+        const messagesEnd = inputMessage.length - input.getLength();
+        const writeKey = input.shiftN(writeKeyLength);
+        const writeKeyIsValid = deps.storage.validateWriteKey(binaryOwnerId, writeKey);
+        if (!writeKeyIsValid)
+            return ok(createProtocolMessageBuffer(ownerId, {
+                errorCode: ProtocolErrorCode.WriteKeyError,
+            }).unwrap());
+        if (broadcast) {
+            // Instead of encoding a new protocol message, we reuse the inputMessage.
+            const broadcastMessage = concatBytes(inputMessage.slice(0, 17), new Uint8Array([ProtocolErrorCode.NoError]), inputMessage.slice(17, messagesEnd));
+            broadcast(ownerId, broadcastMessage);
+        }
+        if (!deps.storage.writeMessages(binaryOwnerId, messages))
+            return ok(createProtocolMessageBuffer(ownerId, {
+                errorCode: ProtocolErrorCode.WriteError,
+            }).unwrap());
+    }
+    const output = createProtocolMessageBuffer(ownerId, {
+        errorCode: ProtocolErrorCode.NoError,
+        totalMaxSize,
+        rangesMaxSize,
+    });
+    return sync(deps)("non-initiator", input, output, binaryOwnerId);
+});
+/**
+ * Wraps Evolu Protocol decoding functions, which use exceptions instead of
+ * {@link Result} to provide stack traces for debugging and reduce allocation
+ * overhead in success cases.
+ */
+const tryDecodeProtocolData = (data, callback) => {
+    try {
+        return callback(createBuffer(data));
+    }
+    catch (error) {
+        if (error instanceof ProtocolDecodeError || error instanceof BufferError)
+            return err({
+                type: "ProtocolInvalidDataError",
+                data,
+                error,
+            });
+        throw error;
+    }
+};
+/**
+ * Error thrown for internal protocol validation failures, such as invalid data
+ * or type errors.
+ */
+class ProtocolDecodeError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = this.constructor.name;
+        Error.captureStackTrace(this, this.constructor);
+    }
+}
+const decodeMessages = (buffer) => {
+    const timestamps = decodeTimestamps(buffer);
+    const messages = [];
+    for (const timestamp of timestamps) {
+        const changeLength = decodeLength(buffer);
+        const change = buffer.shiftN(changeLength);
+        messages.push({ timestamp, change });
+    }
+    return messages;
+};
+const sync = (deps) => (role, input, output, ownerId) => {
+    const ranges = decodeRanges(input);
+    if (!isNonEmptyReadonlyArray(ranges)) {
+        // Nothing to sync.
+        return ok(null);
+    }
+    const binaryOwnerId = binaryOwnerIdToOwnerId(ownerId);
+    const outputInitialSize = output.getSize();
+    const syncFail = () => {
+        // Only the relay (non-initiator) reports sync errors, not the client (initiator).
+        if (role === "initiator") {
+            return ok(null);
+        }
+        const message = createProtocolMessageBuffer(binaryOwnerId, {
+            errorCode: ProtocolErrorCode.SyncError,
+        });
+        return ok(message.unwrap());
+    };
+    const storageSize = deps.storage.getSize(ownerId);
+    if (storageSize == null)
+        return syncFail();
+    let prevUpperBound = null;
+    let prevIndex = 0;
+    let skip = false;
+    let nonSkipRangeAdded = false;
+    const skipRange = (range) => {
+        // The last range, if any non skip was added, must have InfiniteUpperBound.
+        if (nonSkipRangeAdded && range.upperBound === InfiniteUpperBound) {
+            output.addRange({
+                type: RangeType.Skip,
+                upperBound: InfiniteUpperBound,
+            });
+        }
+        else {
+            skip = true;
+        }
+    };
+    const coalesceSkipsBeforeAdd = () => {
+        // Set to true because we are going to add a non skip range.
+        nonSkipRangeAdded = true;
+        if (skip) {
+            skip = false;
+            assert(prevUpperBound != null, "prevUpperBound is null");
+            // There is always a space for a skip range before adding.
+            output.addRange({
+                type: RangeType.Skip,
+                upperBound: prevUpperBound,
+            });
+        }
+    };
+    // When we don't have a space...
+    const addFingerprintForRemainingRange = (begin) => {
+        const fingerprint = deps.storage.fingerprint(ownerId, begin, storageSize);
+        if (!fingerprint)
+            return false;
+        // There is always a space for a ramaining range.
+        output.addRange({
+            type: RangeType.Fingerprint,
+            upperBound: InfiniteUpperBound,
+            fingerprint,
+        });
+        return true;
+    };
+    for (const range of ranges) {
+        const currentUpperBound = range.upperBound;
+        const lower = prevIndex;
+        let upper = deps.storage.findLowerBound(ownerId, prevIndex, storageSize, currentUpperBound);
+        if (upper == null)
+            return syncFail();
+        switch (range.type) {
+            case RangeType.Skip: {
+                skipRange(range);
+                break;
+            }
+            case RangeType.Fingerprint: {
+                const ourFingerprint = deps.storage.fingerprint(ownerId, lower, upper);
+                if (ourFingerprint == null)
+                    return syncFail();
+                if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
+                    skipRange(range);
+                }
+                else {
+                    if (output.canSplitRange()) {
+                        coalesceSkipsBeforeAdd();
+                        splitRange(deps)(ownerId, lower, upper, currentUpperBound, output);
+                    }
+                    else {
+                        if (!addFingerprintForRemainingRange(upper))
+                            return syncFail();
+                        return ok(output.unwrap());
+                    }
+                }
+                break;
+            }
+            case RangeType.Timestamps: {
+                let endBound = currentUpperBound;
+                const timestampsWeNeed = new Map(range.timestamps.map((t) => [t.join(), true]));
+                const ourTimestamps = createTimestampsBuffer();
+                let storageError = false;
+                let exceeded = false;
+                deps.storage.iterate(ownerId, lower, upper, (timestamp, index) => {
+                    const timestampString = timestamp.join();
+                    const timestampBinary = binaryTimestampToTimestamp(timestamp);
+                    let message = null;
+                    if (timestampsWeNeed.has(timestampString)) {
+                        timestampsWeNeed.delete(timestampString);
+                    }
+                    else {
+                        const dbChange = deps.storage.readDbChange(ownerId, timestamp);
+                        if (dbChange == null) {
+                            storageError = true;
+                            return false;
+                        }
+                        message = {
+                            timestamp: timestampBinary,
+                            change: dbChange,
+                        };
+                    }
+                    if (!output.canAddTimestampsRangeAndMessage(ourTimestamps, message)) {
+                        exceeded = true;
+                        endBound = timestamp;
+                        upper = index;
+                        return false;
+                    }
+                    ourTimestamps.add(timestampBinary);
+                    if (message)
+                        output.addMessage(message);
+                    return true;
+                });
+                if (storageError) {
+                    return syncFail();
+                }
+                const addRange = () => {
+                    coalesceSkipsBeforeAdd();
+                    output.addRange({
+                        type: RangeType.Timestamps,
+                        upperBound: endBound,
+                        timestamps: ourTimestamps,
+                    });
+                };
+                if (exceeded) {
+                    addRange();
+                    if (!addFingerprintForRemainingRange(upper)) {
+                        return syncFail();
+                    }
+                    return ok(output.unwrap());
+                }
+                // If we need something, we have to respond with our timestamps.
+                if (timestampsWeNeed.size > 0) {
+                    addRange();
+                }
+                else {
+                    skipRange(range);
+                }
+                break;
+            }
+        }
+        prevIndex = upper;
+        prevUpperBound = currentUpperBound;
+    }
+    // If all ranges were skipped, there are no changes and sync is complete.
+    const hasChange = output.getSize() > outputInitialSize;
+    return ok(hasChange ? output.unwrap() : null);
+};
+const splitRange = (deps) => (ownerId, lower, upper, upperBound, buffer) => {
+    const itemCount = (upper - lower);
+    const buckets = computeBalancedBuckets(itemCount);
+    if (!buckets.ok) {
+        const range = {
+            type: RangeType.Timestamps,
+            upperBound,
+            timestamps: createTimestampsBuffer(),
+        };
+        deps.storage.iterate(ownerId, 0, itemCount, (timestamp) => {
+            range.timestamps.add(binaryTimestampToTimestamp(timestamp));
+            return true;
+        });
+        buffer.addRange(range);
+        return;
+    }
+    // Check Storage.ts `fingerprint` and `fingerprintRanges` docs.
+    const fingerprintRangesBuckets = lower === 0
+        ? buckets.value
+        : [lower, ...buckets.value.map((b) => (b + lower))];
+    const fingerprintRanges = deps.storage.fingerprintRanges(ownerId, fingerprintRangesBuckets, upperBound);
+    // Errors are handled by the storage.
+    if (fingerprintRanges == null)
+        return;
+    const rangesToUse = lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;
+    for (const range of rangesToUse) {
+        buffer.addRange(range);
+    }
+};
+const decodeRanges = (buffer) => {
+    if (buffer.getLength() === 0)
+        return [];
+    const rangesCount = decodeNonNegativeInt(buffer);
+    if (rangesCount === 0)
+        return [];
+    const timestampsCount = (rangesCount - 1);
+    const timestamps = decodeTimestamps(buffer, timestampsCount);
+    const rangeTypes = [];
+    for (let i = 0; i < rangesCount; i++) {
+        const rangeType = decodeNonNegativeInt(buffer);
+        switch (rangeType) {
+            case RangeType.Fingerprint:
+            case RangeType.Skip:
+            case RangeType.Timestamps:
+                rangeTypes.push(rangeType);
+                break;
+            default:
+                throw new ProtocolDecodeError(`Invalid RangeType: ${rangeType}`);
+        }
+    }
+    const ranges = [];
+    for (let i = 0; i < rangesCount; i++) {
+        const upperBound = i < timestampsCount
+            ? timestampToBinaryTimestamp(timestamps[i])
+            : InfiniteUpperBound;
+        const rangeType = rangeTypes[i];
+        switch (rangeType) {
+            case RangeType.Skip:
+                ranges.push({ type: RangeType.Skip, upperBound });
+                break;
+            case RangeType.Fingerprint: {
+                const fingerprint = buffer.shiftN(fingerprintSize);
+                ranges.push({
+                    type: RangeType.Fingerprint,
+                    upperBound,
+                    fingerprint,
+                });
+                break;
+            }
+            case RangeType.Timestamps: {
+                const timestamps = decodeTimestamps(buffer).map(timestampToBinaryTimestamp);
+                ranges.push({
+                    type: RangeType.Timestamps,
+                    upperBound,
+                    timestamps,
+                });
+                break;
+            }
+        }
+    }
+    return ranges;
+};
+/** Decodes an array of sorted timestamps with delta-encoded millis. */
+const decodeTimestamps = (buffer, length) => {
+    length ??= decodeNonNegativeInt(buffer);
+    let previousMillis = 0;
+    const millises = [];
+    for (let i = 0; i < length; i++) {
+        const deltaMillis = decodeNonNegativeInt(buffer);
+        const millis = Millis.from(previousMillis + deltaMillis);
+        if (!millis.ok)
+            throw new Error(millis.error.type);
+        millises.push(millis.value);
+        previousMillis = millis.value;
+    }
+    const counters = [];
+    let counterIndex = 0;
+    while (counterIndex < length) {
+        const counter = Counter.from(decodeNonNegativeInt(buffer));
+        if (!counter.ok)
+            throw new Error(counter.error.type);
+        const runLength = decodeNonNegativeInt(buffer);
+        for (let i = 0; i < runLength; i++) {
+            counters.push(counter.value);
+            counterIndex++;
+        }
+    }
+    const nodeIds = [];
+    let nodeIdIndex = 0;
+    while (nodeIdIndex < length) {
+        const nodeId = decodeNodeId(buffer);
+        const runLength = decodeNonNegativeInt(buffer);
+        for (let i = 0; i < runLength; i++) {
+            nodeIds.push(nodeId);
+            nodeIdIndex++;
+        }
+    }
+    const timestamps = [];
+    for (let i = 0; i < length; i++) {
+        timestamps.push({
+            millis: millises[i],
+            counter: counters[i],
+            nodeId: nodeIds[i],
+        });
+    }
+    return timestamps;
+};
+export const binaryIdLength = 16;
+export const idToBinaryId = (id) => base64Url256ToBytes(id);
+export const binaryIdToId = (binaryId) => decodeId(createBuffer(binaryId));
+export const ownerIdToBinaryOwnerId = (ownerId) => base64Url256ToBytes(ownerId);
+export const binaryOwnerIdToOwnerId = (binaryOwnerId) => decodeOwnerId(createBuffer(binaryOwnerId));
+/**
+ * Base64Url string with maximum length of 256 characters. Encoding strings as
+ * Base64UrlString saves up to 25% in size compared to regular strings.
+ */
+export const Base64Url256 = maxLength(256)(Base64Url);
+/**
+ * Alphabet used for Base64Url encoding. This is copied from the `nanoid`
+ * library to avoid dependency on a specific version of `nanoid`.
+ */
+const urlAlphabet = "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
+/**
+ * Converts a Base64Url string to a Uint8Array for binary storage. This encoding
+ * is more space-efficient than UTF-8 for Base64Url strings.
+ */
+export const base64Url256ToBytes = (string) => {
+    const totalBits = string.length * 6; // 6 bits per character
+    const byteLength = Math.ceil(totalBits / 8);
+    const value = new globalThis.Uint8Array(byteLength);
+    let bitBuffer = 0;
+    let bitsInBuffer = 0;
+    let byteIndex = 0;
+    for (const char of string) {
+        const charValue = urlAlphabet.indexOf(char);
+        bitBuffer = (bitBuffer << 6) | charValue;
+        bitsInBuffer += 6;
+        while (bitsInBuffer >= 8) {
+            bitsInBuffer -= 8;
+            value[byteIndex++] = (bitBuffer >> bitsInBuffer) & 0xff;
+        }
+    }
+    if (bitsInBuffer > 0 && byteIndex < byteLength) {
+        value[byteIndex] = (bitBuffer << (8 - bitsInBuffer)) & 0xff;
+    }
+    return value;
+};
+export const decodeBase64Url256 = (buffer, stringLength) => {
+    const bytes = buffer.shiftN(Math.ceil((stringLength * 6) / 8));
+    let bitBuffer = 0;
+    let bitsInBuffer = 0;
+    let string = "";
+    for (const byte of bytes) {
+        bitBuffer = (bitBuffer << 8) | byte;
+        bitsInBuffer += 8;
+        while (bitsInBuffer >= 6) {
+            bitsInBuffer -= 6;
+            if (string.length < stringLength) {
+                const charValue = (bitBuffer >> bitsInBuffer) & 0x3f;
+                if (charValue < 0 || charValue >= urlAlphabet.length) {
+                    throw new ProtocolDecodeError("invalid charValue");
+                }
+                string += urlAlphabet[charValue];
+            }
+        }
+    }
+    const result = Base64Url256.from(string);
+    if (!result.ok)
+        throw new ProtocolDecodeError(result.error.type);
+    return result.value;
+};
+const decodeId = (buffer) => decodeBase64Url256(buffer, idTypeValueLength);
+/** Not all 16 bytes are valid {@link OwnerId}. */
+const decodeOwnerId = (buffer) => decodeId(buffer);
+/**
+ * Evolu uses MessagePack to handle all number variants except for
+ * NonNegativeInt. For NonNegativeInt, Evolu provides more efficient encoding.
+ */
+export const encodeNumber = (buffer, number) => {
+    buffer.extend(pack(number));
+};
+export const decodeNumber = (buffer) => {
+    let number;
+    let end;
+    unpackMultiple(buffer.unwrap(), (n, _, e) => {
+        number = n;
+        end = e;
+        return false;
+    });
+    const endResult = NonNegativeInt.fromUnknown(end);
+    if (!endResult.ok)
+        throw new ProtocolDecodeError(endResult.error.type);
+    const numberResult = Number.fromUnknown(number);
+    if (!numberResult.ok)
+        throw new ProtocolDecodeError(numberResult.error.type);
+    buffer.shiftN(endResult.value);
+    return numberResult.value;
+};
+export const binaryTimestampToFingerprint = (timestamp) => {
+    const hash = sha256(timestamp).slice(0, fingerprintSize);
+    return hash;
+};
+/**
+ * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
+ * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
+ */
+export const encodeAndEncryptDbChange = (deps) => (change, key) => {
+    const buffer = createBuffer();
+    encodeBase64Url256(buffer, change.table);
+    buffer.extend(idToBinaryId(change.id));
+    const entries = objectToEntries(change.values).map(([column, value]) => {
+        return [column, value];
+    });
+    encodeLength(buffer, entries);
+    for (const [column, value] of entries) {
+        encodeBase64Url256(buffer, column);
+        encodeSqliteValue(buffer, value);
+    }
+    const paddingLength = padmePaddingLength(buffer.getLength());
+    // Add zero bytes as PADMÉ padding - these will be ignored during decoding.
+    buffer.extend(new Uint8Array(paddingLength));
+    const { nonce, ciphertext } = deps.symmetricCrypto.encrypt(buffer.unwrap(), key);
+    buffer.reset();
+    buffer.extend(nonce);
+    encodeLength(buffer, ciphertext);
+    buffer.extend(ciphertext);
+    return buffer.unwrap();
+};
+/**
+ * Decrypts and decodes an {@link EncryptedDbChange} using the provided owner's
+ * encryption key.
+ */
+export const decryptAndDecodeDbChange = (deps) => (change, key) => tryDecodeProtocolData(change, (buffer) => {
+    const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
+    const ciphertextLength = decodeLength(buffer);
+    const ciphertext = buffer.shiftN(ciphertextLength);
+    const plaintextBytes = deps.symmetricCrypto.decrypt(ciphertext, key, nonce);
+    if (!plaintextBytes.ok)
+        return plaintextBytes;
+    buffer.reset();
+    buffer.extend(plaintextBytes.value);
+    const table = decodeBase64Url256WithLength(buffer);
+    const id = decodeId(buffer);
+    const length = decodeLength(buffer);
+    const values = Object.create(null);
+    for (let i = 0; i < length; i++) {
+        const column = decodeBase64Url256WithLength(buffer);
+        const value = decodeSqliteValue(buffer);
+        values[column] = value;
+    }
+    const dbChange = { table, id, values };
+    return ok(dbChange);
+});
+/**
+ * Encodes a non-negative integer into a variable-length integer format. It's
+ * more efficient than encoding via {@link encodeNumber}.
+ *
+ * https://en.wikipedia.org/wiki/Variable-length_quantity
+ */
+export const encodeNonNegativeInt = (buffer, int) => {
+    if (int === 0) {
+        buffer.extend([0]);
+        return;
+    }
+    let remaining = BigInt(int);
+    const bytes = [];
+    while (remaining !== 0n) {
+        const byte = globalThis.Number(remaining & 127n);
+        bytes.push(byte);
+        remaining >>= 7n;
+    }
+    for (let i = 0; i < bytes.length - 1; i++) {
+        bytes[i] |= 128;
+    }
+    buffer.extend(bytes);
+};
+/**
+ * Decodes a non-negative integer from a variable-length integer format.
+ *
+ * https://en.wikipedia.org/wiki/Variable-length_quantity
+ */
+export const decodeNonNegativeInt = (buffer) => {
+    let result = 0n;
+    let shift = 0n;
+    let byte;
+    // 8 is the smallest required count
+    for (let byteCount = 0; byteCount < 8; byteCount++) {
+        byte = buffer.shift();
+        result |= BigInt(byte & 127) << shift;
+        if ((byte & 128) === 0)
+            break;
+        shift += 7n;
+    }
+    const int = NonNegativeInt.from(globalThis.Number(result));
+    if (!int.ok)
+        throw new ProtocolDecodeError(int.error.type);
+    return int.value;
+};
+export const encodeLength = (buffer, value) => {
+    encodeNonNegativeInt(buffer, value.length);
+};
+export const decodeLength = decodeNonNegativeInt;
+export const encodeString = (buffer, value) => {
+    const bytes = utf8ToBytes(value);
+    encodeLength(buffer, bytes);
+    buffer.extend(bytes);
+};
+export const decodeString = (buffer) => {
+    const length = decodeLength(buffer);
+    const bytes = buffer.shiftN(length);
+    return bytesToUtf8(bytes);
+};
+export const encodeNodeId = (buffer, nodeId) => {
+    buffer.extend(hexToBytes(nodeId));
+};
+export const decodeNodeId = (buffer) => {
+    const bytes = buffer.shiftN(8);
+    return bytesToHex(bytes);
+};
+export const encodeBase64Url256 = (buffer, string) => {
+    encodeLength(buffer, string);
+    buffer.extend(base64Url256ToBytes(string));
+};
+export const decodeBase64Url256WithLength = (buffer) => {
+    const length = decodeLength(buffer);
+    return decodeBase64Url256(buffer, length);
+};
+// Small ints are encoded into ProtocolValueType, saving one byte per int.
+const isSmallInt = (value) => value >= 0 && value < 20;
+export const ProtocolValueType = {
+    // 0-19 small ints
+    // SQLite types
+    String: 20,
+    Number: 21,
+    Null: 22,
+    Binary: 23,
+    // We can add more types for other DBs or anything else later.
+    // Optimized types
+    Id: 30,
+    Base64Url256: 31,
+    NonNegativeInt: 32,
+    Json: 33,
+    // new Date().toISOString()   - 24 bytes
+    // encoded with fixed length  - 8 bytes
+    // encode as NonNegativeInt   - 6 bytes (additional 25% reduction)
+    DateIsoWithNonNegativeTime: 34,
+    DateIsoWithNegativeTime: 35, // 9 bytes
+    // TODO: Operations (from 40)
+    // Increment, Decrement, Patch, whatever.
+};
+export const encodeSqliteValue = (buffer, value) => {
+    if (value === null) {
+        encodeNonNegativeInt(buffer, ProtocolValueType.Null);
+        return;
+    }
+    switch (typeof value) {
+        case "string": {
+            const dateIsoString = DateIsoString.from(value);
+            if (dateIsoString.ok) {
+                const time = new Date(dateIsoString.value).getTime();
+                if (NonNegativeInt.is(time)) {
+                    encodeNonNegativeInt(buffer, ProtocolValueType.DateIsoWithNonNegativeTime);
+                    encodeNonNegativeInt(buffer, time);
+                }
+                else {
+                    encodeNonNegativeInt(buffer, ProtocolValueType.DateIsoWithNegativeTime);
+                    encodeNumber(buffer, time);
+                }
+                return;
+            }
+            const base64Url256 = Base64Url256.from(value);
+            if (base64Url256.ok) {
+                if (base64Url256.value.length === idTypeValueLength) {
+                    encodeNonNegativeInt(buffer, ProtocolValueType.Id);
+                    buffer.extend(base64Url256ToBytes(base64Url256.value));
+                    return;
+                }
+                encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url256);
+                encodeBase64Url256(buffer, base64Url256.value);
+                return;
+            }
+            const jsonValue = JsonValueFromString.fromParent(value);
+            if (jsonValue.ok) {
+                const jsonBytes = pack(jsonValue.value);
+                encodeNonNegativeInt(buffer, ProtocolValueType.Json);
+                encodeLength(buffer, jsonBytes);
+                buffer.extend(jsonBytes);
+                return;
+            }
+            encodeNonNegativeInt(buffer, ProtocolValueType.String);
+            encodeString(buffer, value);
+            return;
+        }
+        case "number": {
+            if (NonNegativeInt.is(value)) {
+                if (isSmallInt(value)) {
+                    encodeNonNegativeInt(buffer, value);
+                    return;
+                }
+                encodeNonNegativeInt(buffer, ProtocolValueType.NonNegativeInt);
+                encodeNonNegativeInt(buffer, value);
+                return;
+            }
+            encodeNonNegativeInt(buffer, ProtocolValueType.Number);
+            encodeNumber(buffer, value);
+            return;
+        }
+    }
+    encodeNonNegativeInt(buffer, ProtocolValueType.Binary);
+    encodeLength(buffer, value);
+    buffer.extend(value);
+};
+export const decodeSqliteValue = (buffer) => {
+    const type = decodeNonNegativeInt(buffer);
+    if (isSmallInt(type)) {
+        return type;
+    }
+    switch (type) {
+        case ProtocolValueType.String:
+            return decodeString(buffer);
+        case ProtocolValueType.Number:
+            return decodeNumber(buffer);
+        case ProtocolValueType.Null:
+            return null;
+        case ProtocolValueType.Binary: {
+            const length = decodeLength(buffer);
+            return buffer.shiftN(length);
+        }
+        case ProtocolValueType.Id: {
+            return decodeId(buffer);
+        }
+        case ProtocolValueType.Base64Url256:
+            return decodeBase64Url256WithLength(buffer);
+        case ProtocolValueType.NonNegativeInt:
+            return decodeNonNegativeInt(buffer);
+        case ProtocolValueType.Json: {
+            const length = decodeLength(buffer);
+            const bytes = buffer.shiftN(length);
+            return JSON.stringify(unpack(bytes));
+        }
+        case ProtocolValueType.DateIsoWithNonNegativeTime:
+        case ProtocolValueType.DateIsoWithNegativeTime: {
+            const time = type === ProtocolValueType.DateIsoWithNonNegativeTime
+                ? decodeNonNegativeInt(buffer)
+                : decodeNumber(buffer);
+            const dateIsoString = DateIsoString.fromParent(new Date(time).toISOString());
+            if (!dateIsoString.ok)
+                throw new ProtocolDecodeError(dateIsoString.error.type);
+            return dateIsoString.value;
+        }
+        default:
+            throw new ProtocolDecodeError("invalid ProtocolValueType");
+    }
+};