@evolu/common 6.0.1-preview.8 → 7.0.0

package/dist/src/Evolu/Sync.js

@@ -1,29 +1,478 @@
-import { createWebSocket } from "../WebSocket.js";
-export const createWebSocketSync = (_deps) => (config) => {
+import { firstInArray, isNonEmptyReadonlyArray, } from "../Array.js";
+import { assert } from "../Assert.js";
+import { eqArrayNumber } from "../Eq.js";
+import { createTransferableError } from "../Error.js";
+import { constFalse, constTrue } from "../Function.js";
+import { objectToEntries } from "../Object.js";
+import { createResources } from "../Resources.js";
+import { err, ok } from "../Result.js";
+import { booleanToSqliteBoolean, sql, SqliteBoolean, sqliteBooleanToBoolean, sqliteFalse, } from "../Sqlite.js";
+import { createMutex } from "../Task.js";
+import { idBytesToId, idToIdBytes, } from "../Type.js";
+import { ownerIdBytesToOwnerId, ownerIdToOwnerIdBytes, } from "./Owner.js";
+import { applyProtocolMessageAsClient, createProtocolMessageForSync, createProtocolMessageForUnsubscribe, createProtocolMessageFromCrdtMessages, decryptAndDecodeDbChange, encodeAndEncryptDbChange, SubscriptionFlags, } from "./Protocol.js";
+import { createBaseSqliteStorage, DbChange, getOwnerUsage, getTimestampInsertStrategy, updateOwnerUsage, } from "./Storage.js";
+import { createInitialTimestamp, receiveTimestamp, sendTimestamp, timestampBytesToTimestamp, timestampToDateIso, timestampToTimestampBytes, } from "./Timestamp.js";
+export const createSync = (deps) => (config) => {
+    let isDisposed = false;
+    /** Returns owner data only if actively assigned to at least one transport. */
+    const getSyncOwner = (ownerId) => {
+        if (isDisposed)
+            return null;
+        return transports.getConsumer(ownerId);
+    };
+    const storageResult = createClientStorage({
+        ...deps,
+        getSyncOwner,
+    })(config);
+    if (!storageResult.ok)
+        return storageResult;
+    const storage = storageResult.value;
+    const createResource = (transportConfig) => {
+        const transportKey = createTransportKey(transportConfig);
+        deps.console.log("[sync]", "createWebSocket", {
+            transportKey,
+            url: transportConfig.url,
+        });
+        return deps.createWebSocket(transportConfig.url, {
+            binaryType: "arraybuffer",
+            onOpen: () => {
+                if (isDisposed)
+                    return;
+                const webSocket = transports.getResource(transportKey);
+                if (!webSocket)
+                    return;
+                const ownerIds = transports.getConsumersForResource(transportKey);
+                deps.console.log("[sync]", "onOpen", { transportKey, ownerIds });
+                for (const ownerId of ownerIds) {
+                    const message = createProtocolMessageForSync({ storage })(ownerId, SubscriptionFlags.Subscribe);
+                    if (!message)
+                        continue;
+                    deps.console.log("[sync]", "send", { message });
+                    webSocket.send(message);
+                }
+            },
+            onClose: (event) => {
+                deps.console.log("[sync]", "onClose", {
+                    transportKey,
+                    code: event.code,
+                    reason: event.reason,
+                    wasClean: event.wasClean,
+                });
+            },
+            onError: (error) => {
+                deps.console.warn("[sync]", "onError", { transportKey, error });
+            },
+            onMessage: (data) => {
+                // Only handle ArrayBuffer data for sync messages
+                if (isDisposed || !(data instanceof ArrayBuffer))
+                    return;
+                const webSocket = transports.getResource(transportKey);
+                if (!webSocket)
+                    return;
+                const input = new Uint8Array(data);
+                deps.console.log("[sync]", "onMessage", {
+                    transportKey,
+                    message: input,
+                });
+                applyProtocolMessageAsClient({ storage })(input, {
+                    // No write key, no sync (for a case when an owner was unused).
+                    getWriteKey: (ownerId) => getSyncOwner(ownerId)?.writeKey ?? null,
+                })
+                    .then((message) => {
+                    if (!message.ok) {
+                        config.onError(message.error);
+                        return;
+                    }
+                    switch (message.value.type) {
+                        case "response":
+                            webSocket.send(message.value.message);
+                            break;
+                        case "no-response":
+                            // Sync complete, no response needed
+                            break;
+                        case "broadcast":
+                            // This was a broadcast message, don't affect sync counter
+                            break;
+                    }
+                })
+                    .catch((error) => {
+                    config.onError(createTransferableError(error));
+                });
+            },
+        });
+    };
+    const transports = createResources({
+        createResource,
+        getResourceKey: createTransportKey,
+        getConsumerId: (owner) => owner.id,
+        disposalDelay: config.disposalDelayMs ?? 100,
+        onConsumerAdded: (owner, webSocket) => {
+            deps.console.log("[sync]", "onConsumerAdded", {
+                ownerId: owner.id,
+                isOpen: webSocket.isOpen(),
+            });
+            // The onOpen handler will sync it.
+            if (!webSocket.isOpen())
+                return;
+            const message = createProtocolMessageForSync({ storage })(owner.id, SubscriptionFlags.Subscribe);
+            if (message)
+                webSocket.send(message);
+        },
+        onConsumerRemoved: (owner, webSocket) => {
+            deps.console.log("[sync]", "onConsumerRemoved", {
+                ownerId: owner.id,
+                isOpen: webSocket.isOpen(),
+            });
+            const message = createProtocolMessageForUnsubscribe(owner.id);
+            webSocket.send(message);
+        },
+    });
     const sync = {
-        send: (message) => {
-            /**
-             * We don't need an in-memory queue; apps can be offline for a long time,
-             * and mutations are stored in SQLite. Dropped CRDT messages are synced
-             * when the web socket connection is open.
-             */
-            if (socket.getReadyState() !== "open")
+        useOwner: (use, owner) => {
+            if (isDisposed) {
+                deps.console.warn("[sync]", "useOwner called on disposed Sync instance", { owner });
                 return;
-            socket.send(message);
+            }
+            deps.console.log("[sync]", "useOwner", { use, owner });
+            const transportsToUse = owner.transports ?? config.transports;
+            if (use) {
+                transports.addConsumer(owner, transportsToUse);
+            }
+            else {
+                const result = transports.removeConsumer(owner, transportsToUse);
+                if (!result.ok) {
+                    deps.console.warn("[sync]", "Failed to remove consumer", {
+                        transportsToUse,
+                        ownerId: owner.id,
+                        error: result.error,
+                    });
+                }
+            }
+        },
+        applyChanges: (changes) => {
+            deps.console.log("[sync]", "applyChanges", { changes });
+            let clockTimestamp = deps.clock.get();
+            const ownerMessages = new Map();
+            for (const change of changes) {
+                const nextTimestamp = sendTimestamp(deps)(clockTimestamp);
+                if (!nextTimestamp.ok)
+                    return nextTimestamp;
+                clockTimestamp = nextTimestamp.value;
+                const { ownerId = config.appOwner.id, ...dbChange } = change;
+                const message = {
+                    timestamp: clockTimestamp,
+                    change: dbChange,
+                };
+                const messages = ownerMessages.get(ownerId);
+                if (messages)
+                    messages.push(message);
+                else
+                    ownerMessages.set(ownerId, [message]);
+            }
+            for (const [ownerId, messages] of ownerMessages) {
+                const result = applyMessages({ ...deps, storage })(ownerId, messages);
+                if (!result.ok)
+                    return result;
+                const owner = getSyncOwner(ownerId);
+                if (!owner?.writeKey)
+                    continue;
+                const message = createProtocolMessageFromCrdtMessages(deps)({
+                    id: owner.id,
+                    encryptionKey: owner.encryptionKey,
+                    writeKey: owner.writeKey,
+                }, messages);
+                const transportsToUse = owner.transports ?? config.transports;
+                // Send message to all transports for this owner
+                for (const transportConfig of transportsToUse) {
+                    const transportKey = createTransportKey(transportConfig);
+                    const webSocket = transports.getResource(transportKey);
+                    if (!webSocket)
+                        continue;
+                    if (webSocket.isOpen()) {
+                        deps.console.log("[sync]", "send", { transportKey, message });
+                        webSocket.send(message);
+                    }
+                }
+            }
+            return deps.clock.save(clockTimestamp);
+        },
+        [Symbol.dispose]: () => {
+            if (isDisposed)
+                return;
+            isDisposed = true;
+            transports[Symbol.dispose]();
         },
     };
-    const socket = createWebSocket(config.syncUrl, {
-        binaryType: "arraybuffer",
-        onOpen: () => {
-            config.onOpen(sync.send);
+    return ok(sync);
+};
+export const createClock = (deps) => (initialTimestamp = createInitialTimestamp(deps)) => {
+    let currentTimestamp = initialTimestamp;
+    return {
+        get: () => currentTimestamp,
+        save: (timestamp) => {
+            currentTimestamp = timestamp;
+            const result = deps.sqlite.exec(sql.prepared `
+          update evolu_config
+          set "clock" = ${timestampToTimestampBytes(timestamp)};
+        `);
+            if (!result.ok)
+                return result;
+            return ok();
         },
-        onMessage: (data) => {
-            if (data instanceof ArrayBuffer) {
-                const messages = new Uint8Array(data);
-                config.onMessage(messages, sync.send);
+    };
+};
+const createClientStorage = (deps) => (config) => {
+    const sqliteStorageBase = createBaseSqliteStorage(deps)({
+        onStorageError: config.onError,
+        isOwnerWithinQuota: constTrue, // Clients don't have quota limits
+    });
+    // TODO: Mutex per OwnerId
+    const mutex = createMutex();
+    const storage = {
+        ...sqliteStorageBase,
+        // Not implemented yet.
+        validateWriteKey: constFalse,
+        setWriteKey: constFalse,
+        writeMessages: async (ownerIdBytes, encryptedMessages) => {
+            const ownerId = ownerIdBytesToOwnerId(ownerIdBytes);
+            // Everything is sync now, but we will need async crypto in the future.
+            const writeResult = await mutex.withLock(async () => {
+                const owner = deps.getSyncOwner(ownerId);
+                // Owner can be removed during syncing.
+                // `ok(true)` means success, we just skipped the write.
+                if (!owner)
+                    return ok(true);
+                // TODO: Add quota checking for collaborative scenarios.
+                // When receiving messages from other owners via relay broadcast,
+                // check if this owner is within quota before accepting the data.
+                // This prevents an owner from exceeding storage limits when receiving
+                // data shared by other collaborators.
+                const messages = [];
+                for (const message of encryptedMessages) {
+                    const change = decryptAndDecodeDbChange(deps)(message, owner.encryptionKey);
+                    if (!change.ok)
+                        return change;
+                    messages.push({
+                        timestamp: message.timestamp,
+                        change: change.value,
+                    });
+                }
+                const transaction = deps.sqlite.transaction(() => {
+                    let clockTimestamp = deps.clock.get();
+                    for (const message of messages) {
+                        const nextTimestamp = receiveTimestamp(deps)(clockTimestamp, message.timestamp);
+                        if (!nextTimestamp.ok)
+                            return nextTimestamp;
+                        clockTimestamp = nextTimestamp.value;
+                    }
+                    if (isNonEmptyReadonlyArray(messages)) {
+                        const applyMessagesResult = applyMessages({ ...deps, storage })(owner.id, messages);
+                        if (!applyMessagesResult.ok)
+                            return applyMessagesResult;
+                    }
+                    // // Apply local mutations atomically with approved messages
+                    // for (const change of localMutations) {
+                    //   const result = applyLocalOnlyChange(deps)(change);
+                    //   if (!result.ok) return result;
+                    // }
+                    return deps.clock.save(clockTimestamp);
+                });
+                if (!transaction.ok)
+                    return transaction;
+                return ok(true);
+            })();
+            if (!writeResult.ok) {
+                if (writeResult.error.type !== "AbortError") {
+                    config.onError(writeResult.error);
+                }
+                return err({ type: "StorageWriteError", ownerId });
             }
+            config.onReceive();
+            return ok();
         },
-    });
-    return sync;
+        readDbChange: (ownerId, timestamp) => {
+            const owner = deps.getSyncOwner(ownerIdBytesToOwnerId(ownerId));
+            // Owner can be removed to stop syncing.
+            if (!owner)
+                return null;
+            const result = deps.sqlite.exec(sql `
+          select "table", "id", "column", "value"
+          from evolu_history
+          where "ownerId" = ${ownerId} and "timestamp" = ${timestamp};
+        `);
+            if (!result.ok) {
+                config.onError(result.error);
+                return null;
+            }
+            const { rows } = result.value;
+            assert(rows.length > 0, "Rows must not be empty");
+            const { table, id } = rows[0];
+            const values = {};
+            let isInsert = false;
+            let isDelete = null;
+            for (const r of rows) {
+                assert(r.table === table, "All rows must have the same table");
+                assert(eqArrayNumber(r.id, id), "All rows must have the same Id");
+                switch (r.column) {
+                    case "createdAt":
+                        isInsert = true;
+                        break;
+                    case "isDeleted":
+                        assert(SqliteBoolean.is(r.value), "isDeleted column must contain a valid SqliteBoolean (0 or 1)");
+                        isDelete = sqliteBooleanToBoolean(r.value);
+                        break;
+                    default:
+                        values[r.column] = r.value;
+                }
+            }
+            const message = {
+                timestamp: timestampBytesToTimestamp(timestamp),
+                change: DbChange.orThrow({
+                    table: rows[0].table,
+                    id: idBytesToId(rows[0].id),
+                    values,
+                    isInsert,
+                    isDelete,
+                }),
+            };
+            return encodeAndEncryptDbChange(deps)(message, owner.encryptionKey);
+        },
+    };
+    return ok(storage);
+};
+/** Creates a unique identifier for a transport configuration. */
+const createTransportKey = (transportConfig) => {
+    return `${transportConfig.type}:${transportConfig.url}`;
+};
+export const applyLocalOnlyChange = (deps) => (change) => {
+    if (change.isDelete) {
+        const result = deps.sqlite.exec(sql `
+        delete from ${sql.identifier(change.table)}
+        where id = ${change.id};
+      `);
+        if (!result.ok)
+            return result;
+    }
+    else {
+        const now = deps.time.nowIso();
+        const ownerId = deps.appOwner.id;
+        let entries = objectToEntries(change.values);
+        if (change.isDelete !== null) {
+            entries = [...entries, ["isDeleted", sqliteFalse]];
+        }
+        for (const [column, value] of entries) {
+            const result = deps.sqlite.exec(sql.prepared `
+          insert into ${sql.identifier(change.table)}
+            ("ownerId", "id", ${sql.identifier(column)}, createdAt, updatedAt)
+          values (${ownerId}, ${change.id}, ${value}, ${now}, ${now})
+          on conflict ("ownerId", "id") do update
+            set
+              ${sql.identifier(column)} = ${value},
+              updatedAt = ${now};
+        `);
+            if (!result.ok)
+                return result;
+        }
+    }
+    return ok();
+};
+const applyMessages = (deps) => (ownerId, messages) => {
+    const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
+    const usageResult = getOwnerUsage(deps)(ownerIdBytes, timestampToTimestampBytes(firstInArray(messages).timestamp));
+    if (!usageResult.ok)
+        return usageResult;
+    let { firstTimestamp, lastTimestamp } = usageResult.value;
+    for (const message of messages) {
+        const date = timestampToDateIso(message.timestamp);
+        const result1 = applyMessageToAppTable(deps)(ownerIdBytes, message, date);
+        if (!result1.ok)
+            return result1;
+        const timestamp = timestampToTimestampBytes(message.timestamp);
+        let strategy;
+        [strategy, firstTimestamp, lastTimestamp] = getTimestampInsertStrategy(timestamp, firstTimestamp, lastTimestamp);
+        const result2 = applyMessageToTimestampAndHistoryTables(deps)(ownerIdBytes, message, strategy, date);
+        if (!result2.ok)
+            return result2;
+    }
+    /**
+     * TODO: Implement proper storedBytes tracking for client using encrypted
+     * message sizes (need to figure out how to reuse received or postpone
+     * client...).
+     */
+    const updateUsage = updateOwnerUsage(deps)(ownerIdBytes, 1, // Placeholder until proper tracking implemented
+    firstTimestamp, lastTimestamp);
+    if (!updateUsage.ok)
+        return updateUsage;
+    return ok();
+};
+const applyMessageToAppTable = (deps) => (ownerIdBytes, message, date) => {
+    const ownerId = ownerIdBytesToOwnerId(ownerIdBytes);
+    const columns = dbChangeToColumns(message.change, date);
+    for (const [column, value] of columns) {
+        const result = deps.sqlite.exec(sql.prepared `
+        with
+          existingTimestamp as (
+            select 1
+            from evolu_history
+            where
+              "ownerId" = ${ownerIdBytes}
+              and "table" = ${message.change.table}
+              and "id" = ${idToIdBytes(message.change.id)}
+              and "column" = ${column}
+              and "timestamp" >= ${timestampToTimestampBytes(message.timestamp)}
+            limit 1
+          )
+        insert into ${sql.identifier(message.change.table)}
+          ("ownerId", "id", ${sql.identifier(column)}, updatedAt)
+        select ${ownerId}, ${message.change.id}, ${value}, ${date}
+        where not exists (select 1 from existingTimestamp)
+        on conflict ("ownerId", "id") do update
+          set
+            ${sql.identifier(column)} = ${value},
+            updatedAt = ${date}
+          where not exists (select 1 from existingTimestamp);
+      `);
+        if (!result.ok)
+            return result;
+    }
+    return ok();
+};
+export const applyMessageToTimestampAndHistoryTables = (deps) => (ownerId, message, strategy, date) => {
+    const timestamp = timestampToTimestampBytes(message.timestamp);
+    const id = idToIdBytes(message.change.id);
+    const result = deps.storage.insertTimestamp(ownerId, timestamp, strategy);
+    if (!result.ok)
+        return result;
+    const columns = dbChangeToColumns(message.change, date);
+    for (const [column, value] of columns) {
+        const result = deps.sqlite.exec(sql.prepared `
+        insert into evolu_history
+          ("ownerId", "table", "id", "column", "value", "timestamp")
+        values
+          (
+            ${ownerId},
+            ${message.change.table},
+            ${id},
+            ${column},
+            ${value},
+            ${timestamp}
+          )
+        on conflict do nothing;
+      `);
+        if (!result.ok)
+            return result;
+    }
+    return ok();
+};
+const dbChangeToColumns = (change, date) => {
+    const entries = [...objectToEntries(change.values)];
+    if (change.isInsert) {
+        entries.push(["createdAt", date]);
+    }
+    if (change.isDelete !== null) {
+        entries.push(["isDeleted", booleanToSqliteBoolean(change.isDelete)]);
+    }
+    return entries;
 };
 export const initialSyncState = { type: "SyncStateInitial" };