@tldraw/sync-core 4.3.0-canary.cf5673a789a1 → 4.3.0-canary.d428e9e9a7c6

package/dist-esm/lib/TLSyncRoom.mjs

@@ -1,126 +1,45 @@
-import { transact, transaction } from "@tldraw/state";
 import {
-  AtomMap,
-  MigrationFailureReason
+  AtomMap
 } from "@tldraw/store";
-import { DocumentRecordType, PageRecordType, TLDOCUMENT_ID } from "@tldraw/tlschema";
 import {
-  Result,
   assert,
   assertExists,
   exhaustiveSwitchError,
   getOwnProperty,
-  hasOwnProperty,
   isEqual,
   isNativeStructuredClone,
   objectMapEntriesIterable,
-  structuredClone
+  Result
 } from "@tldraw/utils";
 import { createNanoEvents } from "nanoevents";
+import {
+  applyObjectDiff,
+  diffRecord,
+  RecordOpType,
+  ValueOpType
+} from "./diff.mjs";
+import { interval } from "./interval.mjs";
+import {
+  getTlsyncProtocolVersion,
+  TLIncompatibilityReason
+} from "./protocol.mjs";
+import { applyAndDiffRecord, diffAndValidateRecord, validateRecord } from "./recordDiff.mjs";
 import {
   RoomSessionState,
   SESSION_IDLE_TIMEOUT,
   SESSION_REMOVAL_WAIT_TIME,
   SESSION_START_WAIT_TIME
 } from "./RoomSession.mjs";
-import { TLSyncErrorCloseEventCode, TLSyncErrorCloseEventReason } from "./TLSyncClient.mjs";
+import { TLSyncError, TLSyncErrorCloseEventCode, TLSyncErrorCloseEventReason } from "./TLSyncClient.mjs";
 import {
-  RecordOpType,
-  ValueOpType,
-  applyObjectDiff,
-  diffRecord
-} from "./diff.mjs";
-import { findMin } from "./findMin.mjs";
-import { interval } from "./interval.mjs";
-import {
-  TLIncompatibilityReason,
-  getTlsyncProtocolVersion
-} from "./protocol.mjs";
-const MAX_TOMBSTONES = 3e3;
-const TOMBSTONE_PRUNE_BUFFER_SIZE = 300;
+  toNetworkDiff
+} from "./TLSyncStorage.mjs";
 const DATA_MESSAGE_DEBOUNCE_INTERVAL = 1e3 / 60;
 const timeSince = (time) => Date.now() - time;
-class DocumentState {
-  constructor(state, lastChangedClock, recordType) {
-    this.state = state;
-    this.lastChangedClock = lastChangedClock;
-    this.recordType = recordType;
-  }
-  /**
-   * Create a DocumentState instance without validating the record data.
-   * Used for performance when validation has already been performed.
-   *
-   * @param state - The record data
-   * @param lastChangedClock - Clock value when this record was last modified
-   * @param recordType - The record type definition for validation
-   * @returns A new DocumentState instance
-   */
-  static createWithoutValidating(state, lastChangedClock, recordType) {
-    return new DocumentState(state, lastChangedClock, recordType);
-  }
-  /**
-   * Create a DocumentState instance with validation of the record data.
-   *
-   * @param state - The record data to validate
-   * @param lastChangedClock - Clock value when this record was last modified
-   * @param recordType - The record type definition for validation
-   * @returns Result containing the DocumentState or validation error
-   */
-  static createAndValidate(state, lastChangedClock, recordType) {
-    try {
-      recordType.validate(state);
-    } catch (error) {
-      return Result.err(error);
-    }
-    return Result.ok(new DocumentState(state, lastChangedClock, recordType));
-  }
-  /**
-   * Replace the current state with new state and calculate the diff.
-   *
-   * @param state - The new record state
-   * @param clock - The new clock value
-   * @param legacyAppendMode - If true, string append operations will be converted to Put operations
-   * @returns Result containing the diff and new DocumentState, or null if no changes, or validation error
-   */
-  replaceState(state, clock, legacyAppendMode = false) {
-    const diff = diffRecord(this.state, state, legacyAppendMode);
-    if (!diff) return Result.ok(null);
-    try {
-      this.recordType.validate(state);
-    } catch (error) {
-      return Result.err(error);
-    }
-    return Result.ok([diff, new DocumentState(state, clock, this.recordType)]);
-  }
-  /**
-   * Apply a diff to the current state and return the resulting changes.
-   *
-   * @param diff - The object diff to apply
-   * @param clock - The new clock value
-   * @param legacyAppendMode - If true, string append operations will be converted to Put operations
-   * @returns Result containing the final diff and new DocumentState, or null if no changes, or validation error
-   */
-  mergeDiff(diff, clock, legacyAppendMode = false) {
-    const newState = applyObjectDiff(this.state, diff);
-    return this.replaceState(newState, clock, legacyAppendMode);
-  }
-}
-function getDocumentClock(snapshot) {
-  if (typeof snapshot.documentClock === "number") {
-    return snapshot.documentClock;
-  }
-  let max = 0;
-  for (const doc of snapshot.documents) {
-    max = Math.max(max, doc.lastChangedClock);
-  }
-  for (const tombstone of Object.values(snapshot.tombstones ?? {})) {
-    max = Math.max(max, tombstone);
-  }
-  return max;
-}
 class TLSyncRoom {
   // A table of connected clients
   sessions = /* @__PURE__ */ new Map();
+  lastDocumentClock = 0;
   // eslint-disable-next-line local/prefer-class-methods
   pruneSessions = () => {
     for (const client of this.sessions.values()) {
@@ -152,6 +71,7 @@ class TLSyncRoom {
       }
     }
   };
+  presenceStore = new PresenceStore();
   disposables = [interval(this.pruneSessions, 2e3)];
   _isClosed = false;
   /**
@@ -174,17 +94,8 @@ class TLSyncRoom {
     return this._isClosed;
   }
   events = createNanoEvents();
-  // Values associated with each uid (must be serializable).
-  /** @internal */
-  documents;
-  tombstones;
-  // this clock should start higher than the client, to make sure that clients who sync with their
-  // initial lastServerClock value get the full state
-  // in this case clients will start with 0, and the server will start with 1
-  clock;
-  documentClock;
-  tombstoneHistoryStartsAtClock;
-  // map from record id to clock upon deletion
+  // Storage layer for documents, tombstones, and clocks
+  storage;
   serializedSchema;
   documentTypes;
   presenceType;
@@ -192,10 +103,9 @@ class TLSyncRoom {
   schema;
   constructor(opts) {
     this.schema = opts.schema;
-    let snapshot = opts.snapshot;
     this.log = opts.log;
-    this.onDataChange = opts.onDataChange;
     this.onPresenceChange = opts.onPresenceChange;
+    this.storage = opts.storage;
     assert(
       isNativeStructuredClone,
       "TLSyncRoom is supposed to run either on Cloudflare Workersor on a 18+ version of Node.js, which both support the native structuredClone API"
@@ -213,194 +123,31 @@ class TLSyncRoom {
       );
     }
     this.presenceType = presenceTypes.values().next()?.value ?? null;
-    if (!snapshot) {
-      snapshot = {
-        clock: 0,
-        documentClock: 0,
-        documents: [
-          {
-            state: DocumentRecordType.create({ id: TLDOCUMENT_ID }),
-            lastChangedClock: 0
-          },
-          {
-            state: PageRecordType.create({ name: "Page 1", index: "a1" }),
-            lastChangedClock: 0
-          }
-        ]
-      };
-    }
-    this.clock = snapshot.clock;
-    let didIncrementClock = false;
-    const ensureClockDidIncrement = (_reason) => {
-      if (!didIncrementClock) {
-        didIncrementClock = true;
-        this.clock++;
-      }
-    };
-    this.tombstones = new AtomMap(
-      "room tombstones",
-      objectMapEntriesIterable(snapshot.tombstones ?? {})
-    );
-    this.documents = new AtomMap(
-      "room documents",
-      function* () {
-        for (const doc of snapshot.documents) {
-          if (this.documentTypes.has(doc.state.typeName)) {
-            yield [
-              doc.state.id,
-              DocumentState.createWithoutValidating(
-                doc.state,
-                doc.lastChangedClock,
-                assertExists(getOwnProperty(this.schema.types, doc.state.typeName))
-              )
-            ];
-          } else {
-            ensureClockDidIncrement("doc type was not doc type");
-            this.tombstones.set(doc.state.id, this.clock);
-          }
-        }
-      }.call(this)
-    );
-    this.tombstoneHistoryStartsAtClock = snapshot.tombstoneHistoryStartsAtClock ?? findMin(this.tombstones.values()) ?? this.clock;
-    if (this.tombstoneHistoryStartsAtClock === 0) {
-      this.tombstoneHistoryStartsAtClock++;
-    }
-    transact(() => {
-      const schema = snapshot.schema ?? this.schema.serializeEarliestVersion();
-      const migrationsToApply = this.schema.getMigrationsSince(schema);
-      assert(migrationsToApply.ok, "Failed to get migrations");
-      if (migrationsToApply.value.length > 0) {
-        const store = {};
-        for (const [k, v] of this.documents.entries()) {
-          store[k] = v.state;
-        }
-        const migrationResult = this.schema.migrateStoreSnapshot(
-          { store, schema },
-          { mutateInputStore: true }
-        );
-        if (migrationResult.type === "error") {
-          throw new Error("Failed to migrate: " + migrationResult.reason);
-        }
-        for (const id in migrationResult.value) {
-          if (!Object.prototype.hasOwnProperty.call(migrationResult.value, id)) {
-            continue;
-          }
-          const r = migrationResult.value[id];
-          const existing = this.documents.get(id);
-          if (!existing || !isEqual(existing.state, r)) {
-            ensureClockDidIncrement("record was added or updated during migration");
-            this.documents.set(
-              r.id,
-              DocumentState.createWithoutValidating(
-                r,
-                this.clock,
-                assertExists(getOwnProperty(this.schema.types, r.typeName))
-              )
-            );
-          }
-        }
-        for (const id of this.documents.keys()) {
-          if (!migrationResult.value[id]) {
-            ensureClockDidIncrement("record was removed during migration");
-            this.tombstones.set(id, this.clock);
-            this.documents.delete(id);
-          }
-        }
-      }
-      this.pruneTombstones();
+    const { documentClock } = this.storage.transaction((txn) => {
+      this.schema.migrateStorage(txn);
     });
-    if (didIncrementClock) {
-      this.documentClock = this.clock;
-      opts.onDataChange?.();
-    } else {
-      this.documentClock = getDocumentClock(snapshot);
-    }
-  }
-  didSchedulePrune = true;
-  // eslint-disable-next-line local/prefer-class-methods
-  pruneTombstones = () => {
-    this.didSchedulePrune = false;
-    if (this.tombstones.size > MAX_TOMBSTONES) {
-      const entries = Array.from(this.tombstones.entries());
-      entries.sort((a, b) => a[1] - b[1]);
-      let idx = entries.length - 1 - MAX_TOMBSTONES + TOMBSTONE_PRUNE_BUFFER_SIZE;
-      const cullClock = entries[idx++][1];
-      while (idx < entries.length && entries[idx][1] === cullClock) {
-        idx++;
-      }
-      const keysToDelete = entries.slice(0, idx).map(([key]) => key);
-      this.tombstoneHistoryStartsAtClock = cullClock + 1;
-      this.tombstones.deleteMany(keysToDelete);
-    }
-  };
-  getDocument(id) {
-    return this.documents.get(id);
-  }
-  addDocument(id, state, clock) {
-    if (this.tombstones.has(id)) {
-      this.tombstones.delete(id);
-    }
-    const createResult = DocumentState.createAndValidate(
-      state,
-      clock,
-      assertExists(getOwnProperty(this.schema.types, state.typeName))
+    this.lastDocumentClock = documentClock;
+    this.disposables.push(
+      this.storage.onChange(({ id }) => {
+        if (id !== this.internalTxnId) {
+          this.broadcastExternalStorageChanges();
+        }
+      })
     );
-    if (!createResult.ok) return createResult;
-    this.documents.set(id, createResult.value);
-    return Result.ok(void 0);
   }
-  removeDocument(id, clock) {
-    this.documents.delete(id);
-    this.tombstones.set(id, clock);
-    if (!this.didSchedulePrune) {
-      this.didSchedulePrune = true;
-      setTimeout(this.pruneTombstones, 0);
-    }
-  }
-  /**
-   * Get a complete snapshot of the current room state that can be persisted
-   * and later used to restore the room.
-   *
-   * @returns Room snapshot containing all documents, tombstones, and metadata
-   * @example
-   * ```ts
-   * const snapshot = room.getSnapshot()
-   * await database.saveRoomSnapshot(roomId, snapshot)
-   *
-   * // Later, restore from snapshot
-   * const restoredRoom = new TLSyncRoom({
-   *   schema: mySchema,
-   *   snapshot: snapshot
-   * })
-   * ```
-   */
-  getSnapshot() {
-    const tombstones = Object.fromEntries(this.tombstones.entries());
-    const documents = [];
-    for (const doc of this.documents.values()) {
-      if (this.documentTypes.has(doc.state.typeName)) {
-        documents.push({
-          state: doc.state,
-          lastChangedClock: doc.lastChangedClock
-        });
-      }
-    }
-    return {
-      clock: this.clock,
-      documentClock: this.documentClock,
-      tombstones,
-      tombstoneHistoryStartsAtClock: this.tombstoneHistoryStartsAtClock,
-      schema: this.serializedSchema,
-      documents
-    };
+  broadcastExternalStorageChanges() {
+    this.storage.transaction((txn) => {
+      this.broadcastChanges(txn);
+      this.lastDocumentClock = txn.getClock();
+    });
   }
   /**
    * Send a message to a particular client. Debounces data events
    *
    * @param sessionId - The id of the session to send the message to.
-   * @param message - The message to send.
+   * @param message - The message to send. UNSAFE Any diffs must have been downgraded already if necessary
    */
-  sendMessage(sessionId, message) {
+  _unsafe_sendMessage(sessionId, message) {
     const session = this.sessions.get(sessionId);
     if (!session) {
       this.log?.warn?.("Tried to send message to unknown session", message.type);
@@ -452,7 +199,6 @@ class TLSyncRoom {
       return;
     }
     this.sessions.delete(sessionId);
-    const presence = this.getDocument(session.presenceId ?? "");
     try {
       if (fatalReason) {
         session.socket.close(TLSyncErrorCloseEventCode, fatalReason);
@@ -461,11 +207,12 @@ class TLSyncRoom {
       }
     } catch {
     }
+    const presence = this.presenceStore.get(session.presenceId ?? "");
     if (presence) {
-      this.documents.delete(session.presenceId);
+      this.presenceStore.delete(session.presenceId);
       this.broadcastPatch({
-        diff: { [session.presenceId]: [RecordOpType.Remove] },
-        sourceSessionId: sessionId
+        puts: {},
+        deletes: [session.presenceId]
       });
     }
     this.events.emit("session_removed", { sessionId, meta: session.meta });
@@ -498,24 +245,18 @@ class TLSyncRoom {
     } catch {
     }
   }
+  internalTxnId = "TLSyncRoom.txn";
   /**
    * Broadcast a patch to all connected clients except the one with the sessionId provided.
-   * Automatically handles schema migration for clients on different versions.
    *
-   * @param message - The broadcast message
-   *   - diff - The network diff to broadcast to all clients
-   *   - sourceSessionId - Optional ID of the session that originated this change (excluded from broadcast)
-   * @returns This room instance for method chaining
-   * @example
-   * ```ts
-   * room.broadcastPatch({
-   *   diff: { 'shape:123': [RecordOpType.Put, newShapeData] },
-   *   sourceSessionId: 'user-456' // This user won't receive the broadcast
-   * })
-   * ```
+   * @param diff - The TLSyncForwardDiff with full records (used for migration)
+   * @param networkDiff - Optional pre-computed NetworkDiff for sessions not needing migration.
+   *                      If not provided, will be computed from recordsDiff.
+   * @param sourceSessionId - Optional session ID to exclude from the broadcast
    */
-  broadcastPatch(message) {
-    const { diff, sourceSessionId } = message;
+  broadcastPatch(diff, networkDiff, sourceSessionId) {
+    const unmigrated = networkDiff ?? toNetworkDiff(diff);
+    if (!unmigrated) return this;
     this.sessions.forEach((session) => {
       if (session.state !== RoomSessionState.Connected) return;
       if (sourceSessionId === session.sessionId) return;
@@ -523,18 +264,17 @@ class TLSyncRoom {
         this.cancelSession(session.sessionId);
         return;
       }
-      const res = this.migrateDiffForSession(session.serializedSchema, diff);
-      if (!res.ok) {
-        this.rejectSession(
-          session.sessionId,
-          res.error === MigrationFailureReason.TargetVersionTooNew ? TLSyncErrorCloseEventReason.SERVER_TOO_OLD : TLSyncErrorCloseEventReason.CLIENT_TOO_OLD
-        );
-        return;
-      }
-      this.sendMessage(session.sessionId, {
+      const diffResult = this.migrateDiffOrRejectSession(
+        session.sessionId,
+        session.serializedSchema,
+        session.requiresDownMigrations,
+        diff
+      );
+      if (!diffResult.ok) return;
+      this._unsafe_sendMessage(session.sessionId, {
         type: "patch",
-        diff: res.value,
-        serverClock: this.clock
+        diff: diffResult.value,
+        serverClock: this.lastDocumentClock
       });
     });
     return this;
@@ -562,7 +302,7 @@ class TLSyncRoom {
    * ```
    */
   sendCustomMessage(sessionId, data) {
-    this.sendMessage(sessionId, { type: "custom", data });
+    this._unsafe_sendMessage(sessionId, { type: "custom", data });
   }
   /**
    * Register a new client session with the room. The session will be in an awaiting
@@ -621,34 +361,54 @@ class TLSyncRoom {
   }
   /**
    * When we send a diff to a client, if that client is on a lower version than us, we need to make
-   * the diff compatible with their version. At the moment this means migrating each affected record
-   * to the client's version and sending the whole record again. We can optimize this later by
-   * keeping the previous versions of records around long enough to recalculate these diffs for
-   * older client versions.
+   * the diff compatible with their version. This method takes a TLSyncForwardDiff (which has full
+   * records) and migrates all records down to the client's schema version, returning a NetworkDiff.
+   *
+   * For updates (entries with [before, after] tuples), both records are migrated and a patch is
+   * computed from the migrated versions, preserving efficient patch semantics even across versions.
+   *
+   * If a migration fails, the session will be rejected.
+   *
+   * @param sessionId - The session ID (for rejection on migration failure)
+   * @param serializedSchema - The client's schema to migrate to
+   * @param requiresDownMigrations - Whether the client needs down migrations
+   * @param diff - The TLSyncForwardDiff containing full records to migrate
+   * @param unmigrated - Optional pre-computed NetworkDiff for when no migration is needed
+   * @returns A NetworkDiff with migrated records, or a migration failure
    */
-  migrateDiffForSession(serializedSchema, diff) {
-    if (serializedSchema === this.serializedSchema) {
-      return Result.ok(diff);
+  migrateDiffOrRejectSession(sessionId, serializedSchema, requiresDownMigrations, diff, unmigrated) {
+    if (!requiresDownMigrations) {
+      return Result.ok(unmigrated ?? toNetworkDiff(diff) ?? {});
     }
     const result = {};
-    for (const [id, op] of objectMapEntriesIterable(diff)) {
-      if (op[0] === RecordOpType.Remove) {
-        result[id] = op;
-        continue;
-      }
-      const doc = this.getDocument(id);
-      if (!doc) {
-        return Result.err(MigrationFailureReason.TargetVersionTooNew);
-      }
-      const migrationResult = this.schema.migratePersistedRecord(
-        doc.state,
-        serializedSchema,
-        "down"
-      );
-      if (migrationResult.type === "error") {
-        return Result.err(migrationResult.reason);
+    for (const [id, put] of objectMapEntriesIterable(diff.puts)) {
+      if (Array.isArray(put)) {
+        const [from, to] = put;
+        const fromResult = this.schema.migratePersistedRecord(from, serializedSchema, "down");
+        if (fromResult.type === "error") {
+          this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
+          return Result.err(fromResult.reason);
+        }
+        const toResult = this.schema.migratePersistedRecord(to, serializedSchema, "down");
+        if (toResult.type === "error") {
+          this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
+          return Result.err(toResult.reason);
+        }
+        const patch = diffRecord(fromResult.value, toResult.value);
+        if (patch) {
+          result[id] = [RecordOpType.Patch, patch];
+        }
+      } else {
+        const migrationResult = this.schema.migratePersistedRecord(put, serializedSchema, "down");
+        if (migrationResult.type === "error") {
+          this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
+          return Result.err(migrationResult.reason);
+        }
+        result[id] = [RecordOpType.Put, migrationResult.value];
       }
-      result[id] = [RecordOpType.Put, migrationResult.value];
+    }
+    for (const id of diff.deletes) {
+      result[id] = [RecordOpType.Remove];
     }
     return Result.ok(result);
   }
@@ -673,21 +433,29 @@ class TLSyncRoom {
       this.log?.warn?.("Received message from unknown session");
       return;
     }
-    switch (message.type) {
-      case "connect": {
-        return this.handleConnectRequest(session, message);
-      }
-      case "push": {
-        return this.handlePushRequest(session, message);
-      }
-      case "ping": {
-        if (session.state === RoomSessionState.Connected) {
-          session.lastInteractionTime = Date.now();
+    try {
+      switch (message.type) {
+        case "connect": {
+          return this.handleConnectRequest(session, message);
+        }
+        case "push": {
+          return this.handlePushRequest(session, message);
+        }
+        case "ping": {
+          if (session.state === RoomSessionState.Connected) {
+            session.lastInteractionTime = Date.now();
+          }
+          return this._unsafe_sendMessage(session.sessionId, { type: "pong" });
+        }
+        default: {
+          exhaustiveSwitchError(message);
         }
-        return this.sendMessage(session.sessionId, { type: "pong" });
       }
-      default: {
-        exhaustiveSwitchError(message);
+    } catch (e) {
+      if (e instanceof TLSyncError) {
+        this.rejectSession(session.sessionId, e.reason);
+      } else {
+        throw e;
       }
     }
   }
@@ -744,6 +512,22 @@ class TLSyncRoom {
       this.removeSession(sessionId, fatalReason);
     }
   }
+  forceAllReconnect() {
+    for (const session of this.sessions.values()) {
+      this.removeSession(session.sessionId);
+    }
+  }
+  broadcastChanges(txn) {
+    const changes = txn.getChangesSince(this.lastDocumentClock);
+    if (!changes) return;
+    const { wipeAll, diff } = changes;
+    this.lastDocumentClock = txn.getClock();
+    if (wipeAll) {
+      this.forceAllReconnect();
+      return;
+    }
+    this.broadcastPatch(diff);
+  }
   handleConnectRequest(session, message) {
     let theirProtocolVersion = message.protocolVersion;
     if (theirProtocolVersion === 5) {
@@ -769,11 +553,12 @@ class TLSyncRoom {
       return;
     }
     const migrations = this.schema.getMigrationsSince(message.schema);
-    if (!migrations.ok || migrations.value.some((m) => m.scope === "store" || !m.down)) {
+    if (!migrations.ok || migrations.value.some((m) => m.scope !== "record" || !m.down)) {
       this.rejectSession(session.sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
       return;
     }
     const sessionSchema = isEqual(message.schema, this.serializedSchema) ? this.serializedSchema : message.schema;
+    const requiresDownMigrations = migrations.value.length > 0;
     const connect = async (msg) => {
       this.sessions.set(session.sessionId, {
         state: RoomSessionState.Connected,
@@ -781,6 +566,7 @@ class TLSyncRoom {
         presenceId: session.presenceId,
         socket: session.socket,
         serializedSchema: sessionSchema,
+        requiresDownMigrations,
         lastInteractionTime: Date.now(),
         debounceTimer: null,
         outstandingDataMessages: [],
@@ -789,76 +575,49 @@ class TLSyncRoom {
         isReadonly: session.isReadonly,
         requiresLegacyRejection: session.requiresLegacyRejection
       });
-      this.sendMessage(session.sessionId, msg);
+      this._unsafe_sendMessage(session.sessionId, msg);
     };
-    transaction((rollback) => {
-      if (
-        // if the client requests changes since a time before we have tombstone history, send them the full state
-        message.lastServerClock < this.tombstoneHistoryStartsAtClock || // similarly, if they ask for a time we haven't reached yet, send them the full state
-        // this will only happen if the DB is reset (or there is no db) and the server restarts
-        // or if the server exits/crashes with unpersisted changes
-        message.lastServerClock > this.clock
-      ) {
-        const diff = {};
-        for (const [id, doc] of this.documents.entries()) {
-          if (id !== session.presenceId) {
-            diff[id] = [RecordOpType.Put, doc.state];
-          }
-        }
-        const migrated = this.migrateDiffForSession(sessionSchema, diff);
-        if (!migrated.ok) {
-          rollback();
-          this.rejectSession(
-            session.sessionId,
-            migrated.error === MigrationFailureReason.TargetVersionTooNew ? TLSyncErrorCloseEventReason.SERVER_TOO_OLD : TLSyncErrorCloseEventReason.CLIENT_TOO_OLD
-          );
-          return;
-        }
-        connect({
-          type: "connect",
-          connectRequestId: message.connectRequestId,
-          hydrationType: "wipe_all",
-          protocolVersion: getTlsyncProtocolVersion(),
-          schema: this.schema.serialize(),
-          serverClock: this.clock,
-          diff: migrated.value,
-          isReadonly: session.isReadonly
-        });
-      } else {
-        const diff = {};
-        for (const doc of this.documents.values()) {
-          if (doc.lastChangedClock > message.lastServerClock) {
-            diff[doc.state.id] = [RecordOpType.Put, doc.state];
-          } else if (this.presenceType?.isId(doc.state.id) && doc.state.id !== session.presenceId) {
-            diff[doc.state.id] = [RecordOpType.Put, doc.state];
-          }
-        }
-        for (const [id, deletedAtClock] of this.tombstones.entries()) {
-          if (deletedAtClock > message.lastServerClock) {
-            diff[id] = [RecordOpType.Remove];
-          }
-        }
-        const migrated = this.migrateDiffForSession(sessionSchema, diff);
-        if (!migrated.ok) {
-          rollback();
-          this.rejectSession(
-            session.sessionId,
-            migrated.error === MigrationFailureReason.TargetVersionTooNew ? TLSyncErrorCloseEventReason.SERVER_TOO_OLD : TLSyncErrorCloseEventReason.CLIENT_TOO_OLD
-          );
-          return;
+    const { documentClock, result } = this.storage.transaction((txn) => {
+      this.broadcastChanges(txn);
+      const docChanges = txn.getChangesSince(message.lastServerClock);
+      const presenceDiff = this.migrateDiffOrRejectSession(
+        session.sessionId,
+        sessionSchema,
+        requiresDownMigrations,
+        {
+          puts: Object.fromEntries([...this.presenceStore.values()].map((p) => [p.id, p])),
+          deletes: []
         }
-        connect({
-          type: "connect",
-          connectRequestId: message.connectRequestId,
-          hydrationType: "wipe_presence",
-          schema: this.schema.serialize(),
-          protocolVersion: getTlsyncProtocolVersion(),
-          serverClock: this.clock,
-          diff: migrated.value,
-          isReadonly: session.isReadonly
-        });
+      );
+      if (!presenceDiff.ok) return null;
+      let docDiff = null;
+      if (docChanges && sessionSchema !== this.serializedSchema) {
+        const migrated = this.migrateDiffOrRejectSession(
+          session.sessionId,
+          sessionSchema,
+          requiresDownMigrations,
+          docChanges.diff
+        );
+        if (!migrated.ok) return null;
+        docDiff = migrated.value;
+      } else if (docChanges) {
+        docDiff = toNetworkDiff(docChanges.diff);
       }
+      return {
+        type: "connect",
+        connectRequestId: message.connectRequestId,
+        hydrationType: docChanges?.wipeAll ? "wipe_all" : "wipe_presence",
+        protocolVersion: getTlsyncProtocolVersion(),
+        schema: this.schema.serialize(),
+        serverClock: txn.getClock(),
+        diff: { ...presenceDiff.value, ...docDiff },
+        isReadonly: session.isReadonly
+      };
     });
+    this.lastDocumentClock = documentClock;
+    if (result) {
+      connect(result);
+    }
   }
   handlePushRequest(session, message) {
     if (session && session.state !== RoomSessionState.Connected) {
@@ -867,203 +626,198 @@ class TLSyncRoom {
     if (session) {
       session.lastInteractionTime = Date.now();
     }
-    this.clock++;
-    const initialDocumentClock = this.documentClock;
-    let didPresenceChange = false;
-    transaction((rollback) => {
-      const legacyAppendMode = !this.getCanEmitStringAppend();
-      const docChanges = { diff: null };
-      const presenceChanges = { diff: null };
-      const propagateOp = (changes, id, op) => {
-        if (!changes.diff) changes.diff = {};
-        changes.diff[id] = op;
-      };
-      const fail = (reason, underlyingError) => {
-        rollback();
-        if (session) {
-          this.rejectSession(session.sessionId, reason);
-        } else {
-          throw new Error("failed to apply changes: " + reason, underlyingError);
-        }
-        if (typeof process !== "undefined" && process.env.NODE_ENV !== "test") {
-          this.log?.error?.("failed to apply push", reason, message, underlyingError);
-        }
-        return Result.err(void 0);
-      };
-      const addDocument = (changes, id, _state) => {
-        const res = session ? this.schema.migratePersistedRecord(_state, session.serializedSchema, "up") : { type: "success", value: _state };
-        if (res.type === "error") {
-          return fail(
-            res.reason === MigrationFailureReason.TargetVersionTooOld ? TLSyncErrorCloseEventReason.SERVER_TOO_OLD : TLSyncErrorCloseEventReason.CLIENT_TOO_OLD
-          );
-        }
-        const { value: state } = res;
-        const doc = this.getDocument(id);
-        if (doc) {
-          const diff = doc.replaceState(state, this.clock, legacyAppendMode);
-          if (!diff.ok) {
-            return fail(TLSyncErrorCloseEventReason.INVALID_RECORD);
-          }
-          if (diff.value) {
-            this.documents.set(id, diff.value[1]);
-            propagateOp(changes, id, [RecordOpType.Patch, diff.value[0]]);
-          }
-        } else {
-          const result = this.addDocument(id, state, this.clock);
-          if (!result.ok) {
-            return fail(TLSyncErrorCloseEventReason.INVALID_RECORD);
-          }
-          propagateOp(changes, id, [RecordOpType.Put, state]);
+    const legacyAppendMode = !this.getCanEmitStringAppend();
+    const propagateOp = (changes2, id, op, before, after) => {
+      if (!changes2.diffs) changes2.diffs = { networkDiff: {}, diff: { puts: {}, deletes: [] } };
+      changes2.diffs.networkDiff[id] = op;
+      switch (op[0]) {
+        case RecordOpType.Put:
+          changes2.diffs.diff.puts[id] = op[1];
+          break;
+        case RecordOpType.Patch:
+          assert(before && after, "before and after are required for patches");
+          changes2.diffs.diff.puts[id] = [before, after];
+          break;
+        case RecordOpType.Remove:
+          changes2.diffs.diff.deletes.push(id);
+          break;
+        default:
+          exhaustiveSwitchError(op[0]);
+      }
+    };
+    const addDocument = (storage, changes2, id, _state) => {
+      const res = session ? this.schema.migratePersistedRecord(_state, session.serializedSchema, "up") : { type: "success", value: _state };
+      if (res.type === "error") {
+        throw new TLSyncError(res.reason, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
+      }
+      const { value: state } = res;
+      const doc = storage.get(id);
+      if (doc) {
+        const recordType = assertExists(getOwnProperty(this.schema.types, doc.typeName));
+        const diff = diffAndValidateRecord(doc, state, recordType);
+        if (diff) {
+          storage.set(id, state);
+          propagateOp(changes2, id, [RecordOpType.Patch, diff], doc, state);
         }
-        return Result.ok(void 0);
-      };
-      const patchDocument = (changes, id, patch) => {
-        const doc = this.getDocument(id);
-        if (!doc) return Result.ok(void 0);
-        const downgraded = session ? this.schema.migratePersistedRecord(doc.state, session.serializedSchema, "down") : { type: "success", value: doc.state };
-        if (downgraded.type === "error") {
-          return fail(TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
+      } else {
+        const recordType = assertExists(getOwnProperty(this.schema.types, state.typeName));
+        validateRecord(state, recordType);
+        storage.set(id, state);
+        propagateOp(changes2, id, [RecordOpType.Put, state], void 0, void 0);
+      }
+      return Result.ok(void 0);
+    };
+    const patchDocument = (storage, changes2, id, patch) => {
+      const doc = storage.get(id);
+      if (!doc) return;
+      const recordType = assertExists(getOwnProperty(this.schema.types, doc.typeName));
+      const downgraded = session ? this.schema.migratePersistedRecord(doc, session.serializedSchema, "down") : { type: "success", value: doc };
+      if (downgraded.type === "error") {
+        throw new TLSyncError(downgraded.reason, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
+      }
+      if (downgraded.value === doc) {
+        const diff = applyAndDiffRecord(doc, patch, recordType, legacyAppendMode);
+        if (diff) {
+          storage.set(id, diff[1]);
+          propagateOp(changes2, id, [RecordOpType.Patch, diff[0]], doc, diff[1]);
         }
-        if (downgraded.value === doc.state) {
-          const diff = doc.mergeDiff(patch, this.clock, legacyAppendMode);
-          if (!diff.ok) {
-            return fail(TLSyncErrorCloseEventReason.INVALID_RECORD);
-          }
-          if (diff.value) {
-            this.documents.set(id, diff.value[1]);
-            propagateOp(changes, id, [RecordOpType.Patch, diff.value[0]]);
-          }
-        } else {
-          const patched = applyObjectDiff(downgraded.value, patch);
-          const upgraded = session ? this.schema.migratePersistedRecord(patched, session.serializedSchema, "up") : { type: "success", value: patched };
-          if (upgraded.type === "error") {
-            return fail(TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
-          }
-          const diff = doc.replaceState(upgraded.value, this.clock, legacyAppendMode);
-          if (!diff.ok) {
-            return fail(TLSyncErrorCloseEventReason.INVALID_RECORD);
-          }
-          if (diff.value) {
-            this.documents.set(id, diff.value[1]);
-            propagateOp(changes, id, [RecordOpType.Patch, diff.value[0]]);
-          }
+      } else {
+        const patched = applyObjectDiff(downgraded.value, patch);
+        const upgraded = session ? this.schema.migratePersistedRecord(patched, session.serializedSchema, "up") : { type: "success", value: patched };
+        if (upgraded.type === "error") {
+          throw new TLSyncError(upgraded.reason, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD);
         }
-        return Result.ok(void 0);
-      };
-      const { clientClock } = message;
-      if (this.presenceType && session?.presenceId && "presence" in message && message.presence) {
-        if (!session) throw new Error("session is required for presence pushes");
-        const id = session.presenceId;
-        const [type, val] = message.presence;
-        const { typeName } = this.presenceType;
-        switch (type) {
-          case RecordOpType.Put: {
-            const res = addDocument(presenceChanges, id, { ...val, id, typeName });
-            if (!res.ok) return;
-            break;
-          }
-          case RecordOpType.Patch: {
-            const res = patchDocument(presenceChanges, id, {
-              ...val,
-              id: [ValueOpType.Put, id],
-              typeName: [ValueOpType.Put, typeName]
-            });
-            if (!res.ok) return;
-            break;
-          }
+        const diff = diffAndValidateRecord(doc, upgraded.value, recordType, legacyAppendMode);
+        if (diff) {
+          storage.set(id, upgraded.value);
+          propagateOp(changes2, id, [RecordOpType.Patch, diff], doc, upgraded.value);
         }
       }
-      if (message.diff && !session?.isReadonly) {
-        for (const [id, op] of objectMapEntriesIterable(message.diff)) {
-          switch (op[0]) {
+    };
+    const { result, documentClock, changes } = this.storage.transaction(
+      (txn) => {
+        this.broadcastChanges(txn);
+        const docChanges = { diffs: null };
+        const presenceChanges = { diffs: null };
+        if (this.presenceType && session?.presenceId && "presence" in message && message.presence) {
+          if (!session) throw new Error("session is required for presence pushes");
+          const id = session.presenceId;
+          const [type, val] = message.presence;
+          const { typeName } = this.presenceType;
+          switch (type) {
             case RecordOpType.Put: {
-              if (!this.documentTypes.has(op[1].typeName)) {
-                return fail(TLSyncErrorCloseEventReason.INVALID_RECORD);
-              }
-              const res = addDocument(docChanges, id, op[1]);
-              if (!res.ok) return;
+              addDocument(this.presenceStore, presenceChanges, id, {
+                ...val,
+                id,
+                typeName
+              });
               break;
             }
             case RecordOpType.Patch: {
-              const res = patchDocument(docChanges, id, op[1]);
-              if (!res.ok) return;
-              break;
-            }
-            case RecordOpType.Remove: {
-              const doc = this.getDocument(id);
-              if (!doc) {
-                continue;
-              }
-              this.removeDocument(id, this.clock);
-              propagateOp(docChanges, id, op);
+              patchDocument(this.presenceStore, presenceChanges, id, {
+                ...val,
+                id: [ValueOpType.Put, id],
+                typeName: [ValueOpType.Put, typeName]
+              });
               break;
             }
           }
         }
-      }
-      if (
-        // if there was only a presence push, the client doesn't need to do anything aside from
-        // shift the push request.
-        !message.diff || isEqual(docChanges.diff, message.diff)
-      ) {
-        if (session) {
-          this.sendMessage(session.sessionId, {
-            type: "push_result",
-            serverClock: this.clock,
-            clientClock,
-            action: "commit"
-          });
-        }
-      } else if (!docChanges.diff) {
-        if (session) {
-          this.sendMessage(session.sessionId, {
-            type: "push_result",
-            serverClock: this.clock,
-            clientClock,
-            action: "discard"
-          });
-        }
-      } else {
-        if (session) {
-          const migrateResult = this.migrateDiffForSession(
-            session.serializedSchema,
-            docChanges.diff
-          );
-          if (!migrateResult.ok) {
-            return fail(
-              migrateResult.error === MigrationFailureReason.TargetVersionTooNew ? TLSyncErrorCloseEventReason.SERVER_TOO_OLD : TLSyncErrorCloseEventReason.CLIENT_TOO_OLD
-            );
+        if (message.diff && !session?.isReadonly) {
+          for (const [id, op] of objectMapEntriesIterable(message.diff)) {
+            switch (op[0]) {
+              case RecordOpType.Put: {
+                if (!this.documentTypes.has(op[1].typeName)) {
+                  throw new TLSyncError(
+                    "invalid record",
+                    TLSyncErrorCloseEventReason.INVALID_RECORD
+                  );
+                }
+                addDocument(txn, docChanges, id, op[1]);
+                break;
+              }
+              case RecordOpType.Patch: {
+                patchDocument(txn, docChanges, id, op[1]);
+                break;
+              }
+              case RecordOpType.Remove: {
+                const doc = txn.get(id);
+                if (!doc) {
+                  continue;
+                }
+                txn.delete(id);
+                propagateOp(docChanges, id, op, doc, void 0);
+                break;
+              }
+            }
           }
-          this.sendMessage(session.sessionId, {
-            type: "push_result",
-            serverClock: this.clock,
-            clientClock,
-            action: { rebaseWithDiff: migrateResult.value }
-          });
         }
+        return { docChanges, presenceChanges };
+      },
+      { id: this.internalTxnId, emitChanges: "when-different" }
+    );
+    this.lastDocumentClock = documentClock;
+    let pushResult;
+    if (changes && session) {
+      result.docChanges.diffs = { networkDiff: toNetworkDiff(changes) ?? {}, diff: changes };
+    }
+    if (isEqual(result.docChanges.diffs?.networkDiff, message.diff)) {
+      pushResult = {
+        type: "push_result",
+        clientClock: message.clientClock,
+        serverClock: documentClock,
+        action: "commit"
+      };
+    } else if (!result.docChanges.diffs?.networkDiff) {
+      pushResult = {
+        type: "push_result",
+        clientClock: message.clientClock,
+        serverClock: documentClock,
+        action: "discard"
+      };
+    } else if (session) {
+      const diff = this.migrateDiffOrRejectSession(
+        session.sessionId,
+        session.serializedSchema,
+        session.requiresDownMigrations,
+        result.docChanges.diffs.diff,
+        result.docChanges.diffs.networkDiff
+      );
+      if (diff.ok) {
+        pushResult = {
+          type: "push_result",
+          clientClock: message.clientClock,
+          serverClock: documentClock,
+          action: { rebaseWithDiff: diff.value }
+        };
       }
-      if (docChanges.diff || presenceChanges.diff) {
-        this.broadcastPatch({
-          sourceSessionId: session?.sessionId,
-          diff: {
-            ...docChanges.diff,
-            ...presenceChanges.diff
-          }
-        });
-      }
-      if (docChanges.diff) {
-        this.documentClock = this.clock;
-      }
-      if (presenceChanges.diff) {
-        didPresenceChange = true;
-      }
-      return;
-    });
-    if (this.documentClock !== initialDocumentClock) {
-      this.onDataChange?.();
     }
-    if (didPresenceChange) {
-      this.onPresenceChange?.();
+    if (session && pushResult) {
+      this._unsafe_sendMessage(session.sessionId, pushResult);
+    }
+    if (result.docChanges.diffs || result.presenceChanges.diffs) {
+      this.broadcastPatch(
+        {
+          puts: {
+            ...result.docChanges.diffs?.diff.puts,
+            ...result.presenceChanges.diffs?.diff.puts
+          },
+          deletes: [
+            ...(result.docChanges.diffs?.diff.deletes ?? []),
+            ...(result.presenceChanges.diffs?.diff.deletes ?? [])
+          ]
+        },
+        {
+          ...result.docChanges.diffs?.networkDiff,
+          ...result.presenceChanges.diffs?.networkDiff
+        },
+        session?.sessionId
+      );
+    }
+    if (result.presenceChanges.diffs) {
+      queueMicrotask(() => {
+        this.onPresenceChange?.();
+      });
     }
   }
   /**
@@ -1081,111 +835,25 @@ class TLSyncRoom {
   handleClose(sessionId) {
     this.cancelSession(sessionId);
   }
-  /**
-   * Apply changes to the room's store in a transactional way. Changes are
-   * automatically synchronized to all connected clients.
-   *
-   * @param updater - Function that receives store methods to make changes
-   * @returns Promise that resolves when the transaction is complete
-   * @example
-   * ```ts
-   * // Add multiple shapes atomically
-   * await room.updateStore((store) => {
-   *   store.put(createShape({ type: 'geo', x: 100, y: 100 }))
-   *   store.put(createShape({ type: 'text', x: 200, y: 200 }))
-   * })
-   *
-   * // Async operations are supported
-   * await room.updateStore(async (store) => {
-   *   const template = await loadTemplate()
-   *   template.shapes.forEach(shape => store.put(shape))
-   * })
-   * ```
-   */
-  async updateStore(updater) {
-    if (this._isClosed) {
-      throw new Error("Cannot update store on a closed room");
-    }
-    const context = new StoreUpdateContext(
-      Object.fromEntries(this.getSnapshot().documents.map((d) => [d.state.id, d.state]))
-    );
-    try {
-      await updater(context);
-    } finally {
-      context.close();
-    }
-    const diff = context.toDiff();
-    if (Object.keys(diff).length === 0) {
-      return;
-    }
-    this.handlePushRequest(null, { type: "push", diff, clientClock: 0 });
-  }
 }
-class StoreUpdateContext {
-  constructor(snapshot) {
-    this.snapshot = snapshot;
-  }
-  updates = {
-    puts: {},
-    deletes: /* @__PURE__ */ new Set()
-  };
-  put(record) {
-    if (this._isClosed) throw new Error("StoreUpdateContext is closed");
-    if (record.id in this.snapshot && isEqual(this.snapshot[record.id], record)) {
-      delete this.updates.puts[record.id];
-    } else {
-      this.updates.puts[record.id] = structuredClone(record);
-    }
-    this.updates.deletes.delete(record.id);
-  }
-  delete(recordOrId) {
-    if (this._isClosed) throw new Error("StoreUpdateContext is closed");
-    const id = typeof recordOrId === "string" ? recordOrId : recordOrId.id;
-    delete this.updates.puts[id];
-    if (this.snapshot[id]) {
-      this.updates.deletes.add(id);
-    }
-  }
+class PresenceStore {
+  presences = new AtomMap("presences");
   get(id) {
-    if (this._isClosed) throw new Error("StoreUpdateContext is closed");
-    if (hasOwnProperty(this.updates.puts, id)) {
-      return structuredClone(this.updates.puts[id]);
-    }
-    if (this.updates.deletes.has(id)) {
-      return null;
-    }
-    return structuredClone(this.snapshot[id] ?? null);
+    return this.presences.get(id);
   }
-  getAll() {
-    if (this._isClosed) throw new Error("StoreUpdateContext is closed");
-    const result = Object.values(this.updates.puts);
-    for (const [id, record] of Object.entries(this.snapshot)) {
-      if (!this.updates.deletes.has(id) && !hasOwnProperty(this.updates.puts, id)) {
-        result.push(record);
-      }
-    }
-    return structuredClone(result);
+  set(id, state) {
+    this.presences.set(id, state);
   }
-  toDiff() {
-    const diff = {};
-    for (const [id, record] of Object.entries(this.updates.puts)) {
-      diff[id] = [RecordOpType.Put, record];
-    }
-    for (const id of this.updates.deletes) {
-      diff[id] = [RecordOpType.Remove];
-    }
-    return diff;
+  delete(id) {
+    this.presences.delete(id);
   }
-  _isClosed = false;
-  close() {
-    this._isClosed = true;
+  values() {
+    return this.presences.values();
   }
 }
 export {
   DATA_MESSAGE_DEBOUNCE_INTERVAL,
-  DocumentState,
-  MAX_TOMBSTONES,
-  TLSyncRoom,
-  TOMBSTONE_PRUNE_BUFFER_SIZE
+  PresenceStore,
+  TLSyncRoom
 };
 //# sourceMappingURL=TLSyncRoom.mjs.map