@doubling/compound-sync 1.10.5 → 1.12.0

package/yjs-binding-state.js

@@ -0,0 +1,61 @@
+// Per-file Y.Doc binary state persistence (DOU-209).
+//
+// On daemon restart we need to merge offline edits made on disk with
+// offline edits made via the web. Yjs converges deterministically when
+// both sides apply their ops to a common baseline. The daemon's last
+// view of the file (== ytext at last write, == disk content at last
+// write) is that baseline. Persisting the Y.Doc binary state per file
+// lets the daemon restore that baseline on startup so:
+//   1. ytext == lastDisk after restore
+//   2. fast-diff(ytext, currentDisk) gives positions valid in ytext
+//   3. cloud yupdates apply on top via CRDT merge (idempotent for ops
+//      already in the loaded state, additive for new cloud ops).
+//
+// Storage layout: one file per Y.Doc, keyed by fileId, under
+// `<localPath>/.compound-yjs-binding/<fileId>.yjs`. The binary blob is
+// the output of Y.encodeStateAsUpdate(doc); the daemon applies it via
+// Y.applyUpdate(doc, blob) on restore.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const STATE_DIR = '.compound-yjs-binding';
+
+function statePathFor(localPath, fileId) {
+  return path.join(localPath, STATE_DIR, `${fileId}.yjs`);
+}
+
+// Load the encoded Y.Doc state for a file. Returns null if no state
+// exists yet (first run, or this fileId hasn't been synced before) or
+// if the file is unreadable for any reason (corruption, permissions).
+// Treating unreadable as "no state" is safe: the binding falls back to
+// its first-run behavior, which writes ytext to disk on initial flush.
+export function loadBindingState(localPath, fileId) {
+  try {
+    return fs.readFileSync(statePathFor(localPath, fileId));
+  } catch {
+    return null;
+  }
+}
+
+// Persist the encoded Y.Doc state for a file. Atomic via write-to-tmp +
+// rename so a crash mid-write can't leave a half-written .yjs file that
+// would corrupt the next startup. Best-effort: a failed save costs us a
+// cold-start reconciliation next launch, never data.
+export function saveBindingState(localPath, fileId, encoded) {
+  const finalPath = statePathFor(localPath, fileId);
+  const tmpPath = `${finalPath}.${process.pid}.${Date.now()}.tmp`;
+  try {
+    fs.mkdirSync(path.dirname(finalPath), { recursive: true });
+    fs.writeFileSync(tmpPath, encoded);
+    fs.renameSync(tmpPath, finalPath);
+  } catch {
+    try { fs.unlinkSync(tmpPath); } catch { /* ignore */ }
+  }
+}
+
+// Best-effort cleanup when a file is deleted from the workspace, so
+// stale .yjs blobs don't accumulate. Failing to delete is not fatal.
+export function deleteBindingState(localPath, fileId) {
+  try { fs.unlinkSync(statePathFor(localPath, fileId)); } catch { /* ignore */ }
+}

package/yjs-file-binding.js

@@ -0,0 +1,236 @@
+// YjsFileBinding: per-file glue that keeps a local Markdown file
+// converged with the per-file Y.Doc that lives in Firestore.
+//
+// Two directions, both flowing through the Y.Doc:
+//   cloud -> disk: provider applies remote yupdates to ytext; an
+//     observer debounces the resulting text into a single disk write
+//     so a burst of web keystrokes coalesces into one fs.writeFile.
+//   disk -> cloud: when the local file changes, caller invokes
+//     applyDiskText(newText); we diff against ytext (fast-diff for a
+//     char-level edit script) and apply inserts/deletes in one
+//     transaction so the provider's local-update listener emits a
+//     single yupdate carrying just the changed regions, preserving
+//     concurrent web edits to unrelated regions.
+//
+// Disk I/O is injected via onDiskWrite so tests can record without
+// touching the filesystem and so the daemon can plumb its existing
+// suppressLocal set (avoids the disk write feeding back through
+// chokidar as a fake user edit).
+
+import * as Y from 'yjs';
+import fastDiff from 'fast-diff';
+import { YjsProvider } from './yjs-provider.js';
+
+export class YjsFileBinding {
+  constructor(store, {
+    debounceMs = 300, onDiskWrite, onDocStateChange,
+    initialDocState = null, currentDiskText = null,
+  }) {
+    if (!onDiskWrite) throw new Error('YjsFileBinding: onDiskWrite is required');
+    this.doc = new Y.Doc();
+    this.ytext = this.doc.getText('content');
+    this.store = store;
+    this.provider = new YjsProvider(this.doc, store);
+    this.debounceMs = debounceMs;
+    this.onDiskWrite = onDiskWrite;
+    // Called after every Y.Doc mutation reaches a stable state, with
+    // the encoded Y.Doc binary state. The daemon persists this to
+    // disk so the next startup can restore the baseline and merge
+    // offline disk edits without losing offline cloud edits (DOU-209).
+    this.onDocStateChange = onDocStateChange ?? null;
+    // Restore options for offline-merge (DOU-209).
+    this.initialDocState = initialDocState;
+    this.currentDiskText = currentDiskText;
+    this.flushTimer = null;
+    this.lastDiskText = null;
+    this.observer = null;
+    this.stopped = false;
+  }
+
+  async start() {
+    // DOU-209: offline-merge startup. Three-step process that turns the
+    // daemon's restart into a deterministic CRDT merge:
+    //
+    //   1. Restore Y.Doc from the daemon's last-saved binary state (if
+    //      any). After this, ytext equals what disk was at the daemon's
+    //      last write. This is the common ancestor for any divergence
+    //      that happened while the daemon was offline.
+    //   2. If the current disk content differs from the restored ytext,
+    //      the difference is disk-only edits made during downtime.
+    //      Apply them as Y.Doc operations (positions are valid in the
+    //      restored ytext since restored-ytext == lastDisk == fast-diff
+    //      input).
+    //   3. provider.loadInitial() pulls cloud's yupdates on top. Y.applyUpdate
+    //      is idempotent (Yjs's state-vector dedup), so already-known
+    //      ops are no-ops. Only NEW cloud ops (offline web edits) take
+    //      effect. They CRDT-merge with the disk ops from step 2.
+    //
+    // Without step 1, fast-diff(post-loadInitial-ytext, currentDisk)
+    // treats web's offline insertions as "things present in ytext but
+    // missing from disk" and DELETES them. That's the bug this fixes.
+    // Seed lastDiskText from the disk content we observed at construction
+    // time so the initial flush is a no-op when ytext already matches
+    // what's on disk. Without this, the binding's initial flush always
+    // wrote ytext to disk on startup, which clobbered any content the
+    // daemon's legacy contentHash + Storage blob pull (the fallthrough
+    // path in handleFirestoreChange) may have written first for non-Yjs
+    // cloud edits. Regression case: "after restart, pulls a web edit
+    // made while stopped despite a fresh local mtime" in
+    // sync/org-sync.test.js.
+    this.lastDiskText = this.currentDiskText;
+
+    if (this.initialDocState) {
+      Y.applyUpdate(this.doc, this.initialDocState);
+    }
+
+    // Disk-merge ops emitted below need to reach the store so other
+    // clients see the offline disk edits. The provider's local-update
+    // handler isn't attached yet (it's wired in startSubscription,
+    // which we can't call until after loadInitial), so we capture the
+    // updates synchronously here and push them after the subscription
+    // is set up. Capture by Y.Doc 'update' event with a 'disk' origin
+    // filter so we only grab the ops from our own _applyTextDiff and
+    // not anything spurious.
+    //
+    // The disk-merge is only safe when we have an `initialDocState`:
+    // that gives ytext a known baseline (the daemon's last view of the
+    // file) before we diff against currentDiskText, so fast-diff
+    // positions are valid AND the resulting ops are scoped to JUST the
+    // disk-only changes. Without a baseline, applying currentDiskText
+    // as fresh inserts on top of an empty ytext would then collide
+    // with whatever loadInitial pulls from the cloud: Yjs would see
+    // both the daemon's "fresh insert" and the cloud's "original
+    // insert" as concurrent ops, producing duplicated content. So on
+    // first contact (no saved state) we skip the disk merge and let
+    // loadInitial + initial flush behave the same as before DOU-209.
+    const capturedDiskUpdates = [];
+    if (this.initialDocState && this.currentDiskText != null) {
+      const restored = this.ytext.toString();
+      if (this.currentDiskText !== restored) {
+        const captureHandler = (update, origin) => {
+          if (origin === 'disk') capturedDiskUpdates.push(update);
+        };
+        this.doc.on('update', captureHandler);
+        // Same diff-and-apply shape as applyDiskText, but we call the
+        // internal helper directly: applyDiskText is the live-chokidar
+        // path and also touches lastDiskText / _emitDocState in ways
+        // that aren't appropriate during the startup-merge phase.
+        this._applyTextDiff(restored, this.currentDiskText);
+        this.doc.off('update', captureHandler);
+      }
+    }
+
+    // DOU-204: two-phase provider start. Phase 1 loads snapshot +
+    // initial updates without subscribing. We then attach the
+    // Y.Text observer that drives the debounced disk write. Phase 2
+    // subscribes to remote updates; the first listener fire is the
+    // initial query result (re-applies the already-loaded docs
+    // idempotently), and any update that lands after loadInitial
+    // routes through the now-attached observer instead of mutating
+    // Y.Text silently between provider.start and observer attach.
+    await this.provider.loadInitial();
+    this.observer = () => {
+      if (this.stopped) return;
+      if (this.flushTimer) clearTimeout(this.flushTimer);
+      this.flushTimer = setTimeout(() => this._flushNow(), this.debounceMs);
+    };
+    this.ytext.observe(this.observer);
+    this.provider.startSubscription();
+
+    // Push the disk-side updates captured above so the store records
+    // the offline disk edits and other clients receive them. Y.applyUpdate
+    // is idempotent so the daemon's own subscription echo (when the
+    // store fires its own append back through subscribeUpdates) is a
+    // no-op on our local Y.Doc.
+    for (const update of capturedDiskUpdates) {
+      try { await this.store.appendUpdate(update); }
+      catch (err) { console.error('YjsFileBinding: disk-merge appendUpdate failed:', err && err.message); }
+    }
+
+    // Initial flush: ytext may be non-empty after replay (file has
+    // existing Yjs state). Writes the loaded content to disk so the
+    // local file is up-to-date the moment the binding is wired in.
+    if (this.ytext.length > 0) this._flushNow();
+    // Persist the merged baseline so the next restart has the right
+    // anchor (the binding's state right now == disk after the flush
+    // above; future disk-side edits should diff against this state).
+    this._emitDocState();
+  }
+
+  _applyTextDiff(fromText, toText) {
+    if (fromText === toText) return;
+    const diffs = fastDiff(fromText, toText);
+    this.doc.transact(() => {
+      let idx = 0;
+      for (const [op, str] of diffs) {
+        if (op === 0) {
+          idx += str.length;
+        } else if (op === 1) {
+          this.ytext.insert(idx, str);
+          idx += str.length;
+        } else {
+          this.ytext.delete(idx, str.length);
+        }
+      }
+    }, 'disk');
+  }
+
+  _emitDocState() {
+    if (this.stopped || !this.onDocStateChange) return;
+    try {
+      this.onDocStateChange(Y.encodeStateAsUpdate(this.doc));
+    } catch (err) {
+      console.error('YjsFileBinding: onDocStateChange failed:', err && err.message);
+    }
+  }
+
+  // Sync flush used by both the debounced observer and the final
+  // drain in stop().
+  _flushNow() {
+    if (this.stopped) return;
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    const text = this.ytext.toString();
+    if (text === this.lastDiskText) return;
+    this.lastDiskText = text;
+    this.onDiskWrite(text);
+    // The disk now matches ytext: snapshot the Y.Doc state so the next
+    // restart has the right baseline for offline-merge (DOU-209).
+    this._emitDocState();
+  }
+
+  // The local disk file changed (chokidar). Diff against the current
+  // ytext and apply as a sequence of inserts/deletes inside one
+  // transaction. Using a distinct origin ('disk') keeps the provider's
+  // local-update listener pushing the resulting yupdate to Firestore.
+  applyDiskText(newText) {
+    if (this.stopped) return;
+    const oldText = this.ytext.toString();
+    if (oldText === newText) return;
+    this._applyTextDiff(oldText, newText);
+    // Suppress the debounced echo: lastDiskText now matches the disk
+    // content the caller just observed, so the observer's pending
+    // flush is a no-op.
+    this.lastDiskText = newText;
+    // Snapshot the Y.Doc state so the next restart's baseline includes
+    // this disk-originated edit (DOU-209).
+    this._emitDocState();
+  }
+
+  async stop() {
+    if (this.stopped) return;
+    this.stopped = true;
+    if (this.observer) {
+      this.ytext.unobserve(this.observer);
+      this.observer = null;
+    }
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    this.provider.stop();
+    this.doc.destroy();
+  }
+}

package/yjs-firestore-update-store.js

@@ -0,0 +1,117 @@
+// FirestoreUpdateStore: Firestore-backed store for per-file Yjs
+// CRDT updates. Ported from web/src/yjs/firestore-update-store.ts so
+// the published @doubling/compound-sync package does not reach into
+// the web source tree. Keep the API identical to the web copy; a
+// follow-up will unify the two into a shared workspace package once
+// PR 4 finishes retiring the legacy blob path.
+
+import {
+  collection,
+  doc,
+  addDoc,
+  getDoc,
+  getDocs,
+  setDoc,
+  query,
+  where,
+  orderBy,
+  onSnapshot,
+  serverTimestamp,
+  writeBatch,
+  Bytes,
+  Timestamp,
+} from 'firebase/firestore';
+
+function formatId(createdAt, docId) {
+  return `${createdAt.toMillis().toString().padStart(16, '0')}_${docId}`;
+}
+
+function parseTimestampFromId(id) {
+  return Timestamp.fromMillis(Number(id.split('_')[0]));
+}
+
+export class FirestoreUpdateStore {
+  constructor(db, orgId, fileId) {
+    this.updatesCol = collection(db, 'orgs', orgId, 'files', fileId, 'yupdates');
+    this.snapshotDoc = doc(db, 'orgs', orgId, 'files', fileId, 'yjs', 'state');
+  }
+
+  async loadSnapshot() {
+    const snap = await getDoc(this.snapshotDoc);
+    if (!snap.exists()) return null;
+    const data = snap.data();
+    return {
+      state: data.state.toUint8Array(),
+      baselineId: data.baselineId ?? null,
+    };
+  }
+
+  async loadUpdatesAfter(baselineId) {
+    const q =
+      baselineId === null
+        ? query(this.updatesCol, orderBy('createdAt', 'asc'))
+        : query(
+            this.updatesCol,
+            where('createdAt', '>', parseTimestampFromId(baselineId)),
+            orderBy('createdAt', 'asc'),
+          );
+    const snap = await getDocs(q);
+    return snap.docs
+      .map((d) => {
+        const data = d.data();
+        if (!data.createdAt) return null;
+        return { id: formatId(data.createdAt, d.id), update: data.update.toUint8Array() };
+      })
+      .filter((r) => r !== null);
+  }
+
+  async appendUpdate(update) {
+    const docRef = await addDoc(this.updatesCol, {
+      update: Bytes.fromUint8Array(update),
+      createdAt: serverTimestamp(),
+    });
+    const fresh = await getDoc(docRef);
+    const data = fresh.data();
+    if (!data || !data.createdAt) {
+      throw new Error('FirestoreUpdateStore.appendUpdate: createdAt unresolved');
+    }
+    return { id: formatId(data.createdAt, docRef.id), update };
+  }
+
+  subscribeUpdates(onUpdate) {
+    const q = query(this.updatesCol, orderBy('createdAt', 'asc'));
+    // includeMetadataChanges + emitted-id dedup: see the web copy for
+    // the full rationale. Skipping pending writes alone drops the
+    // server-ack delivery because that fire is metadata-only.
+    const emitted = new Set();
+    return onSnapshot(q, { includeMetadataChanges: true }, (snap) => {
+      for (const change of snap.docChanges({ includeMetadataChanges: true })) {
+        if (change.type !== 'added' && change.type !== 'modified') continue;
+        const data = change.doc.data();
+        if (!data.createdAt) continue;
+        if (emitted.has(change.doc.id)) continue;
+        emitted.add(change.doc.id);
+        onUpdate({
+          id: formatId(data.createdAt, change.doc.id),
+          update: data.update.toUint8Array(),
+        });
+      }
+    });
+  }
+
+  async saveSnapshotAndCompact(state, newBaselineId) {
+    const baselineTs = parseTimestampFromId(newBaselineId);
+    await setDoc(this.snapshotDoc, {
+      state: Bytes.fromUint8Array(state),
+      baselineId: newBaselineId,
+      baselineCreatedAt: baselineTs,
+      updatedAt: serverTimestamp(),
+    });
+    const q = query(this.updatesCol, where('createdAt', '<=', baselineTs));
+    const stale = await getDocs(q);
+    if (stale.empty) return;
+    const batch = writeBatch(this.snapshotDoc.firestore);
+    for (const d of stale.docs) batch.delete(d.ref);
+    await batch.commit();
+  }
+}

package/yjs-provider.js

@@ -0,0 +1,85 @@
+// YjsProvider: binds a Y.Doc to an UpdateStore. Ported from
+// web/src/yjs/provider.ts; see that file for the full design notes
+// (echo handling, snapshot replay, compaction semantics, two-phase
+// start lifecycle from DOU-204). API kept identical so the two copies
+// can be unified into a shared workspace package in a follow-up.
+
+import * as Y from 'yjs';
+
+export class YjsProvider {
+  constructor(doc, store) {
+    this.doc = doc;
+    this.store = store;
+    this.unsubscribeStore = null;
+    this.updateHandler = null;
+    this.loaded = false;
+    this.subscribed = false;
+    this.stopped = false;
+  }
+
+  /** Phase 1: snapshot + initial updates → Y.Doc. No remote
+   * subscription yet. Callers that intend to attach a Y.Text observer
+   * (e.g. a disk-write debouncer here, yCollab on the web) should
+   * install that observer between `loadInitial()` and
+   * `startSubscription()` so the first remote update routes through
+   * to it instead of mutating Y.Text silently in the gap (DOU-204). */
+  async loadInitial() {
+    if (this.loaded) throw new Error('YjsProvider.loadInitial() called twice');
+    this.loaded = true;
+
+    const snapshot = await this.store.loadSnapshot();
+    if (snapshot) Y.applyUpdate(this.doc, snapshot.state, this);
+
+    const initial = await this.store.loadUpdatesAfter(snapshot?.baselineId ?? null);
+    for (const record of initial) Y.applyUpdate(this.doc, record.update, this);
+  }
+
+  /** Phase 2: subscribe to remote updates and push local Y.Doc
+   * mutations back to the store. */
+  startSubscription() {
+    if (!this.loaded) throw new Error('YjsProvider.startSubscription() before loadInitial()');
+    if (this.subscribed) throw new Error('YjsProvider.startSubscription() called twice');
+    this.subscribed = true;
+
+    this.unsubscribeStore = this.store.subscribeUpdates((record) => {
+      if (this.stopped) return;
+      Y.applyUpdate(this.doc, record.update, this);
+    });
+
+    this.updateHandler = (update, origin) => {
+      if (this.stopped) return;
+      if (origin === this) return;
+      this.store.appendUpdate(update).catch((err) => {
+        console.error('YjsProvider: appendUpdate failed:', err && err.message);
+      });
+    };
+    this.doc.on('update', this.updateHandler);
+  }
+
+  /** Convenience: load + subscribe in one call. Use only when no
+   * view-layer observer needs to attach between the phases. */
+  async start() {
+    await this.loadInitial();
+    this.startSubscription();
+  }
+
+  stop() {
+    this.stopped = true;
+    if (this.unsubscribeStore) {
+      this.unsubscribeStore();
+      this.unsubscribeStore = null;
+    }
+    if (this.updateHandler) {
+      this.doc.off('update', this.updateHandler);
+      this.updateHandler = null;
+    }
+  }
+
+  async compact() {
+    const state = Y.encodeStateAsUpdate(this.doc);
+    const records = await this.store.loadUpdatesAfter(null);
+    const last = records[records.length - 1];
+    if (!last) return;
+    await this.store.saveSnapshotAndCompact(state, last.id);
+  }
+}