@pylonsync/loro 0.22.38 → 0.22.40

package/src/outbox.ts

@@ -0,0 +1,214 @@
+// ---------------------------------------------------------------------------
+// CrdtOutbox — sends a doc's local edits to the server
+//
+// Every edit made on this copy of a row's doc (by an editor binding, a
+// hook, or app code) has to reach `POST /api/crdt/<entity>/<id>`. The
+// outbox listens for local commits and sends the edits the server has not
+// confirmed:
+//
+//   - One request at a time, so edits arrive in order. Commits made while
+//     a request is in flight go out together in the next one.
+//   - What it sends is every op of this peer after the last confirmed one,
+//     so a failed request loses nothing: the next send carries it again.
+//     Ops of other peers are never sent back; they came from the server.
+//   - A network error or a 5xx, 408 or 429 retries with backoff. Any other
+//     4xx (policy denied, row gone, bad update) stops the outbox: sending
+//     the same edits again would fail the same way.
+//
+// The server merges idempotently, so an edit sent twice (a response lost
+// after the server applied it) does no harm.
+// ---------------------------------------------------------------------------
+
+import { VersionVector, type LoroDoc } from "loro-crdt";
+
+/** Send one update. Resolve when the server accepted it; reject otherwise.
+ *  A rejection with a numeric `status` is treated as the HTTP status. */
+export type OutboxSend = (update: Uint8Array, options: { keepalive: boolean }) => Promise<void>;
+
+/**
+ * - `idle`: the server has every local edit.
+ * - `sending`: a request is in flight.
+ * - `retrying`: the last request failed and a retry is scheduled.
+ * - `rejected`: the server refused the edits; nothing more is sent.
+ */
+export type OutboxState = "idle" | "sending" | "retrying" | "rejected";
+
+export interface OutboxOptions {
+  /** First retry delay. Doubles per failure up to `retryMaxMs`. */
+  retryBaseMs?: number;
+  retryMaxMs?: number;
+  /** Timer functions, replaceable in tests. */
+  setTimer?: (fn: () => void, ms: number) => unknown;
+  clearTimer?: (handle: unknown) => void;
+}
+
+/** True for an error the same request could get past by trying again. */
+export function isRetryable(err: unknown): boolean {
+  const status = (err as { status?: unknown } | null)?.status;
+  if (typeof status !== "number") return true; // network error, abort, timeout
+  return status >= 500 || status === 408 || status === 429;
+}
+
+export class CrdtOutbox {
+  private readonly doc: LoroDoc;
+  private readonly send: OutboxSend;
+  private readonly retryBaseMs: number;
+  private readonly retryMaxMs: number;
+  private readonly setTimer: (fn: () => void, ms: number) => unknown;
+  private readonly clearTimer: (handle: unknown) => void;
+  private readonly stopListening: () => void;
+  private readonly listeners = new Set<() => void>();
+
+  /** This peer's op counter the server has confirmed (exclusive end). */
+  private confirmed = 0;
+  private inFlight = false;
+  private retryHandle: unknown = null;
+  private failures = 0;
+  private disposed = false;
+  private _state: OutboxState = "idle";
+  private _error: unknown = null;
+
+  constructor(doc: LoroDoc, send: OutboxSend, options: OutboxOptions = {}) {
+    this.doc = doc;
+    this.send = send;
+    this.retryBaseMs = options.retryBaseMs ?? 500;
+    this.retryMaxMs = options.retryMaxMs ?? 10_000;
+    this.setTimer = options.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+    this.clearTimer = options.clearTimer ?? ((h) => clearTimeout(h as ReturnType<typeof setTimeout>));
+    this.stopListening = doc.subscribeLocalUpdates(() => this.schedule());
+    // Edits made before the outbox existed go out now.
+    this.schedule();
+  }
+
+  get state(): OutboxState {
+    return this._state;
+  }
+
+  /** The error that made the outbox `rejected` or `retrying`. */
+  get error(): unknown {
+    return this._error;
+  }
+
+  /** True while some local edit has not been confirmed by the server. */
+  hasUnsent(): boolean {
+    return this.ownCounter() > this.confirmed;
+  }
+
+  /** Call `listener` on every state change. Returns the unsubscribe. */
+  subscribe(listener: () => void): () => void {
+    this.listeners.add(listener);
+    return () => this.listeners.delete(listener);
+  }
+
+  /** Send unconfirmed edits now, unless a request is already in flight. */
+  schedule(): void {
+    if (this.disposed || this._state === "rejected") return;
+    if (this.inFlight || this.retryHandle !== null) return; // picked up when that settles
+    void this.flush(false);
+  }
+
+  /** Skip the backoff wait (the browser came back online). */
+  retryNow(): void {
+    if (this.retryHandle === null) return;
+    this.clearTimer(this.retryHandle);
+    this.retryHandle = null;
+    this.schedule();
+  }
+
+  /**
+   * Last chance before the page goes away: send what is unconfirmed with
+   * `keepalive`, so the browser finishes the request after the page is gone.
+   * Runs even while a request is in flight; the server merges duplicates.
+   */
+  flushOnHide(): void {
+    if (this.disposed || this._state === "rejected") return;
+    const update = this.unsent();
+    if (!update) return;
+    const target = this.ownCounter();
+    this.send(update, { keepalive: true }).then(
+      () => this.confirm(target),
+      () => {
+        /* the regular path retries if the page stays */
+      },
+    );
+  }
+
+  /** Stop listening and cancel a pending retry. Edits not sent are dropped. */
+  dispose(): void {
+    if (this.disposed) return;
+    this.disposed = true;
+    this.stopListening();
+    if (this.retryHandle !== null) {
+      this.clearTimer(this.retryHandle);
+      this.retryHandle = null;
+    }
+    this.listeners.clear();
+  }
+
+  private ownCounter(): number {
+    return this.doc.oplogVersion().get(this.doc.peerIdStr) ?? 0;
+  }
+
+  /** This peer's ops after `confirmed`, or null when there are none. */
+  private unsent(): Uint8Array | null {
+    if (!this.hasUnsent()) return null;
+    // Start from everything the doc has, then roll this peer back to what
+    // the server confirmed: the export is exactly this peer's new ops.
+    const from = this.doc.oplogVersion().toJSON();
+    if (this.confirmed > 0) from.set(this.doc.peerIdStr, this.confirmed);
+    else from.delete(this.doc.peerIdStr);
+    return this.doc.export({ mode: "update", from: VersionVector.parseJSON(from) });
+  }
+
+  private confirm(counter: number): void {
+    if (counter > this.confirmed) this.confirmed = counter;
+  }
+
+  private async flush(keepalive: boolean): Promise<void> {
+    const update = this.unsent();
+    if (!update) {
+      this.setState("idle", null);
+      return;
+    }
+    const target = this.ownCounter();
+    this.inFlight = true;
+    this.setState("sending", null);
+    try {
+      await this.send(update, { keepalive });
+      this.confirm(target);
+      this.failures = 0;
+    } catch (err) {
+      this.inFlight = false;
+      if (this.disposed) return;
+      if (!isRetryable(err)) {
+        this.setState("rejected", err);
+        return;
+      }
+      this.failures += 1;
+      const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** (this.failures - 1));
+      this.setState("retrying", err);
+      this.retryHandle = this.setTimer(() => {
+        this.retryHandle = null;
+        this.schedule();
+      }, delay);
+      return;
+    }
+    this.inFlight = false;
+    if (this.disposed) return;
+    if (this.hasUnsent()) void this.flush(false);
+    else this.setState("idle", null);
+  }
+
+  private setState(state: OutboxState, error: unknown): void {
+    if (this._state === state && this._error === error) return;
+    this._state = state;
+    this._error = error;
+    for (const listener of this.listeners) {
+      try {
+        listener();
+      } catch (err) {
+        console.warn("[loro] outbox listener threw:", err);
+      }
+    }
+  }
+}

package/src/{prosemirror.test.ts → prosemirror/index.test.ts}

@@ -11,7 +11,7 @@ import {
   readProseMirror,
   waitForProseMirrorContainer,
   writeProseMirror,
-} from "./prosemirror";
+} from "./index";
 
 // The basic schema plus a mark with an id, like a suggestion.
 const schema = new Schema({

package/src/{prosemirror.ts → prosemirror/index.ts}

@@ -15,7 +15,7 @@
  *   const doc = useLoroDoc("Doc", id);
  *   configureProseMirrorMarks(doc, schema);
  *   const containerId = await waitForProseMirrorContainer(doc, "body");
- *   const plugin = LoroSyncPlugin({ doc, containerId });
+ *   const plugin = ProseMirrorSyncPlugin({ doc, containerId }); // ./sync
  *
  * In a server function, read and change the document without an editor
  * with `readProseMirror` and `writeProseMirror`.

package/src/prosemirror/react.ts

@@ -0,0 +1,31 @@
+/**
+ * React binding for a ProseMirror (Tiptap) field.
+ *
+ *   const { doc, containerId } = useProseMirrorDoc("Doc", id, "body", schema);
+ *   if (!containerId) return <Loading />;
+ *   const plugins = [ProseMirrorSyncPlugin({ doc, containerId }), ProseMirrorUndoPlugin({ doc })];
+ *
+ * `containerId` is null until the server's snapshot brings the field's
+ * container; mount the editor after that (see ./index.ts for why a client
+ * never creates it). Edits the editor makes reach the server through the
+ * row's outbox, like every edit on a `useLoroDoc` doc.
+ */
+import type { ContainerID, LoroDoc } from "loro-crdt";
+import type { Schema } from "prosemirror-model";
+import { useMemo } from "react";
+import { useLoroDoc } from "../index";
+import { configureProseMirrorMarks, findProseMirrorContainer } from "./index";
+
+export function useProseMirrorDoc(
+  entity: string,
+  id: string,
+  field: string,
+  schema: Schema,
+): { doc: LoroDoc; containerId: ContainerID | null } {
+  const doc = useLoroDoc(entity, id);
+  // Before any binding writes a mark. Idempotent, so once per doc is enough.
+  useMemo(() => configureProseMirrorMarks(doc, schema), [doc, schema]);
+  // useLoroDoc re-renders on every applied frame, so this picks up the
+  // container as soon as the snapshot lands.
+  return { doc, containerId: findProseMirrorContainer(doc, field) };
+}

package/src/prosemirror/sync.test.ts

@@ -0,0 +1,193 @@
+// ProseMirrorSyncPlugin with two real editors (happy-dom) on two LoroDocs.
+// `sync()` stands in for the network: it moves each doc's new ops to the
+// other, the way the outbox and the server broadcast do.
+import { GlobalRegistrator } from "@happy-dom/global-registrator";
+if (!globalThis.document) GlobalRegistrator.register();
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { LoroDoc, LoroList, LoroMap, type ContainerID } from "loro-crdt";
+import { undo } from "loro-prosemirror";
+import { Schema } from "prosemirror-model";
+import { schema as basic } from "prosemirror-schema-basic";
+import { EditorState, TextSelection } from "prosemirror-state";
+import { EditorView } from "prosemirror-view";
+import { configureProseMirrorMarks, findProseMirrorContainer } from "./index";
+import { ProseMirrorSyncPlugin, ProseMirrorUndoPlugin } from "./sync";
+
+const schema = new Schema({ nodes: basic.spec.nodes, marks: basic.spec.marks });
+const tick = () => new Promise((r) => setTimeout(r, 0));
+
+const views: EditorView[] = [];
+afterEach(() => {
+  for (const v of views.splice(0)) v.destroy();
+});
+
+/** A server doc with the field's container, and two clients synced to it. */
+function setup() {
+  // The empty document the server makes on insert (crates/crdt prosemirror.rs).
+  const server = new LoroDoc();
+  const body = server.getMap("row").setContainer("body", new LoroMap());
+  body.set("nodeName", "doc");
+  body.setContainer("attributes", new LoroMap());
+  body.setContainer("children", new LoroList());
+  server.commit();
+  const docs = [new LoroDoc(), new LoroDoc()];
+  for (const d of docs) {
+    d.import(server.export({ mode: "snapshot" }));
+    configureProseMirrorMarks(d, schema);
+  }
+  const containerId = findProseMirrorContainer(docs[0], "body") as ContainerID;
+  const make = (doc: LoroDoc) => {
+    const view = new EditorView(document.createElement("div"), {
+      state: EditorState.create({
+        schema,
+        plugins: [ProseMirrorSyncPlugin({ doc, containerId }), ProseMirrorUndoPlugin({ doc })],
+      }),
+    });
+    views.push(view);
+    return view;
+  };
+  const [a, b] = [make(docs[0]), make(docs[1])];
+  const sync = () => {
+    for (const from of docs) {
+      const bytes = from.export({ mode: "update", from: server.oplogVersion() });
+      server.import(bytes);
+    }
+    const all = server.export({ mode: "snapshot" });
+    for (const to of docs) to.import(all);
+  };
+  return { a, b, docs, sync };
+}
+
+const text = (v: EditorView) => v.state.doc.textBetween(0, v.state.doc.content.size, "\n");
+const type = (v: EditorView, s: string) => {
+  for (const ch of s) v.dispatch(v.state.tr.insertText(ch));
+};
+const caretAt = (v: EditorView, pos: number) =>
+  v.dispatch(v.state.tr.setSelection(TextSelection.create(v.state.doc, pos)));
+
+describe("ProseMirrorSyncPlugin", () => {
+  test("typing in one editor reaches the other", async () => {
+    const { a, b, sync } = setup();
+    await tick();
+    type(a, "Hello");
+    sync();
+    expect(text(b)).toBe("Hello");
+  });
+
+  test("a remote edit before the caret keeps the caret on its words", async () => {
+    const { a, b, sync } = setup();
+    await tick();
+    type(a, "world");
+    sync();
+    caretAt(b, 6); // after "world"
+    caretAt(a, 1);
+    type(a, "hello ");
+    sync();
+    expect(text(b)).toBe("hello world");
+    expect(b.state.selection.head).toBe(12);
+  });
+
+  test("typing right after a remote change goes at the caret, not the end", async () => {
+    const { a, b, sync } = setup();
+    await tick();
+    type(a, "one three");
+    sync();
+    caretAt(b, 4); // after "one"
+    caretAt(a, 10);
+    type(a, " four");
+    sync();
+    type(b, " two"); // no timer in between
+    sync();
+    expect(text(a)).toBe("one two three four");
+    expect(text(b)).toBe("one two three four");
+  });
+
+  test("two people typing at the same spot keep their words whole", async () => {
+    const { a, b, sync } = setup();
+    await tick();
+    type(a, "Hi");
+    sync();
+    caretAt(a, 3);
+    caretAt(b, 3);
+    const left = " [from a]";
+    const right = " [from b]";
+    for (let i = 0; i < left.length; i++) {
+      type(a, left[i]);
+      type(b, right[i]);
+      sync();
+    }
+    expect(text(a)).toBe(text(b));
+    expect([`Hi${left}${right}`, `Hi${right}${left}`]).toContain(text(a));
+  });
+
+  test("a paragraph split elsewhere does not move the caret", async () => {
+    const { a, b, sync } = setup();
+    await tick();
+    type(a, "first second");
+    sync();
+    caretAt(b, 13); // end of "second"
+    caretAt(a, 6); // after "first"
+    a.dispatch(a.state.tr.split(6));
+    sync();
+    type(b, "!");
+    sync();
+    expect(text(a)).toBe("first\n second!");
+    expect(text(b)).toBe("first\n second!");
+  });
+
+  test("undo takes back a burst of typing as one step", async () => {
+    const { a, sync } = setup();
+    await tick();
+    type(a, "kept");
+    sync();
+    await new Promise((r) => setTimeout(r, 600));
+    type(a, " typed together");
+    undo(a.state, a.dispatch);
+    expect(text(a)).toBe("kept");
+  });
+
+  // Loro does not merge an undo step across a remote change in the same
+  // container (loro-internal undo.rs, push_with_merge), so while two people
+  // type in one paragraph each undo takes back a smaller step. Each step is
+  // still only this person's.
+  test("undo while someone else types removes only this person's typing", async () => {
+    const { a, b, sync } = setup();
+    await tick();
+    type(a, "x");
+    sync();
+    await new Promise((r) => setTimeout(r, 600));
+    caretAt(a, 2);
+    caretAt(b, 1);
+    for (const [ca, cb] of [["1", "a"], ["2", "b"], ["3", "c"], ["4", "d"]]) {
+      type(a, ca);
+      type(b, cb);
+      sync();
+    }
+    undo(b.state, b.dispatch);
+    sync();
+    expect(text(a)).toContain("1234");
+    expect(text(a)).not.toContain("d");
+    for (let i = 0; i < 4; i++) undo(b.state, b.dispatch);
+    sync();
+    expect(text(a)).toBe("x1234");
+    expect(text(b)).toBe("x1234");
+  });
+
+  test("undo removes only this editor's edit", async () => {
+    const { a, b, sync } = setup();
+    await tick();
+    type(a, "base");
+    sync();
+    caretAt(b, 5);
+    type(b, " by b");
+    sync();
+    caretAt(a, 1);
+    type(a, "A:");
+    sync();
+    undo(b.state, b.dispatch);
+    sync();
+    expect(text(a)).toBe("A:base");
+    expect(text(b)).toBe("A:base");
+  });
+});