@kerebron/extension-yjs 0.6.7 → 0.7.1

package/src/binding/DiffViewer.ts

@@ -0,0 +1,138 @@
+import * as Y from 'yjs';
+
+import { Node } from 'prosemirror-model';
+import { EditorState } from 'prosemirror-state';
+
+import { isVisible } from '../utils.js';
+import { YjsData } from './PmYjsBinding.js';
+import { createNodeFromYElement } from './createNodeFromYElement.js';
+
+export class DiffViewer {
+  prevSnapshot?: Y.Snapshot;
+  snapshot?: Y.Snapshot;
+
+  private historyYXmlFragment: Y.XmlFragment | undefined;
+  private historyDoc: Y.Doc | undefined;
+
+  constructor() {
+  }
+
+  isActive() {
+    return !!this.snapshot || !!this.prevSnapshot;
+  }
+
+  setSnapshot(yjs: YjsData, uSnapshot: Uint8Array, uPrevSnapshot?: Uint8Array) {
+    this.prevSnapshot = undefined;
+    if (uPrevSnapshot) {
+      this.prevSnapshot = Y.decodeSnapshotV2(uPrevSnapshot);
+    }
+
+    this.snapshot = Y.decodeSnapshotV2(uSnapshot);
+
+    this.historyDoc = Y.createDocFromSnapshot(
+      yjs.ydoc,
+      this.snapshot,
+      new Y.Doc({ gc: false }),
+    );
+
+    if (yjs.xmlFragment._item === null) {
+      /**
+       * If is a root type, we need to find the root key in the initial document
+       * and use it to get the history type.
+       */
+      const share: Map<string, Y.AbstractType<Y.YEvent<any>>> = yjs.ydoc.share;
+      const rootKey = Array.from(share.keys()).find(
+        (key) => share.get(key) === yjs.xmlFragment as Y.AbstractType<any>,
+      );
+      this.historyYXmlFragment = this.historyDoc.getXmlFragment(rootKey);
+    } else {
+      /**
+       * If it is a sub type, we use the item id to find the history type.
+       */
+      const historyStructs =
+        this.historyDoc.store.clients.get(yjs.xmlFragment._item.id.client) ??
+          [];
+      const itemIndex = Y.findIndexSS(
+        historyStructs,
+        yjs.xmlFragment._item.id.clock,
+      );
+      if (historyStructs[itemIndex] instanceof Y.GC) {
+        throw new Error('Incorrect type Y.GC');
+      }
+      const item: Y.Item = historyStructs[itemIndex];
+      const content: Y.ContentType = item.content;
+      this.historyYXmlFragment = content.type as Y.XmlFragment;
+    }
+  }
+
+  reset() {
+    this.snapshot = undefined;
+    this.prevSnapshot = undefined;
+    this.historyDoc = undefined;
+    this.historyYXmlFragment = undefined;
+  }
+
+  getFragmentContent(
+    state: EditorState,
+    ytr: Y.Transaction,
+  ): Node[] | undefined {
+    if (!this.historyYXmlFragment || !this.snapshot) {
+      return;
+    }
+
+    // before rendering, we are going to sanitize ops and split deleted ops
+    // if they were deleted by seperate users.
+    const pud: Y.PermanentUserData | undefined = undefined;
+    // pluginState.permanentUserData;
+    if (pud) {
+      pud.dss.forEach((ds) => {
+        Y.iterateDeletedStructs(ytr, ds, (_item) => {});
+      });
+    }
+    const computeYChange = (type: 'removed' | 'added', id: Y.ID) => {
+      const user = pud
+        ? (type === 'added'
+          ? pud.getUserByClientId(id.client)
+          : pud.getUserByDeletedId(id))
+        : undefined;
+      return {
+        user,
+        type,
+      };
+    };
+
+    const deleteSet = this.prevSnapshot
+      ? this.prevSnapshot.ds
+      : Y.emptySnapshot.ds;
+
+    // Create document fragment and render
+    const fragmentContent = Y.typeListToArraySnapshot(
+      this.historyYXmlFragment,
+      new Y.Snapshot(deleteSet, this.snapshot.sv),
+    ).map((t) => {
+      if (
+        !t._item.deleted || isVisible(t._item, this.snapshot) ||
+        isVisible(t._item, this.prevSnapshot)
+      ) {
+        return createNodeFromYElement(
+          t,
+          state.schema,
+          { mapping: new Map(), isOverlappingMark: new Map() },
+          this.snapshot,
+          this.prevSnapshot,
+          computeYChange,
+        );
+      } else {
+        // No need to render elements that are not visible by either snapshot.
+        // If a client adds and deletes content in the same snapshot the element is not visible by either snapshot.
+        return null;
+      }
+    }).filter((n) => n !== null);
+
+    return fragmentContent;
+  }
+
+  getHistoryDoc(): Y.Doc | undefined {
+    return this.historyDoc;
+  }
+}

package/src/binding/PmYjsBinding.ts

@@ -0,0 +1,360 @@
+import * as Y from 'yjs';
+
+import { Fragment, Slice } from 'prosemirror-model';
+import { Transaction } from 'prosemirror-state';
+
+import { CoreEditor } from '@kerebron/editor';
+import { User } from '@kerebron/editor/user';
+
+import { CreateYjsProvider, YjsProvider } from '../YjsProvider.js';
+import { ProsemirrorMapping } from '../lib.js';
+import { SelectionStash } from '../ui/selection.js';
+import { ySyncPluginKey } from '../keys.js';
+
+import { yXmlFragmentToProseMirrorRootNode } from './convertUtils.js';
+import { updateYFragment } from './updateYFragment.js';
+import { BindingMetadata } from './BindingMetadata.js';
+import { createNodeIfNotExists } from './createNodeFromYElement.js';
+import { DiffViewer } from './DiffViewer.js';
+
+type Mutex = (f: () => void, g?: () => void) => void;
+
+export const createMutex = (): Mutex => {
+  let token = true;
+  return (f: () => void, g?: () => void) => {
+    if (token) {
+      token = false;
+      try {
+        f();
+      } finally {
+        token = true;
+      }
+    } else if (g !== undefined) {
+      g();
+    }
+  };
+};
+
+type ConnectionState = 'idle' | 'joining' | 'synced' | 'leaving' | 'error';
+
+export type YjsData = { ydoc: Y.Doc; xmlFragment: Y.XmlFragment };
+
+export class PmYjsBinding extends EventTarget {
+  private provider: YjsProvider | undefined;
+  private connectionState: ConnectionState = 'idle';
+
+  private yjs: YjsData | undefined;
+  private selectionStash: SelectionStash | undefined;
+  public readonly diffViewer: DiffViewer;
+
+  public addToYjsHistory = true;
+  private hasImported = false;
+  private mux: Mutex;
+  private bindingMetadata: BindingMetadata;
+
+  private syncedHandler: (
+    event: CustomEvent<{
+      state: boolean;
+    }>,
+  ) => void;
+  private _observeFunction: (
+    events: Array<Y.YEvent<any>>,
+    transaction: Y.Transaction,
+  ) => void;
+
+  constructor(private readonly editor: CoreEditor) {
+    super();
+
+    this.diffViewer = new DiffViewer();
+    this.bindingMetadata = { mapping: new Map(), isOverlappingMark: new Map() };
+
+    this.mux = createMutex();
+
+    this.syncedHandler = (
+      event: CustomEvent<{
+        state: boolean;
+      }>,
+    ) => {
+      const synced = event.detail.state;
+      if (synced && !this.hasImported) {
+        this.importRemoteYdoc();
+      } else {
+        // this.ydocChanged();
+      }
+    };
+
+    this._observeFunction = (events, transaction) => {
+      this.yXmlChanged(events, transaction);
+    };
+  }
+
+  destroy() {
+    this.connectionState = 'idle';
+    const tr = this.editor.state.tr;
+    this.leaveRoom(tr);
+    this.editor.dispatchTransaction(tr);
+    this.diffViewer.reset();
+    this.selectionStash?.destroy();
+  }
+
+  changeUser(user: User) {
+    if (!this.provider) {
+      return;
+    }
+    this.provider.awareness.setLocalStateField('kerebron:user', user);
+  }
+
+  changeRoom(
+    roomId: string,
+    createYjsProvider: CreateYjsProvider,
+    tr: Transaction,
+  ) {
+    this.connectionState = 'joining';
+    this.hasImported = false;
+
+    if (this.provider) {
+      this.provider.removeEventListener('synced', this.syncedHandler);
+      this.provider.destroy();
+      this.provider = undefined;
+    }
+
+    const [provider, ydoc] = createYjsProvider(
+      roomId,
+    );
+
+    this.addToYjsHistory = true;
+    this.provider = provider;
+
+    this.bindingMetadata.mapping.clear();
+    this.bindingMetadata.isOverlappingMark.clear();
+    this.diffViewer.reset();
+
+    const fieldName = 'kerebron:' + this.editor.schema.topNodeType.name;
+    this.yjs = { ydoc, xmlFragment: ydoc.getXmlFragment(fieldName) };
+    this.selectionStash = new SelectionStash(
+      this.yjs,
+      this.getMapping(),
+      this.editor,
+    );
+    this.yjs.xmlFragment.observeDeep(this._observeFunction);
+
+    this.provider.addEventListener('synced', this.syncedHandler);
+
+    tr.setMeta('yjs:setAwareness', provider.awareness);
+    tr.setMeta('setYjs', this.yjs);
+  }
+
+  leaveRoom(tr: Transaction) {
+    this.connectionState = 'leaving';
+    if (this.provider) {
+      this.provider.removeEventListener('synced', this.syncedHandler);
+      this.provider.destroy();
+    }
+
+    this.addToYjsHistory = true;
+    this.provider = undefined;
+
+    this.bindingMetadata.mapping.clear();
+    this.bindingMetadata.isOverlappingMark.clear();
+    this.diffViewer.reset();
+
+    this.selectionStash?.destroy();
+    this.selectionStash = undefined;
+    this.yjs?.xmlFragment.unobserveDeep(this._observeFunction);
+    this.yjs = undefined;
+
+    this.hasImported = false;
+
+    tr.setMeta('addToYjsHistory', false);
+    tr.setMeta('yjs:removeAwareness', true);
+    tr.setMeta('clearYjs', true);
+  }
+
+  importRemoteYdoc() {
+    if (!this.yjs) {
+      return;
+    }
+
+    this.addToYjsHistory = this.editor.state.doc.content.size > 2; // Non empty para
+
+    if (this.yjs.xmlFragment.length === 0) {
+      this.createFromProseMirror(this.yjs);
+    } else {
+      this.overwriteProseMirror(this.yjs.xmlFragment);
+    }
+
+    this.connectionState = 'synced';
+    this.hasImported = true;
+  }
+
+  yXmlChanged(events: Array<Y.YEvent<any>>, ytr: Y.Transaction) {
+    this.mux(() => {
+      if (!this.yjs) {
+        return;
+      }
+
+      if (this.diffViewer.isActive()) {
+        return;
+      }
+
+      const state = this.editor.state;
+
+      const { xmlFragment } = this.yjs;
+
+      const mapping = this.bindingMetadata.mapping;
+      const delType = (_: any, type: Y.AbstractType<any>) =>
+        mapping.delete(type);
+      Y.iterateDeletedStructs(
+        ytr,
+        ytr.deleteSet,
+        (struct) => {
+          if (struct.constructor === Y.Item) {
+            const type = (struct as Y.Item).content.type;
+            type && mapping.delete(type);
+          }
+        },
+      );
+      ytr.changed.forEach(delType);
+      ytr.changedParentTypes.forEach(delType);
+
+      const fragmentContent = xmlFragment.toArray().map((t) =>
+        createNodeIfNotExists(
+          t as Y.XmlElement,
+          state.schema,
+          this.bindingMetadata,
+        )
+      ).filter((n) => n !== null);
+
+      const tr = state.tr;
+      tr.setMeta('addToYjsHistory', false);
+      tr.replace(
+        0,
+        state.doc.content.size,
+        new Slice(Fragment.from(fragmentContent), 0, 0),
+      );
+
+      this.selectionStash?.restore(tr);
+      this.editor.dispatchTransaction(tr);
+    });
+  }
+
+  pmChanged() {
+    if (!this.hasImported) {
+      return;
+    }
+
+    if (!this.yjs) {
+      return;
+    }
+
+    if (this.diffViewer.isActive()) {
+      return;
+    }
+
+    const doc = this.editor.state.doc;
+    const { ydoc, xmlFragment } = this.yjs;
+
+    this.mux(() => {
+      const origin = ySyncPluginKey;
+
+      ydoc.transact((ytr) => {
+        ytr.meta.set('addToYjsHistory', this.addToYjsHistory);
+        updateYFragment(
+          ydoc,
+          xmlFragment,
+          doc,
+          this.bindingMetadata,
+          this.addToYjsHistory,
+        );
+        this.selectionStash?.store();
+      }, origin);
+    });
+  }
+
+  createFromProseMirror(yjs: YjsData) {
+    const doc = this.editor.state.doc;
+    const { ydoc, xmlFragment } = yjs;
+    updateYFragment(
+      ydoc,
+      xmlFragment,
+      doc,
+      this.bindingMetadata,
+      this.addToYjsHistory,
+    );
+  }
+
+  overwriteProseMirror(yXmlFragment: Y.XmlFragment) {
+    const state = this.editor.state;
+    const newRoot = yXmlFragmentToProseMirrorRootNode(
+      yXmlFragment,
+      state.schema,
+    );
+
+    const tr = state.tr;
+    tr.setMeta('addToYjsHistory', false);
+    tr.replaceWith(
+      0,
+      state.doc.content.size,
+      newRoot,
+    );
+
+    this.editor.dispatchTransaction(tr);
+  }
+
+  getYjs(): YjsData | undefined {
+    return this.yjs;
+  }
+
+  getMapping(): ProsemirrorMapping {
+    return this.bindingMetadata.mapping;
+  }
+
+  getSelectionStash(): SelectionStash | undefined {
+    return this.selectionStash;
+  }
+
+  isEditable() {
+    return !this.diffViewer.isActive();
+  }
+
+  setSnapshot(
+    snapshot: Uint8Array<ArrayBufferLike>,
+    prevSnapshot: Uint8Array<ArrayBufferLike> | undefined,
+  ) {
+    if (!this.yjs) {
+      return;
+    }
+    this.diffViewer.setSnapshot(this.yjs, snapshot, prevSnapshot);
+
+    // clear mapping because we are going to rerender
+    this.bindingMetadata.mapping.clear();
+    this.bindingMetadata.isOverlappingMark.clear();
+
+    this.mux(() => {
+      const historyDoc = this.diffViewer.getHistoryDoc();
+      if (!historyDoc) {
+        return;
+      }
+
+      historyDoc.transact((ytr) => {
+        const state = this.editor.state;
+        const fragmentContent = this.diffViewer.getFragmentContent(state, ytr);
+
+        if (!fragmentContent) {
+          return;
+        }
+
+        const tr = state.tr;
+        tr.replace(
+          0,
+          state.doc.content.size,
+          new Slice(Fragment.from(fragmentContent), 0, 0),
+        );
+
+        tr.setMeta('addToHistory', false);
+        tr.setMeta(ySyncPluginKey, { isChangeOrigin: true }),
+          this.editor.dispatchTransaction(tr);
+      }, ySyncPluginKey);
+    });
+  }
+}

package/src/binding/convertUtils.ts

@@ -0,0 +1,124 @@
+import * as Y from 'yjs';
+
+import { Fragment, Node, type Schema } from 'prosemirror-model';
+
+import { TransactFunc } from '../lib.js';
+
+import { updateYFragment } from './updateYFragment.js';
+import { createNodeFromYElement } from './createNodeFromYElement.js';
+import { BindingMetadata } from './BindingMetadata.js';
+
+export const createEmptyMeta = (): BindingMetadata => ({
+  mapping: new Map(),
+  isOverlappingMark: new Map(),
+});
+
+/**
+ * Utility function for converting an Y.Fragment to a ProseMirror fragment.
+ */
+export const yXmlFragmentToProseMirrorFragment = (
+  yXmlFragment: Y.XmlFragment,
+  schema: Schema,
+) => {
+  const elements: Y.XmlElement[] = yXmlFragment.toArray().filter((i) =>
+    i instanceof Y.XmlElement
+  );
+  const fragmentContent = elements.map((t) =>
+    createNodeFromYElement(
+      t,
+      schema,
+      createEmptyMeta(),
+    )
+  ).filter((n) => n !== null);
+  return Fragment.fromArray(fragmentContent);
+};
+
+/**
+ * Utility function for converting an Y.Fragment to a ProseMirror node.
+ */
+export const yXmlFragmentToProseMirrorRootNode = (
+  yXmlFragment: Y.XmlFragment,
+  schema: Schema,
+) =>
+  schema.topNodeType.create(
+    null,
+    yXmlFragmentToProseMirrorFragment(yXmlFragment, schema),
+  );
+
+/**
+ * Utility method to convert a Prosemirror Doc Node into a Y.Doc.
+ *
+ * This can be used when importing existing content to Y.Doc for the first time,
+ * note that this should not be used to rehydrate a Y.Doc from a database once
+ * collaboration has begun as all history will be lost
+ */
+export function prosemirrorToYDoc(
+  doc: Node,
+  xmlFragment: string = 'prosemirror',
+): Y.Doc {
+  const ydoc = new Y.Doc();
+  const type: Y.XmlFragment = ydoc.get(xmlFragment, Y.XmlFragment);
+  if (!type.doc) {
+    return ydoc;
+  }
+
+  prosemirrorToYXmlFragment(doc, type);
+  return type.doc;
+}
+
+/**
+ * Utility method to update an empty Y.XmlFragment with content from a Prosemirror Doc Node.
+ *
+ * This can be used when importing existing content to Y.Doc for the first time,
+ * note that this should not be used to rehydrate a Y.Doc from a database once
+ * collaboration has begun as all history will be lost
+ *
+ * Note: The Y.XmlFragment does not need to be part of a Y.Doc document at the time that this
+ * method is called, but it must be added before any other operations are performed on it.
+ */
+export function prosemirrorToYXmlFragment(
+  doc: Node,
+  xmlFragment: Y.XmlFragment,
+): Y.XmlFragment {
+  const type = xmlFragment || new Y.XmlFragment();
+  const ydoc: { transact: TransactFunc<void> } = type.doc
+    ? type.doc
+    : { transact: (transaction) => transaction(undefined) };
+  updateYFragment(ydoc, type, doc, {
+    mapping: new Map(),
+    isOverlappingMark: new Map(),
+  });
+  return type;
+}
+
+/**
+ * Utility method to convert Prosemirror compatible JSON into a Y.Doc.
+ *
+ * This can be used when importing existing content to Y.Doc for the first time,
+ * note that this should not be used to rehydrate a Y.Doc from a database once
+ * collaboration has begun as all history will be lost
+ */
+export function prosemirrorJSONToYDoc(
+  schema: Schema,
+  state: any,
+  xmlFragment: string = 'prosemirror',
+): Y.Doc {
+  const doc = Node.fromJSON(schema, state);
+  return prosemirrorToYDoc(doc, xmlFragment);
+}
+
+/**
+ * Utility method to convert Prosemirror compatible JSON to a Y.XmlFragment
+ *
+ * This can be used when importing existing content to Y.Doc for the first time,
+ * note that this should not be used to rehydrate a Y.Doc from a database once
+ * collaboration has begun as all history will be lost
+ */
+export function prosemirrorJSONToYXmlFragment(
+  schema: Schema,
+  state: any,
+  xmlFragment: Y.XmlFragment,
+): Y.XmlFragment {
+  const doc = Node.fromJSON(schema, state);
+  return prosemirrorToYXmlFragment(doc, xmlFragment);
+}

package/src/{createNodeFromYElement.ts → binding/createNodeFromYElement.ts}

@@ -2,9 +2,9 @@ import * as PModel from 'prosemirror-model';
 import { Mark, Schema } from 'prosemirror-model';
 import * as Y from 'yjs';
 
-import type { BindingMetadata } from './ProsemirrorBinding.js';
-import { ySyncPluginKey } from './keys.js';
-import { isVisible } from './utils.js';
+import type { BindingMetadata } from './BindingMetadata.js';
+import { ySyncPluginKey } from '../keys.js';
+import { isVisible } from '../utils.js';
 import { yattr2markname } from './updateYFragment.js';
 
 export const attributesToMarks = (
@@ -51,7 +51,7 @@ export const createNodeFromYElement = (
   meta: BindingMetadata,
   snapshot?: Y.Snapshot,
   prevSnapshot?: Y.Snapshot,
-  computeYChange?: (arg0: 'removed' | 'added', arg1: Y.ID) => any,
+  computeYChange?: (changeType: 'removed' | 'added', yid: Y.ID) => any,
 ): PModel.Node | null => {
   const children: PModel.Node[] = [];
   const createChildren = (type: Y.XmlElement | Y.XmlText) => {

package/src/{updateYFragment.ts → binding/updateYFragment.ts}

@@ -4,10 +4,10 @@ import { Mark, Node, Schema } from 'prosemirror-model';
 
 import { simpleDiff } from 'lib0/diff';
 
-import { ySyncPluginKey } from './keys.js';
-import * as utils from './utils.js';
-import type { BindingMetadata } from './ProsemirrorBinding.js';
-import { TransactFunc } from './ySyncPlugin.js';
+import { ySyncPluginKey } from '../keys.js';
+import * as utils from '../utils.js';
+import type { BindingMetadata } from './BindingMetadata.js';
+import { TransactFunc } from '../lib.js';
 
 const hashedMarkNameRegex = /(.*)(--[a-zA-Z0-9+/=]{8})$/;
 export const yattr2markname = (attrName: string) =>
@@ -21,8 +21,11 @@ const marksToAttributes = (
   marks.forEach((mark) => {
     if (mark.type.name !== 'ychange') {
       let isOverlapping = true;
-      if (!meta.isOMark.has(mark.type.name)) {
-        meta.isOMark.set(mark.type.name, !mark.type.excludes(mark.type));
+      if (!meta.isOverlappingMark.has(mark.type.name)) {
+        meta.isOverlappingMark.set(
+          mark.type.name,
+          !mark.type.excludes(mark.type),
+        );
         isOverlapping = false;
       }
 
@@ -278,7 +281,9 @@ export const updateYFragment = (
   yDomFragment: Y.XmlFragment,
   pNode: Node,
   meta: BindingMetadata,
+  addToYjsHistory?: boolean,
 ) => {
+  const origin = ySyncPluginKey;
   if (
     yDomFragment instanceof Y.XmlElement &&
     yDomFragment.nodeName !== pNode.type.name
@@ -342,7 +347,9 @@ export const updateYFragment = (
       }
     }
   }
-  ydoc.transact(() => {
+  ydoc.transact((ytr) => {
+    ytr.meta.set('updateYFragment', 1);
+    ytr.meta.set('addToYjsHistory', addToYjsHistory);
     // try to compare and update
     while (yChildCnt - left - right > 0 && pChildCnt - left - right > 0) {
       const leftY: Y.XmlElement | Y.XmlText = yChildren[left];
@@ -435,5 +442,5 @@ export const updateYFragment = (
       }
       yDomFragment.insert(left, ins);
     }
-  }, ySyncPluginKey);
+  }, origin);
 };