@dxos/react-ui-editor 0.8.3 → 0.8.4-main.1da679c

package/src/extensions/preview/preview.ts

@@ -2,21 +2,14 @@
 // Copyright 2023 DXOS.org
 //
 
-import '@dxos/lit-ui/dx-ref-tag.pcss';
-
 import { syntaxTree } from '@codemirror/language';
-import {
-  type EditorState,
-  type Extension,
-  type RangeSet,
-  RangeSetBuilder,
-  StateField,
-  type Transaction,
-} from '@codemirror/state';
+import { type EditorState, type Extension, RangeSetBuilder, StateField } from '@codemirror/state';
 import { Decoration, type DecorationSet, EditorView, WidgetType } from '@codemirror/view';
 import { type SyntaxNode } from '@lezer/common';
 
 export type PreviewLinkRef = {
+  /** @deprecated */
+  // TODO(burdon): Remove?
   suggest?: boolean;
   block?: boolean;
   label: string;
@@ -29,10 +22,6 @@ export type PreviewLinkTarget = {
   object?: any;
 };
 
-// TODO(wittjosiah): Remove.
-// TODO(burdon): Handle error.
-export type PreviewLookup = (link: PreviewLinkRef) => Promise<PreviewLinkTarget | null | undefined>;
-
 export type PreviewOptions = {
   addBlockContainer?: (link: PreviewLinkRef, el: HTMLElement) => void;
   removeBlockContainer?: (link: PreviewLinkRef) => void;
@@ -44,10 +33,16 @@ export type PreviewOptions = {
 export const preview = (options: PreviewOptions = {}): Extension => {
   return [
     // NOTE: Atomic block decorations must be created from a state field, now a widget, otherwise it results in the following error:
-    // "Block decorations may not be specified via plugins"
+    // "Block decorations may not be specified via plugins".
     StateField.define<DecorationSet>({
       create: (state) => buildDecorations(state, options),
-      update: (_: RangeSet<Decoration>, tr: Transaction) => buildDecorations(tr.state, options),
+      update: (decorations, tr) => {
+        if (tr.docChanged) {
+          return buildDecorations(tr.state, options);
+        }
+
+        return decorations.map(tr.changes);
+      },
       provide: (field) => [
         EditorView.decorations.from(field),
         EditorView.atomicRanges.of((view) => view.state.field(field)),
@@ -56,42 +51,19 @@ export const preview = (options: PreviewOptions = {}): Extension => {
   ];
 };
 
-/**
- * Link references.
- *
- *  [Label][dxn:echo:123]    Inline reference
- * ![Label][dxn:echo:123]    Block reference
- * ![Label][?dxn:echo:123]   Suggestion
- */
-export const getLinkRef = (state: EditorState, node: SyntaxNode): PreviewLinkRef | undefined => {
-  const mark = node.getChild('LinkMark');
-  const label = node.getChild('LinkLabel');
-  if (mark && label) {
-    const ref = state.sliceDoc(label.from + 1, label.to - 1);
-    return {
-      suggest: ref.startsWith('?'),
-      block: state.sliceDoc(mark.from, mark.from + 1) === '!',
-      label: state.sliceDoc(mark.to, label.from - 1),
-      ref: ref.startsWith('?') ? ref.slice(1) : ref,
-    };
-  }
-};
-
 /**
  * Echo references are represented as markdown reference links.
  * https://www.markdownguide.org/basic-syntax/#reference-style-links
- * [Label|block][dxn:echo:123]
- * [Label|inline][dxn:echo:123]
  */
-const buildDecorations = (state: EditorState, options: PreviewOptions) => {
+const buildDecorations = (state: EditorState, options: PreviewOptions): DecorationSet => {
   const builder = new RangeSetBuilder<Decoration>();
 
   syntaxTree(state).iterate({
     enter: (node) => {
       switch (node.name) {
         //
-        // Decoration.
-        // [Label][dxn:echo:123]
+        // Inline widget.
+        // [Label](dxn:echo:123)
         //
         case 'Link': {
           const link = getLinkRef(state, node.node);
@@ -101,29 +73,32 @@ const buildDecorations = (state: EditorState, options: PreviewOptions) => {
               node.to,
               Decoration.replace({
                 widget: new PreviewInlineWidget(options, link),
+                side: 1,
               }),
             );
           }
-          break;
+          return false;
         }
+
         //
-        // Block widget.
-        // ![Label][dxn:echo:123]
+        // Block widget (transclusion).
+        // ![Label](dxn:echo:123)
         //
         case 'Image': {
-          const link = getLinkRef(state, node.node);
-          if (options.addBlockContainer && options.removeBlockContainer && link) {
-            builder.add(
-              node.from,
-              node.to,
-              Decoration.replace({
-                block: true,
-                // atomic: true,
-                widget: new PreviewBlockWidget(options, link),
-              }),
-            );
+          if (options.addBlockContainer && options.removeBlockContainer) {
+            const link = getLinkRef(state, node.node);
+            if (link) {
+              builder.add(
+                node.from,
+                node.to,
+                Decoration.replace({
+                  block: true,
+                  widget: new PreviewBlockWidget(options, link),
+                }),
+              );
+            }
           }
-          break;
+          return false;
         }
       }
     },
@@ -132,9 +107,30 @@ const buildDecorations = (state: EditorState, options: PreviewOptions) => {
   return builder.finish();
 };
 
+/**
+ * Link references.
+ *  [Label](dxn:echo:123) Inline reference
+ * ![Label](dxn:echo:123) Block reference
+ */
+export const getLinkRef = (state: EditorState, node: SyntaxNode): PreviewLinkRef | undefined => {
+  const mark = node.getChildren('LinkMark');
+  const urlNode = node.getChild('URL');
+  if (mark && urlNode) {
+    const url = state.sliceDoc(urlNode.from, urlNode.to);
+    if (url.startsWith('dxn:')) {
+      const label = state.sliceDoc(mark[0].to, mark[1].from);
+      return {
+        block: state.sliceDoc(mark[0].from, mark[0].from + 1) === '!',
+        label,
+        ref: url,
+      };
+    }
+  }
+};
+
 /**
  * Inline widget.
- * [Label][dxn:echo:123]
+ *  [Label](dxn:echo:123)
  */
 class PreviewInlineWidget extends WidgetType {
   constructor(
@@ -152,8 +148,9 @@ class PreviewInlineWidget extends WidgetType {
     return this._link.ref === other._link.ref && this._link.label === other._link.label;
   }
 
-  override toDOM(view: EditorView): HTMLElement {
-    const root = document.createElement('dx-ref-tag');
+  override toDOM(_view: EditorView): HTMLElement {
+    const root = document.createElement('dx-anchor');
+    root.classList.add('dx-tag--anchor');
     root.textContent = this._link.label;
     root.setAttribute('refId', this._link.ref);
     return root;
@@ -161,7 +158,7 @@ class PreviewInlineWidget extends WidgetType {
 }
 
 /**
- * Block widget.
+ * Block widget (e.g., for surfaces).
  * ![Label][dxn:echo:123]
  */
 class PreviewBlockWidget extends WidgetType {
@@ -180,7 +177,7 @@ class PreviewBlockWidget extends WidgetType {
     return this._link.ref === other._link.ref;
   }
 
-  override toDOM(view: EditorView): HTMLDivElement {
+  override toDOM(_view: EditorView): HTMLDivElement {
     const root = document.createElement('div');
     root.classList.add('cm-preview-block', 'density-coarse');
     this._options.addBlockContainer?.(this._link, root);

package/src/extensions/tags/extended-markdown.test.ts

@@ -0,0 +1,261 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { syntaxTree } from '@codemirror/language';
+import { EditorState } from '@codemirror/state';
+import { type SyntaxNode } from '@lezer/common';
+import { describe, test } from 'vitest';
+
+import { trim } from '@dxos/util';
+
+import { extendedMarkdown } from './extended-markdown';
+import { nodeToJson } from './xml-util';
+
+describe('extended-markdown', () => {
+  const createEditorState = (doc: string) => {
+    return EditorState.create({
+      doc,
+      extensions: [extendedMarkdown()],
+    });
+  };
+
+  test('tree', async ({ expect }) => {
+    const doc = trim`
+      <prompt>
+        Hello
+      </prompt>
+
+      Hi there!
+
+      What can I do for you?
+
+      <suggestion>Summarize tools</suggestion>
+
+      <choice>
+        <option>Summarize tools</option>
+        <option>Retry</option>
+      </choice>
+
+      <toolkit />
+    `;
+
+    const nodes: SyntaxNode[] = [];
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+    tree.iterate({
+      enter: (node) => {
+        if (node.type.name === 'Element') {
+          nodes.push(node.node);
+          return false; // Stop traversal.
+        }
+      },
+    });
+
+    expect(nodes).toHaveLength(4);
+    expect(
+      nodes.map((node) => ({
+        content: doc.slice(node.from, node.to),
+        data: nodeToJson(state, node.node),
+      })),
+    ).toEqual([
+      {
+        content: '<prompt>\n  Hello\n</prompt>',
+        data: {
+          _tag: 'prompt',
+          children: ['Hello'],
+        },
+      },
+      {
+        content: '<suggestion>Summarize tools</suggestion>',
+        data: {
+          _tag: 'suggestion',
+          children: ['Summarize tools'],
+        },
+      },
+      {
+        content: '<choice>\n  <option>Summarize tools</option>\n  <option>Retry</option>\n</choice>',
+        data: {
+          _tag: 'choice',
+          children: [
+            {
+              _tag: 'option',
+              children: ['Summarize tools'],
+            },
+            {
+              _tag: 'option',
+              children: ['Retry'],
+            },
+          ],
+        },
+      },
+      {
+        content: '<toolkit />',
+        data: {
+          _tag: 'toolkit',
+        },
+      },
+    ]);
+  });
+
+  test('setext heading disabled', () => {
+    const doc = trim`
+      This should NOT be a heading
+      =
+      
+      Another line that should NOT be a heading
+      -
+    `;
+
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+
+    tree.iterate({
+      enter: (node) => {
+        if (node.type.name === 'SetextHeading') {
+          throw new Error('SetextHeading should be disabled!');
+        }
+      },
+    });
+  });
+
+  //
+  // TODO(burdon): All tests below should test the tree.
+  //
+
+  test('should parse standard markdown elements', ({ expect }) => {
+    const doc = trim`
+      # Heading 1
+      ## Heading 2
+      ### Heading 3
+
+      This is a paragraph with **bold** and *italic* text.
+
+      \`\`\`javascript
+      const code = 'block';
+      \`\`\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify the document is parsed without errors.
+    expect(tree).toBe(doc);
+    expect(state.doc.lines).toBeGreaterThan(1);
+  });
+
+  test('should parse custom XML tags', ({ expect }) => {
+    const doc = trim`
+      # Document with Custom Tags
+
+      <prompt>This is a custom prompt block</prompt>
+
+      Regular paragraph text.
+
+      <prompt />
+
+      Regular paragraph text.
+
+      <prompt>
+        Multi-line
+        prompt content
+      </prompt>
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify the document contains custom tags.
+    expect(tree).toContain('<prompt>');
+    expect(tree).toContain('<prompt />');
+    expect(tree).toContain('</prompt>');
+  });
+
+  test('should handle mixed markdown and XML content', ({ expect }) => {
+    const doc = trim`
+      # Mixed Content
+
+      This is a paragraph with a <prompt>inline prompt</prompt> tag.
+
+      ## Section 2
+
+      <prompt>
+        This prompt contains **markdown** formatting
+      </prompt>
+
+      \`\`\`
+      <prompt>This should not be parsed as XML inside code block</prompt>
+      \`\`\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify mixed content is preserved.
+    expect(tree).toBe(doc);
+  });
+
+  test('should not parse XML tags in code blocks', ({ expect }) => {
+    const doc = trim`
+      \`\`\`xml
+      <prompt>This is inside a code block</prompt>
+      \`\`\`
+
+      \`<prompt>inline code</prompt>\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify code blocks are preserved as-is.
+    expect(tree).toContain('```xml');
+    expect(tree).toContain('<prompt>This is inside a code block</prompt>');
+  });
+
+  test('should handle nested and complex structures', ({ expect }) => {
+    const doc = trim`
+      # Complex Document
+
+      <prompt>
+        First prompt with some content
+      </prompt>
+
+      ## Lists and Prompts
+
+      - Item 1
+      - Item 2 with <prompt>embedded prompt</prompt>
+      - Item 3
+
+      <prompt>
+        Another prompt after the list
+      </prompt>
+
+      ### Code Example
+
+      \`\`\`typescript
+      const example = '<prompt>not parsed</prompt>';
+      \`\`\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify complex structures are handled.
+    expect(tree).toBe(doc);
+    expect(tree).toMatch(/Item 2 with <prompt>embedded prompt<\/prompt>/);
+  });
+
+  test('should handle empty and edge cases', ({ expect }) => {
+    const emptyDoc = '';
+    const emptyState = createEditorState(emptyDoc);
+    expect(emptyState.doc.length).toBe(0);
+
+    const onlyPromptDoc = '<prompt>Only prompt content</prompt>';
+    const onlyPromptState = createEditorState(onlyPromptDoc);
+    expect(onlyPromptState.sliceDoc(0)).toBe(onlyPromptDoc);
+
+    const unclosedPromptDoc = '<prompt>Unclosed prompt';
+    const unclosedState = createEditorState(unclosedPromptDoc);
+    expect(unclosedState.sliceDoc(0)).toBe(unclosedPromptDoc);
+  });
+});

package/src/extensions/tags/extended-markdown.ts

@@ -0,0 +1,78 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { xmlLanguage } from '@codemirror/lang-xml';
+import { type Extension } from '@codemirror/state';
+import { type ParseWrapper, parseMixed } from '@lezer/common';
+
+import { createMarkdownExtensions } from '../markdown';
+
+import { type XmlWidgetRegistry } from './xml-tags';
+
+export type ExtendedMarkdownOptions = {
+  registry?: XmlWidgetRegistry;
+};
+
+/**
+ * Extended markdown parser with mixed parser for custom rendering of XML tags.
+ */
+export const extendedMarkdown = ({ registry }: ExtendedMarkdownOptions = {}): Extension => {
+  return [
+    createMarkdownExtensions({
+      extensions: [
+        {
+          wrap: mixedParser(registry),
+          parseBlock: [
+            // Disable SetextHeading since it causes flickering when parsing/rendering tasks in chunks.
+            {
+              name: 'SetextHeading',
+              parse: () => false,
+            },
+          ],
+        },
+      ],
+    }),
+  ];
+};
+
+/**
+ * Configure mixed parser to recognize custom tags.
+ */
+const mixedParser = (registry?: XmlWidgetRegistry): ParseWrapper => {
+  const customTags = Object.keys(registry ?? {});
+  const tagPattern = new RegExp(`<(${customTags.join('|')})`);
+
+  return parseMixed((node, input) => {
+    switch (node.name) {
+      // Ignore XML inside of fenced and inline code.
+      case 'FencedCode':
+      case 'InlineCode': {
+        return null;
+      }
+
+      // Parse multi-line HTML blocks.
+      // case 'XMLBlock':
+      case 'HTMLBlock': {
+        return {
+          parser: xmlLanguage.parser,
+        };
+      }
+
+      // Parse paragraphs that contain custom XML tags.
+      // TODO(burdon): Entire paragraph should be parsed as XML.
+      case 'Paragraph': {
+        const content = input.read(node.from, node.to);
+        if (tagPattern.test(content)) {
+          return {
+            parser: xmlLanguage.parser,
+          };
+        }
+
+        return null;
+      }
+    }
+
+    return null;
+  });
+};

package/src/extensions/tags/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './extended-markdown';
+export * from './streamer';
+export * from './xml-tags';