@dxos/react-ui-editor 0.8.4-main.c1de068 → 0.8.4-main.e098934

package/src/extensions/tags/extended-markdown.test.ts

@@ -0,0 +1,261 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { syntaxTree } from '@codemirror/language';
+import { EditorState } from '@codemirror/state';
+import { type SyntaxNode } from '@lezer/common';
+import { describe, test } from 'vitest';
+
+import { trim } from '@dxos/util';
+
+import { extendedMarkdown } from './extended-markdown';
+import { nodeToJson } from './xml-util';
+
+describe('extended-markdown', () => {
+  const createEditorState = (doc: string) => {
+    return EditorState.create({
+      doc,
+      extensions: [extendedMarkdown()],
+    });
+  };
+
+  test('tree', async ({ expect }) => {
+    const doc = trim`
+      <prompt>
+        Hello
+      </prompt>
+
+      Hi there!
+
+      What can I do for you?
+
+      <suggestion>Summarize tools</suggestion>
+
+      <choice>
+        <option>Summarize tools</option>
+        <option>Retry</option>
+      </choice>
+
+      <toolkit />
+    `;
+
+    const nodes: SyntaxNode[] = [];
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+    tree.iterate({
+      enter: (node) => {
+        if (node.type.name === 'Element') {
+          nodes.push(node.node);
+          return false; // Stop traversal.
+        }
+      },
+    });
+
+    expect(nodes).toHaveLength(4);
+    expect(
+      nodes.map((node) => ({
+        content: doc.slice(node.from, node.to),
+        data: nodeToJson(state, node.node),
+      })),
+    ).toEqual([
+      {
+        content: '<prompt>\n  Hello\n</prompt>',
+        data: {
+          _tag: 'prompt',
+          children: ['Hello'],
+        },
+      },
+      {
+        content: '<suggestion>Summarize tools</suggestion>',
+        data: {
+          _tag: 'suggestion',
+          children: ['Summarize tools'],
+        },
+      },
+      {
+        content: '<choice>\n  <option>Summarize tools</option>\n  <option>Retry</option>\n</choice>',
+        data: {
+          _tag: 'choice',
+          children: [
+            {
+              _tag: 'option',
+              children: ['Summarize tools'],
+            },
+            {
+              _tag: 'option',
+              children: ['Retry'],
+            },
+          ],
+        },
+      },
+      {
+        content: '<toolkit />',
+        data: {
+          _tag: 'toolkit',
+        },
+      },
+    ]);
+  });
+
+  test('setext heading disabled', () => {
+    const doc = trim`
+      This should NOT be a heading
+      =
+      
+      Another line that should NOT be a heading
+      -
+    `;
+
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+
+    tree.iterate({
+      enter: (node) => {
+        if (node.type.name === 'SetextHeading') {
+          throw new Error('SetextHeading should be disabled!');
+        }
+      },
+    });
+  });
+
+  //
+  // TODO(burdon): All tests below should test the tree.
+  //
+
+  test('should parse standard markdown elements', ({ expect }) => {
+    const doc = trim`
+      # Heading 1
+      ## Heading 2
+      ### Heading 3
+
+      This is a paragraph with **bold** and *italic* text.
+
+      \`\`\`javascript
+      const code = 'block';
+      \`\`\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify the document is parsed without errors.
+    expect(tree).toBe(doc);
+    expect(state.doc.lines).toBeGreaterThan(1);
+  });
+
+  test('should parse custom XML tags', ({ expect }) => {
+    const doc = trim`
+      # Document with Custom Tags
+
+      <prompt>This is a custom prompt block</prompt>
+
+      Regular paragraph text.
+
+      <prompt />
+
+      Regular paragraph text.
+
+      <prompt>
+        Multi-line
+        prompt content
+      </prompt>
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify the document contains custom tags.
+    expect(tree).toContain('<prompt>');
+    expect(tree).toContain('<prompt />');
+    expect(tree).toContain('</prompt>');
+  });
+
+  test('should handle mixed markdown and XML content', ({ expect }) => {
+    const doc = trim`
+      # Mixed Content
+
+      This is a paragraph with a <prompt>inline prompt</prompt> tag.
+
+      ## Section 2
+
+      <prompt>
+        This prompt contains **markdown** formatting
+      </prompt>
+
+      \`\`\`
+      <prompt>This should not be parsed as XML inside code block</prompt>
+      \`\`\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify mixed content is preserved.
+    expect(tree).toBe(doc);
+  });
+
+  test('should not parse XML tags in code blocks', ({ expect }) => {
+    const doc = trim`
+      \`\`\`xml
+      <prompt>This is inside a code block</prompt>
+      \`\`\`
+
+      \`<prompt>inline code</prompt>\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify code blocks are preserved as-is.
+    expect(tree).toContain('```xml');
+    expect(tree).toContain('<prompt>This is inside a code block</prompt>');
+  });
+
+  test('should handle nested and complex structures', ({ expect }) => {
+    const doc = trim`
+      # Complex Document
+
+      <prompt>
+        First prompt with some content
+      </prompt>
+
+      ## Lists and Prompts
+
+      - Item 1
+      - Item 2 with <prompt>embedded prompt</prompt>
+      - Item 3
+
+      <prompt>
+        Another prompt after the list
+      </prompt>
+
+      ### Code Example
+
+      \`\`\`typescript
+      const example = '<prompt>not parsed</prompt>';
+      \`\`\`
+    `;
+
+    const state = createEditorState(doc);
+    const tree = state.sliceDoc(0, state.doc.length);
+
+    // Verify complex structures are handled.
+    expect(tree).toBe(doc);
+    expect(tree).toMatch(/Item 2 with <prompt>embedded prompt<\/prompt>/);
+  });
+
+  test('should handle empty and edge cases', ({ expect }) => {
+    const emptyDoc = '';
+    const emptyState = createEditorState(emptyDoc);
+    expect(emptyState.doc.length).toBe(0);
+
+    const onlyPromptDoc = '<prompt>Only prompt content</prompt>';
+    const onlyPromptState = createEditorState(onlyPromptDoc);
+    expect(onlyPromptState.sliceDoc(0)).toBe(onlyPromptDoc);
+
+    const unclosedPromptDoc = '<prompt>Unclosed prompt';
+    const unclosedState = createEditorState(unclosedPromptDoc);
+    expect(unclosedState.sliceDoc(0)).toBe(unclosedPromptDoc);
+  });
+});

package/src/extensions/tags/extended-markdown.ts

@@ -0,0 +1,78 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { xmlLanguage } from '@codemirror/lang-xml';
+import { type Extension } from '@codemirror/state';
+import { type ParseWrapper, parseMixed } from '@lezer/common';
+
+import { createMarkdownExtensions } from '../markdown';
+
+import { type XmlWidgetRegistry } from './xml-tags';
+
+export type ExtendedMarkdownOptions = {
+  registry?: XmlWidgetRegistry;
+};
+
+/**
+ * Extended markdown parser with mixed parser for custom rendering of XML tags.
+ */
+export const extendedMarkdown = ({ registry }: ExtendedMarkdownOptions = {}): Extension => {
+  return [
+    createMarkdownExtensions({
+      extensions: [
+        {
+          wrap: mixedParser(registry),
+          parseBlock: [
+            // Disable SetextHeading since it causes flickering when parsing/rendering tasks in chunks.
+            {
+              name: 'SetextHeading',
+              parse: () => false,
+            },
+          ],
+        },
+      ],
+    }),
+  ];
+};
+
+/**
+ * Configure mixed parser to recognize custom tags.
+ */
+const mixedParser = (registry?: XmlWidgetRegistry): ParseWrapper => {
+  const customTags = Object.keys(registry ?? {});
+  const tagPattern = new RegExp(`<(${customTags.join('|')})`);
+
+  return parseMixed((node, input) => {
+    switch (node.name) {
+      // Ignore XML inside of fenced and inline code.
+      case 'FencedCode':
+      case 'InlineCode': {
+        return null;
+      }
+
+      // Parse multi-line HTML blocks.
+      // case 'XMLBlock':
+      case 'HTMLBlock': {
+        return {
+          parser: xmlLanguage.parser,
+        };
+      }
+
+      // Parse paragraphs that contain custom XML tags.
+      // TODO(burdon): Entire paragraph should be parsed as XML.
+      case 'Paragraph': {
+        const content = input.read(node.from, node.to);
+        if (tagPattern.test(content)) {
+          return {
+            parser: xmlLanguage.parser,
+          };
+        }
+
+        return null;
+      }
+    }
+
+    return null;
+  });
+};

package/src/extensions/tags/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './extended-markdown';
+export * from './streamer';
+export * from './xml-tags';

package/src/extensions/tags/streamer.ts

@@ -0,0 +1,244 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Extension, StateEffect, StateField } from '@codemirror/state';
+import { Decoration, type DecorationSet, EditorView, ViewPlugin, type ViewUpdate, WidgetType } from '@codemirror/view';
+
+import { isNotFalsy } from '@dxos/util';
+
+import { Domino } from '../../util';
+
+const BLINK_RATE = 2_000;
+
+export type StreamerOptions = {
+  cursor?: boolean;
+  // When true, uses defaults. When object, allows configuring removal delay.
+  fadeIn?: boolean | { removalDelay?: number };
+};
+
+/**
+ * Extension that adds a blinking cursor widget at the end of the document.
+ */
+export const streamer = (options: StreamerOptions = {}): Extension => {
+  return [
+    options.cursor && cursor(),
+    options.fadeIn && fadeIn(typeof options.fadeIn === 'object' ? options.fadeIn : {}),
+  ].filter(isNotFalsy);
+};
+
+/**
+ * State field to manage the cursor widget decoration.
+ */
+const cursor = (): Extension => {
+  const hideCursor = StateEffect.define();
+
+  // State field to track if cursor should be shown.
+  const showCursor = StateField.define<boolean>({
+    create: () => true,
+    update: (value, tr) => {
+      for (const effect of tr.effects) {
+        if (effect.is(hideCursor)) {
+          return false;
+        }
+      }
+      if (tr.docChanged) {
+        return true;
+      }
+
+      return value;
+    },
+  });
+
+  // View plugin to manage timer and dispatch effects.
+  const timerPlugin = ViewPlugin.fromClass(
+    class {
+      timer: any;
+
+      constructor(private view: EditorView) {}
+
+      update(update: ViewUpdate) {
+        if (update.docChanged) {
+          clearTimeout(this.timer);
+          this.timer = setTimeout(() => {
+            this.view.dispatch({
+              effects: hideCursor.of(null),
+            });
+          }, BLINK_RATE);
+        }
+      }
+
+      destroy() {
+        clearTimeout(this.timer);
+      }
+    },
+  );
+
+  // Decoration field that uses the showCursor state.
+  const cursorDecoration = StateField.define<DecorationSet>({
+    create: () => Decoration.none,
+    update: (_decorations, tr) => {
+      const show = tr.state.field(showCursor);
+      if (!show) {
+        return Decoration.none;
+      }
+
+      // Always place cursor at the end of the document.
+      const endPos = tr.state.doc.length;
+      return Decoration.set([
+        Decoration.widget({
+          widget: new CursorWidget(),
+          side: 1, // Place after the position.
+        }).range(endPos),
+      ]);
+    },
+    provide: (f) => EditorView.decorations.from(f),
+  });
+
+  return [showCursor, timerPlugin, cursorDecoration];
+};
+
+/**
+ * Widget class for the cursor at the end of the document.
+ * Half
+ */
+class CursorWidget extends WidgetType {
+  toDOM() {
+    return Domino.of('span')
+      .style({ opacity: '0.8' })
+      .child(Domino.of('span').text('\u258F').style({ animation: 'blink 2s infinite' }))
+      .build();
+  }
+}
+
+/**
+ * State field to detect and decorate appended text with a fade-in effect.
+ * Also schedules removal of the last appended decoration after a delay.
+ */
+const fadeIn = (options: { removalDelay?: number } = {}): Extension => {
+  const FADE_IN_DURATION = 1_000; // ms.
+  const DEFAULT_REMOVAL_DELAY = 5_000; // ms.
+  const removalDelay = options.removalDelay ?? DEFAULT_REMOVAL_DELAY;
+
+  // Effect to remove a specific decoration by range.
+  const removeDecoration = StateEffect.define<{ from: number; to: number }>();
+
+  // Decoration field that adds fade-in marks for appended content and responds to removal effects.
+  const fadeField = StateField.define<DecorationSet>({
+    create: () => Decoration.none,
+    update: (decorations, tr) => {
+      let next = decorations;
+
+      // Apply removals first, if any.
+      for (const effect of tr.effects) {
+        if (effect.is(removeDecoration)) {
+          const target = effect.value;
+          next = next.update({
+            filter: (from, to) => !(from === target.from && to === target.to),
+          });
+        }
+      }
+
+      if (!tr.docChanged) {
+        return next;
+      }
+
+      // Reset decorations if the entire content was replaced.
+      let isReset = tr.state.doc.length === 0;
+      if (!isReset) {
+        tr.changes.iterChanges((fromA, toA) => {
+          if (fromA === 0 && toA === tr.startState.doc.length) {
+            isReset = true;
+          }
+        });
+      }
+      if (isReset) {
+        return Decoration.none;
+      }
+
+      // Add fade-in decorations for appended content at the end only.
+      const add: any[] = [];
+      tr.changes.iterChanges((fromA, toA, fromB, toB, inserted) => {
+        // Don't fade in initial content.
+        if (fromB === 0 && toB === inserted.length) {
+          return;
+        }
+        // At-the-end append.
+        if (toA === tr.startState.doc.length && inserted.length > 0) {
+          add.push(Decoration.mark({ class: 'cm-fade-in' }).range(fromB, toB));
+        }
+      });
+
+      return next.update({ add });
+    },
+    provide: (f) => EditorView.decorations.from(f),
+  });
+
+  // View plugin that tracks appended ranges and schedules their removal.
+  const timerPlugin = ViewPlugin.fromClass(
+    class {
+      // Map a simple key "from-to" to timer id.
+      _timers = new Map<string, any>();
+
+      constructor(private view: EditorView) {}
+
+      update(update: ViewUpdate) {
+        if (!update.docChanged) {
+          return;
+        }
+
+        update.changes.iterChanges((fromA, toA, fromB, toB, inserted) => {
+          // Only consider appends at the end.
+          if (toA !== update.startState.doc.length || inserted.length === 0) {
+            return;
+          }
+
+          const key = `${fromB}-${toB}`;
+          // Clear any prior timer for this exact range.
+          if (this._timers.has(key)) {
+            clearTimeout(this._timers.get(key));
+          }
+
+          const totalDelay = FADE_IN_DURATION + removalDelay;
+          const id = setTimeout(() => {
+            this.view.dispatch({ effects: removeDecoration.of({ from: fromB, to: toB }) });
+            this._timers.delete(key);
+          }, totalDelay);
+
+          this._timers.set(key, id);
+        });
+      }
+
+      destroy() {
+        for (const id of this._timers.values()) {
+          clearTimeout(id);
+        }
+        this._timers.clear();
+      }
+    },
+  );
+
+  return [
+    fadeField,
+    timerPlugin,
+    EditorView.theme({
+      '.cm-line > span': {
+        opacity: '0.8',
+      },
+      '.cm-fade-in': {
+        animation: 'fade-in 3s ease-out forwards',
+      },
+      '@keyframes fade-in': {
+        '0%': {
+          opacity: '0',
+        },
+        '80%': {
+          opacity: '1',
+        },
+        '100%': {
+          opacity: '0.8',
+        },
+      },
+    }),
+  ];
+};