@dxos/ui-editor 0.0.0 → 0.8.4-main.05e74ebcff

package/src/extensions/scrolling/crawler.ts

@@ -0,0 +1,263 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { StateEffect } from '@codemirror/state';
+import { EditorView, ViewPlugin } from '@codemirror/view';
+
+import { log } from '@dxos/log';
+
+/**
+ * Parameters for the scroll to line effect.
+ */
+export type ScrollToProps = {
+  /**
+   * Zero-based line number; -1 for end of document.
+   * NOTE: view.state.doc.lineAt() uses 1-based line numbers.
+   */
+  line: number;
+
+  /**
+   * Additional offset from the target line in pixels.
+   * Positive values scroll past the line, negative values stop before it.
+   * @default 0
+   */
+  offset?: number;
+
+  /**
+   * Position of the target line in the viewport.
+   * - 'start': Line appears at the start (top) of the screen
+   * - 'end': Line appears at the end (bottom) of the screen
+   * @default 'start'
+   */
+  position?: 'start' | 'end';
+
+  /**
+   * Whether to use smooth scrolling.
+   * @default 'instant'
+   */
+  behavior?: ScrollBehavior;
+};
+
+/** Scroll to a specific line. */
+export const crawlerLineEffect = StateEffect.define<ScrollToProps>();
+
+/** Start/stop crawling the end of the document. */
+export const crawlerActiveEffect = StateEffect.define<boolean>();
+
+/**
+ * Helper function to scroll to a specific line.
+ * This is a convenience function that can be used directly with an EditorView.
+ */
+export const scrollToLine = (view: EditorView, options: ScrollToProps) => {
+  view.dispatch({
+    effects: crawlerLineEffect.of(options),
+  });
+};
+
+export type CrawlerOptions = {
+  /** Threshold in px to trigger scroll from bottom. */
+  overScroll?: number;
+};
+
+/**
+ * Imperative scroll-control primitive for streaming editor views.
+ *
+ * Owns the scroll-related effects (`crawlerLineEffect`, `crawlerActiveEffect`), the spring
+ * crawler that follows the bottom of the document, and the `.cm-scroller` theme (overflow,
+ * scrollbar styling, and the `::after` overscroll spacer).
+ *
+ * Use directly for jump-to-line navigation, or pair with `autoScroll` for a pin-to-bottom
+ * streaming policy. The composite `streamScroll` bundles both for the common case.
+ */
+export const crawler = ({ overScroll = 0 }: CrawlerOptions = {}) => {
+  // ViewPlugin to manage scroll animations.
+  const crawlerPlugin = ViewPlugin.fromClass(
+    class CrawlerPlugin {
+      private readonly crawler: ReturnType<typeof createCrawler>;
+      constructor(private readonly view: EditorView) {
+        this.crawler = createCrawler(this.view);
+      }
+
+      // No-op.
+      destroy() {
+        this.crawler.cancel();
+      }
+
+      cancel() {
+        this.crawler.cancel();
+      }
+
+      crawl(start = false) {
+        if (start) {
+          this.crawler.scroll();
+        } else {
+          this.crawler.cancel();
+        }
+      }
+
+      scroll({ line, offset = 0, position, behavior = 'instant' }: ScrollToProps) {
+        const { scrollTop, scrollHeight, clientHeight } = this.view.scrollDOM;
+        const scrollerRect = this.view.scrollDOM.getBoundingClientRect();
+        const doc = this.view.state.doc;
+
+        let targetScrollTop = scrollHeight - clientHeight + offset;
+        if (line >= 0 && line <= doc.lines - 1) {
+          const lineStart = doc.line(line + 1).from;
+          const coords = this.view.coordsAtPos(lineStart);
+          if (coords) {
+            // Calculate target scroll position based on position option.
+            const currentScrollTop = scrollTop;
+            const maxScrollTop = scrollHeight - clientHeight;
+
+            if (position === 'end') {
+              // Position line at end (bottom) of viewport.
+              // Calculate how far down we need to scroll so the line's bottom aligns with viewport bottom.
+              targetScrollTop = currentScrollTop + coords.bottom - scrollerRect.bottom + offset;
+            } else {
+              // Default: position line at start (top) of viewport.
+              targetScrollTop = currentScrollTop + coords.top - scrollerRect.top + offset;
+            }
+
+            // Clamp to valid scroll range.
+            targetScrollTop = Math.max(0, Math.min(targetScrollTop, maxScrollTop));
+          }
+        }
+
+        // TODO(burdon): Smooth scrolling doesn't work when the document is being streamed into.
+        requestAnimationFrame(() => {
+          this.view.scrollDOM.scrollTo({ top: targetScrollTop }); //, behavior });
+        });
+      }
+    },
+  );
+
+  return [
+    crawlerPlugin,
+
+    // Listen for effect.
+    EditorView.updateListener.of((update) => {
+      update.transactions.forEach((transaction) => {
+        try {
+          const plugin = update.view.plugin(crawlerPlugin);
+          if (plugin) {
+            for (const effect of transaction.effects) {
+              if (effect.is(crawlerActiveEffect)) {
+                plugin.crawl(effect.value);
+              } else if (effect.is(crawlerLineEffect)) {
+                plugin.scroll(effect.value);
+              }
+            }
+          }
+        } catch (err) {
+          log.catch(err);
+        }
+      });
+    }),
+
+    // Styles.
+    EditorView.theme({
+      '.cm-scroller': {
+        overflowY: 'scroll',
+        // Browser scroll-anchoring: when widgets above the viewport resize (e.g. tool blocks
+        // expanding their TogglePanel), the browser picks a stable element near the viewport
+        // top and adjusts `scrollTop` so the user's view doesn't jump. Auto-scroll's pinning
+        // logic still has the final word when pinned (forces scrollTop to scrollHeight).
+        overflowAnchor: 'auto',
+      },
+      '.cm-scroller.cm-hide-scrollbar::-webkit-scrollbar': {
+        display: 'none',
+      },
+      '.cm-scroller::-webkit-scrollbar-thumb': {
+        background: 'transparent',
+        transition: 'background 0.15s',
+      },
+      '&:hover .cm-scroller::-webkit-scrollbar-thumb': {
+        background: 'var(--color-scrollbar-thumb)',
+      },
+      // Spacer below the last text line. Implemented as a real block pseudo-element
+      // (rather than `padding-bottom` on `.cm-content`) so it materializes in the
+      // scroller's `scrollHeight` regardless of how `padding` is reset by the base
+      // theme or downstream classes — this is what gives auto-scroll its head-room
+      // so the last line stays `overScroll` px above the viewport bottom.
+      '.cm-content::after': {
+        content: '""',
+        display: 'block',
+        height: `${overScroll}px`,
+      },
+      '.cm-scroll-button': {
+        position: 'absolute',
+        bottom: '0.5rem',
+        right: '1rem',
+      },
+    }),
+  ];
+};
+
+/**
+ * Creates a smooth crawler that follows the live bottom of a CodeMirror 6 EditorView.
+ *
+ * Uses a critically-damped spring: each frame applies a restoring force toward the
+ * target and a damping force opposing current velocity. With damping = 2·ω, the
+ * system is critically damped — fastest approach without overshoot. The spring
+ * naturally sprints when far behind and eases as it approaches, so streaming
+ * content is followed tightly without the jerk of explicit accel/decel state.
+ *
+ * Integration uses real elapsed wall-clock time so the perceived speed stays
+ * constant when requestAnimationFrame is throttled (e.g. low-power mode dropping
+ * from 60Hz to 30Hz).
+ *
+ * @param omega Spring stiffness in rad/s. Higher = snappier follow. ~12–18 feels good.
+ * @param snapThreshold Snap-to-target distance threshold in px.
+ * @param snapVelocity Snap-to-target velocity threshold in px/s.
+ */
+export function createCrawler(view: EditorView, omega = 5, snapThreshold = 5, snapVelocity = 50) {
+  const el = view.scrollDOM;
+
+  let currentTop = 0;
+  let velocity = 0;
+  let rafId: number | null = null;
+  let lastTime = 0;
+
+  function frame(now: number) {
+    // Clamp dt to handle long pauses (tab backgrounded) and the first frame.
+    const dt = lastTime === 0 ? 1 / 60 : Math.min(0.1, (now - lastTime) / 1000);
+    lastTime = now;
+
+    const targetTop = el.scrollHeight - el.clientHeight;
+    const delta = targetTop - currentTop;
+    if (Math.abs(delta) < snapThreshold && Math.abs(velocity) < snapVelocity) {
+      el.scrollTop = targetTop;
+      currentTop = targetTop;
+      velocity = 0;
+      rafId = null;
+      lastTime = 0;
+      return;
+    }
+
+    // Critically-damped spring: a = ω²·delta − 2ω·v.
+    const accel = omega * omega * delta - 2 * omega * velocity;
+    velocity += accel * dt;
+    currentTop += velocity * dt;
+    el.scrollTop = currentTop;
+    rafId = requestAnimationFrame(frame);
+  }
+
+  return {
+    scroll: () => {
+      if (rafId === null) {
+        currentTop = el.scrollTop;
+        lastTime = 0;
+        rafId = requestAnimationFrame(frame);
+      }
+    },
+    cancel: () => {
+      if (rafId !== null) {
+        cancelAnimationFrame(rafId);
+        velocity = 0;
+        lastTime = 0;
+        rafId = null;
+      }
+    },
+  };
+}

package/src/extensions/scrolling/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './auto-scroll';
+export * from './crawler';
+export * from './scroll-past-end';
+export * from './scroller';

package/src/extensions/scrolling/scroll-past-end.ts

@@ -0,0 +1,32 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Extension } from '@codemirror/state';
+import { EditorView, ViewPlugin } from '@codemirror/view';
+
+/**
+ * Custom scroll-past-end extension that accounts for the actual height of the last line.
+ * The built-in CodeMirror `scrollPastEnd` uses `defaultLineHeight` which doesn't account
+ * for taller elements like headings, block widgets, etc.
+ */
+const scrollPastEndPlugin = ViewPlugin.fromClass(
+  class {
+    _height = 1_000;
+    _attrs: { style: string } | null = { style: `padding-bottom: ${this._height}px` };
+
+    update({ view }: { view: EditorView }) {
+      const lastLineBlock = view.lineBlockAt(view.state.doc.length);
+      const height = view.dom.clientHeight - lastLineBlock.height - view.documentPadding.top - 0.5;
+      if (height >= 0 && height !== this._height) {
+        this._height = height;
+        this._attrs = { style: `padding-bottom: ${height}px` };
+      }
+    }
+  },
+);
+
+export const scrollPastEnd = (): Extension => [
+  scrollPastEndPlugin,
+  EditorView.contentAttributes.of((view) => view.plugin(scrollPastEndPlugin)?._attrs ?? null),
+];

package/src/extensions/scrolling/scroller.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Extension } from '@codemirror/state';
+
+import { isTruthy } from '@dxos/util';
+
+import { type AutoScrollProps, autoScroll as autoScrollExtension } from './auto-scroll';
+import { type CrawlerOptions, crawler } from './crawler';
+
+export type ScrollerOptions = CrawlerOptions &
+  AutoScrollProps & {
+    /**
+     * Include the auto-scroll policy (pin-to-bottom, user-scroll unpin, scroll-to-bottom button).
+     * Set to `false` to get only the crawler primitive (line jumps, theme, overscroll spacer).
+     * @default true
+     */
+    autoScroll?: boolean;
+  };
+
+/**
+ * Composite scroll extension for streaming editor views (chat threads, transcripts, logs).
+ */
+export const scroller = ({ overScroll, scrollOnResize, autoScroll = true }: ScrollerOptions = {}): Extension[] => {
+  return [crawler({ overScroll }), autoScroll && autoScrollExtension({ scrollOnResize })].filter(isTruthy);
+};

package/src/extensions/selection.ts

@@ -31,7 +31,7 @@ export type EditorStateStore = {
   getState: (id: string) => EditorSelectionState | undefined;
 };
 
-const stateRestoreAnnotation = 'dxos.org/cm/state-restore';
+const stateRestoreAnnotation = 'org.dxos.cm.state-restore';
 
 export const createEditorStateTransaction = ({ scrollTo, selection }: EditorSelectionState): TransactionSpec => {
   return {

package/src/extensions/snippets.ts

@@ -0,0 +1,67 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type Extension } from '@codemirror/state';
+import { keymap } from '@codemirror/view';
+
+const defaultItems = ['hello world!', 'this is a test.', 'this is [DXOS](https://dxos.org)'];
+
+export type SnippetsOptions = {
+  delay?: number;
+  items?: string[];
+};
+
+/**
+ * Configurable plugin that lets users cycle through pre-configured input snippets.
+ */
+// TODO(burdon): Review https://github.com/sergeche/codemirror-movie?tab=readme-ov-file
+export const snippets = ({ delay = 75, items = defaultItems }: SnippetsOptions = {}): Extension => {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let index = 0; // TODO(burdon): Make global.
+
+  return [
+    keymap.of([
+      {
+        // Reset.
+        key: "alt-meta-'",
+        run: () => {
+          clearTimeout(timer);
+          index = 0;
+          return true;
+        },
+      },
+      {
+        // Next snippet.
+        // TODO(burdon): Press 1-9 to select snippet?
+        key: "Shift-Meta-'",
+        run: (view) => {
+          clearTimeout(timer);
+          // TODO(burdon): Add space if needed.
+          const text = items[index++];
+          if (index === items?.length) {
+            index = 0;
+          }
+
+          let offset = 0;
+          const insert = (delayMs = 0) => {
+            timer = setTimeout(() => {
+              const pos = view.state.selection.main.head;
+              view.dispatch({
+                changes: { from: pos, insert: text[offset++] },
+                selection: { anchor: pos + 1 },
+              });
+
+              if (offset < text.length) {
+                insert(Math.random() * delay * (text[offset] === ' ' ? 2 : 1));
+              }
+            }, delayMs);
+          };
+
+          insert();
+          return true;
+        },
+      },
+    ]),
+  ];
+};

package/src/extensions/tags/extended-markdown.test.ts

@@ -12,11 +12,24 @@ import { trim } from '@dxos/util';
 import { extendedMarkdown } from './extended-markdown';
 import { nodeToJson } from './xml-util';
 
+const testRegistry = {
+  prompt: { block: true },
+  suggestion: { block: true },
+  choice: { block: true },
+  toolkit: { block: true },
+  toolCall: { block: true },
+  summary: { block: true },
+  reasoning: { block: true },
+  stats: { block: true },
+  reference: { block: false },
+  select: { block: true },
+};
+
 describe('extended-markdown', () => {
-  const createEditorState = (doc: string) => {
+  const createEditorState = (doc: string, registry?: Record<string, any>) => {
     return EditorState.create({
       doc,
-      extensions: [extendedMarkdown()],
+      extensions: [extendedMarkdown({ registry: registry ?? testRegistry })],
     });
   };
 
@@ -121,6 +134,111 @@ describe('extended-markdown', () => {
     });
   });
 
+  test('self-closing tags with attributes should not consume subsequent content', ({ expect }) => {
+    const doc = trim`
+      <toolCall id="toolu_01ABC123" />
+
+      ## Architecture
+
+      Regular paragraph.
+    `;
+
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+
+    const nodeNames: string[] = [];
+    tree.iterate({
+      enter: (node) => {
+        nodeNames.push(node.name);
+      },
+    });
+
+    // The heading must be parsed as ATXHeading2, not swallowed into an HTMLBlock.
+    expect(nodeNames).toContain('ATXHeading2');
+  });
+
+  test('self-closing tags without attributes should parse correctly', ({ expect }) => {
+    const doc = trim`
+      <toolkit />
+
+      ## Heading After Self-Close
+
+      Paragraph text.
+    `;
+
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+
+    const nodeNames: string[] = [];
+    tree.iterate({
+      enter: (node) => {
+        nodeNames.push(node.name);
+      },
+    });
+
+    expect(nodeNames).toContain('ATXHeading2');
+  });
+
+  test('self-closing tags with trailing text should not consume subsequent content', ({ expect }) => {
+    const doc = trim`
+      <toolkit /> trailing text
+
+      ## Heading After Trailing
+    `;
+
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+
+    const nodeNames: string[] = [];
+    tree.iterate({
+      enter: (node) => {
+        nodeNames.push(node.name);
+      },
+    });
+
+    // The trailing text line should not be swallowed as an HTMLBlock.
+    expect(nodeNames).toContain('ATXHeading2');
+  });
+
+  test('mixed self-closing and block tags should not break markdown parsing', ({ expect }) => {
+    const doc = trim`
+      <prompt>What is markdown?</prompt>
+
+      <reasoning>
+      The user is asking about markdown.
+      </reasoning>
+
+      Some explanation text.
+
+      <toolCall id="toolu_01ABC123analyze" />
+
+      <summary>Analyzed 42 files.</summary>
+
+      ## Architecture
+
+      The project follows a modular pattern.
+
+      <stats>3 tool uses · 12.4k tokens</stats>
+
+      ### Sub-heading
+
+      More text.
+    `;
+
+    const state = createEditorState(doc);
+    const tree = syntaxTree(state);
+
+    const nodeNames: string[] = [];
+    tree.iterate({
+      enter: (node) => {
+        nodeNames.push(node.name);
+      },
+    });
+
+    expect(nodeNames).toContain('ATXHeading2');
+    expect(nodeNames).toContain('ATXHeading3');
+  });
+
   //
   // TODO(burdon): All tests below should test the tree.
   //

package/src/extensions/tags/extended-markdown.ts

@@ -5,15 +5,18 @@
 import { xmlLanguage } from '@codemirror/lang-xml';
 import { type Extension } from '@codemirror/state';
 import { type ParseWrapper, parseMixed } from '@lezer/common';
+import { type BlockParser } from '@lezer/markdown';
 
 import { createMarkdownExtensions } from '../markdown';
-
 import { type XmlWidgetRegistry } from './xml-tags';
 
 export type ExtendedMarkdownOptions = {
   registry?: XmlWidgetRegistry;
 };
 
+/** Escapes a string for safe embedding in RegExp source. */
+const escapeRegExpSource = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
 /**
  * Extended markdown parser with mixed parser for custom rendering of XML tags.
  */
@@ -29,6 +32,9 @@ export const extendedMarkdown = ({ registry }: ExtendedMarkdownOptions = {}): Ex
               name: 'SetextHeading',
               parse: () => false,
             },
+            // Custom XML block parser that keeps registered tags as a single HTMLBlock
+            // even when their content contains blank lines.
+            ...xmlBlockParsers(registry),
           ],
         },
       ],
@@ -36,6 +42,79 @@ export const extendedMarkdown = ({ registry }: ExtendedMarkdownOptions = {}): Ex
   ];
 };
 
+/**
+ * Creates block parsers for registered XML tags that may contain blank lines.
+ *
+ * By default, the markdown parser treats custom HTML tags as type 6 HTML blocks,
+ * which end at blank lines. This causes tags like `<reasoning>...\n\n...</reasoning>`
+ * to be split into separate blocks, preventing the XML mixed parser from seeing the
+ * full element. This custom parser consumes all lines (including blanks) until the
+ * matching closing tag, emitting a single HTMLBlock.
+ */
+const xmlBlockParsers = (registry?: XmlWidgetRegistry): BlockParser[] => {
+  const customTags = Object.keys(registry ?? {});
+  if (customTags.length === 0) {
+    return [];
+  }
+
+  const tagPattern = customTags.map(escapeRegExpSource).join('|');
+  const selfClosePattern = new RegExp(`^\\s*<(${tagPattern})(\\s[^>]*)?\\/>\\s*$`);
+  const openPattern = new RegExp(`^\\s*<(${tagPattern})(\\s[^>]*)?\\/?>`);
+
+  return [
+    {
+      name: 'XMLBlock',
+      before: 'HTMLBlock',
+      parse: (cx, line) => {
+        const match = openPattern.exec(line.text);
+        if (!match) {
+          return false;
+        }
+
+        // Self-closing tag (e.g., `<tag />` or `<tag id="x" />`).
+        if (selfClosePattern.test(line.text)) {
+          const end = cx.lineStart + line.text.length;
+          cx.addElement(cx.elt('HTMLBlock', cx.lineStart, end));
+          cx.nextLine();
+          return true;
+        }
+
+        // Self-closing tag with trailing text (e.g., `<tag /> text`).
+        // Let markdown parse it as a normal paragraph.
+        if (match[0].trimEnd().endsWith('/>')) {
+          return false;
+        }
+
+        const tagName = match[1];
+        const closeTag = `</${tagName}>`;
+        const start = cx.lineStart;
+
+        // Check if closing tag is on the same line.
+        if (line.text.includes(closeTag)) {
+          cx.addElement(cx.elt('HTMLBlock', start, start + line.text.length));
+          cx.nextLine();
+          return true;
+        }
+
+        // Consume lines (including blank lines) until the closing tag.
+        let end = cx.lineStart + line.text.length;
+        while (cx.nextLine()) {
+          end = cx.lineStart + line.text.length;
+          if (line.text.includes(closeTag)) {
+            cx.addElement(cx.elt('HTMLBlock', start, end));
+            cx.nextLine();
+            return true;
+          }
+        }
+
+        // Unclosed tag (e.g., still streaming) — emit what we have.
+        cx.addElement(cx.elt('HTMLBlock', start, end));
+        return true;
+      },
+    },
+  ];
+};
+
 /**
  * Configure mixed parser to recognize custom tags.
  */