@dxos/plugin-assistant 0.8.4-main.3eb6e50203 → 0.8.4-main.422d1c7879

package/src/components/ChatThread/registry.tsx

@@ -5,56 +5,65 @@
 import React from 'react';
 
 import { log } from '@dxos/log';
-import { ToggleContainer } from '@dxos/react-ui-components';
+import { ContentBlock, type Message } from '@dxos/types';
+import { type XmlWidgetRegistry, getXmlTextChild } from '@dxos/ui-editor';
+
+import { type BlockRenderer, type MessageThreadContext } from './sync';
+import { applyToolBlockToWidgetState } from './tool-widget-state';
 import {
+  FallbackWidget,
   PromptWidget,
+  ReasoningWidget,
   ReferenceWidget,
   SelectWidget,
   SuggestionWidget,
+  StatsWidget,
   SummaryWidget,
-} from '@dxos/react-ui-components';
-import { Json } from '@dxos/react-ui-syntax-highlighter';
-import { ContentBlock, type Message } from '@dxos/types';
-import { type XmlWidgetProps, type XmlWidgetRegistry, getXmlTextChild } from '@dxos/ui-editor';
-
-import { ToolBlock } from '../ToolBlock';
-
-import { type BlockRenderer, type MessageThreadContext } from './sync';
-
-const Fallback = ({ _tag, ...props }: XmlWidgetProps<MessageThreadContext>) => {
-  return (
-    <ToggleContainer.Root classNames='rounded-sm'>
-      <ToggleContainer.Header classNames='bg-groupSurface'>{_tag}</ToggleContainer.Header>
-      <ToggleContainer.Content classNames='bg-modalSurface'>
-        <Json classNames='!p-2 text-sm' data={props} />
-      </ToggleContainer.Content>
-    </ToggleContainer.Root>
-  );
-};
+  ToolWidget,
+  StatusWidget,
+} from './widgets';
 
 /**
  * Custom XML tags registry.
  */
 export const componentRegistry: XmlWidgetRegistry = {
   //
-  // Widgets
+  // DOM Widgets
   //
 
-  ['prompt' as const]: {
+  prompt: {
     block: true,
     factory: ({ children }) => {
       const text = getXmlTextChild(children);
       return text ? new PromptWidget(text) : null;
     },
   },
-  ['reference' as const]: {
+  synthetic: {
+    block: true,
+    factory: ({ children, range }) => {
+      const text = getXmlTextChild(children);
+      // TODO(dmaretskyi): Synthetic blocks are "user"-turn messages generated by the system, i.e. and event from a trigger, an email-encoded to JSON, etc.
+      //                   They are meant to start agent, to process the event without looking like user-authored prompt.
+      //                   Create a new widget type for them.
+      return text ? new ReasoningWidget(text, range.from) : null;
+    },
+  },
+  reasoning: {
+    block: true,
+    streaming: true,
+    factory: ({ children, range }) => {
+      const text = getXmlTextChild(children);
+      return text ? new ReasoningWidget(text, range.from) : null;
+    },
+  },
+  reference: {
     block: false,
     factory: ({ children, ref }) => {
       const text = getXmlTextChild(children);
       return text && ref ? new ReferenceWidget(text, ref) : null;
     },
   },
-  ['select' as const]: {
+  select: {
     block: true,
     factory: ({ children }) => {
       const options = children
@@ -63,45 +72,61 @@ export const componentRegistry: XmlWidgetRegistry = {
       return options?.length ? new SelectWidget(options) : null;
     },
   },
-  ['suggestion' as const]: {
+  suggestion: {
     block: true,
     factory: ({ children }) => {
       const text = getXmlTextChild(children);
       return text ? new SuggestionWidget(text) : null;
     },
   },
-  ['summary' as const]: {
+  stats: {
     block: true,
     factory: ({ children }) => {
       const text = getXmlTextChild(children);
-      return text ? new SummaryWidget(text) : null;
+      return text ? new StatsWidget(text) : null;
+    },
+  },
+  status: {
+    block: true,
+    streaming: true,
+    factory: ({ children, range }) => {
+      const text = getXmlTextChild(children);
+      return text ? new StatusWidget(text, range.from) : null;
     },
   },
 
   //
-  // React
+  // React Widgets (portaled outside of the editor)
   //
 
-  ['toolCall' as const]: {
+  summary: {
+    block: true,
+    Component: SummaryWidget,
+  },
+  toolCall: {
     block: true,
-    Component: ToolBlock,
+    Component: (props) => (
+      <div role='none' className='py-2'>
+        <ToolWidget {...props} />
+      </div>
+    ),
   },
-  ['toolResult' as const]: {
+  toolResult: {
     block: true,
-    Component: Fallback,
+    Component: FallbackWidget,
   },
-  ['toolkit' as const]: {
+  toolkit: {
     block: true,
-    Component: Fallback,
+    Component: FallbackWidget,
   },
 
   //
   // Fallback
   //
 
-  ['json' as const]: {
+  json: {
     block: true,
-    Component: Fallback,
+    Component: FallbackWidget,
   },
 };
 
@@ -126,7 +151,11 @@ const blockToMarkdownImpl = (context: MessageThreadContext, message: Message.Mes
   switch (block._tag) {
     case 'text': {
       if (message.sender.role === 'user') {
-        return `<prompt>${block.text}</prompt>`;
+        if (block.disposition === 'synthetic') {
+          return renderXMLBlock('synthetic', { content: block.text, pending: block.pending });
+        } else {
+          return `<prompt>${block.text}</prompt>`;
+        }
       } else {
         const text = block.text.trim();
         if (text.length > 0) {
@@ -135,40 +164,84 @@ const blockToMarkdownImpl = (context: MessageThreadContext, message: Message.Mes
       }
       break;
     }
+
     case 'reference': {
+      if (block.pending) {
+        return;
+      }
       const dxn = block.reference.dxn;
       return `<reference ref="${dxn.toString()}">${context.getObjectLabel(dxn)}</reference>`;
     }
+
     case 'suggestion': {
       if (block.pending) {
         return;
       }
       return `<suggestion>${block.text}</suggestion>`;
     }
+
     case 'select': {
       if (block.pending || block.options.length === 0) {
         return;
       }
       return `<select>${block.options.map((option) => `<option>${option}</option>`).join('')}</select>`;
     }
+
     case 'toolCall': {
-      context.updateWidget<{ blocks: ContentBlock.Any[] }>(block.toolCallId, {
-        blocks: [block],
-      });
+      applyToolBlockToWidgetState(context, block);
       return `<toolCall id="${block.toolCallId}" />`;
     }
+
     case 'toolResult': {
-      context.updateWidget<{ blocks: ContentBlock.Any[] }>(block.toolCallId, ({ blocks = [] }) => ({
-        blocks: [...blocks, block],
-      }));
+      // TODO(dmaretskyi): the parameter could be undefined, perhaps tool blocks are not arriving in order.
+      applyToolBlockToWidgetState(context, block);
       break;
     }
+
+    case 'stats': {
+      return '';
+    }
+
+    case 'reasoning': {
+      let text = block.reasoningText ?? block.redactedText;
+      if (!text) {
+        return;
+      }
+      return renderXMLBlock('reasoning', { content: text, pending: block.pending });
+    }
+
     case 'summary': {
-      return `<summary>${ContentBlock.createSummaryMessage(block)}</summary>`;
+      return renderXMLBlock('summary', { content: block.content, pending: block.pending });
+    }
+
+    case 'status': {
+      return renderXMLBlock('status', { content: block.statusText, pending: block.pending });
     }
+
     default: {
       // TODO(burdon): Needs stable ID.
       return `<json id="${message.id}">\n${JSON.stringify(block)}\n</json>`;
     }
   }
 };
+
+/**
+ * Escape text embedded in generated XML so the mixed XML parser does not treat `&`, `<`, `>` as markup.
+ */
+const escapeXmlTextContent = (raw: string): string =>
+  raw.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+const renderXMLBlock = (tag: string, opts: { content?: string; pending?: boolean; attributes?: string }) => {
+  // Replace paragraph breaks so that the markdown parser does not split the content into multiple paragraphs.
+  // NOTE: do not strip leading list-marker prefixes (`-`, `*`, `1. `, ...) per line — the streaming
+  // tag's range is replaced atomically by the XML widget so the inner markdown is never user-visible,
+  // and stripping can collapse a line to empty mid-stream (e.g. when only `1. ` has arrived), breaking
+  // the prefix-extension contract that `MessageSyncer` relies on for incremental appends.
+  const content = escapeXmlTextContent((opts.content ?? '').replace(/\n\n/g, ' ').trim());
+
+  if (opts.pending) {
+    return `<${tag}${opts.attributes ? ` ${opts.attributes}` : ''}>${content}`;
+  } else {
+    return `<${tag}${opts.attributes ? ` ${opts.attributes}` : ''}>${content}</${tag}>`;
+  }
+};

package/src/components/ChatThread/sync.test.ts

@@ -1,3 +1,5 @@
+// @vitest-environment jsdom
+
 //
 // Copyright 2025 DXOS.org
 //
@@ -8,25 +10,25 @@ import * as Effect from 'effect/Effect';
 
 import { Obj } from '@dxos/echo';
 import { type Mutable } from '@dxos/echo/internal';
-import { type ContentBlock } from '@dxos/types';
+import { type ContentBlock, type Message } from '@dxos/types';
 
-import { createMessage } from '../../testing';
+import { createMessage } from '#testing';
 
 import { blockToMarkdown } from './registry';
-import { MessageSyncer, type TextModel } from './sync';
+import { type BlockRenderer, MessageSyncer, type MessageThreadContext, type TextModel } from './sync';
 
 class TestDocument implements TextModel {
   private readonly _view = new EditorView({ extensions: [] });
 
-  get view() {
-    return this._view;
+  get length() {
+    return this._view.state.doc.length;
   }
 
   get content() {
     return this._view.state.doc.toString();
   }
 
-  async reset(text: string) {
+  async setContent(text: string) {
     this._view.dispatch({
       changes: { from: 0, to: this._view.state.doc.length, insert: text },
     });
@@ -53,13 +55,13 @@ describe('reducers', () => {
         createMessage('assistant', [{ _tag: 'text', text: 'Hi there!' }]),
       ];
 
-      syncer.append(messages);
+      syncer.update(messages);
       expect(doc.content).toEqual(['<prompt>Hello</prompt>', 'Hi there!', ''].join('\n'));
 
-      Obj.change(messages[1], (m) => {
-        m.blocks.push({ _tag: 'text', text: 'How can I help?' });
+      Obj.change(messages[1], (obj) => {
+        obj.blocks.push({ _tag: 'text', text: 'How can I help?' });
       });
-      syncer.append(messages);
+      syncer.update(messages);
       expect(doc.content).toEqual(['<prompt>Hello</prompt>', 'Hi there!', 'How can I help?', ''].join('\n'));
     }),
   );
@@ -75,23 +77,115 @@ describe('reducers', () => {
         createMessage('assistant', [{ _tag: 'text', text: 'Hi there!', pending: true }]),
       ];
 
-      syncer.append(messages);
+      syncer.update(messages);
       expect(doc.content).toEqual(['<prompt>Hello</prompt>', 'Hi there!'].join('\n'));
 
-      Obj.change(messages[1], (m) => {
-        const block = m.blocks[0] as Mutable<ContentBlock.Text>;
+      Obj.change(messages[1], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Text>;
         block.text = 'Hi there! How are you?';
         block.pending = false;
       });
-      syncer.append(messages);
+      syncer.update(messages);
 
-      Obj.change(messages[1], (m) => {
-        m.blocks.push({ _tag: 'text', text: 'How can I help?' });
+      Obj.change(messages[1], (obj) => {
+        obj.blocks.push({ _tag: 'text', text: 'How can I help?' });
       });
-      syncer.append(messages);
+      syncer.update(messages);
       expect(doc.content).toEqual(
         ['<prompt>Hello</prompt>', 'Hi there! How are you?', 'How can I help?', ''].join('\n'),
       );
     }),
   );
+
+  // Regression: streaming reasoning text that passes through a bare list-marker state
+  // (e.g. `"…\n1. "`) used to make `stripBulletLikeLinePrefixes` collapse the line to empty,
+  // breaking the prefix-diff invariant in `MessageSyncer` and producing a duplicate `<reasoning>`
+  // opening tag in the document.
+  it.effect(
+    'streaming reasoning with list-marker transitions does not duplicate opening tag',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+      const syncer = new MessageSyncer(doc, blockToMarkdown);
+
+      const setReasoning = (message: Message.Message, text: string, pending: boolean) => {
+        Obj.change(message, (message) => {
+          const block = message.blocks[0] as Mutable<ContentBlock.Reasoning>;
+          block.reasoningText = text;
+          block.pending = pending;
+        });
+      };
+
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'abc\n1', pending: true }])];
+
+      // Tick 1: `"abc\n1"` — `\d+[.)]\s` regex does not match (no dot/space yet).
+      syncer.update(messages);
+
+      // Tick 2: `"abc\n1."` — still no match (no trailing whitespace).
+      setReasoning(messages[0], 'abc\n1.', true);
+      syncer.update(messages);
+
+      // Tick 3: `"abc\n1. "` — line `"1. "` matches and is stripped to empty.
+      setReasoning(messages[0], 'abc\n1. ', true);
+      syncer.update(messages);
+
+      // Tick 4: `"abc\n1. foo"` — list item with content.
+      setReasoning(messages[0], 'abc\n1. foo', true);
+      syncer.update(messages);
+
+      // Tick 5: finalize.
+      setReasoning(messages[0], 'abc\n1. foo', false);
+      syncer.update(messages);
+
+      const openTagCount = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeTagCount = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openTagCount).toBe(1);
+      expect(closeTagCount).toBe(1);
+    }),
+  );
+
+  // Direct test of `MessageSyncer`'s tolerance for non-monotonic renderer output —
+  // any renderer that produces a shorter string for the same streaming block (e.g., due
+  // to whitespace normalisation or future transforms) must not produce duplicate output.
+  it.effect(
+    'non-monotonic renderer output does not duplicate previously-emitted content',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+
+      const renderer: BlockRenderer = (_ctx: MessageThreadContext, _msg: Message.Message, block: ContentBlock.Any) => {
+        if (block._tag !== 'reasoning') {
+          return undefined;
+        }
+        const text = block.reasoningText ?? '';
+        // Simulate a non-monotonic transform: collapse a line that is a sole digit+dot+space.
+        const normalised = text
+          .split(/\r?\n/)
+          .map((line) => line.replace(/^\s*\d+[.)]\s$/, ''))
+          .join('\n')
+          .trim();
+        return block.pending ? `<reasoning>${normalised}` : `<reasoning>${normalised}</reasoning>\n`;
+      };
+
+      const syncer = new MessageSyncer(doc, renderer);
+
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'abc\n1.', pending: true }])];
+
+      syncer.update(messages);
+      Obj.change(messages[0], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Reasoning>;
+        block.reasoningText = 'abc\n1. ';
+      });
+      syncer.update(messages);
+      Obj.change(messages[0], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Reasoning>;
+        block.reasoningText = 'abc\n1. tail';
+        block.pending = false;
+      });
+      syncer.update(messages);
+
+      const openTagCount = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeTagCount = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openTagCount).toBe(1);
+      expect(closeTagCount).toBe(1);
+    }),
+  );
 });

package/src/components/ChatThread/sync.ts

@@ -3,15 +3,31 @@
 //
 
 import { type DXN } from '@dxos/echo';
-import { log } from '@dxos/log';
 import { type MarkdownStreamController } from '@dxos/react-ui-components';
 import { type ContentBlock, type Message } from '@dxos/types';
 import { type StateDispatch, type XmlWidgetStateManager } from '@dxos/ui-editor';
 
+import { rehydrateToolWidgetsFromMessages } from './tool-widget-state';
+
 /**
  * Update document.
  */
-export type TextModel = Pick<MarkdownStreamController, 'view' | 'reset' | 'append' | 'updateWidget'>;
+export type TextModel = Pick<MarkdownStreamController, 'length' | 'setContent' | 'append' | 'updateWidget'>;
+
+/**
+ * Renders a block to markdown.
+ *
+ * Contract: for any block whose lifetime spans multiple invocations (i.e. a streaming block
+ * with `pending: true` whose content grows over time, transitioning to `pending: false`), the
+ * sequence of returned strings must be monotonically extending — each subsequent value must be
+ * a string-extension of the previous. The {@link MessageSyncer} relies on this invariant to
+ * compute appendable deltas without diffing.
+ */
+export type BlockRenderer = (
+  context: MessageThreadContext,
+  message: Message.Message,
+  block: ContentBlock.Any,
+) => string | undefined;
 
 /**
  * Thread context passed to renderer.
@@ -23,132 +39,112 @@ export class MessageThreadContext implements Pick<MarkdownStreamController, 'upd
     this._widgetState?.updateWidget(id, value);
   }
 
-  // TODO(burdon): Resolve from hypergraph.
+  // TODO(burdon): Resolve name from hypergraph.
   getObjectLabel(_id: DXN) {
     return 'Object';
   }
 }
 
-/**
- * Renders a block to markdown.
- */
-export type BlockRenderer = (
-  context: MessageThreadContext,
-  message: Message.Message,
-  block: ContentBlock.Any,
-) => string | undefined;
-
 /**
  * Syncs messages with the editor.
+ *
+ * Reflects the AI streaming contract:
+ * - Messages and their blocks are appended in order.
+ * - Only the last block in `messages` may be `pending`; all earlier blocks are finalized.
+ * - The renderer's output for a streaming block grows monotonically (see {@link BlockRenderer}).
+ * - The document is read-only outside this syncer.
+ *
+ * Under those rules the syncer needs only:
+ * - `_completed`: the flat-block index past which everything has been fully appended.
+ * - `_trailing`: chars of the in-flight block (at index `_completed`) already appended.
+ * - `_threadId`: identity sentinel; if `messages[0]?.id` changes, the document is replaced.
  */
 export class MessageSyncer {
-  private _initialMessageId?: string;
-  private _currentMessageIndex = 0;
-  private _currentBlockIndex = 0;
-  private _currentBlockContent?: string;
+  private _threadId?: string;
+  private _completed = 0;
+  private _trailing = 0;
 
   private readonly _context: MessageThreadContext;
 
   constructor(
-    private readonly _model: TextModel,
-    private readonly _blockRenderer: BlockRenderer,
+    private readonly _document: TextModel,
+    private readonly _renderer: BlockRenderer,
   ) {
-    this._context = new MessageThreadContext(this._model);
+    this._context = new MessageThreadContext(this._document);
   }
 
   get context() {
     return this._context;
   }
 
-  reset() {
-    log('reset');
-    this._initialMessageId = undefined;
-    this._currentMessageIndex = 0;
-    this._currentBlockIndex = 0;
-    this._currentBlockContent = undefined;
-    void this._model.reset('');
+  /**
+   * Replace the document with the rendering of `messages`. Use on mount, on thread switch,
+   * and from {@link update} when it detects an identity change in `messages[0]`.
+   */
+  reset(messages: Message.Message[] = []): void {
+    this._threadId = messages[0]?.id;
+    this._completed = 0;
+    this._trailing = 0;
+    const buffer = this._walk(messages);
+    // Match the pre-rewrite behaviour: rendering from a steady state (initial mount with
+    // non-empty messages, or thread switch) lands the entire content via `setContent` — which
+    // uses `wireBypass`, so the editor jumps straight to the final text and `update()` returns
+    // `true` so the caller can scroll to the bottom. Live streaming partials that arrive
+    // *after* this initial render flow through `update()`'s incremental path → `_document.append`
+    // → wire's drip filter, preserving the char-by-char typewriter for incoming text.
+    void this._document.setContent(buffer).then(() => {
+      rehydrateToolWidgetsFromMessages(this._context, messages);
+    });
   }
 
   /**
-   * Syncs messages with the editor.
+   * Stream the suffix of the rendered messages into the document.
+   * Returns `true` if the document was replaced (initial mount or thread switch), `false`
+   * if the call was a streaming append (or a no-op).
    */
-  append(messages: Message.Message[], flush = false): boolean {
-    // Check if new set of messages.
-    if (this._initialMessageId !== messages[0]?.id) {
-      this.reset();
-      this._initialMessageId = messages[0]?.id;
-    }
-
-    if (flush && this._model.view?.state.doc.length === 0) {
-      const buffer: string[] = [];
-      this.process(messages, (content) => {
-        buffer.push(content);
-      });
-
-      const content = buffer.join('');
-      this._model.view?.dispatch({
-        changes: [
-          {
-            from: 0,
-            to: this._model.view?.state.doc.length ?? 0,
-            insert: content,
-          },
-        ],
-        selection: { anchor: content.length },
-      });
-
+  update(messages: Message.Message[]): boolean {
+    if (messages[0]?.id !== this._threadId) {
+      this.reset(messages);
       return true;
-    } else {
-      this.process(messages, (content) => {
-        void this._model.append(content);
-      });
-
-      return false;
     }
+    const buffer = this._walk(messages);
+    if (buffer.length > 0) {
+      void this._document.append(buffer);
+    }
+    return false;
   }
 
-  private process(messages: Message.Message[], append: (content: string) => void) {
-    log('sync', {
-      doc: this._model.view?.state.doc.length,
-      messages: messages.map((message) => message.blocks.length),
-      currentMessageIndex: this._currentMessageIndex,
-      currentBlockIndex: this._currentBlockIndex,
-      currentBlockContent: this._currentBlockContent,
-    });
-
-    let i = this._currentMessageIndex;
-    for (const message of messages.slice(this._currentMessageIndex)) {
-      if (i > this._currentMessageIndex) {
-        this._currentBlockIndex = 0;
-      }
-
-      this._currentMessageIndex = i;
-      let j = this._currentBlockIndex;
-      for (const block of message.blocks.slice(this._currentBlockIndex)) {
-        this._currentBlockIndex = j;
-        const currentBlockContent = this._blockRenderer(this._context, message, block);
-        if (currentBlockContent) {
-          let content: string = '';
-          if (this._currentBlockContent && currentBlockContent.startsWith(this._currentBlockContent)) {
-            content = currentBlockContent.slice(this._currentBlockContent.length);
-          } else {
-            content = currentBlockContent;
-          }
-
-          log('append', { message: i, block: j, content });
-          this._currentBlockContent = currentBlockContent;
-          append(content);
+  /**
+   * Walk flat blocks starting at `_completed`, advancing the cursors and returning the chars
+   * to append. Blocks before `_completed` are skipped — their renderer is never re-invoked,
+   * which preserves single-shot side effects (e.g. tool widget state mutation).
+   */
+  _walk(messages: Message.Message[]): string {
+    let buffer = '';
+    let index = 0;
+    outer: for (const message of messages) {
+      for (const block of message.blocks) {
+        if (index < this._completed) {
+          index++;
+          continue;
+        }
+        const rendered = this._renderer(this._context, message, block) ?? '';
+        if (rendered.length > this._trailing) {
+          buffer += rendered.slice(this._trailing);
         }
-
         if (block.pending) {
-          return;
-        } else {
-          this._currentBlockContent = undefined;
-          this._currentBlockIndex++;
+          // Stay on this block; record how far we've appended so the next call can resume.
+          // `Math.max`-style guard against a non-monotonic renderer output without shrinking the doc.
+          if (rendered.length > this._trailing) {
+            this._trailing = rendered.length;
+          }
+          break outer;
         }
-        j++;
+        this._completed = index + 1;
+        this._trailing = 0;
+        index++;
       }
-      i++;
     }
+    return buffer;
   }
 }