@dxos/plugin-assistant 0.8.4-main.c4373fc → 0.8.4-main.d9fc60f731

package/src/components/ChatThread/registry.tsx

@@ -5,128 +5,200 @@
 import React from 'react';
 
 import { log } from '@dxos/log';
+import { ContentBlock, type Message } from '@dxos/types';
+import { type XmlWidgetRegistry, getXmlTextChild } from '@dxos/ui-editor';
+
+import { type Assistant } from '../../types';
+import { type BlockRenderer, type MessageThreadContext } from './sync';
+import { applyToolBlockToWidgetState } from './tool-widget-state';
 import {
-  PromptWidget,
+  FallbackWidget,
+  ReasoningWidget,
   ReferenceWidget,
   SelectWidget,
   SuggestionWidget,
+  StatsWidget,
   SummaryWidget,
-  ToggleContainer,
-} from '@dxos/react-ui-components';
-import { type XmlWidgetProps, type XmlWidgetRegistry, getXmlTextChild } from '@dxos/react-ui-editor';
-import { Json } from '@dxos/react-ui-syntax-highlighter';
-import { ContentBlock, type DataType } from '@dxos/schema';
-
-import { ToolBlock } from '../ToolBlock';
-
-import { type BlockRenderer, type MessageThreadContext } from './sync';
-
-const Fallback = ({ _tag, ...props }: XmlWidgetProps<MessageThreadContext>) => {
-  return (
-    <ToggleContainer.Root classNames='rounded-sm'>
-      <ToggleContainer.Header classNames='bg-groupSurface'>{_tag}</ToggleContainer.Header>
-      <ToggleContainer.Content classNames='bg-modalSurface'>
-        <Json classNames='!p-2 text-sm' data={props} />
-      </ToggleContainer.Content>
-    </ToggleContainer.Root>
-  );
-};
+  ToolWidget,
+  StatusWidget,
+} from './widgets';
 
 /**
  * Custom XML tags registry.
+ *
+ * NOTE: `<prompt>` has no widget — it is rendered via mark + line decorations
+ * (`xmlBlockDecoration`) in MarkdownStream so the prompt text remains part of the document
+ * and can be highlighted by `xmlFormatting`. It is still listed here so the markdown block
+ * parser (`xmlBlockParsers`) knows to keep `<prompt>...</prompt>` as a single HTMLBlock
+ * — otherwise an unregistered prompt opens a Paragraph that lazy-continues into the
+ * following lines and breaks parsing of subsequent multi-line tags (e.g. a `<reasoning>`
+ * block whose content contains an ordered list).
  */
 export const componentRegistry: XmlWidgetRegistry = {
   //
-  // Widgets
+  // Block-only (no widget — see note above).
   //
 
-  ['prompt' as const]: {
+  prompt: {
     block: true,
-    factory: (props) => {
-      const text = getXmlTextChild(props.children);
-      return text ? new PromptWidget(text) : null;
+  },
+
+  //
+  // DOM Widgets
+  //
+
+  synthetic: {
+    block: true,
+    factory: ({ children, range }) => {
+      const text = getXmlTextChild(children);
+      // TODO(dmaretskyi): Synthetic blocks are "user"-turn messages generated by the system, i.e. and event from a trigger, an email-encoded to JSON, etc.
+      //                   They are meant to start agent, to process the event without looking like user-authored prompt.
+      //                   Create a new widget type for them.
+      return text ? new ReasoningWidget(text, range.from) : null;
     },
   },
-  ['reference' as const]: {
+  reasoning: {
+    block: true,
+    streaming: true,
+    factory: ({ children, range }) => {
+      const text = getXmlTextChild(children);
+      return text ? new ReasoningWidget(text, range.from) : null;
+    },
+  },
+  reference: {
     block: false,
-    factory: (props) => {
-      const text = getXmlTextChild(props.children);
-      return text && props.ref ? new ReferenceWidget(text, props.ref) : null;
+    factory: ({ children, ref }) => {
+      const text = getXmlTextChild(children);
+      return text && ref ? new ReferenceWidget(text, ref) : null;
     },
   },
-  ['select' as const]: {
+  select: {
     block: true,
-    factory: (props) => {
-      const options = props.children
+    factory: ({ children }) => {
+      const options = children
         ?.map((option: any) => option._tag === 'option' && getXmlTextChild(option.children))
         .filter(Boolean);
       return options?.length ? new SelectWidget(options) : null;
     },
   },
-  ['suggestion' as const]: {
+  suggestion: {
     block: true,
-    factory: (props) => {
-      const text = getXmlTextChild(props.children);
+    factory: ({ children }) => {
+      const text = getXmlTextChild(children);
       return text ? new SuggestionWidget(text) : null;
     },
   },
-  ['summary' as const]: {
+  stats: {
+    block: true,
+    factory: ({ children }) => {
+      const text = getXmlTextChild(children);
+      return text ? new StatsWidget(text) : null;
+    },
+  },
+  status: {
     block: true,
-    factory: (props) => {
-      const text = getXmlTextChild(props.children);
-      return text ? new SummaryWidget(text) : null;
+    streaming: true,
+    factory: ({ children }) => {
+      const text = getXmlTextChild(children);
+      return text ? new StatusWidget(text) : null;
     },
   },
 
   //
-  // React
+  // React Widgets (portaled outside of the editor)
   //
 
-  ['toolCall' as const]: {
+  summary: {
     block: true,
-    Component: ToolBlock,
+    Component: SummaryWidget,
   },
-  ['toolResult' as const]: {
+  toolCall: {
     block: true,
-    Component: Fallback,
+    Component: (props) => (
+      <div className='py-2'>
+        <ToolWidget {...props} />
+      </div>
+    ),
   },
-  ['toolkit' as const]: {
+  toolResult: {
     block: true,
-    Component: Fallback,
+    Component: FallbackWidget,
+  },
+  toolkit: {
+    block: true,
+    Component: FallbackWidget,
   },
 
   //
   // Fallback
   //
 
-  ['json' as const]: {
+  json: {
     block: true,
-    Component: Fallback,
+    Component: FallbackWidget,
   },
 };
 
 /**
  * Convert block to markdown.
  */
-export const blockToMarkdown: BlockRenderer = (
-  context: MessageThreadContext,
-  message: DataType.Message,
+export const blockToMarkdown: BlockRenderer = createBlockRenderer('normal');
+
+/**
+ * Create a {@link BlockRenderer} that filters blocks based on the chat view type.
+ *
+ * - `summary`: only user prompts.
+ * - `normal`: user prompts + assistant text + tool calls + summary/select/suggestion/reference + status. Hides reasoning.
+ * - `thinking`: same as normal plus reasoning.
+ * - `debug`: renders every block (raw fallbacks visible).
+ */
+export function createBlockRenderer(viewType: Assistant.ChatView | undefined): BlockRenderer {
+  return (context, message, block) => {
+    if (!isBlockVisible(viewType, message, block)) {
+      return;
+    }
+    let str = blockToMarkdownImpl(context, message, block);
+    if (str && !block.pending) {
+      // Use a blank line as the block separator so each rendered block parses as its own
+      // markdown block. A single newline lets CommonMark absorb a following `<prompt>` (an
+      // HTML type-7 tag, which can't interrupt an open paragraph) into the previous
+      // paragraph, rendering the prompt bubble inline next to the prior assistant text.
+      return (str += '\n');
+    }
+    return str;
+  };
+}
+
+const isBlockVisible = (
+  viewType: Assistant.ChatView | undefined,
+  message: Message.Message,
   block: ContentBlock.Any,
-) => {
-  let str = _blockToMarkdown(context, message, block);
-  if (str && !block.pending) {
-    return (str += '\n');
+): boolean => {
+  switch (viewType) {
+    case 'debug':
+      return true;
+    case 'normal':
+      return block._tag !== 'reasoning';
+    case 'summary':
+      // Show only conversational text (user prompts + assistant replies); hide reasoning,
+      // tool calls, status, stats, and synthetic system messages.
+      return block._tag === 'text' && (block as ContentBlock.Text).disposition !== 'synthetic';
+    case 'thinking':
+    default:
+      return true;
   }
-
-  return str;
 };
 
-const _blockToMarkdown = (context: MessageThreadContext, message: DataType.Message, block: ContentBlock.Any) => {
+const blockToMarkdownImpl = (context: MessageThreadContext, message: Message.Message, block: ContentBlock.Any) => {
   log('blockToMarkdown', { block: JSON.stringify(block) });
   switch (block._tag) {
     case 'text': {
       if (message.sender.role === 'user') {
-        return `\n<prompt>${block.text}</prompt>\n`;
+        if (block.disposition === 'synthetic') {
+          return renderXMLBlock('synthetic', { content: block.text, pending: block.pending });
+        } else {
+          return `\n<prompt>${block.text}</prompt>`;
+        }
       } else {
         const text = block.text.trim();
         if (text.length > 0) {
@@ -136,49 +208,83 @@ const _blockToMarkdown = (context: MessageThreadContext, message: DataType.Messa
       break;
     }
 
+    case 'reference': {
+      if (block.pending) {
+        return;
+      }
+      const dxn = block.reference.uri;
+      return `<reference ref="${dxn.toString()}">${context.getObjectLabel(dxn)}</reference>`;
+    }
+
     case 'suggestion': {
       if (block.pending) {
         return;
       }
       return `<suggestion>${block.text}</suggestion>`;
     }
+
     case 'select': {
       if (block.pending || block.options.length === 0) {
         return;
       }
-
       return `<select>${block.options.map((option) => `<option>${option}</option>`).join('')}</select>`;
     }
 
-    // TODO(burdon): Need label.
-    case 'reference': {
-      return `<reference ref="${block.reference.dxn.toString()}">Ref</reference>`;
-    }
-
-    // case 'toolkit': {
-    //   return `<toolkit />`;
-    // }
-
     case 'toolCall': {
-      context.updateWidget(block.toolCallId, {
-        blocks: [block],
-      });
+      applyToolBlockToWidgetState(context, block);
       return `<toolCall id="${block.toolCallId}" />`;
     }
+
     case 'toolResult': {
-      context.updateWidget(block.toolCallId, ({ blocks = [] }: { blocks: ContentBlock.Any[] }) => ({
-        blocks: [...blocks, block],
-      }));
+      // TODO(dmaretskyi): the parameter could be undefined, perhaps tool blocks are not arriving in order.
+      applyToolBlockToWidgetState(context, block);
       break;
     }
 
+    case 'stats': {
+      return '';
+    }
+
+    case 'reasoning': {
+      let text = block.reasoningText ?? block.redactedText;
+      if (!text) {
+        return;
+      }
+      return renderXMLBlock('reasoning', { content: text, pending: block.pending });
+    }
+
     case 'summary': {
-      return `<summary>${ContentBlock.createSummaryMessage(block)}</summary>`;
+      return renderXMLBlock('summary', { content: block.content, pending: block.pending });
+    }
+
+    case 'status': {
+      return renderXMLBlock('status', { content: block.statusText, pending: block.pending });
     }
 
-    // TODO(burdon): Need stable ID.
     default: {
+      // TODO(burdon): Needs stable ID.
       return `<json id="${message.id}">\n${JSON.stringify(block)}\n</json>`;
     }
   }
 };
+
+/**
+ * Escape text embedded in generated XML so the mixed XML parser does not treat `&`, `<`, `>` as markup.
+ */
+const escapeXmlTextContent = (raw: string): string =>
+  raw.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+const renderXMLBlock = (tag: string, opts: { content?: string; pending?: boolean; attributes?: string }) => {
+  // Replace paragraph breaks so that the markdown parser does not split the content into multiple paragraphs.
+  // NOTE: do not strip leading list-marker prefixes (`-`, `*`, `1. `, ...) per line — the streaming
+  // tag's range is replaced atomically by the XML widget so the inner markdown is never user-visible,
+  // and stripping can collapse a line to empty mid-stream (e.g. when only `1. ` has arrived), breaking
+  // the prefix-extension contract that `MessageSyncer` relies on for incremental appends.
+  const content = escapeXmlTextContent((opts.content ?? '').replace(/\n\n/g, ' ').trim());
+
+  if (opts.pending) {
+    return `<${tag}${opts.attributes ? ` ${opts.attributes}` : ''}>${content}`;
+  } else {
+    return `<${tag}${opts.attributes ? ` ${opts.attributes}` : ''}>${content}</${tag}>`;
+  }
+};

package/src/components/ChatThread/sync.test.ts

@@ -2,30 +2,42 @@
 // Copyright 2025 DXOS.org
 //
 
+// @vitest-environment jsdom
+
 import { EditorView } from '@codemirror/view';
 import { describe, it } from '@effect/vitest';
 import * as Effect from 'effect/Effect';
 
-import { type ContentBlock } from '@dxos/schema';
+import { Obj } from '@dxos/echo';
+import { type Mutable } from '@dxos/echo/internal';
+import { type ContentBlock, type Message } from '@dxos/types';
 
-import { createMessage } from '../../testing';
+import { createMessage } from '#testing';
 
-import { blockToMarkdown } from './registry';
-import { MessageSyncer, type TextModel } from './sync';
+import { createBlockRenderer } from './registry';
+import { type BlockRenderer, MessageSyncer, type MessageThreadContext, type TextModel } from './sync';
 
 class TestDocument implements TextModel {
   private readonly _view = new EditorView({ extensions: [] });
 
+  get length() {
+    return this._view.state.doc.length;
+  }
+
   get content() {
     return this._view.state.doc.toString();
   }
 
-  async reset(text: string) {
-    this._view.dispatch({ changes: { from: 0, to: this._view.state.doc.length, insert: text } });
+  async setContent(text: string) {
+    this._view.dispatch({
+      changes: { from: 0, to: this._view.state.doc.length, insert: text },
+    });
   }
 
   async append(text: string) {
-    this._view.dispatch({ changes: { from: this._view.state.doc.length, insert: text } });
+    this._view.dispatch({
+      changes: { from: this._view.state.doc.length, insert: text },
+    });
   }
 
   updateWidget(_id: string, _value: any) {}
@@ -36,19 +48,21 @@ describe('reducers', () => {
     'basic sync',
     Effect.fn(function* ({ expect }) {
       const doc = new TestDocument();
-      const syncer = new MessageSyncer(doc, blockToMarkdown);
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
 
       const messages = [
         createMessage('user', [{ _tag: 'text', text: 'Hello' }]),
         createMessage('assistant', [{ _tag: 'text', text: 'Hi there!' }]),
       ];
 
-      syncer.sync(messages);
-      expect(doc.content).toEqual(['\n<prompt>Hello</prompt>\n', 'Hi there!', ''].join('\n'));
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there!\n');
 
-      messages[1].blocks.push({ _tag: 'text', text: 'How can I help?' });
-      syncer.sync(messages);
-      expect(doc.content).toEqual(['\n<prompt>Hello</prompt>\n', 'Hi there!', 'How can I help?', ''].join('\n'));
+      Obj.update(messages[1], (obj) => {
+        obj.blocks.push({ _tag: 'text', text: 'How can I help?' });
+      });
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there!\nHow can I help?\n');
     }),
   );
 
@@ -56,26 +70,188 @@ describe('reducers', () => {
     'sync with partial updates',
     Effect.fn(function* ({ expect }) {
       const doc = new TestDocument();
-      const syncer = new MessageSyncer(doc, blockToMarkdown);
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
 
       const messages = [
         createMessage('user', [{ _tag: 'text', text: 'Hello' }]),
         createMessage('assistant', [{ _tag: 'text', text: 'Hi there!', pending: true }]),
       ];
 
-      syncer.sync(messages);
-      expect(doc.content).toEqual(['\n<prompt>Hello</prompt>\n', 'Hi there!'].join('\n'));
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there!');
+
+      Obj.update(messages[1], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Text>;
+        block.text = 'Hi there! How are you?';
+        block.pending = false;
+      });
+      syncer.update(messages);
+
+      Obj.update(messages[1], (obj) => {
+        obj.blocks.push({ _tag: 'text', text: 'How can I help?' });
+      });
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there! How are you?\nHow can I help?\n');
+    }),
+  );
+
+  // Regression: streaming reasoning text that passes through a bare list-marker state
+  // (e.g. `"…\n1. "`) used to make `stripBulletLikeLinePrefixes` collapse the line to empty,
+  // breaking the prefix-diff invariant in `MessageSyncer` and producing a duplicate `<reasoning>`
+  // opening tag in the document.
+  it.effect(
+    'streaming reasoning with list-marker transitions does not duplicate opening tag',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
+
+      const setReasoning = (message: Message.Message, text: string, pending: boolean) => {
+        Obj.update(message, (message) => {
+          const block = message.blocks[0] as Mutable<ContentBlock.Reasoning>;
+          block.reasoningText = text;
+          block.pending = pending;
+        });
+      };
+
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'abc\n1', pending: true }])];
+
+      // Tick 1: `"abc\n1"` — `\d+[.)]\s` regex does not match (no dot/space yet).
+      syncer.update(messages);
+
+      // Tick 2: `"abc\n1."` — still no match (no trailing whitespace).
+      setReasoning(messages[0], 'abc\n1.', true);
+      syncer.update(messages);
+
+      // Tick 3: `"abc\n1. "` — line `"1. "` matches and is stripped to empty.
+      setReasoning(messages[0], 'abc\n1. ', true);
+      syncer.update(messages);
+
+      // Tick 4: `"abc\n1. foo"` — list item with content.
+      setReasoning(messages[0], 'abc\n1. foo', true);
+      syncer.update(messages);
+
+      // Tick 5: finalize.
+      setReasoning(messages[0], 'abc\n1. foo', false);
+      syncer.update(messages);
+
+      const openTagCount = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeTagCount = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openTagCount).toBe(1);
+      expect(closeTagCount).toBe(1);
+    }),
+  );
+
+  // Regression: prompt "respond with your name inside an xml tag" produces a streamed
+  // reasoning block followed by a text block containing a non-registered XML tag
+  // (`<name>Claude</name>`). The text block must appear in the document AFTER the
+  // closed reasoning block — the bug symptom is the reasoning tag rendering with no
+  // follow-up response visible.
+  it.effect(
+    'reasoning block followed by text containing a non-registered xml tag',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
+
+      const setReasoning = (message: Message.Message, text: string, pending: boolean) => {
+        Obj.update(message, (message) => {
+          const block = message.blocks[0] as Mutable<ContentBlock.Reasoning>;
+          block.reasoningText = text;
+          block.pending = pending;
+        });
+      };
+
+      const setText = (message: Message.Message, text: string, pending: boolean) => {
+        Obj.update(message, (message) => {
+          const block = message.blocks[1] as Mutable<ContentBlock.Text>;
+          block.text = text;
+          block.pending = pending;
+        });
+      };
+
+      // Tick 1: reasoning starts streaming.
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'Thinking', pending: true }])];
+      syncer.update(messages);
+
+      // Tick 2: reasoning grows.
+      setReasoning(messages[0], 'Thinking about the answer', true);
+      syncer.update(messages);
+
+      // Tick 3: reasoning closes.
+      setReasoning(messages[0], 'Thinking about the answer', false);
+      syncer.update(messages);
+
+      // Tick 4: text block appears, pending and empty (model has started emitting but text is still '').
+      Obj.update(messages[0], (message) => {
+        message.blocks.push({ _tag: 'text', text: '', pending: true });
+      });
+      syncer.update(messages);
+
+      // Tick 5: partial text — opening tag only.
+      setText(messages[0], '<name>', true);
+      syncer.update(messages);
+
+      // Tick 6: full text streamed.
+      setText(messages[0], '<name>Claude</name>', true);
+      syncer.update(messages);
+
+      // Tick 7: text finalised.
+      setText(messages[0], '<name>Claude</name>', false);
+      syncer.update(messages);
+
+      // Both the closed reasoning tag and the response text must be present.
+      expect(doc.content).toContain('<reasoning>Thinking about the answer</reasoning>');
+      expect(doc.content).toContain('<name>Claude</name>');
+
+      const openReasoning = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeReasoning = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openReasoning).toBe(1);
+      expect(closeReasoning).toBe(1);
+    }),
+  );
+
+  // Direct test of `MessageSyncer`'s tolerance for non-monotonic renderer output —
+  // any renderer that produces a shorter string for the same streaming block (e.g., due
+  // to whitespace normalisation or future transforms) must not produce duplicate output.
+  it.effect(
+    'non-monotonic renderer output does not duplicate previously-emitted content',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+
+      const renderer: BlockRenderer = (_ctx: MessageThreadContext, _msg: Message.Message, block: ContentBlock.Any) => {
+        if (block._tag !== 'reasoning') {
+          return undefined;
+        }
+        const text = block.reasoningText ?? '';
+        // Simulate a non-monotonic transform: collapse a line that is a sole digit+dot+space.
+        const normalised = text
+          .split(/\r?\n/)
+          .map((line) => line.replace(/^\s*\d+[.)]\s$/, ''))
+          .join('\n')
+          .trim();
+        return block.pending ? `<reasoning>${normalised}` : `<reasoning>${normalised}</reasoning>\n`;
+      };
+
+      const syncer = new MessageSyncer(doc, renderer);
+
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'abc\n1.', pending: true }])];
 
-      const block = messages[1].blocks[0] as ContentBlock.Text;
-      block.text = 'Hi there! How are you?';
-      block.pending = false;
-      syncer.sync(messages);
+      syncer.update(messages);
+      Obj.update(messages[0], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Reasoning>;
+        block.reasoningText = 'abc\n1. ';
+      });
+      syncer.update(messages);
+      Obj.update(messages[0], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Reasoning>;
+        block.reasoningText = 'abc\n1. tail';
+        block.pending = false;
+      });
+      syncer.update(messages);
 
-      messages[1].blocks.push({ _tag: 'text', text: 'How can I help?' });
-      syncer.sync(messages);
-      expect(doc.content).toEqual(
-        ['\n<prompt>Hello</prompt>\n', 'Hi there! How are you?', 'How can I help?', ''].join('\n'),
-      );
+      const openTagCount = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeTagCount = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openTagCount).toBe(1);
+      expect(closeTagCount).toBe(1);
     }),
   );
 });