@dxos/plugin-assistant 0.8.4-main.4a85c3132b → 0.8.4-main.4f23b4e393

package/src/components/ChatThread/registry.tsx

@@ -5,63 +5,64 @@
 import React from 'react';
 
 import { log } from '@dxos/log';
-import { ToggleContainer } from '@dxos/react-ui-components';
+import { ContentBlock, type Message } from '@dxos/types';
+import { type XmlWidgetRegistry, getXmlTextChild } from '@dxos/ui-editor';
+
+import { type Assistant } from '../../types';
+import { type BlockRenderer, type MessageThreadContext } from './sync';
+import { applyToolBlockToWidgetState } from './tool-widget-state';
 import {
-  PromptWidget,
+  FallbackWidget,
   ReasoningWidget,
   ReferenceWidget,
   SelectWidget,
   SuggestionWidget,
+  StatsWidget,
   SummaryWidget,
-} from '@dxos/react-ui-components';
-import { Json } from '@dxos/react-ui-syntax-highlighter';
-import { ContentBlock, type Message } from '@dxos/types';
-import { type XmlWidgetProps, type XmlWidgetRegistry, getXmlTextChild } from '@dxos/ui-editor';
-
-import { ToolBlock } from '../ToolBlock';
-
-import { type BlockRenderer, type MessageThreadContext } from './sync';
-
-const Fallback = ({ _tag, ...props }: XmlWidgetProps<MessageThreadContext>) => {
-  return (
-    <ToggleContainer.Root classNames='rounded-xs'>
-      <ToggleContainer.Header classNames='bg-group-surface'>{_tag}</ToggleContainer.Header>
-      <ToggleContainer.Content classNames='bg-modal-surface'>
-        <Json classNames='p-2! text-sm' data={props} />
-      </ToggleContainer.Content>
-    </ToggleContainer.Root>
-  );
-};
-
-const Summary = ({ text }: { text: string }) => {
-  return (
-    <ToggleContainer.Root classNames='rounded-sm'>
-      <ToggleContainer.Header classNames='bg-group-surface'>Conversation summarized</ToggleContainer.Header>
-      <ToggleContainer.Content classNames='bg-modal-surface'>{text}</ToggleContainer.Content>
-    </ToggleContainer.Root>
-  );
-};
+  ToolWidget,
+  StatusWidget,
+} from './widgets';
 
 /**
  * Custom XML tags registry.
+ *
+ * NOTE: `<prompt>` has no widget — it is rendered via mark + line decorations
+ * (`xmlBlockDecoration`) in MarkdownStream so the prompt text remains part of the document
+ * and can be highlighted by `xmlFormatting`. It is still listed here so the markdown block
+ * parser (`xmlBlockParsers`) knows to keep `<prompt>...</prompt>` as a single HTMLBlock
+ * — otherwise an unregistered prompt opens a Paragraph that lazy-continues into the
+ * following lines and breaks parsing of subsequent multi-line tags (e.g. a `<reasoning>`
+ * block whose content contains an ordered list).
  */
 export const componentRegistry: XmlWidgetRegistry = {
   //
-  // Widgets
+  // Block-only (no widget — see note above).
   //
 
   prompt: {
     block: true,
-    factory: ({ children }) => {
+  },
+
+  //
+  // DOM Widgets
+  //
+
+  synthetic: {
+    block: true,
+    factory: ({ children, range }) => {
       const text = getXmlTextChild(children);
-      return text ? new PromptWidget(text) : null;
+      // TODO(dmaretskyi): Synthetic blocks are "user"-turn messages generated by the system, i.e. and event from a trigger, an email-encoded to JSON, etc.
+      //                   They are meant to start agent, to process the event without looking like user-authored prompt.
+      //                   Create a new widget type for them.
+      return text ? new ReasoningWidget(text, range.from) : null;
     },
   },
   reasoning: {
     block: true,
-    factory: ({ children }) => {
+    streaming: true,
+    factory: ({ children, range }) => {
       const text = getXmlTextChild(children);
-      return text ? new ReasoningWidget(text) : null;
+      return text ? new ReasoningWidget(text, range.from) : null;
     },
   },
   reference: {
@@ -91,29 +92,41 @@ export const componentRegistry: XmlWidgetRegistry = {
     block: true,
     factory: ({ children }) => {
       const text = getXmlTextChild(children);
-      return text ? new SummaryWidget(text) : null;
+      return text ? new StatsWidget(text) : null;
+    },
+  },
+  status: {
+    block: true,
+    streaming: true,
+    factory: ({ children }) => {
+      const text = getXmlTextChild(children);
+      return text ? new StatusWidget(text) : null;
     },
   },
 
   //
-  // React
+  // React Widgets (portaled outside of the editor)
   //
 
+  summary: {
+    block: true,
+    Component: SummaryWidget,
+  },
   toolCall: {
     block: true,
-    Component: ToolBlock,
+    Component: (props) => (
+      <div className='py-2'>
+        <ToolWidget {...props} />
+      </div>
+    ),
   },
   toolResult: {
     block: true,
-    Component: Fallback,
+    Component: FallbackWidget,
   },
   toolkit: {
     block: true,
-    Component: Fallback,
-  },
-  summary: {
-    block: true,
-    Component: Summary,
+    Component: FallbackWidget,
   },
 
   //
@@ -122,24 +135,58 @@ export const componentRegistry: XmlWidgetRegistry = {
 
   json: {
     block: true,
-    Component: Fallback,
+    Component: FallbackWidget,
   },
 };
 
 /**
  * Convert block to markdown.
  */
-export const blockToMarkdown: BlockRenderer = (
-  context: MessageThreadContext,
+export const blockToMarkdown: BlockRenderer = createBlockRenderer('normal');
+
+/**
+ * Create a {@link BlockRenderer} that filters blocks based on the chat view type.
+ *
+ * - `summary`: only user prompts.
+ * - `normal`: user prompts + assistant text + tool calls + summary/select/suggestion/reference + status. Hides reasoning.
+ * - `thinking`: same as normal plus reasoning.
+ * - `debug`: renders every block (raw fallbacks visible).
+ */
+export function createBlockRenderer(viewType: Assistant.ChatView | undefined): BlockRenderer {
+  return (context, message, block) => {
+    if (!isBlockVisible(viewType, message, block)) {
+      return;
+    }
+    let str = blockToMarkdownImpl(context, message, block);
+    if (str && !block.pending) {
+      // Use a blank line as the block separator so each rendered block parses as its own
+      // markdown block. A single newline lets CommonMark absorb a following `<prompt>` (an
+      // HTML type-7 tag, which can't interrupt an open paragraph) into the previous
+      // paragraph, rendering the prompt bubble inline next to the prior assistant text.
+      return (str += '\n');
+    }
+    return str;
+  };
+}
+
+const isBlockVisible = (
+  viewType: Assistant.ChatView | undefined,
   message: Message.Message,
   block: ContentBlock.Any,
-) => {
-  let str = blockToMarkdownImpl(context, message, block);
-  if (str && !block.pending) {
-    return (str += '\n');
+): boolean => {
+  switch (viewType) {
+    case 'debug':
+      return true;
+    case 'normal':
+      return block._tag !== 'reasoning';
+    case 'summary':
+      // Show only conversational text (user prompts + assistant replies); hide reasoning,
+      // tool calls, status, stats, and synthetic system messages.
+      return block._tag === 'text' && (block as ContentBlock.Text).disposition !== 'synthetic';
+    case 'thinking':
+    default:
+      return true;
   }
-
-  return str;
 };
 
 const blockToMarkdownImpl = (context: MessageThreadContext, message: Message.Message, block: ContentBlock.Any) => {
@@ -147,7 +194,11 @@ const blockToMarkdownImpl = (context: MessageThreadContext, message: Message.Mes
   switch (block._tag) {
     case 'text': {
       if (message.sender.role === 'user') {
-        return `<prompt>${block.text}</prompt>`;
+        if (block.disposition === 'synthetic') {
+          return renderXMLBlock('synthetic', { content: block.text, pending: block.pending });
+        } else {
+          return `\n<prompt>${block.text}</prompt>`;
+        }
       } else {
         const text = block.text.trim();
         if (text.length > 0) {
@@ -156,52 +207,84 @@ const blockToMarkdownImpl = (context: MessageThreadContext, message: Message.Mes
       }
       break;
     }
+
     case 'reference': {
+      if (block.pending) {
+        return;
+      }
       const dxn = block.reference.dxn;
       return `<reference ref="${dxn.toString()}">${context.getObjectLabel(dxn)}</reference>`;
     }
+
     case 'suggestion': {
       if (block.pending) {
         return;
       }
       return `<suggestion>${block.text}</suggestion>`;
     }
+
     case 'select': {
       if (block.pending || block.options.length === 0) {
         return;
       }
       return `<select>${block.options.map((option) => `<option>${option}</option>`).join('')}</select>`;
     }
+
     case 'toolCall': {
-      context.updateWidget<{ blocks: ContentBlock.Any[] }>(block.toolCallId, {
-        blocks: [block],
-      });
+      applyToolBlockToWidgetState(context, block);
       return `<toolCall id="${block.toolCallId}" />`;
     }
+
     case 'toolResult': {
       // TODO(dmaretskyi): the parameter could be undefined, perhaps tool blocks are not arriving in order.
-      context.updateWidget<{ blocks: ContentBlock.Any[] }>(block.toolCallId, ({ blocks = [] } = { blocks: [] }) => ({
-        blocks: [...blocks, block],
-      }));
+      applyToolBlockToWidgetState(context, block);
       break;
     }
+
     case 'stats': {
-      return `<stats>${ContentBlock.createStatsMessage(block)}</stats>`;
+      return '';
     }
+
     case 'reasoning': {
-      const text = block.reasoningText ?? block.redactedText;
+      let text = block.reasoningText ?? block.redactedText;
       if (!text) {
         return;
       }
-      // TODO(dmaretskyi): The mixed Markdown/XML parser does not support parsing multi-line XML tags.
-      return `<reasoning>${text.replace(/\n/g, ' ').trim()}</reasoning>`;
+      return renderXMLBlock('reasoning', { content: text, pending: block.pending });
     }
+
     case 'summary': {
-      return `<summary>${block.content}</summary>`;
+      return renderXMLBlock('summary', { content: block.content, pending: block.pending });
     }
+
+    case 'status': {
+      return renderXMLBlock('status', { content: block.statusText, pending: block.pending });
+    }
+
     default: {
       // TODO(burdon): Needs stable ID.
       return `<json id="${message.id}">\n${JSON.stringify(block)}\n</json>`;
     }
   }
 };
+
+/**
+ * Escape text embedded in generated XML so the mixed XML parser does not treat `&`, `<`, `>` as markup.
+ */
+const escapeXmlTextContent = (raw: string): string =>
+  raw.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+const renderXMLBlock = (tag: string, opts: { content?: string; pending?: boolean; attributes?: string }) => {
+  // Replace paragraph breaks so that the markdown parser does not split the content into multiple paragraphs.
+  // NOTE: do not strip leading list-marker prefixes (`-`, `*`, `1. `, ...) per line — the streaming
+  // tag's range is replaced atomically by the XML widget so the inner markdown is never user-visible,
+  // and stripping can collapse a line to empty mid-stream (e.g. when only `1. ` has arrived), breaking
+  // the prefix-extension contract that `MessageSyncer` relies on for incremental appends.
+  const content = escapeXmlTextContent((opts.content ?? '').replace(/\n\n/g, ' ').trim());
+
+  if (opts.pending) {
+    return `<${tag}${opts.attributes ? ` ${opts.attributes}` : ''}>${content}`;
+  } else {
+    return `<${tag}${opts.attributes ? ` ${opts.attributes}` : ''}>${content}</${tag}>`;
+  }
+};

package/src/components/ChatThread/sync.test.ts

@@ -1,34 +1,34 @@
-// @vitest-environment jsdom
-
 //
 // Copyright 2025 DXOS.org
 //
 
+// @vitest-environment jsdom
+
 import { EditorView } from '@codemirror/view';
 import { describe, it } from '@effect/vitest';
 import * as Effect from 'effect/Effect';
 
 import { Obj } from '@dxos/echo';
 import { type Mutable } from '@dxos/echo/internal';
-import { type ContentBlock } from '@dxos/types';
+import { type ContentBlock, type Message } from '@dxos/types';
 
-import { createMessage } from '../../testing';
+import { createMessage } from '#testing';
 
-import { blockToMarkdown } from './registry';
-import { MessageSyncer, type TextModel } from './sync';
+import { createBlockRenderer } from './registry';
+import { type BlockRenderer, MessageSyncer, type MessageThreadContext, type TextModel } from './sync';
 
 class TestDocument implements TextModel {
   private readonly _view = new EditorView({ extensions: [] });
 
-  get view() {
-    return this._view;
+  get length() {
+    return this._view.state.doc.length;
   }
 
   get content() {
     return this._view.state.doc.toString();
   }
 
-  async reset(text: string) {
+  async setContent(text: string) {
     this._view.dispatch({
       changes: { from: 0, to: this._view.state.doc.length, insert: text },
     });
@@ -48,21 +48,21 @@ describe('reducers', () => {
     'basic sync',
     Effect.fn(function* ({ expect }) {
       const doc = new TestDocument();
-      const syncer = new MessageSyncer(doc, blockToMarkdown);
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
 
       const messages = [
         createMessage('user', [{ _tag: 'text', text: 'Hello' }]),
         createMessage('assistant', [{ _tag: 'text', text: 'Hi there!' }]),
       ];
 
-      syncer.append(messages);
-      expect(doc.content).toEqual(['<prompt>Hello</prompt>', 'Hi there!', ''].join('\n'));
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there!\n');
 
-      Obj.change(messages[1], (m) => {
-        m.blocks.push({ _tag: 'text', text: 'How can I help?' });
+      Obj.update(messages[1], (obj) => {
+        obj.blocks.push({ _tag: 'text', text: 'How can I help?' });
       });
-      syncer.append(messages);
-      expect(doc.content).toEqual(['<prompt>Hello</prompt>', 'Hi there!', 'How can I help?', ''].join('\n'));
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there!\nHow can I help?\n');
     }),
   );
 
@@ -70,30 +70,188 @@ describe('reducers', () => {
     'sync with partial updates',
     Effect.fn(function* ({ expect }) {
       const doc = new TestDocument();
-      const syncer = new MessageSyncer(doc, blockToMarkdown);
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
 
       const messages = [
         createMessage('user', [{ _tag: 'text', text: 'Hello' }]),
         createMessage('assistant', [{ _tag: 'text', text: 'Hi there!', pending: true }]),
       ];
 
-      syncer.append(messages);
-      expect(doc.content).toEqual(['<prompt>Hello</prompt>', 'Hi there!'].join('\n'));
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there!');
 
-      Obj.change(messages[1], (m) => {
-        const block = m.blocks[0] as Mutable<ContentBlock.Text>;
+      Obj.update(messages[1], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Text>;
         block.text = 'Hi there! How are you?';
         block.pending = false;
       });
-      syncer.append(messages);
+      syncer.update(messages);
+
+      Obj.update(messages[1], (obj) => {
+        obj.blocks.push({ _tag: 'text', text: 'How can I help?' });
+      });
+      syncer.update(messages);
+      expect(doc.content).toEqual('\n<prompt>Hello</prompt>\nHi there! How are you?\nHow can I help?\n');
+    }),
+  );
+
+  // Regression: streaming reasoning text that passes through a bare list-marker state
+  // (e.g. `"…\n1. "`) used to make `stripBulletLikeLinePrefixes` collapse the line to empty,
+  // breaking the prefix-diff invariant in `MessageSyncer` and producing a duplicate `<reasoning>`
+  // opening tag in the document.
+  it.effect(
+    'streaming reasoning with list-marker transitions does not duplicate opening tag',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
+
+      const setReasoning = (message: Message.Message, text: string, pending: boolean) => {
+        Obj.update(message, (message) => {
+          const block = message.blocks[0] as Mutable<ContentBlock.Reasoning>;
+          block.reasoningText = text;
+          block.pending = pending;
+        });
+      };
+
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'abc\n1', pending: true }])];
+
+      // Tick 1: `"abc\n1"` — `\d+[.)]\s` regex does not match (no dot/space yet).
+      syncer.update(messages);
+
+      // Tick 2: `"abc\n1."` — still no match (no trailing whitespace).
+      setReasoning(messages[0], 'abc\n1.', true);
+      syncer.update(messages);
+
+      // Tick 3: `"abc\n1. "` — line `"1. "` matches and is stripped to empty.
+      setReasoning(messages[0], 'abc\n1. ', true);
+      syncer.update(messages);
+
+      // Tick 4: `"abc\n1. foo"` — list item with content.
+      setReasoning(messages[0], 'abc\n1. foo', true);
+      syncer.update(messages);
+
+      // Tick 5: finalize.
+      setReasoning(messages[0], 'abc\n1. foo', false);
+      syncer.update(messages);
+
+      const openTagCount = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeTagCount = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openTagCount).toBe(1);
+      expect(closeTagCount).toBe(1);
+    }),
+  );
+
+  // Regression: prompt "respond with your name inside an xml tag" produces a streamed
+  // reasoning block followed by a text block containing a non-registered XML tag
+  // (`<name>Claude</name>`). The text block must appear in the document AFTER the
+  // closed reasoning block — the bug symptom is the reasoning tag rendering with no
+  // follow-up response visible.
+  it.effect(
+    'reasoning block followed by text containing a non-registered xml tag',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+      const syncer = new MessageSyncer(doc, createBlockRenderer('thinking'));
+
+      const setReasoning = (message: Message.Message, text: string, pending: boolean) => {
+        Obj.update(message, (message) => {
+          const block = message.blocks[0] as Mutable<ContentBlock.Reasoning>;
+          block.reasoningText = text;
+          block.pending = pending;
+        });
+      };
+
+      const setText = (message: Message.Message, text: string, pending: boolean) => {
+        Obj.update(message, (message) => {
+          const block = message.blocks[1] as Mutable<ContentBlock.Text>;
+          block.text = text;
+          block.pending = pending;
+        });
+      };
+
+      // Tick 1: reasoning starts streaming.
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'Thinking', pending: true }])];
+      syncer.update(messages);
+
+      // Tick 2: reasoning grows.
+      setReasoning(messages[0], 'Thinking about the answer', true);
+      syncer.update(messages);
+
+      // Tick 3: reasoning closes.
+      setReasoning(messages[0], 'Thinking about the answer', false);
+      syncer.update(messages);
+
+      // Tick 4: text block appears, pending and empty (model has started emitting but text is still '').
+      Obj.update(messages[0], (message) => {
+        message.blocks.push({ _tag: 'text', text: '', pending: true });
+      });
+      syncer.update(messages);
+
+      // Tick 5: partial text — opening tag only.
+      setText(messages[0], '<name>', true);
+      syncer.update(messages);
+
+      // Tick 6: full text streamed.
+      setText(messages[0], '<name>Claude</name>', true);
+      syncer.update(messages);
 
-      Obj.change(messages[1], (m) => {
-        m.blocks.push({ _tag: 'text', text: 'How can I help?' });
+      // Tick 7: text finalised.
+      setText(messages[0], '<name>Claude</name>', false);
+      syncer.update(messages);
+
+      // Both the closed reasoning tag and the response text must be present.
+      expect(doc.content).toContain('<reasoning>Thinking about the answer</reasoning>');
+      expect(doc.content).toContain('<name>Claude</name>');
+
+      const openReasoning = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeReasoning = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openReasoning).toBe(1);
+      expect(closeReasoning).toBe(1);
+    }),
+  );
+
+  // Direct test of `MessageSyncer`'s tolerance for non-monotonic renderer output —
+  // any renderer that produces a shorter string for the same streaming block (e.g., due
+  // to whitespace normalisation or future transforms) must not produce duplicate output.
+  it.effect(
+    'non-monotonic renderer output does not duplicate previously-emitted content',
+    Effect.fn(function* ({ expect }) {
+      const doc = new TestDocument();
+
+      const renderer: BlockRenderer = (_ctx: MessageThreadContext, _msg: Message.Message, block: ContentBlock.Any) => {
+        if (block._tag !== 'reasoning') {
+          return undefined;
+        }
+        const text = block.reasoningText ?? '';
+        // Simulate a non-monotonic transform: collapse a line that is a sole digit+dot+space.
+        const normalised = text
+          .split(/\r?\n/)
+          .map((line) => line.replace(/^\s*\d+[.)]\s$/, ''))
+          .join('\n')
+          .trim();
+        return block.pending ? `<reasoning>${normalised}` : `<reasoning>${normalised}</reasoning>\n`;
+      };
+
+      const syncer = new MessageSyncer(doc, renderer);
+
+      const messages = [createMessage('assistant', [{ _tag: 'reasoning', reasoningText: 'abc\n1.', pending: true }])];
+
+      syncer.update(messages);
+      Obj.update(messages[0], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Reasoning>;
+        block.reasoningText = 'abc\n1. ';
       });
-      syncer.append(messages);
-      expect(doc.content).toEqual(
-        ['<prompt>Hello</prompt>', 'Hi there! How are you?', 'How can I help?', ''].join('\n'),
-      );
+      syncer.update(messages);
+      Obj.update(messages[0], (obj) => {
+        const block = obj.blocks[0] as Mutable<ContentBlock.Reasoning>;
+        block.reasoningText = 'abc\n1. tail';
+        block.pending = false;
+      });
+      syncer.update(messages);
+
+      const openTagCount = (doc.content.match(/<reasoning>/g) ?? []).length;
+      const closeTagCount = (doc.content.match(/<\/reasoning>/g) ?? []).length;
+      expect(openTagCount).toBe(1);
+      expect(closeTagCount).toBe(1);
     }),
   );
 });