@dxos/plugin-assistant 0.8.2-main.f11618f → 0.8.2-staging.42af850

package/src/components/Thread/ThreadContainer.stories.tsx

@@ -7,6 +7,7 @@ import '@dxos-theme';
 import { type StoryObj, type Meta } from '@storybook/react';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
+import { AIServiceEdgeClient, Message } from '@dxos/ai';
 import {
   Capabilities,
   Events,
@@ -17,22 +18,22 @@ import {
   useIntentDispatcher,
 } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
-import { Message, type Tool } from '@dxos/artifact';
-import { genericTools, localServiceEndpoints, type IsObject } from '@dxos/artifact-testing';
-import { AIServiceEdgeClient } from '@dxos/assistant';
-import { create, ObjectId } from '@dxos/echo-schema';
+import { remoteServiceEndpoints } from '@dxos/artifact-testing';
+import { Type, type Obj } from '@dxos/echo';
+import { createQueueDxn, create, Query, Filter } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';
-import { DXN } from '@dxos/keys';
 import { ChessPlugin } from '@dxos/plugin-chess';
 import { ChessType } from '@dxos/plugin-chess/types';
 import { ClientPlugin } from '@dxos/plugin-client';
 import { InboxPlugin } from '@dxos/plugin-inbox';
 import { MapPlugin } from '@dxos/plugin-map';
+import { MarkdownPlugin } from '@dxos/plugin-markdown';
 import { SpacePlugin } from '@dxos/plugin-space';
 import { TablePlugin } from '@dxos/plugin-table';
-import { createQueueDxn, useQueue, useSpace } from '@dxos/react-client/echo';
+import { useClient } from '@dxos/react-client';
+import { useQueue, useQuery } from '@dxos/react-client/echo';
 import { IconButton, Input, Toolbar } from '@dxos/react-ui';
-import { mx } from '@dxos/react-ui-theme';
+import { descriptionMessage, mx } from '@dxos/react-ui-theme';
 import { withLayout, withTheme } from '@dxos/storybook-utils';
 
 import { Thread, type ThreadProps } from './Thread';
@@ -40,18 +41,22 @@ import { ChatProcessor } from '../../hooks';
 import { createProcessorOptions } from '../../testing';
 import translations from '../../translations';
 
-const endpoints = localServiceEndpoints;
+// TODO(burdon): Configure for local with ollama/LM studio.
+// const endpoints = localServiceEndpoints;
+const endpoints = remoteServiceEndpoints;
 
 type RenderProps = {
-  items?: IsObject[];
+  items?: Obj.Any[];
   prompts?: string[];
 } & Pick<ThreadProps, 'debug'>;
 
 // TODO(burdon): Use ChatContainer.
 const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) => {
-  const space = useSpace();
+  const client = useClient();
+  const space = client.spaces.default;
+
   const artifactDefinitions = useCapabilities(Capabilities.ArtifactDefinition);
-  const tools = useMemo<Tool[]>(() => [...genericTools], []);
+  const tools = useCapabilities(Capabilities.Tools);
 
   const [aiClient] = useState(() => new AIServiceEdgeClient({ endpoint: endpoints.ai }));
   const { dispatchPromise: dispatch } = useIntentDispatcher();
@@ -65,7 +70,7 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
 
     return new ChatProcessor(
       aiClient,
-      tools,
+      tools.flat(),
       artifactDefinitions,
       {
         space,
@@ -76,11 +81,17 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
   }, [aiClient, tools, space, dispatch, artifactDefinitions]);
 
   // Queue.
-  const [queueDxn, setQueueDxn] = useState<string>(() => createQueueDxn().toString());
-  const queue = useQueue<Message>(DXN.tryParse(queueDxn));
+  const [queueDxn, setQueueDxn] = useState<string>(() => createQueueDxn(space.id).toString());
+  const queue = useQueue<Message>(Type.DXN.tryParse(queueDxn));
+
+  useEffect(() => {
+    if (space) {
+      setQueueDxn(createQueueDxn(space.id).toString());
+    }
+  }, [space]);
 
   useEffect(() => {
-    if (queue?.items.length === 0 && !queue.isLoading && prompts.length > 0) {
+    if (queue?.objects.length === 0 && !queue.isLoading && prompts.length > 0) {
       queue.append([
         create(Message, {
           role: 'assistant',
@@ -95,48 +106,52 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
         }),
       ]);
     }
-  }, [queueDxn, prompts, queue?.items.length, queue?.isLoading]);
+  }, [queueDxn, prompts, queue?.objects.length, queue?.isLoading]);
 
   // State.
-  const artifactItems: any[] = []; // TODO(burdon): Query from space.
-  const messages = [...(queue?.items ?? []), ...(processor?.messages.value ?? [])];
-
-  const handleSubmit = processor
-    ? (message: string) => {
-        requestAnimationFrame(async () => {
-          invariant(processor);
-          if (processor.streaming.value) {
-            await processor.cancel();
-          }
-
-          invariant(queue);
-          await processor.request(message, {
-            history: queue.items,
-            onComplete: (messages) => {
-              queue.append(messages);
-            },
-          });
-        });
+  const query = useMemo(
+    () => Query.select(Filter.or(...artifactDefinitions.map((definition) => Filter.type(definition.schema)))),
+    [artifactDefinitions],
+  );
+  const artifactItems = useQuery(space, query);
+  const messages = [...(queue?.objects ?? []), ...(processor?.messages.value ?? [])];
+
+  const handleSubmit = useCallback(
+    (message: string) => {
+      requestAnimationFrame(async () => {
+        if (!processor || !queue) {
+          return;
+        }
+
+        if (processor.streaming.value) {
+          await processor.cancel();
+        }
 
-        return true;
-      }
-    : undefined;
+        await processor.request(message, {
+          history: queue.objects,
+          onComplete: (messages) => {
+            queue.append(messages);
+          },
+        });
+      });
 
-  const handlePrompt = useCallback(
-    (text: string) => {
-      void handleSubmit?.(text);
+      return true;
     },
-    [handleSubmit],
+    [processor, queue],
   );
 
   const handleDelete = useCallback(
     (id: string) => {
-      invariant(ObjectId.isValid(id), 'Invalid message id');
+      invariant(Type.ObjectId.isValid(id), 'Invalid message id');
       void queue?.delete([id]);
     },
     [queue],
   );
 
+  if (!space) {
+    return <></>;
+  }
+
   return (
     <div className='grid grid-cols-2 w-full h-full divide-x divide-separator overflow-hidden'>
       {/* Thread */}
@@ -159,7 +174,7 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
               iconOnly
               label='Clear history'
               icon='ph--trash--regular'
-              onClick={() => setQueueDxn(createQueueDxn().toString())}
+              onClick={() => setQueueDxn(createQueueDxn(space.id).toString())}
             />
             <IconButton iconOnly label='Stop' icon='ph--stop--regular' onClick={() => processor?.cancel()} />
           </Input.Root>
@@ -172,7 +187,7 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
           error={processor?.error.value}
           tools={processor?.tools}
           onSubmit={processor ? handleSubmit : undefined}
-          onPrompt={processor ? handlePrompt : undefined}
+          onPrompt={processor ? handleSubmit : undefined}
           onDelete={processor ? handleDelete : undefined}
           {...props}
         />
@@ -182,7 +197,7 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
       <div className='overflow-hidden grid grid-rows-[2fr_1fr] divide-y divide-separator'>
         {artifactItems.length > 0 && (
           <div className={mx('flex grow overflow-hidden', artifactItems.length === 1 && 'row-span-2')}>
-            <Surface role='canvas-node' limit={1} data={artifactItems[0]} />
+            <Surface role='article' limit={1} data={{ subject: artifactItems[0] }} fallback={Fallback} />
           </div>
         )}
 
@@ -190,7 +205,7 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
           <div className='flex shrink-0 overflow-hidden divide-x divide-separator'>
             <div className='flex flex-1 h-full'>
               {artifactItems.slice(1, 3).map((item, idx) => (
-                <Surface key={idx} role='canvas-node' limit={1} data={item} />
+                <Surface key={idx} role='article' limit={1} data={{ subject: item }} fallback={Fallback} />
               ))}
             </div>
           </div>
@@ -201,7 +216,7 @@ const DefaultStory = ({ items: _items, prompts = [], ...props }: RenderProps) =>
 };
 
 const meta: Meta<typeof DefaultStory> = {
-  title: 'plugins/plugin-automation/ThreadContainer',
+  title: 'plugins/plugin-assistant/ThreadContainer',
   render: DefaultStory,
   decorators: [
     withPluginManager({
@@ -219,12 +234,13 @@ const meta: Meta<typeof DefaultStory> = {
         ChessPlugin(),
         InboxPlugin(),
         MapPlugin(),
+        MarkdownPlugin(),
         TablePlugin(),
       ],
       fireEvents: [Events.SetupArtifactDefinition],
     }),
     withTheme,
-    withLayout({ fullscreen: true, tooltips: true }),
+    withLayout({ fullscreen: true }),
   ],
   parameters: {
     translations,
@@ -252,3 +268,17 @@ export const WithInitialItems: Story = {
     ],
   },
 };
+
+const Fallback = ({ error }: { error?: Error }) => {
+  const errorString = error?.toString() ?? '';
+  return (
+    <div role='none' className='overflow-auto p-8 attention-surface grid place-items-center'>
+      <p
+        role='alert'
+        className={mx(descriptionMessage, 'break-words rounded-lg p-8', errorString.length < 256 && 'text-lg')}
+      >
+        {error ? errorString : 'error'}
+      </p>
+    </div>
+  );
+};

package/src/components/Thread/ThreadContainer.tsx

@@ -8,7 +8,7 @@ import { CollaborationActions, createIntent, useIntentDispatcher } from '@dxos/a
 import { type AssociatedArtifact } from '@dxos/artifact';
 import { invariant } from '@dxos/invariant';
 import { DXN } from '@dxos/keys';
-import { makeRef, refFromDXN } from '@dxos/live-object';
+import { refFromDXN } from '@dxos/live-object';
 import { log } from '@dxos/log';
 import { getSpace } from '@dxos/react-client/echo';
 import { type ThemedClassName } from '@dxos/react-ui';
@@ -22,7 +22,7 @@ export type ThreadContainerProps = {
   settings?: AssistantSettingsProps;
   part?: 'deck' | 'dialog';
   associatedArtifact?: AssociatedArtifact;
-} & Pick<ThreadProps, 'debug' | 'transcription' | 'onOpenChange'>;
+} & Pick<ThreadProps, 'debug' | 'transcription' | 'onOpenChange' | 'onAddToGraph'>;
 
 // TODO(burdon): Since this only wraps Thread, just separate out hook?
 export const ThreadContainer: FC<ThemedClassName<ThreadContainerProps>> = ({
@@ -40,18 +40,17 @@ export const ThreadContainer: FC<ThemedClassName<ThreadContainerProps>> = ({
   const messageQueue = useMessageQueue(chat);
   const { dispatchPromise: dispatch } = useIntentDispatcher();
   // TODO(thure): This will be referentially new on every render, is it causing overreactivity?
-  const messages = [...(messageQueue?.items ?? []), ...processor.messages.value];
+  const messages = [...(messageQueue?.objects ?? []), ...processor.messages.value];
 
   // Post last message to document.
   useEffect(() => {
-    if (!processor.streaming.value && messageQueue?.items) {
-      const message = messageQueue.items[messageQueue.items.length - 1];
+    if (!processor.streaming.value && messageQueue?.objects) {
+      const message = messageQueue.objects[messageQueue.objects.length - 1];
       if (space && chat && message && dispatch && associatedArtifact) {
         void dispatch(
           createIntent(CollaborationActions.InsertContent, {
-            spaceId: space.id,
-            target: makeRef(associatedArtifact),
-            object: refFromDXN(new DXN(DXN.kind.QUEUE, [...chat.assistantChatQueue.dxn.parts, message.id])),
+            target: associatedArtifact,
+            object: refFromDXN(new DXN(DXN.kind.QUEUE, [...chat.queue.dxn.parts, message.id])),
             label: 'View proposal',
           }),
         );
@@ -71,7 +70,7 @@ export const ThreadContainer: FC<ThemedClassName<ThreadContainerProps>> = ({
 
       invariant(messageQueue);
       void processor.request(text, {
-        history: messageQueue.items,
+        history: messageQueue.objects,
         onComplete: (messages) => {
           messageQueue.append(messages);
         },

package/src/components/Thread/ThreadMessage.tsx

@@ -2,9 +2,11 @@
 // Copyright 2025 DXOS.org
 //
 
-import React, { type PropsWithChildren, type FC } from 'react';
+import React, { type FC, type PropsWithChildren } from 'react';
 
-import { type MessageContentBlock, type Message, type ToolType } from '@dxos/artifact';
+import { type MessageContentBlock, type Message, type Tool } from '@dxos/ai';
+import { Surface } from '@dxos/app-framework';
+import type { BaseEchoObject } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';
 import { type Space } from '@dxos/react-client/echo';
 import { Button, Icon, IconButton, type ThemedClassName } from '@dxos/react-ui';
@@ -20,8 +22,8 @@ import { safeParseJson } from '@dxos/util';
 import { ToolBlock, isToolMessage } from './ToolInvocations';
 import { ToolboxContainer } from '../Toolbox';
 
-const panelClassNames = 'flex flex-col w-full px-2 bg-groupSurface rounded-md';
-const userClassNames = 'bg-[--user-fill] text-inverse';
+const panelClassNames = 'flex flex-col w-full px-2 bg-activeSurface rounded-md';
+const userClassNames = 'bg-[--user-fill] text-accentSurfaceText';
 
 const ToggleContainer = (props: ToggleContainerProps) => {
   return <NativeToggleContainer {...props} classNames={mx(panelClassNames, props.classNames)} />;
@@ -43,12 +45,20 @@ export type ThreadMessageProps = ThemedClassName<{
   space?: Space;
   message: Message;
   debug?: boolean;
-  tools?: ToolType[];
+  tools?: Tool[];
   onPrompt?: (text: string) => void;
   onDelete?: (id: string) => void;
+  onAddToGraph?: (object: BaseEchoObject) => void;
 }>;
 
-export const ThreadMessage: FC<ThreadMessageProps> = ({ classNames, space, message, tools, onPrompt }) => {
+export const ThreadMessage: FC<ThreadMessageProps> = ({
+  classNames,
+  space,
+  message,
+  tools,
+  onPrompt,
+  onAddToGraph,
+}) => {
   const { role, content = [] } = message;
 
   // TODO(burdon): Restructure types to make check unnecessary.
@@ -74,13 +84,18 @@ export const ThreadMessage: FC<ThreadMessageProps> = ({ classNames, space, messa
         classNames={mx(classNames, 'animate-[fadeIn_0.5s]')}
         user={block.type === 'text' && role === 'user'}
       >
-        <Component space={space} block={block} onPrompt={onPrompt} />
+        <Component space={space} block={block} onPrompt={onPrompt} onAddToGraph={onAddToGraph} />
       </MessageContainer>
     );
   });
 };
 
-type BlockComponent = FC<{ space?: Space; block: MessageContentBlock; onPrompt?: (text: string) => void }>;
+type BlockComponent = FC<{
+  space?: Space;
+  block: MessageContentBlock;
+  onPrompt?: (text: string) => void;
+  onAddToGraph?: (object: BaseEchoObject) => void;
+}>;
 
 const components: Record<string, BlockComponent> = {
   //
@@ -124,7 +139,7 @@ const components: Record<string, BlockComponent> = {
   //
   // JSON
   //
-  ['json' as const]: ({ space, block, onPrompt }) => {
+  ['json' as const]: ({ space, block, onPrompt, onAddToGraph }) => {
     invariant(block.type === 'json');
 
     switch (block.disposition) {
@@ -158,6 +173,26 @@ const components: Record<string, BlockComponent> = {
         );
       }
 
+      case 'graph': {
+        return (
+          <div className='flex flex-wrap gap-1'>
+            <Surface
+              role='card'
+              data={{ subject: JSON.parse(block.json ?? '{}') }}
+              limit={1}
+              fallback={<div className='font-mono text-xs text-pre'>{block.json}</div>}
+            />
+            {onAddToGraph && (
+              <IconButton
+                icon='ph--plus--regular'
+                label='Add to graph'
+                onClick={() => onAddToGraph?.(JSON.parse(block.json ?? '{}'))}
+              />
+            )}
+          </div>
+        );
+      }
+
       default: {
         const title = block.disposition ? titles[block.disposition] : undefined;
         return (

package/src/components/Thread/ToolInvocations.tsx

@@ -4,7 +4,7 @@
 
 import React, { type FC, useEffect, useMemo, useRef, useState } from 'react';
 
-import { type Message, type ToolType } from '@dxos/artifact';
+import { type AgentStatus, type Message, type Tool } from '@dxos/ai';
 import { log } from '@dxos/log';
 import { type ThemedClassName } from '@dxos/react-ui';
 import { NumericTabs, StatusRoll, ToggleContainer } from '@dxos/react-ui-components';
@@ -17,22 +17,22 @@ export const isToolMessage = (message: Message) => {
   return message.content.some((block) => block.type === 'tool_use' || block.type === 'tool_result');
 };
 
-const getToolName = (tool: ToolType) => {
+const getToolName = (tool: Tool) => {
   return tool.namespace && tool.function ? `${tool.namespace}:${tool.function}` : tool.name.split('_').pop();
 };
 
-const getToolCaption = (tool: ToolType | undefined) => {
+const getToolCaption = (tool: Tool | undefined, status: AgentStatus | undefined) => {
   if (!tool) {
     return 'Calling tool...';
   }
 
-  return tool.caption ?? `Calling ${getToolName(tool)}...`;
+  return status?.message ?? tool.caption ?? `Calling ${getToolName(tool)}...`;
 };
 
 export const ToolBlock: FC<ThemedClassName<ThreadMessageProps>> = ({ classNames, message, tools }) => {
   const { content = [] } = message;
 
-  let request: { tool: ToolType | undefined; block: any } | undefined;
+  let request: { tool: Tool | undefined; block: any } | undefined;
   const blocks = content.filter((block) => block.type === 'tool_use' || block.type === 'tool_result');
   const items = blocks
     .map((block) => {
@@ -43,8 +43,10 @@ export const ToolBlock: FC<ThemedClassName<ThreadMessageProps>> = ({ classNames,
             return null;
           }
 
+          log.info('tool_use', { tool: request?.tool, status: block.currentStatus });
+
           request = { tool: tools?.find((tool) => tool.name === block.name), block };
-          return { title: getToolCaption(request.tool), block };
+          return { title: getToolCaption(request.tool, block.currentStatus), block };
         }
 
         case 'tool_result': {
@@ -53,7 +55,7 @@ export const ToolBlock: FC<ThemedClassName<ThreadMessageProps>> = ({ classNames,
             return { title: 'Error', block };
           }
 
-          return { title: `${getToolCaption(request.tool)} (Success)`, block };
+          return { title: `${getToolCaption(request.tool, undefined)} (Success)`, block };
         }
 
         default: {

package/src/components/Thread/reducer.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type Message } from '@dxos/artifact';
+import { type Message } from '@dxos/ai';
 
 // TODO(burdon): Move to util?
 type Reducer<R, I> = (acc: R, value: I) => R;

package/src/components/Toolbox/Toolbox.stories.tsx

@@ -32,7 +32,7 @@ const DefaultStory = (props: ToolboxProps) => {
 };
 
 const meta: Meta<typeof Toolbox> = {
-  title: 'plugins/plugin-automation/Toolbox',
+  title: 'plugins/plugin-assistant/Toolbox',
   component: Toolbox,
   render: DefaultStory,
   decorators: [
@@ -54,11 +54,11 @@ const meta: Meta<typeof Toolbox> = {
       capabilities,
     }),
     withTheme,
-    withLayout({ tooltips: true }),
+    withLayout(),
   ],
   parameters: {
-    translations,
     layout: 'centered',
+    translations,
   },
 };
 

package/src/components/Toolbox/Toolbox.tsx

@@ -4,9 +4,10 @@
 
 import React, { useState, useEffect, Fragment, type FC } from 'react';
 
+import { parseToolName, type Tool } from '@dxos/ai';
 import { Capabilities, useCapabilities } from '@dxos/app-framework';
-import { parseToolName, type ArtifactDefinition, type Tool } from '@dxos/artifact';
-import { FunctionType } from '@dxos/functions/types';
+import { type ArtifactDefinition } from '@dxos/artifact';
+import { FunctionType } from '@dxos/functions';
 import { log } from '@dxos/log';
 import { Filter, type Space, useQuery } from '@dxos/react-client/echo';
 import { type ThemedClassName } from '@dxos/react-ui';
@@ -93,7 +94,7 @@ export const ToolboxContainer = ({ classNames, space }: ThemedClassName<{ space?
   const artifactDefinitions = useCapabilities(Capabilities.ArtifactDefinition);
 
   // Registered services.
-  const services = useQuery(space, Filter.schema(ServiceType));
+  const services = useQuery(space, Filter.type(ServiceType));
   const [serviceTools, setServiceTools] = useState<{ service: ServiceType; tools: Tool[] }[]>([]);
   useEffect(() => {
     log('creating service tools...', { services: services.length });
@@ -107,7 +108,7 @@ export const ToolboxContainer = ({ classNames, space }: ThemedClassName<{ space?
   }, [services]);
 
   // Deployed functions.
-  const functions = useQuery(space, Filter.schema(FunctionType));
+  const functions = useQuery(space, Filter.type(FunctionType));
 
   return (
     <Toolbox classNames={classNames} artifacts={artifactDefinitions} services={serviceTools} functions={functions} />

package/src/components/index.ts

@@ -4,7 +4,9 @@
 
 import { lazy } from 'react';
 
+export * from './AmbientDialog';
 export * from './AssistantSettings';
+export * from './Prompt';
 export * from './PromptSettings';
 export * from './TemplateEditor';
 export * from './Thread';

package/src/experimental/transcription/index.ts

@@ -0,0 +1,3 @@
+//
+// Copyright 2025 DXOS.org
+//

package/src/hooks/index.ts

@@ -5,7 +5,6 @@
 export * from './processor';
 
 export * from './useChatProcessor';
-export * from './useLocalTriggerManager';
 export * from './useMessageQueue';
 export * from './useServices';
 export * from './useTextInputEvents';