@dxos/plugin-automation 0.7.5-main.9d26e3a → 0.7.5-main.b19bfc8

package/src/components/Thread/ThreadContainer.stories.tsx

@@ -0,0 +1,258 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type StoryObj, type Meta } from '@storybook/react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+
+import {
+  Capabilities,
+  IntentPlugin,
+  SettingsPlugin,
+  Surface,
+  useCapabilities,
+  useIntentDispatcher,
+} from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { Message, type Tool } from '@dxos/artifact';
+import { capabilities, genericTools, localServiceEndpoints, type IsObject } from '@dxos/artifact-testing';
+import { AIServiceClientImpl } from '@dxos/assistant';
+import { createStatic, ObjectId } from '@dxos/echo-schema';
+import { EdgeHttpClient } from '@dxos/edge-client';
+import { invariant } from '@dxos/invariant';
+import { DXN, QueueSubspaceTags, SpaceId } from '@dxos/keys';
+import { ChessPlugin } from '@dxos/plugin-chess';
+import { ChessType } from '@dxos/plugin-chess/types';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { MapPlugin } from '@dxos/plugin-map';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { TablePlugin } from '@dxos/plugin-table';
+import { useSpace } from '@dxos/react-client/echo';
+import { withClientProvider } from '@dxos/react-client/testing';
+import { useQueue } from '@dxos/react-edge-client';
+import { IconButton, Input, Toolbar } from '@dxos/react-ui';
+import { mx } from '@dxos/react-ui-theme';
+import { withLayout, withSignals, withTheme } from '@dxos/storybook-utils';
+
+import { Thread, type ThreadProps } from './Thread';
+import { ChatProcessor } from '../../hooks';
+import { createProcessorOptions } from '../../testing';
+import translations from '../../translations';
+
+const endpoints = localServiceEndpoints;
+
+type RenderProps = {
+  items?: IsObject[];
+  prompts?: string[];
+} & Pick<ThreadProps, 'debug'>;
+
+// TODO(burdon): Use ChatContainer.
+const Render = ({ items: _items, prompts = [], ...props }: RenderProps) => {
+  const space = useSpace();
+  const artifactDefinitions = useCapabilities(Capabilities.ArtifactDefinition);
+  const tools = useMemo<Tool[]>(
+    () => [...genericTools, ...artifactDefinitions.flatMap((definition) => definition.tools)],
+    [genericTools, artifactDefinitions],
+  );
+
+  const [aiClient] = useState(() => new AIServiceClientImpl({ endpoint: endpoints.ai }));
+  const [edgeClient] = useState(() => new EdgeHttpClient(endpoints.edge));
+  const { dispatchPromise: dispatch } = useIntentDispatcher();
+  const processor = useMemo<ChatProcessor | undefined>(() => {
+    if (!space) {
+      return;
+    }
+
+    return new ChatProcessor(
+      aiClient,
+      tools,
+      {
+        space,
+        dispatch,
+      },
+      createProcessorOptions(artifactDefinitions.map((definition) => definition.instructions)),
+    );
+  }, [aiClient, tools, space, dispatch, artifactDefinitions]);
+
+  // Queue.
+  const [queueDxn, setQueueDxn] = useState(() => randomQueueDxn());
+  const queue = useQueue<Message>(edgeClient, DXN.tryParse(queueDxn));
+
+  useEffect(() => {
+    if (queue?.items.length === 0 && !queue.isLoading && prompts.length > 0) {
+      queue.append([
+        createStatic(Message, {
+          role: 'assistant',
+          content: prompts.map(
+            (prompt) =>
+              ({
+                type: 'json',
+                disposition: 'suggest',
+                json: JSON.stringify({ text: prompt }),
+              }) as const,
+          ),
+        }),
+      ]);
+    }
+  }, [queueDxn, prompts, queue?.items.length, queue?.isLoading]);
+
+  // State.
+  const artifactItems: any[] = []; // TODO(burdon): Query from space.
+  const messages = [...(queue?.items ?? []), ...(processor?.messages.value ?? [])];
+
+  const handleSubmit = processor
+    ? (message: string) => {
+        requestAnimationFrame(async () => {
+          invariant(processor);
+          if (processor.streaming.value) {
+            await processor.cancel();
+          }
+
+          invariant(queue);
+          await processor.request(message, {
+            history: queue.items,
+            onComplete: (messages) => {
+              queue.append(messages);
+            },
+          });
+        });
+
+        return true;
+      }
+    : undefined;
+
+  const handlePrompt = useCallback(
+    (text: string) => {
+      void handleSubmit?.(text);
+    },
+    [handleSubmit],
+  );
+
+  const handleDelete = useCallback(
+    (id: string) => {
+      invariant(ObjectId.isValid(id), 'Invalid message id');
+      void queue?.delete([id]);
+    },
+    [queue],
+  );
+
+  return (
+    <div className='grid grid-cols-2 w-full h-full divide-x divide-separator overflow-hidden'>
+      {/* Thread */}
+      <div className='flex flex-col gap-4 overflow-hidden'>
+        <Toolbar.Root classNames='p-2'>
+          <Input.Root>
+            <Input.TextInput
+              spellCheck={false}
+              placeholder='Queue DXN'
+              value={queueDxn}
+              onChange={(ev) => setQueueDxn(ev.target.value)}
+            />
+            <IconButton
+              iconOnly
+              label='Copy DXN'
+              icon='ph--copy--regular'
+              onClick={() => navigator.clipboard.writeText(queueDxn)}
+            />
+            <IconButton
+              iconOnly
+              label='Clear history'
+              icon='ph--trash--regular'
+              onClick={() => setQueueDxn(randomQueueDxn())}
+            />
+            <IconButton iconOnly label='Stop' icon='ph--stop--regular' onClick={() => processor?.cancel()} />
+          </Input.Root>
+        </Toolbar.Root>
+
+        {/* TODO(burdon): Replace with ThreadContainer. */}
+        <Thread
+          messages={messages}
+          processing={processor?.streaming.value}
+          onSubmit={processor ? handleSubmit : undefined}
+          onPrompt={processor ? handlePrompt : undefined}
+          onDelete={processor ? handleDelete : undefined}
+          {...props}
+        />
+      </div>
+
+      {/* Artifacts Deck/Mosaic */}
+      <div className='overflow-hidden grid grid-rows-[2fr_1fr] divide-y divide-separator'>
+        {artifactItems.length > 0 && (
+          <div className={mx('flex grow overflow-hidden', artifactItems.length === 1 && 'row-span-2')}>
+            <Surface role='canvas-node' limit={1} data={artifactItems[0]} />
+          </div>
+        )}
+
+        {artifactItems.length > 1 && (
+          <div className='flex shrink-0 overflow-hidden divide-x divide-separator'>
+            <div className='flex flex-1 h-full'>
+              {artifactItems.slice(1, 3).map((item, idx) => (
+                <Surface key={idx} role='canvas-node' limit={1} data={item} />
+              ))}
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+};
+
+const randomQueueDxn = () =>
+  new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, SpaceId.random(), ObjectId.random()]).toString();
+
+const meta: Meta<typeof Render> = {
+  title: 'plugins/plugin-automation/ThreadContainer',
+  render: Render,
+  decorators: [
+    withSignals,
+    withClientProvider({
+      createIdentity: true,
+      createSpace: true,
+    }),
+    withPluginManager({
+      plugins: [
+        ClientPlugin({
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+          },
+        }),
+        SpacePlugin({ observability: false }),
+        SettingsPlugin(),
+        IntentPlugin(),
+        ChessPlugin(),
+        MapPlugin(),
+        TablePlugin(),
+      ],
+      capabilities,
+    }),
+    withTheme,
+    withLayout({ fullscreen: true, tooltips: true }),
+  ],
+  parameters: {
+    translations,
+  },
+};
+
+export default meta;
+
+type Story = StoryObj<typeof Render>;
+
+export const Default: Story = {
+  args: {
+    debug: true,
+    prompts: ['Ask me a question', 'Show me a chess puzzle'],
+  },
+};
+
+export const WithInitialItems: Story = {
+  args: {
+    debug: true,
+    items: [
+      createStatic(ChessType, {
+        fen: 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1',
+      }),
+    ],
+  },
+};

package/src/components/Thread/ThreadContainer.tsx

@@ -0,0 +1,78 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React, { type FC, useCallback } from 'react';
+
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { getSpace } from '@dxos/react-client/echo';
+import { type ThemedClassName } from '@dxos/react-ui';
+
+import { Thread, type ThreadProps } from './Thread';
+import { useChatProcessor, useMessageQueue } from '../../hooks';
+import { type AIChatType, type AutomationSettingsProps } from '../../types';
+
+export type ThreadContainerProps = {
+  chat?: AIChatType;
+  settings?: AutomationSettingsProps;
+} & Pick<ThreadProps, 'debug' | 'transcription' | 'onOpenChange'>;
+
+// TODO(burdon): Since this only wraps Thread, just separate out hook?
+export const ThreadContainer: FC<ThemedClassName<ThreadContainerProps>> = ({
+  classNames,
+  chat,
+  settings,
+  onOpenChange,
+  ...props
+}) => {
+  // Push up capabilities hooks out of components.
+  const space = getSpace(chat);
+  const processor = useChatProcessor(space, settings);
+  const messageQueue = useMessageQueue(chat);
+  const messages = [...(messageQueue?.items ?? []), ...processor.messages.value];
+
+  const handleSubmit = useCallback(
+    (text: string) => {
+      // Don't accept input if still processing.
+      if (processor.streaming.value) {
+        log.warn('ignoring submit; still processing.');
+        return false;
+      }
+
+      onOpenChange?.(true);
+
+      invariant(messageQueue);
+      void processor.request(text, {
+        history: messageQueue.items,
+        onComplete: (messages) => {
+          messageQueue.append(messages);
+        },
+      });
+
+      return true;
+    },
+    [processor, messageQueue, onOpenChange],
+  );
+
+  const handleCancel = useCallback(() => {
+    if (processor.streaming.value) {
+      void processor.cancel();
+    }
+  }, [processor]);
+
+  return (
+    <Thread
+      classNames={classNames}
+      space={space}
+      messages={messages}
+      processing={processor.streaming.value}
+      error={processor.error.value}
+      onSubmit={handleSubmit}
+      onCancel={handleCancel}
+      onPrompt={handleSubmit}
+      onOpenChange={onOpenChange}
+      {...props}
+    />
+  );
+};

package/src/components/Thread/ThreadMessage.tsx

@@ -0,0 +1,187 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React, { type PropsWithChildren, type FC } from 'react';
+
+import { type MessageContentBlock, type Message } from '@dxos/artifact';
+import { invariant } from '@dxos/invariant';
+import { type Space } from '@dxos/react-client/echo';
+import { Button, ButtonGroup, Icon, IconButton, type ThemedClassName } from '@dxos/react-ui';
+import {
+  MarkdownViewer,
+  ToggleContainer as NativeToggleContainer,
+  type ToggleContainerProps,
+} from '@dxos/react-ui-components';
+import { Json } from '@dxos/react-ui-syntax-highlighter';
+import { mx } from '@dxos/react-ui-theme';
+import { safeParseJson } from '@dxos/util';
+
+import { ToolBlock, isToolMessage } from './ToolInvocations';
+import { ToolboxContainer } from '../Toolbox';
+
+const panelClassNames = 'flex flex-col w-full px-2 bg-groupSurface rounded-md';
+const userClassNames = 'bg-[--user-fill]';
+
+const ToggleContainer = (props: ToggleContainerProps) => {
+  return <NativeToggleContainer {...props} classNames={mx(panelClassNames, props.classNames)} />;
+};
+
+const MessageContainer = ({ children, classNames, user }: ThemedClassName<PropsWithChildren<{ user?: boolean }>>) => {
+  if (!children) {
+    return null;
+  }
+
+  return (
+    <div role='list-item' className={mx('flex w-full', user && 'justify-end', classNames)}>
+      <div className={mx(user ? ['px-2 py-1 rounded-md', userClassNames] : 'w-full')}>{children}</div>
+    </div>
+  );
+};
+
+export type ThreadMessageProps = ThemedClassName<{
+  space?: Space;
+  message: Message;
+  debug?: boolean;
+  onPrompt?: (text: string) => void;
+  onDelete?: (id: string) => void;
+}>;
+
+export const ThreadMessage: FC<ThreadMessageProps> = ({ classNames, space, message, onPrompt }) => {
+  const { role, content = [] } = message;
+
+  // TODO(burdon): Restructure types to make check unnecessary.
+  if (isToolMessage(message)) {
+    return (
+      <MessageContainer classNames={classNames}>
+        <ToolBlock space={space} classNames={panelClassNames} message={message} />
+      </MessageContainer>
+    );
+  }
+
+  return content.map((block, idx) => {
+    // TODO(burdon): Filter empty messages.
+    if (block.type === 'text' && block.text.replaceAll(/\s+/g, '').length === 0) {
+      return null;
+    }
+
+    const Component = components[block.type] ?? components.default;
+
+    return (
+      <MessageContainer key={idx} classNames={classNames} user={block.type === 'text' && role === 'user'}>
+        <Component space={space} block={block} onPrompt={onPrompt} />
+      </MessageContainer>
+    );
+  });
+};
+
+type BlockComponent = FC<{ space?: Space; block: MessageContentBlock; onPrompt?: (text: string) => void }>;
+
+const components: Record<string, BlockComponent> = {
+  //
+  // Text
+  //
+  ['text' as const]: ({ block }) => {
+    invariant(block.type === 'text');
+    // const [open, setOpen] = useState(block.disposition === 'cot' && block.pending);
+    const title = block.disposition ? titles[block.disposition] : undefined;
+    if (!title) {
+      return <MarkdownViewer content={block.text} />;
+    }
+
+    // TOOD(burdon): Store last time user opened/closed COT.
+    // Autoclose when streaming ends.
+    // useEffect(() => {
+    //   if (block.disposition === 'cot' && !block.pending) {
+    //     setOpen(false);
+    //   }
+    // }, [block.disposition, block.pending]);
+
+    return (
+      <ToggleContainer
+        // open={open}
+        defaultOpen={block.disposition === 'cot' && block.pending}
+        title={title}
+        icon={
+          block.pending ? (
+            <Icon icon={'ph--circle-notch--regular'} classNames='text-subdued ml-2 animate-spin' size={4} />
+          ) : undefined
+        }
+      >
+        <MarkdownViewer
+          content={block.text}
+          classNames={['pbe-2', block.disposition === 'cot' && 'text-sm text-subdued']}
+        />
+      </ToggleContainer>
+    );
+  },
+
+  //
+  // JSON
+  //
+  ['json' as const]: ({ space, block, onPrompt }) => {
+    invariant(block.type === 'json');
+
+    switch (block.disposition) {
+      case 'tool_list': {
+        return (
+          <ToggleContainer title={titles[block.disposition]} defaultOpen={true}>
+            <ToolboxContainer space={space} classNames='pbe-2' />
+          </ToggleContainer>
+        );
+      }
+
+      case 'suggest': {
+        const { text = '' }: { text: string } = safeParseJson(block.json ?? '{}') ?? ({} as any);
+        return <IconButton icon='ph--lightning--regular' label={text} onClick={() => onPrompt?.(text)} />;
+      }
+
+      case 'select': {
+        const { options = [] }: { options: string[] } = safeParseJson(block.json ?? '{}') ?? ({} as any);
+        return (
+          <ButtonGroup>
+            {options.map((option) => (
+              <Button key={option} onClick={() => onPrompt?.(option)}>
+                {option}
+              </Button>
+            ))}
+          </ButtonGroup>
+        );
+      }
+
+      default: {
+        const title = block.disposition ? titles[block.disposition] : undefined;
+        return (
+          <ToggleContainer title={title ?? 'JSON'}>
+            <Json data={safeParseJson(block.json ?? block)} classNames='!p-1 text-xs' />
+          </ToggleContainer>
+        );
+      }
+    }
+  },
+
+  //
+  // Default
+  //
+  default: ({ block }) => {
+    let title = titles[block.type];
+    if (block.type === 'tool_use') {
+      title = `Tool [${block.name}]`; // TODO(burdon): Get label from tool.
+    }
+
+    return (
+      <ToggleContainer title={title ?? 'JSON'}>
+        <Json data={block} classNames='!p-1 text-xs' />
+      </ToggleContainer>
+    );
+  },
+};
+
+// TODO(burdon): Translations.
+const titles: Record<string, string> = {
+  ['cot' as const]: 'Chain of thought',
+  ['artifact' as const]: 'Artifact',
+  ['tool_use' as const]: 'Tool request',
+  ['tool_result' as const]: 'Tool result',
+  ['tool_list' as const]: 'Tools',
+};

package/src/components/Thread/ToolInvocations.tsx

@@ -0,0 +1,93 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React, { type FC, useEffect, useMemo, useRef, useState } from 'react';
+
+import { type Message, type MessageContentBlock } from '@dxos/artifact';
+import { log } from '@dxos/log';
+import { type ThemedClassName } from '@dxos/react-ui';
+import { NumericTabs, StatusRoll, ToggleContainer } from '@dxos/react-ui-components';
+import { Json } from '@dxos/react-ui-syntax-highlighter';
+import { isNonNullable, isNotFalsy } from '@dxos/util';
+
+import { type ThreadMessageProps } from './ThreadMessage';
+
+export const isToolMessage = (message: Message) => {
+  return message.content.some((block) => block.type === 'tool_use' || block.type === 'tool_result');
+};
+
+export const ToolBlock: FC<ThemedClassName<ThreadMessageProps>> = ({ classNames, message }) => {
+  const { content = [] } = message;
+
+  let request: (MessageContentBlock & { type: 'tool_use' }) | undefined;
+  const blocks = content.filter((block) => block.type === 'tool_use' || block.type === 'tool_result');
+  const items = blocks
+    .map((block, i) => {
+      switch (block.type) {
+        case 'tool_use': {
+          // TODO(burdon): Skip these updates?
+          if (block.id === request?.id && block.pending) {
+            return null;
+          }
+
+          request = block;
+          // TODO(burdon): Get plugin name.
+          return { title: `Calling ${block.name}...`, block };
+        }
+
+        case 'tool_result': {
+          if (!request) {
+            log.warn('unexpected message', { block });
+            return { title: 'Error', block };
+          }
+
+          return { title: `Processed ${request.name}`, block };
+        }
+
+        default: {
+          request = undefined;
+          return { title: 'Error', block };
+        }
+      }
+    })
+    .filter(isNonNullable);
+
+  return <ToolContainer classNames={classNames} items={items} />;
+};
+
+export const ToolContainer: FC<ThemedClassName<{ items: { title: string; block: any }[] }>> = ({
+  classNames,
+  items,
+}) => {
+  const tabsRef = useRef<HTMLDivElement>(null);
+  const [selected, setSelected] = useState(0);
+  const [open, setOpen] = useState(false);
+  useEffect(() => {
+    if (open) {
+      tabsRef.current?.focus();
+    }
+  }, [open]);
+
+  const handleSelect = (index: number) => {
+    if (index === selected) {
+      setOpen(false);
+    } else {
+      setSelected(index);
+    }
+  };
+
+  const title = useMemo(() => {
+    const lines = items.map((item) => item.title).filter(isNotFalsy);
+    return <StatusRoll key='status-roll' lines={lines} autoAdvance />;
+  }, [items]);
+
+  return (
+    <ToggleContainer classNames={['flex flex-col', classNames]} title={title} open={open} onChangeOpen={setOpen}>
+      <div className='grid grid-cols-[32px_1fr]'>
+        <NumericTabs ref={tabsRef} length={items.length} selected={selected} onSelect={handleSelect} />
+        <Json data={items[selected].block} classNames='!p-1 text-xs' />
+      </div>
+    </ToggleContainer>
+  );
+};

package/src/components/Thread/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './Thread';
+export * from './ThreadContainer';

package/src/components/Thread/reducer.ts

@@ -0,0 +1,52 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Message } from '@dxos/artifact';
+
+// TODO(burdon): Move to util?
+type Reducer<R, I> = (acc: R, value: I) => R;
+
+/**
+ * Reducer that collapses related message blocks into single messages.
+ * For example, combines tool request/response pairs into a single message.
+ */
+export const messageReducer: Reducer<{ messages: Message[]; current?: Message }, Message> = (
+  { current, messages },
+  message,
+) => {
+  let i = 0;
+  for (const block of message.content) {
+    switch (block.type) {
+      case 'tool_use':
+      case 'tool_result': {
+        if (current) {
+          current.content.push(block);
+        } else {
+          current = {
+            id: [message.id, i].join('_'),
+            role: message.role,
+            content: [block],
+          };
+          messages.push(current);
+        }
+        break;
+      }
+
+      case 'text':
+      default: {
+        current = undefined;
+        messages.push({
+          id: [message.id, i].join('_'),
+          role: message.role,
+          content: [block],
+        });
+        break;
+      }
+    }
+
+    i++;
+  }
+
+  return { current, messages };
+};