@dxos/plugin-assistant 0.8.1-staging.391c573 → 0.8.1-staging.97aedb1

package/src/capabilities/react-surface.tsx

@@ -2,11 +2,13 @@
 // Copyright 2025 DXOS.org
 //
 
-import React from 'react';
+import React, { useMemo } from 'react';
 
 import { Capabilities, contributes, createSurface } from '@dxos/app-framework';
+import { isInstanceOf } from '@dxos/echo-schema';
+import { getTypename } from '@dxos/live-object';
 import { SettingsStore } from '@dxos/local-storage';
-import { getSpace, isSpace } from '@dxos/react-client/echo';
+import { fullyQualifiedId, isReactiveObject, isSpace, type Space } from '@dxos/react-client/echo';
 
 import { AssistantDialog, AssistantSettings, ChatContainer, ServiceRegistry, TemplateContainer } from '../components';
 import { ASSISTANT_PLUGIN, ASSISTANT_DIALOG } from '../meta';
@@ -15,7 +17,7 @@ import { AIChatType, type AssistantSettingsProps, TemplateType } from '../types'
 export default () =>
   contributes(Capabilities.ReactSurface, [
     createSurface({
-      id: `${ASSISTANT_PLUGIN}/settings`,
+      id: `${ASSISTANT_PLUGIN}/plugin-settings`,
       role: 'article',
       filter: (data): data is { subject: SettingsStore<AssistantSettingsProps> } =>
         data.subject instanceof SettingsStore && data.subject.prefix === ASSISTANT_PLUGIN,
@@ -30,20 +32,36 @@ export default () =>
     createSurface({
       id: `${ASSISTANT_PLUGIN}/chat`,
       role: 'article',
-      filter: (data): data is { subject: AIChatType } => data.subject instanceof AIChatType,
+      filter: (data): data is { subject: AIChatType; variant: undefined } =>
+        isInstanceOf(AIChatType, data.subject) && data.variant !== 'assistant-chat',
       component: ({ data, role }) => <ChatContainer role={role} chat={data.subject} />,
     }),
+    createSurface({
+      id: `${ASSISTANT_PLUGIN}/object-chat`,
+      role: 'article',
+      filter: (data): data is { companionTo: AIChatType; subject: 'assistant-chat' } =>
+        isReactiveObject(data.companionTo) && data.companionTo.assistantChatQueue && data.subject === 'assistant-chat',
+      component: ({ data, role }) => {
+        const associatedArtifact = useMemo(
+          () => ({
+            id: fullyQualifiedId(data.companionTo),
+            typename: getTypename(data.companionTo) ?? 'unknown',
+          }),
+          [data.companionTo],
+        );
+        return <ChatContainer role={role} chat={data.companionTo} associatedArtifact={associatedArtifact} />;
+      },
+    }),
     createSurface({
       id: `${ASSISTANT_PLUGIN}/template`,
       role: 'article',
-      filter: (data): data is { subject: TemplateType } => data.subject instanceof TemplateType,
+      filter: (data): data is { subject: TemplateType } => isInstanceOf(TemplateType, data.subject),
       component: ({ data, role }) => <TemplateContainer role={role} template={data.subject} />,
     }),
     createSurface({
       id: `${ASSISTANT_PLUGIN}/service-registry`,
-      role: 'complementary--service-registry',
-      component: ({ data }) => (
-        <ServiceRegistry space={isSpace(data.subject) ? data.subject : getSpace(data.subject)!} />
-      ),
+      role: 'deck-companion--service-registry',
+      filter: (data): data is { subject: Space } => isSpace(data.subject),
+      component: ({ data }) => <ServiceRegistry space={data.subject} />,
     }),
   ]);

package/src/components/AmbientDialog/AmbientDialog.stories.tsx

@@ -65,7 +65,9 @@ const meta: Meta<typeof AmbientDialog> = {
       createIdentity: true,
       createSpace: true,
     }),
-    withPluginManager({ plugins: [IntentPlugin()] }),
+    withPluginManager({
+      plugins: [IntentPlugin()],
+    }),
     withTheme,
     withLayout({ fullscreen: true, tooltips: true }),
   ],

package/src/components/AmbientDialog/AmbientDialog.tsx

@@ -12,21 +12,21 @@ const preventDefault = (event: Event) => event.preventDefault();
 
 const minSize = 5;
 
-// TODO(burdon): Factor out.
-export const AmbientDialog = ({
-  children,
-  open: _open,
-  title,
-  onOpenChange,
-}: PropsWithChildren<{ open?: boolean; onOpenChange?: (open: boolean) => void; title?: string }>) => {
+export type AmbientDialogProps = PropsWithChildren<{
+  open?: boolean;
+  title?: string;
+  onOpenChange?: (open: boolean) => void;
+}>;
+
+export const AmbientDialog = ({ children, open: controlledOpen, title, onOpenChange }: AmbientDialogProps) => {
   const [resizeKey, setReizeKey] = useState(0);
   const [size, setSize] = useState<Size>('min-content');
-  const [open, setOpen] = useState(_open);
+  const [open, setOpen] = useState(controlledOpen);
 
   // Update controlled value.
   useEffect(() => {
-    setOpen(_open);
-  }, [_open]);
+    setOpen(controlledOpen);
+  }, [controlledOpen]);
 
   // Update size and key.
   useEffect(() => {
@@ -52,7 +52,7 @@ export const AmbientDialog = ({
         {...resizeAttributes}
         style={{
           ...sizeStyle(size, 'vertical', true),
-          maxBlockSize: 'calc(100dvh - env(safe-area-inset-bottom) - env(safe-area-inset-top) - 8rem)',
+          maxBlockSize: 'calc(100dvh - env(safe-area-inset-bottom) - env(safe-area-inset-top) - 9rem)',
         }}
         onInteractOutside={preventDefault}
       >

package/src/components/AssistantDialog.tsx

@@ -15,7 +15,7 @@ import { type AssistantSettingsProps, type AIChatType } from '../types';
 
 export const AssistantDialog: FC<{ chat?: AIChatType }> = ({ chat }) => {
   const { t } = useTranslation(ASSISTANT_PLUGIN);
-  const transcription = useCapabilities(TranscriptionCapabilities.Transcription).length > 0;
+  const transcription = useCapabilities(TranscriptionCapabilities.Transcriber).length > 0;
   const settings = useCapability(Capabilities.SettingsStore).getStore<AssistantSettingsProps>(ASSISTANT_PLUGIN)?.value;
 
   // TODO(burdon): Refocus when open.
@@ -23,7 +23,13 @@ export const AssistantDialog: FC<{ chat?: AIChatType }> = ({ chat }) => {
 
   return (
     <AmbientDialog open={open} onOpenChange={setOpen} title={t('assistant dialog title')}>
-      <ThreadContainer chat={chat} onOpenChange={setOpen} settings={settings} transcription={transcription} />
+      <ThreadContainer
+        chat={chat}
+        onOpenChange={setOpen}
+        settings={settings}
+        part={'dialog'}
+        transcription={transcription}
+      />
     </AmbientDialog>
   );
 };

package/src/components/AssistantSettings/AssistantSettings.tsx

@@ -4,13 +4,16 @@
 
 import React from 'react';
 
-import { DEFAULT_LLM_MODELS } from '@dxos/assistant';
+import { DEFAULT_EDGE_MODELS, DEFAULT_OLLAMA_MODELS } from '@dxos/assistant';
 import { Input, Select, useTranslation } from '@dxos/react-ui';
 import { DeprecatedFormContainer, DeprecatedFormInput } from '@dxos/react-ui-form';
 
 import { ASSISTANT_PLUGIN } from '../../meta';
 import { type AssistantSettingsProps } from '../../types';
 
+// TODO(burdon): Factor out.
+const DEFAULT_VALUE = '__default';
+
 export const AssistantSettings = ({ settings }: { settings: AssistantSettingsProps }) => {
   const { t } = useTranslation(ASSISTANT_PLUGIN);
 
@@ -23,18 +26,49 @@ export const AssistantSettings = ({ settings }: { settings: AssistantSettingsPro
         />
       </DeprecatedFormInput>
 
-      <DeprecatedFormInput label={t('settings llm model label')}>
+      <DeprecatedFormInput label={t('settings llm provider label')}>
+        <Input.Switch
+          checked={settings.llmProvider === 'ollama'}
+          onCheckedChange={(checked) => (settings.llmProvider = checked ? 'ollama' : 'edge')}
+        />
+      </DeprecatedFormInput>
+
+      <DeprecatedFormInput label={t('settings edge llm model label')}>
+        <Select.Root
+          value={settings.edgeModel ?? DEFAULT_VALUE}
+          onValueChange={(value) => {
+            settings.edgeModel = value === DEFAULT_VALUE ? undefined : value;
+          }}
+        >
+          <Select.TriggerButton placeholder={t('settings default llm model label')} />
+          <Select.Portal>
+            <Select.Content>
+              <Select.Viewport>
+                <Select.Option value={DEFAULT_VALUE}>{t('settings default label')}</Select.Option>
+                {DEFAULT_EDGE_MODELS.map((model) => (
+                  <Select.Option key={model} value={model}>
+                    {model}
+                  </Select.Option>
+                ))}
+              </Select.Viewport>
+            </Select.Content>
+          </Select.Portal>
+        </Select.Root>
+      </DeprecatedFormInput>
+
+      <DeprecatedFormInput label={t('settings ollama llm model label')}>
         <Select.Root
-          value={settings.llmModel ?? 'default'}
+          value={settings.ollamaModel ?? DEFAULT_VALUE}
           onValueChange={(value) => {
-            settings.llmModel = value;
+            settings.ollamaModel = value === DEFAULT_VALUE ? undefined : value;
           }}
         >
           <Select.TriggerButton placeholder={t('settings default llm model label')} />
           <Select.Portal>
             <Select.Content>
               <Select.Viewport>
-                {DEFAULT_LLM_MODELS.map((model) => (
+                <Select.Option value={DEFAULT_VALUE}>{t('settings default label')}</Select.Option>
+                {DEFAULT_OLLAMA_MODELS.map((model) => (
                   <Select.Option key={model} value={model}>
                     {model}
                   </Select.Option>

package/src/components/ChatContainer.tsx

@@ -5,6 +5,7 @@
 import React from 'react';
 
 import { Capabilities, useCapabilities, useCapability } from '@dxos/app-framework';
+import { type AssociatedArtifact } from '@dxos/artifact';
 import { TranscriptionCapabilities } from '@dxos/plugin-transcription';
 import { StackItem } from '@dxos/react-ui-stack';
 
@@ -13,13 +14,26 @@ import { ASSISTANT_PLUGIN } from '../meta';
 import { type AssistantSettingsProps, type AIChatType } from '../types';
 
 // TODO(burdon): Attention.
-export const ChatContainer = ({ chat, role }: { chat: AIChatType; role: string }) => {
-  const transcription = useCapabilities(TranscriptionCapabilities.Transcription).length > 0;
+export const ChatContainer = ({
+  role,
+  chat,
+  associatedArtifact,
+}: {
+  role: string;
+  chat: AIChatType;
+  associatedArtifact?: AssociatedArtifact;
+}) => {
+  const transcription = useCapabilities(TranscriptionCapabilities.Transcriber).length > 0;
   const settings = useCapability(Capabilities.SettingsStore).getStore<AssistantSettingsProps>(ASSISTANT_PLUGIN)?.value;
 
   return (
-    <StackItem.Content toolbar={false} role={role} classNames='mli-auto w-full max-w-[50rem]'>
-      <ThreadContainer chat={chat} settings={settings} transcription={transcription} />
+    <StackItem.Content role={role} classNames='container-max-width'>
+      <ThreadContainer
+        chat={chat}
+        settings={settings}
+        transcription={transcription}
+        associatedArtifact={associatedArtifact}
+      />
     </StackItem.Content>
   );
 };

package/src/components/Prompt/Prompt.stories.tsx

@@ -11,6 +11,7 @@ import { withTheme, withLayout } from '@dxos/storybook-utils';
 
 import { Prompt } from './Prompt';
 import { PromptBar } from './PromptBar';
+import type { ReferenceData } from './references';
 import translations from '../../translations';
 
 const meta: Meta<typeof Prompt> = {
@@ -77,3 +78,36 @@ export const Toolbar: Story = {
     );
   },
 };
+
+export const Includes: Story = {
+  args: {
+    classNames: 'w-96 p-4 rounded outline outline-gray-200',
+    references: {
+      getReferences: async ({ query }) => {
+        const res = references.filter((i) => i.label.toLowerCase().startsWith(query.toLowerCase()));
+        console.log('getReferences', { query, res });
+        return res;
+      },
+      resolveReference: async ({ uri }) => {
+        const res = references.find((i) => i.uri === uri);
+        console.log('resolveReference', { uri, res });
+        return res ?? null;
+      },
+    },
+  },
+};
+
+const references: ReferenceData[] = [
+  {
+    uri: 'dxn:echo:@:AAAAAAAA',
+    label: 'Meeting Notes',
+  },
+  {
+    uri: 'dxn:echo:@:BBBBBBBB',
+    label: 'Project Plan',
+  },
+  {
+    uri: 'dxn:echo:@:CCCCCCCC',
+    label: 'Meeting Plan',
+  },
+];

package/src/components/Prompt/Prompt.tsx

@@ -2,6 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
+import { Prec } from '@codemirror/state';
 import React, { forwardRef, useImperativeHandle } from 'react';
 
 import { type ThemedClassName, useThemeContext } from '@dxos/react-ui';
@@ -16,6 +17,7 @@ import {
 import { mx } from '@dxos/react-ui-theme';
 
 import { createAutocompleteExtension, type AutocompleteOptions } from './autocomplete';
+import { promptReferences, type ReferencesProvider } from './references';
 
 // TODO(burdon): Handle object references.
 
@@ -27,16 +29,21 @@ export interface PromptController {
 export type PromptProps = ThemedClassName<
   {
     onOpenChange?: (open: boolean) => void;
+    references?: ReferencesProvider;
   } & AutocompleteOptions &
     Pick<UseTextEditorProps, 'autoFocus'> &
     Pick<BasicExtensionsOptions, 'lineWrapping' | 'placeholder'>
 >;
 
 export const Prompt = forwardRef<PromptController, PromptProps>(
-  ({ classNames, autoFocus, lineWrapping = false, placeholder, onSubmit, onSuggest, onOpenChange }, forwardRef) => {
+  (
+    { classNames, autoFocus, lineWrapping = false, placeholder, onSubmit, onSuggest, onOpenChange, references },
+    forwardRef,
+  ) => {
     const { themeMode } = useThemeContext();
     const { parentRef, view } = useTextEditor(
       {
+        debug: true,
         autoFocus,
         extensions: [
           createBasicExtensions({
@@ -45,25 +52,28 @@ export const Prompt = forwardRef<PromptController, PromptProps>(
             placeholder,
           }),
           createThemeExtensions({ themeMode }),
+          references ? promptReferences({ provider: references }) : [],
           createAutocompleteExtension({ onSubmit, onSuggest }),
-          keymap.of([
-            {
-              key: 'Alt-ArrowUp',
-              preventDefault: true,
-              run: (view) => {
-                onOpenChange?.(true);
-                return true;
+          Prec.highest(
+            keymap.of([
+              {
+                key: 'cmd-ArrowUp',
+                preventDefault: true,
+                run: (view) => {
+                  onOpenChange?.(true);
+                  return true;
+                },
               },
-            },
-            {
-              key: 'Alt-ArrowDown',
-              preventDefault: true,
-              run: (view) => {
-                onOpenChange?.(false);
-                return true;
+              {
+                key: 'cmd-ArrowDown',
+                preventDefault: true,
+                run: (view) => {
+                  onOpenChange?.(false);
+                  return true;
+                },
               },
-            },
-          ]),
+            ]),
+          ),
         ],
       },
       [themeMode, onSubmit, onSuggest],
@@ -95,6 +105,6 @@ export const Prompt = forwardRef<PromptController, PromptProps>(
       [view, onSubmit],
     );
 
-    return <div ref={parentRef} className={mx('w-full overflow-hidden', classNames)} />;
+    return <div ref={parentRef} className={mx('w-full', classNames)} />;
   },
 );

package/src/components/Prompt/PromptBar.tsx

@@ -7,13 +7,13 @@ import React, { useRef, useState } from 'react';
 import { useVoiceInput } from '@dxos/plugin-transcription';
 import { Icon, IconButton, type ThemedClassName, Tooltip, useTranslation } from '@dxos/react-ui';
 import { Spinner } from '@dxos/react-ui-sfx';
-import { errorText, mx } from '@dxos/react-ui-theme';
+import { errorMessageColors, errorText, mx } from '@dxos/react-ui-theme';
 
 import { Prompt, type PromptController, type PromptProps } from './Prompt';
 import { ASSISTANT_PLUGIN } from '../../meta';
 
 export type PromptBarProps = ThemedClassName<
-  Pick<PromptProps, 'placeholder' | 'lineWrapping' | 'onSubmit' | 'onSuggest' | 'onOpenChange'> & {
+  Pick<PromptProps, 'placeholder' | 'lineWrapping' | 'onSubmit' | 'onSuggest' | 'onOpenChange' | 'references'> & {
     processing?: boolean;
     error?: Error;
     microphone?: boolean;
@@ -28,6 +28,7 @@ export const PromptBar = ({
   error,
   microphone,
   onCancel,
+  references,
   ...props
 }: PromptBarProps) => {
   const { t } = useTranslation(ASSISTANT_PLUGIN);
@@ -59,7 +60,7 @@ export const PromptBar = ({
             </Tooltip.Trigger>
             <Tooltip.Portal>
               <Tooltip.Content>
-                <div className='text-sm text-error-500'>{error.message}</div>
+                <div className={mx('text-sm', errorMessageColors)}>{error.message}</div>
                 <Tooltip.Arrow />
               </Tooltip.Content>
             </Tooltip.Portal>
@@ -72,6 +73,7 @@ export const PromptBar = ({
         classNames='pbs-2'
         lineWrapping={true}
         placeholder={placeholder ?? t('prompt placeholder')}
+        references={references}
         {...props}
       />
       {(onCancel || microphone) && (

package/src/components/Prompt/references.ts

@@ -0,0 +1,180 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { autocompletion, completionKeymap, type CompletionResult } from '@codemirror/autocomplete';
+import { type Extension, RangeSet } from '@codemirror/state';
+import {
+  Decoration,
+  EditorView,
+  keymap,
+  ViewPlugin,
+  WidgetType,
+  type DecorationSet,
+  type ViewUpdate,
+} from '@codemirror/view';
+
+import { Mutex } from '@dxos/async';
+
+export type ReferenceData = {
+  uri: string;
+  label: string;
+  // TODO(dmaretskyi): Consider adding details renderer for when you hover over the reference.
+};
+
+export interface ReferencesProvider {
+  getReferences({ query }: { query: string }): Promise<ReferenceData[]>;
+
+  resolveReference({ uri }: { uri: string }): Promise<ReferenceData | null>;
+}
+
+export type PromptReferencesOptions = {
+  provider: ReferencesProvider;
+  /**
+   * Will prevent the autocomplete from closing when the user blurs the editor.
+   * @default false
+   */
+  debug?: boolean;
+  /**
+   * @default '@'
+   */
+  triggerCharacter?: string;
+};
+
+/**
+ * Include references into text.
+ */
+export const promptReferences = ({
+  provider,
+  debug = false,
+  triggerCharacter = '@',
+}: PromptReferencesOptions): Extension => {
+  if (triggerCharacter.length !== 1) {
+    throw new Error('triggerCharacter must be a single character');
+  }
+
+  const decorationField = ViewPlugin.fromClass(
+    class ReferenceView {
+      private _mutex = new Mutex();
+
+      decorations: DecorationSet = Decoration.set([]);
+
+      constructor(view: EditorView) {
+        queueMicrotask(async () => {
+          const guard = await this._mutex.acquire();
+          try {
+            this.decorations = await this._computeDecorations(view);
+          } finally {
+            guard.release();
+          }
+        });
+      }
+
+      update(update: ViewUpdate) {
+        if (update.docChanged) {
+          queueMicrotask(async () => {
+            const guard = await this._mutex.acquire();
+            try {
+              this.decorations = await this._computeDecorations(update.view);
+            } finally {
+              guard.release();
+            }
+          });
+        }
+      }
+
+      private async _computeDecorations(view: EditorView): Promise<DecorationSet> {
+        const text = view.state.doc.toString();
+        const references = text.matchAll(new RegExp(`${triggerCharacter}[a-zA-Z0-9@:]+\\s`, 'g'));
+
+        const decorations = [];
+        for (const match of references) {
+          const reference = match[0];
+          const uri = reference.slice(1, -1);
+          const data = await provider.resolveReference({ uri });
+          if (data) {
+            decorations.push(
+              Decoration.replace({
+                widget: new ReferenceWidget(data),
+              }).range(match.index!, match.index! + reference.length),
+            );
+          }
+        }
+
+        return Decoration.set(decorations);
+      }
+    },
+    {
+      decorations: (v) => v.decorations,
+      provide: (plugin) => [
+        EditorView.atomicRanges.of(
+          (view): DecorationSet => view.plugin(decorationField)?.decorations ?? RangeSet.empty,
+        ),
+      ],
+    },
+  );
+
+  return [
+    decorationField,
+
+    EditorView.theme({
+      '.cm-reference-pill': {
+        borderRadius: '0.25rem',
+        borderWidth: '1px',
+        marginRight: '0.25rem',
+        marginLeft: '0.25rem',
+      },
+    }),
+
+    autocompletion({
+      activateOnTyping: true,
+      override: [
+        async (context): Promise<CompletionResult | null> => {
+          const match = context.matchBefore(new RegExp(`${triggerCharacter}[a-zA-Z0-9]+`));
+
+          if (!match || match?.to === match?.from) {
+            return null;
+          }
+
+          const query = match.text.slice(1);
+          const references = await provider.getReferences({ query });
+
+          return {
+            from: match.from,
+            filter: false,
+            options: references.map((reference) => ({
+              label: reference.label,
+              apply: `${triggerCharacter}${reference.uri} `,
+            })),
+          };
+        },
+      ],
+      closeOnBlur: !debug,
+      tooltipClass: () => 'shadow rounded',
+      aboveCursor: true,
+    }),
+
+    keymap.of(completionKeymap),
+  ];
+};
+
+class ReferenceWidget extends WidgetType {
+  constructor(private data: ReferenceData) {
+    super();
+  }
+
+  override toDOM() {
+    const span = document.createElement('span');
+    span.textContent = `@ ${this.data.label}`;
+    span.className = 'cm-reference-pill';
+    return span;
+  }
+
+  override eq(other: ReferenceWidget) {
+    return other.data.uri === this.data.uri;
+  }
+
+  override ignoreEvent() {
+    return true;
+  }
+}