@dxos/plugin-automation 0.7.4-staging.f7e8224 → 0.7.5-labs.071a3e2

package/src/AutomationPlugin.tsx

@@ -2,207 +2,90 @@
 // Copyright 2023 DXOS.org
 //
 
-import React from 'react';
-
-import { parseMetadataResolverPlugin, type PluginDefinition, resolvePlugin } from '@dxos/app-framework';
+import { Capabilities, contributes, createIntent, defineModule, definePlugin, Events } from '@dxos/app-framework';
 import { FunctionTrigger } from '@dxos/functions';
-import { invariant } from '@dxos/invariant';
-import { parseClientPlugin } from '@dxos/plugin-client';
-import { createExtension, toSignal } from '@dxos/plugin-graph';
-import { memoizeQuery } from '@dxos/plugin-space';
-import {
-  getSpace,
-  getTypename,
-  isEchoObject,
-  loadObjectReferences,
-  parseId,
-  SpaceState,
-} from '@dxos/react-client/echo';
-import { translations as formTranslations } from '@dxos/react-ui-form';
+import { ClientCapabilities, ClientEvents } from '@dxos/plugin-client';
+import { DeckCapabilities } from '@dxos/plugin-deck';
+import { RefArray } from '@dxos/react-client/echo';
 
-import { AssistantPanel, AutomationPanel } from './components';
-import meta, { AUTOMATION_PLUGIN } from './meta';
+import { AiClient, AppGraphBuilder, IntentResolver, ReactSurface } from './capabilities';
+import { AUTOMATION_PLUGIN, meta } from './meta';
 import translations from './translations';
-import { type AutomationPluginProvides, ChainPromptType, ChainType } from './types';
+import { AutomationAction, ChainPromptType, ChainType, AIChatType, ServiceType } from './types';
 
-export const AutomationPlugin = (): PluginDefinition<AutomationPluginProvides> => {
-  return {
-    meta,
-    provides: {
-      metadata: {
-        records: {
-          [ChainType.typename]: {
+// TODO(wittjosiah): Rename to AssistantPlugin?
+export const AutomationPlugin = () =>
+  definePlugin(meta, [
+    defineModule({
+      id: `${meta.id}/module/ai-client`,
+      activatesOn: ClientEvents.ClientReady,
+      activate: AiClient,
+    }),
+    defineModule({
+      id: `${meta.id}/module/translations`,
+      activatesOn: Events.SetupTranslations,
+      activate: () => contributes(Capabilities.Translations, translations),
+    }),
+    defineModule({
+      id: `${meta.id}/module/metadata`,
+      activatesOn: Events.SetupMetadata,
+      activate: () => [
+        contributes(Capabilities.Metadata, {
+          id: ChainType.typename,
+          metadata: {
             placeholder: ['object placeholder', { ns: AUTOMATION_PLUGIN }],
             icon: 'ph--magic-wand--regular',
             // TODO(wittjosiah): Move out of metadata.
-            loadReferences: (chain: ChainType) => loadObjectReferences(chain, (chain) => chain.prompts),
-          },
-        },
-      },
-      translations: [...translations, ...formTranslations],
-      echo: {
-        system: [ChainType, ChainPromptType, FunctionTrigger],
-      },
-      complementary: {
-        panels: [
-          {
-            id: 'automation',
-            label: ['open automation panel label', { ns: AUTOMATION_PLUGIN }],
-            icon: 'ph--magic-wand--regular',
+            loadReferences: async (chain: ChainType) => await RefArray.loadAll(chain.prompts ?? []),
           },
-          {
-            id: 'assistant',
-            label: ['open assistant panel label', { ns: AUTOMATION_PLUGIN }],
+        }),
+        contributes(Capabilities.Metadata, {
+          id: AIChatType.typename,
+          metadata: {
+            createObject: (props: { name?: string }) => createIntent(AutomationAction.Create, props),
+            placeholder: ['object placeholder', { ns: AUTOMATION_PLUGIN }],
             icon: 'ph--atom--regular',
           },
-        ],
-      },
-      graph: {
-        builder: (plugins) => {
-          const clientPlugin = resolvePlugin(plugins, parseClientPlugin);
-          const metadataPlugin = resolvePlugin(plugins, parseMetadataResolverPlugin);
-          const resolve = metadataPlugin?.provides.metadata.resolver;
-          const client = clientPlugin?.provides.client;
-          invariant(resolve);
-          invariant(client);
-
-          return [
-            // Create nodes for object settings.
-            createExtension({
-              id: `${AUTOMATION_PLUGIN}/automation-for-subject`,
-              resolver: ({ id }) => {
-                if (!id.endsWith('~automation')) {
-                  return;
-                }
-
-                const type = 'orphan-settings-for-subject';
-                const icon = 'ph--magic-wand--regular';
-
-                const [subjectId] = id.split('~');
-                const { spaceId, objectId } = parseId(subjectId);
-                const spaces = toSignal(
-                  (onChange) => client.spaces.subscribe(() => onChange()).unsubscribe,
-                  () => client.spaces.get(),
-                );
-                const space = spaces?.find(
-                  (space) => space.id === spaceId && space.state.get() === SpaceState.SPACE_READY,
-                );
-                if (!objectId) {
-                  // TODO(burdon): Ref SPACE_PLUGIN ns.
-                  const label = space
-                    ? space.properties.name || ['unnamed space label', { ns: AUTOMATION_PLUGIN }]
-                    : ['unnamed object settings label', { ns: AUTOMATION_PLUGIN }];
-
-                  // TODO(wittjosiah): Support comments for arbitrary subjects.
-                  //   This is to ensure that the comments panel is not stuck on an old object.
-                  return {
-                    id,
-                    type,
-                    data: null,
-                    properties: {
-                      icon,
-                      label,
-                      showResolvedThreads: false,
-                      object: null,
-                      space,
-                    },
-                  };
-                }
-
-                const [object] = memoizeQuery(space, { id: objectId });
-                if (!object || !subjectId) {
-                  return;
-                }
-
-                const meta = resolve(getTypename(object) ?? '');
-                const label = meta.label?.(object) ||
-                  object.name ||
-                  meta.placeholder || ['unnamed object settings label', { ns: AUTOMATION_PLUGIN }];
-
-                return {
-                  id,
-                  type,
-                  data: null,
-                  properties: {
-                    icon,
-                    label,
-                    object,
-                  },
-                };
-              },
-            }),
-            createExtension({
-              id: `${AUTOMATION_PLUGIN}/assistant-for-subject`,
-              resolver: ({ id }) => {
-                // TODO(Zan): Find util (or make one). Effect schema!!
-                if (!id.endsWith('~assistant')) {
-                  return;
-                }
-
-                const [subjectId] = id.split('~');
-                const { spaceId, objectId } = parseId(subjectId);
-                const spaces = toSignal(
-                  (onChange) => client.spaces.subscribe(() => onChange()).unsubscribe,
-                  () => client.spaces.get(),
-                );
-                const space = spaces?.find(
-                  (space) => space.id === spaceId && space.state.get() === SpaceState.SPACE_READY,
-                );
-                if (!objectId) {
-                  // TODO(wittjosiah): Support assistant for arbitrary subjects.
-                  //   This is to ensure that the assistant panel is not stuck on an old object.
-                  return {
-                    id,
-                    type: 'orphan-automation-for-subject',
-                    data: null,
-                    properties: {
-                      icon: 'ph--atom--regular',
-                      label: ['assistant panel label', { ns: AUTOMATION_PLUGIN }],
-                      object: null,
-                      space,
-                    },
-                  };
-                }
-
-                const [object] = memoizeQuery(space, { id: objectId });
-
-                return {
-                  id,
-                  type: 'orphan-automation-for-subject',
-                  data: null,
-                  properties: {
-                    icon: 'ph--atom--regular',
-                    label: ['assistant panel label', { ns: AUTOMATION_PLUGIN }],
-                    object,
-                  },
-                };
-              },
-            }),
-          ];
-        },
-      },
-      surface: {
-        component: ({ data, role }) => {
-          switch (role) {
-            case 'complementary--assistant':
-              return <AssistantPanel subject={data.subject as any} />;
-            case 'complementary--automation': {
-              const object = data.subject;
-              const space = isEchoObject(object) ? getSpace(object) : undefined;
-              if (space) {
-                invariant(isEchoObject(object));
-                return <AutomationPanel space={space} object={object} />;
-              }
-              break;
-            }
-          }
-
-          return null;
-        },
-      },
-      intent: {
-        resolver: (intent) => {},
-      },
-    },
-  };
-};
+        }),
+      ],
+    }),
+    defineModule({
+      id: `${meta.id}/module/schema`,
+      activatesOn: ClientEvents.SetupSchema,
+      activate: () => [
+        contributes(ClientCapabilities.SystemSchema, [ChainType, ChainPromptType, FunctionTrigger, ServiceType]),
+        contributes(ClientCapabilities.Schema, [AIChatType]),
+      ],
+    }),
+    defineModule({
+      id: `${meta.id}/module/complementary-panels`,
+      activatesOn: Events.Startup,
+      activate: () => [
+        contributes(DeckCapabilities.ComplementaryPanel, {
+          id: 'service-registry',
+          label: ['service registry label', { ns: AUTOMATION_PLUGIN }],
+          icon: 'ph--plugs--regular',
+        }),
+        contributes(DeckCapabilities.ComplementaryPanel, {
+          id: 'automation',
+          label: ['automation panel label', { ns: AUTOMATION_PLUGIN }],
+          icon: 'ph--magic-wand--regular',
+        }),
+      ],
+    }),
+    defineModule({
+      id: `${meta.id}/module/react-surface`,
+      activatesOn: Events.SetupSurfaces,
+      activate: ReactSurface,
+    }),
+    defineModule({
+      id: `${meta.id}/module/app-graph-builder`,
+      activatesOn: Events.SetupAppGraph,
+      activate: AppGraphBuilder,
+    }),
+    defineModule({
+      id: `${meta.id}/module/intent-resolver`,
+      activatesOn: Events.SetupIntents,
+      activate: IntentResolver,
+    }),
+  ]);

package/src/artifacts.stories.tsx

@@ -0,0 +1,241 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type StoryObj, type Meta } from '@storybook/react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+
+import { Capabilities, IntentPlugin, Surface, useCapabilities, useIntentDispatcher } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { Message, type Tool } from '@dxos/artifact';
+import { capabilities, genericTools, localServiceEndpoints, type IsObject } from '@dxos/artifact-testing';
+import { AIServiceClientImpl } from '@dxos/assistant';
+import { createStatic, ObjectId } from '@dxos/echo-schema';
+import { EdgeHttpClient } from '@dxos/edge-client';
+import { invariant } from '@dxos/invariant';
+import { DXN, QueueSubspaceTags, SpaceId } from '@dxos/keys';
+import { ChessPlugin } from '@dxos/plugin-chess';
+import { ChessType } from '@dxos/plugin-chess/types';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { MapPlugin } from '@dxos/plugin-map';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { TablePlugin } from '@dxos/plugin-table';
+import { useSpace } from '@dxos/react-client/echo';
+import { useQueue } from '@dxos/react-edge-client';
+import { IconButton, Input, Toolbar } from '@dxos/react-ui';
+import { mx } from '@dxos/react-ui-theme';
+import { withLayout, withSignals, withTheme } from '@dxos/storybook-utils';
+
+import { Thread, type ThreadProps } from './components';
+import { ChatProcessor } from './hooks';
+import { createProcessorOptions } from './testing';
+import translations from './translations';
+
+const endpoints = localServiceEndpoints;
+
+type RenderProps = {
+  items?: IsObject[];
+  prompts?: string[];
+} & Pick<ThreadProps, 'debug'>;
+
+// TODO(burdon): Use ChatContainer.
+const Render = ({ items: _items, prompts = [], ...props }: RenderProps) => {
+  const space = useSpace();
+  const artifactDefinitions = useCapabilities(Capabilities.ArtifactDefinition);
+  const tools = useMemo<Tool[]>(
+    () => [...genericTools, ...artifactDefinitions.flatMap((definition) => definition.tools)],
+    [genericTools, artifactDefinitions],
+  );
+
+  const [aiClient] = useState(() => new AIServiceClientImpl({ endpoint: endpoints.ai }));
+  const [edgeClient] = useState(() => new EdgeHttpClient(endpoints.edge));
+  const { dispatchPromise: dispatch } = useIntentDispatcher();
+  const processor = useMemo<ChatProcessor | undefined>(() => {
+    if (!space) {
+      return;
+    }
+
+    return new ChatProcessor(
+      aiClient,
+      tools,
+      {
+        space,
+        dispatch,
+      },
+      createProcessorOptions(artifactDefinitions.map((definition) => definition.instructions)),
+    );
+  }, [aiClient, tools, space, dispatch, artifactDefinitions]);
+
+  // Queue.
+  const [queueDxn, setQueueDxn] = useState(() => randomQueueDxn());
+  const queue = useQueue<Message>(edgeClient, DXN.tryParse(queueDxn));
+
+  useEffect(() => {
+    if (queue?.items.length === 0 && !queue.isLoading && prompts.length > 0) {
+      queue.append([
+        createStatic(Message, {
+          role: 'assistant',
+          content: prompts.map(
+            (prompt) =>
+              ({
+                type: 'json',
+                disposition: 'suggest',
+                json: JSON.stringify({ text: prompt }),
+              }) as const,
+          ),
+        }),
+      ]);
+    }
+  }, [queueDxn, prompts, queue?.items.length, queue?.isLoading]);
+
+  // State.
+  const artifactItems: any[] = []; // TODO(burdon): Query from space.
+  const messages = [...(queue?.items ?? []), ...(processor?.messages.value ?? [])];
+
+  const handleSubmit = processor
+    ? async (message: string) => {
+        invariant(processor);
+        if (processor.streaming.value) {
+          await processor.cancel();
+        }
+
+        invariant(queue);
+        await processor.request(message, {
+          history: queue.items,
+          onComplete: (messages) => {
+            queue.append(messages);
+          },
+        });
+      }
+    : undefined;
+
+  const handleSuggest = useCallback(
+    (text: string) => {
+      void handleSubmit?.(text);
+    },
+    [handleSubmit],
+  );
+
+  const handleDelete = useCallback(
+    (id: string) => {
+      invariant(ObjectId.isValid(id), 'Invalid message id');
+      void queue?.delete([id]);
+    },
+    [queue],
+  );
+
+  return (
+    <div className='grid grid-cols-2 w-full h-full divide-x divide-separator overflow-hidden'>
+      {/* Thread */}
+      <div className='flex flex-col gap-4 overflow-hidden'>
+        <Toolbar.Root classNames='p-2'>
+          <Input.Root>
+            <Input.TextInput
+              spellCheck={false}
+              placeholder='Queue DXN'
+              value={queueDxn}
+              // onClick={() => setQueueDxn('')} Why?????
+              onChange={(ev) => setQueueDxn(ev.target.value)}
+            />
+            <IconButton
+              iconOnly
+              label='Copy DXN'
+              icon='ph--copy--regular'
+              onClick={() => navigator.clipboard.writeText(queueDxn)}
+            />
+            <IconButton
+              iconOnly
+              label='Clear history'
+              icon='ph--trash--regular'
+              onClick={() => setQueueDxn(randomQueueDxn())}
+            />
+            <IconButton iconOnly label='Stop' icon='ph--stop--regular' onClick={() => processor?.cancel()} />
+          </Input.Root>
+        </Toolbar.Root>
+
+        <Thread
+          messages={messages}
+          streaming={processor?.streaming.value}
+          onSubmit={processor ? handleSubmit : undefined}
+          onSuggest={processor ? handleSuggest : undefined}
+          onDelete={processor ? handleDelete : undefined}
+          {...props}
+        />
+      </div>
+
+      {/* Artifacts Deck/Mosaic */}
+      <div className='overflow-hidden grid grid-rows-[2fr_1fr] divide-y divide-separator'>
+        {artifactItems.length > 0 && (
+          <div className={mx('flex grow overflow-hidden', artifactItems.length === 1 && 'row-span-2')}>
+            <Surface role='canvas-node' limit={1} data={artifactItems[0]} />
+          </div>
+        )}
+
+        {artifactItems.length > 1 && (
+          <div className='flex shrink-0 overflow-hidden divide-x divide-separator'>
+            <div className='flex flex-1 h-full'>
+              {artifactItems.slice(1, 3).map((item, idx) => (
+                <Surface key={idx} role='canvas-node' limit={1} data={item} />
+              ))}
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+};
+
+const randomQueueDxn = () =>
+  new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, SpaceId.random(), ObjectId.random()]).toString();
+
+const meta: Meta<typeof Render> = {
+  title: 'plugins/plugin-automation/artifacts',
+  render: Render,
+  decorators: [
+    withSignals,
+    withPluginManager({
+      plugins: [
+        ClientPlugin({
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+          },
+        }),
+        SpacePlugin({ observability: false }),
+        IntentPlugin(),
+        ChessPlugin(),
+        MapPlugin(),
+        TablePlugin(),
+      ],
+      capabilities,
+    }),
+    withTheme,
+    withLayout({ fullscreen: true, tooltips: true }),
+  ],
+  parameters: {
+    translations,
+  },
+};
+
+export default meta;
+
+type Story = StoryObj<typeof Render>;
+
+export const Default: Story = {
+  args: {
+    debug: true,
+    prompts: ['Ask me a question', 'Show me a chess puzzle'],
+  },
+};
+
+export const WithInitialItems: Story = {
+  args: {
+    debug: true,
+    items: [
+      createStatic(ChessType, {
+        fen: 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1',
+      }),
+    ],
+  },
+};

package/src/capabilities/ai-client.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { contributes, type PluginsContext } from '@dxos/app-framework';
+import { AIServiceClientImpl } from '@dxos/assistant';
+import { ClientCapabilities } from '@dxos/plugin-client';
+
+import { AutomationCapabilities } from './capabilities';
+
+// TODO(wittjosiah): Factor out.
+const DEFAULT_AI_SERVICE_URL = 'http://localhost:8788';
+
+export default (context: PluginsContext) => {
+  const client = context.requestCapability(ClientCapabilities.Client);
+  const endpoint = client.config.values.runtime?.services?.ai?.server ?? DEFAULT_AI_SERVICE_URL;
+  const aiClient = new AIServiceClientImpl({ endpoint });
+  return contributes(AutomationCapabilities.AiClient, aiClient);
+};