@dxos/plugin-automation 0.7.5-main.9d26e3a → 0.7.5-main.b19bfc8

package/src/hooks/useChatProcessor.tsx

@@ -0,0 +1,77 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo, useState } from 'react';
+
+import { Capabilities, useCapabilities, useCapability, useIntentDispatcher } from '@dxos/app-framework';
+import { createSystemPrompt, type Tool } from '@dxos/artifact';
+import { DEFAULT_LLM_MODEL } from '@dxos/assistant';
+import { FunctionType } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { useConfig } from '@dxos/react-client';
+import { Filter, type Space, useQuery } from '@dxos/react-client/echo';
+import { isNonNullable } from '@dxos/util';
+
+import { AutomationCapabilities } from '../capabilities';
+import { ChatProcessor } from '../hooks';
+import { covertFunctionToTool, createToolsFromService } from '../tools';
+import { type AutomationSettingsProps, ServiceType } from '../types';
+
+/**
+ * Configure and create ChatProcessor.
+ */
+export const useChatProcessor = (space?: Space, settings?: AutomationSettingsProps): ChatProcessor => {
+  const aiClient = useCapability(AutomationCapabilities.AiClient);
+  const globalTools = useCapabilities(Capabilities.Tools);
+  const artifactDefinitions = useCapabilities(Capabilities.ArtifactDefinition);
+  const { dispatchPromise: dispatch } = useIntentDispatcher();
+
+  // Services.
+  const services = useQuery(space, Filter.schema(ServiceType));
+  const [serviceTools, setServiceTools] = useState<Tool[]>([]);
+  useEffect(() => {
+    log('creating service tools...');
+    queueMicrotask(async () => {
+      const tools = await Promise.all(services.map((service) => createToolsFromService(service)));
+      setServiceTools(tools.flat());
+    });
+  }, [services]);
+
+  // Tools and context.
+  const config = useConfig();
+  const functions = useQuery(space, Filter.schema(FunctionType));
+  const [tools, extensions] = useMemo(() => {
+    log('creating tools...');
+    const tools = [
+      ...globalTools.flat(),
+      ...artifactDefinitions.flatMap((definition) => definition.tools),
+      ...serviceTools,
+      ...functions
+        .map((fn) => covertFunctionToTool(fn, config.values.runtime?.services?.edge?.url ?? '', space?.id))
+        .filter(isNonNullable),
+    ];
+    const extensions = { space, dispatch };
+    return [tools, extensions];
+  }, [dispatch, globalTools, artifactDefinitions, space, serviceTools, functions]);
+
+  // Prompt.
+  const systemPrompt = useMemo(
+    () => createSystemPrompt({ artifacts: artifactDefinitions.map((definition) => definition.instructions) }),
+    [artifactDefinitions],
+  );
+
+  // Create processor.
+  // TODO(burdon): Updated on each query update above. Should just update current processor.
+  const processor = useMemo(() => {
+    log('creating processor...', { settings });
+    return new ChatProcessor(aiClient, tools, extensions, {
+      // TODO(burdon): Remove defualt (let backend decide if not specified).
+      model: settings?.llmModel ?? DEFAULT_LLM_MODEL,
+      // TOOD(burdon): Query.
+      systemPrompt,
+    });
+  }, [aiClient, tools, extensions, systemPrompt, settings?.llmModel]);
+
+  return processor;
+};

package/src/hooks/useMessageQueue.ts

@@ -0,0 +1,23 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useMemo } from 'react';
+
+import { type Message } from '@dxos/artifact';
+import { DXN, QueueSubspaceTags } from '@dxos/keys';
+import { getSpace } from '@dxos/react-client/echo';
+import { useEdgeClient, useQueue } from '@dxos/react-edge-client';
+
+import { type AIChatType } from '../types';
+
+export const useMessageQueue = (chat?: AIChatType) => {
+  const space = getSpace(chat);
+  const queueDxn = useMemo(() => {
+    const dxn = space && chat?.queue.dxn;
+    return dxn ? new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, space.id, dxn.parts.at(-1)!]) : undefined;
+  }, [space, chat?.queue.dxn]);
+
+  const edgeClient = useEdgeClient();
+  return useQueue<Message>(edgeClient, queueDxn);
+};

package/src/hooks/useServices.ts

@@ -0,0 +1,28 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo, useState } from 'react';
+
+import { type Space } from '@dxos/client/echo';
+
+import { MockServiceRegistry } from '../testing';
+import { type ServiceType, type ServiceQuery } from '../types';
+
+/**
+ * Retrieves matching services from the registry.
+ */
+export const useServices = (space: Space, query?: ServiceQuery): ServiceType[] => {
+  const registry = useMemo(() => new MockServiceRegistry(), []);
+  const [services, setServices] = useState<ServiceType[]>([]);
+  useEffect(() => {
+    const t = setTimeout(async () => {
+      const services = await registry.queryServices(query);
+      setServices(services);
+    });
+
+    return () => clearTimeout(t);
+  }, [query, registry]);
+
+  return services;
+};

package/src/hooks/useTextInputEvents.ts

@@ -0,0 +1,49 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useState, useCallback, type KeyboardEventHandler, type ChangeEventHandler } from 'react';
+
+/**
+ * Hook that handles text input events.
+ */
+// TODO(burdon): Factor out.
+export const useTextInputEvents = ({
+  onEnter,
+  onEscape,
+}: {
+  onEnter?: (text: string) => boolean | void;
+  onEscape?: () => void;
+}) => {
+  const [value, setValue] = useState('');
+
+  const onKeyDown = useCallback<KeyboardEventHandler<HTMLInputElement>>(
+    (ev) => {
+      switch (ev.key) {
+        case 'Escape': {
+          onEscape?.();
+          setValue('');
+          break;
+        }
+
+        case 'Enter': {
+          const value = (ev.target as HTMLInputElement).value.trim();
+          if (value.length > 0) {
+            const reset = onEnter?.(value);
+            if (reset) {
+              setValue('');
+            }
+          }
+          break;
+        }
+      }
+    },
+    [onEnter, onEscape],
+  );
+
+  const onChange = useCallback<ChangeEventHandler<HTMLInputElement>>((ev) => {
+    setValue(ev.target.value);
+  }, []);
+
+  return [{ value, onKeyDown, onChange }, setValue];
+};

package/src/index.ts

@@ -2,11 +2,8 @@
 // Copyright 2023 DXOS.org
 //
 
-import { AutomationPlugin } from './AutomationPlugin';
-
-export default AutomationPlugin;
-
-export * from './AutomationPlugin';
 export * from './components';
-export * from './presets';
+export * from './meta';
 export * from './types';
+
+export * from './AutomationPlugin';

package/src/meta.ts

@@ -6,10 +6,13 @@ import { type PluginMeta } from '@dxos/app-framework';
 
 export const AUTOMATION_PLUGIN = 'dxos.org/plugin/automation';
 
-export default {
+export const ASSISTANT_DIALOG = `${AUTOMATION_PLUGIN}/assistant/dialog`;
+
+export const meta = {
   id: AUTOMATION_PLUGIN,
   name: 'Automation',
-  description: 'Automation workflows.',
+  description:
+    'The Automation tab allows you to trigger pre-defined workflows related to the element you are interacting with inside of Composer.',
   icon: 'ph--magic-wand--regular',
   source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/experimental/plugin-automation',
   tags: ['experimental'],

package/src/testing/index.ts

@@ -2,4 +2,5 @@
 // Copyright 2024 DXOS.org
 //
 
-export * from './testing';
+export * from './test-functions';
+export * from './test-services';

package/src/testing/{testing.ts → test-functions.ts}

@@ -2,12 +2,19 @@
 // Copyright 2024 DXOS.org
 //
 
+import { createSystemPrompt } from '@dxos/artifact';
+import { DEFAULT_LLM_MODEL, type GenerateRequest } from '@dxos/assistant';
 import { AST, S, toJsonSchema } from '@dxos/echo-schema';
 
+export const createProcessorOptions = (artifacts: string[]): Pick<GenerateRequest, 'model' | 'systemPrompt'> => ({
+  model: DEFAULT_LLM_MODEL,
+  systemPrompt: createSystemPrompt({ artifacts }),
+});
+
 export const functions = [
   {
     name: 'example.com/function/chess',
-    version: '0.0.1',
+    version: '0.1.0',
     inputSchema: toJsonSchema(
       S.Struct({
         level: S.Number.annotations({
@@ -18,7 +25,7 @@ export const functions = [
   },
   {
     name: 'example.com/function/forex',
-    version: '0.0.1',
+    version: '0.1.0',
     binding: 'FOREX',
     inputSchema: toJsonSchema(
       S.Struct({

package/src/testing/test-services.ts

@@ -0,0 +1,131 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { createStatic } from '@dxos/echo-schema';
+
+import {
+  type ApiAuthorization,
+  type BaseServiceRegistry,
+  type ServiceQuery,
+  ServiceType,
+  categoryIcons,
+} from '../types';
+
+export class MockServiceRegistry implements BaseServiceRegistry {
+  async queryServices(query?: ServiceQuery): Promise<ServiceType[]> {
+    return TEST_SERVICES;
+  }
+}
+
+// TODO(burdon): Can we generalize credentials?
+
+const AMADEUS_AUTH: ApiAuthorization = {
+  type: 'oauth',
+  clientId: 'BOEnpLd1sMyKjAPGKYeAPFFy60u53QEG',
+  clientSecret: 'n4qldSN7usvD57gm',
+  tokenUrl: 'https://test.api.amadeus.com/v1/security/oauth2/token',
+  grantType: 'client_credentials',
+};
+
+const VISUAL_CROSSING_CREDENTIALS: ApiAuthorization = {
+  type: 'api-key',
+  key: 'FDPRVS953KB4GQQLD25GRT975',
+  placement: {
+    type: 'query',
+    name: 'key',
+  },
+};
+
+// Registries:
+//  - https://apis.guru
+//  - https://rapidapi.com
+//  - https://github.com/konfig-sdks/openapi-examples
+//  - https://publicapis.io/?utm_source=chatgpt.com
+
+// Examples:
+//  - https://petstore.swagger.io/v2/swagger.json (testing)
+//  - https://lichess.org/api
+//  - https://github.com/konfig-sdks/openapi-examples/tree/main/xkcd
+//  - https://api.coindesk.com/v1/bpi/currentprice.json
+//  - https://www.coingecko.com/en/api/documentation
+
+// TODO(burdon): Support yaml endpoints.
+//  - e.g., https://github.com/konfig-sdks/openapi-examples/blob/main/xkcd/openapi.yaml
+
+const TEST_SERVICES: ServiceType[] = [
+  createStatic(ServiceType, {
+    serviceId: 'amadeus.com/service/FlightSearch',
+    name: 'Amadeus Flight Search',
+    description: 'Search for local and international flights.',
+    category: 'travel',
+    interfaces: [
+      {
+        kind: 'api',
+        schemaUrl: 'https://api.apis.guru/v2/specs/amadeus.com/amadeus-flight-availabilities-search/1.0.2/swagger.json',
+        authorization: AMADEUS_AUTH,
+      },
+    ],
+  }),
+
+  createStatic(ServiceType, {
+    serviceId: 'amadeus.com/service/HotelSearch',
+    name: 'Amadeus Hotel Search',
+    description: 'Search for local and international hotels.',
+    category: 'travel',
+    interfaces: [
+      {
+        kind: 'api',
+        schemaUrl: 'https://api.apis.guru/v2/specs/amadeus.com/amadeus-hotel-search/3.0.8/swagger.json',
+        authorization: AMADEUS_AUTH,
+      },
+    ],
+  }),
+
+  createStatic(ServiceType, {
+    serviceId: 'visualcrossing.com/service/Weather',
+    name: 'Visual Crossing Weather',
+    description: 'Search for global weather forecasts.',
+    category: 'weather',
+    interfaces: [
+      {
+        kind: 'api',
+        schemaUrl: 'https://api.apis.guru/v2/specs/visualcrossing.com/weather/4.6/openapi.json',
+        authorization: VISUAL_CROSSING_CREDENTIALS,
+      },
+    ],
+  }),
+
+  // TODO(burdon): Needs auth.
+  createStatic(ServiceType, {
+    serviceId: 'abstractapi.com/service/GeoLocation',
+    name: 'Abstract GeoLocation',
+    description: 'Get the location of any IP address.',
+    category: 'geolocation',
+    interfaces: [
+      {
+        kind: 'api',
+        schemaUrl: 'https://api.apis.guru/v2/specs/abstractapi.com/geolocation/1.0.0/openapi.json',
+      },
+    ],
+  }),
+
+  //
+  // Testing
+  //
+
+  ...Array.from({ length: 20 }, (_, i) =>
+    createStatic(ServiceType, {
+      serviceId: `example.com/service/test-${i}`,
+      name: `Test ${i}`,
+      description: `Test ${i}`,
+      category: Object.keys(categoryIcons)[Math.floor(Math.random() * Object.keys(categoryIcons).length)],
+      interfaces: [
+        {
+          kind: 'api',
+          schemaUrl: 'https://petstore.swagger.io/v2/swagger.json',
+        },
+      ],
+    }),
+  ),
+] as const;

package/src/tools/function.ts

@@ -0,0 +1,47 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { defineTool, ToolResult } from '@dxos/artifact';
+import type { Tool } from '@dxos/artifact';
+import { toEffectSchema } from '@dxos/echo-schema';
+import { getInvocationUrl, getUserFunctionUrlInMetadata } from '@dxos/functions';
+import type { FunctionType } from '@dxos/functions';
+import { log } from '@dxos/log';
+import type { SpaceId } from '@dxos/react-client/echo';
+import { getMeta } from '@dxos/react-client/echo';
+
+export const covertFunctionToTool = (
+  fn: FunctionType,
+  edgeUrl: string,
+  spaceId?: SpaceId | undefined,
+): Tool | undefined => {
+  if (!fn.description || !fn.inputSchema) {
+    return undefined;
+  }
+
+  const existingFunctionUrl = getUserFunctionUrlInMetadata(getMeta(fn));
+  if (!existingFunctionUrl) {
+    return undefined;
+  }
+  const url = getInvocationUrl(existingFunctionUrl, edgeUrl, {
+    spaceId,
+  });
+
+  return defineTool({
+    name: fn.name,
+    description: fn.description,
+    schema: toEffectSchema(fn.inputSchema),
+    execute: async (input) => {
+      log.info('execute function tool', { name: fn.name, url, input });
+      const response = await fetch(url, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(input),
+      });
+      return ToolResult.Success(await response.text());
+    },
+  });
+};

package/src/tools/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './function';
+export * from './openapi';

package/src/tools/openapi.test.ts

@@ -0,0 +1,224 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, test } from 'vitest';
+
+import { AIServiceClientImpl } from '@dxos/assistant';
+import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
+import { log } from '@dxos/log';
+
+import { createToolsFromApi, resolveAuthorization } from './openapi';
+import { ChatProcessor } from '../hooks';
+import type { ApiAuthorization } from '../types';
+
+describe.skip('openapi', () => {
+  describe.skip('mapping', () => {
+    test('amadeus flight availabilities', async () => {
+      const tools = await createToolsFromApi(FLIGHT_SEARCH_API);
+
+      log.info('tools', { tools });
+      // for (const tool of tools) {
+      //   const schema = tool.parameters;
+      //   // log.info('schema', { schema });
+      // }
+    });
+
+    test('amadeus hotel search', async () => {
+      const tools = await createToolsFromApi(HOTEL_SEARCH_API);
+      log.info('tools', { tools });
+    });
+
+    test('amadeus hotel name autocomplete', async () => {
+      const tools = await createToolsFromApi(HOTEL_NAME_AUTOCOMPLETE_API);
+      log.info('tools', { tools });
+    });
+
+    test('weather', async () => {
+      const tools = await createToolsFromApi(WEATHER_API, { authorization: VISUAL_CROSSING_CREDENTIALS });
+      log.info('tools', { tools });
+    });
+  });
+
+  describe.skip('invoke tools', () => {
+    test('amadeus hotel name autocomplete', async () => {
+      const tools = await createToolsFromApi(HOTEL_NAME_AUTOCOMPLETE_API, { authorization: AMADEUS_AUTH });
+
+      const result = await tools[0].execute!({
+        keyword: 'William Vale Brooklyn',
+        subType: ['HOTEL_LEISURE', 'HOTEL_GDS'],
+        countryCode: 'US',
+      });
+
+      log.info('result', { result });
+    });
+
+    test('weather API', async () => {
+      const tools = await createToolsFromApi(WEATHER_API, { authorization: VISUAL_CROSSING_CREDENTIALS });
+      const forecastTool = tools.find((t) => t.name.includes('forecast'));
+
+      const result = await forecastTool?.execute!({
+        locations: 'Brooklyn, NY',
+        aggregateHours: '24',
+      });
+
+      log.info('result', { result });
+    });
+  });
+
+  describe.skip('AI uses tools', () => {
+    test('amadeus flight availabilities', { timeout: 60_000 }, async () => {
+      const tools = await createToolsFromApi(FLIGHT_SEARCH_API, {
+        authorization: AMADEUS_AUTH,
+      });
+
+      const client = new AIServiceClientImpl({
+        endpoint: AI_SERVICE_ENDPOINT.LOCAL,
+      });
+      const processor = new ChatProcessor(client, tools);
+      const reply = await processor.request(
+        `What is the cheapest flight from New York to Paris? going on ${new Date().toISOString()} and returning after a week. 1 adult traveler`,
+      );
+
+      log.info('reply', { reply });
+    });
+
+    // TODO(dmaretskyi): Doesn't work.
+    test('amadeus hotel name autocomplete', { timeout: 60_000 }, async () => {
+      const tools = await createToolsFromApi(HOTEL_NAME_AUTOCOMPLETE_API, { authorization: AMADEUS_AUTH });
+
+      const client = new AIServiceClientImpl({
+        endpoint: AI_SERVICE_ENDPOINT.LOCAL,
+      });
+      const processor = new ChatProcessor(client, tools);
+      const reply = await processor.request('Find me the William Wale in Brooklyn New York');
+
+      log.info('reply', { reply });
+    });
+
+    test.only('weather forecast', { timeout: 60_000 }, async () => {
+      const tools = await createToolsFromApi(WEATHER_API, {
+        authorization: VISUAL_CROSSING_CREDENTIALS,
+        instructions: WEATHER_INSTRUCTIONS,
+      });
+
+      const client = new AIServiceClientImpl({
+        endpoint: AI_SERVICE_ENDPOINT.LOCAL,
+      });
+      const processor = new ChatProcessor(client, tools);
+      const reply = await processor.request(
+        `Today's date is ${new Date().toISOString().split('T')[0]}. Give me weather forecast for Warsaw for next 5 days.`,
+      );
+
+      log.info('reply', { reply });
+    });
+  });
+
+  describe.skip('invoke api directly', { timeout: 10_000 }, () => {
+    test('amadeus flight availabilities', async () => {
+      const response = await fetch('https://test.api.amadeus.com/v1/shopping/availability/flight-availabilities', {
+        method: 'POST',
+        headers: {
+          accept: 'application/vnd.amadeus+json',
+          'X-HTTP-Method-Override': 'GET',
+          'Content-Type': 'application/vnd.amadeus+json',
+          Authorization: await resolveAuthorization(AMADEUS_AUTH),
+        },
+        body: JSON.stringify({
+          originDestinations: [
+            {
+              departureDateTime: {
+                date: new Date().toISOString().split('T')[0],
+                time: new Date().toTimeString().split(' ')[0],
+              },
+              destinationLocationCode: 'MAD',
+              id: '1',
+              originLocationCode: 'BOS',
+            },
+          ],
+          sources: ['GDS'],
+          travelers: [
+            {
+              id: '1',
+              travelerType: 'ADULT',
+            },
+          ],
+        }),
+      });
+
+      log.info('response', { status: response.status, body: await response.json() });
+      expect(response.status).toBe(200);
+    });
+
+    test.only('amadeus hotel name autocomplete', async () => {
+      const response = await fetch(
+        'https://test.api.amadeus.com/v1/reference-data/locations/hotel?keyword=PARI&subtype=HOTEL_LEISURE,HOTEL_GDS',
+        {
+          method: 'GET',
+          headers: {
+            // accept: 'application/vnd.amadeus+json',
+            // 'X-HTTP-Method-Override': 'GET',
+            Authorization: await resolveAuthorization(AMADEUS_AUTH),
+          },
+        },
+      );
+
+      log.info('response', { status: response.status, body: await response.json() });
+      expect(response.status).toBe(200);
+    });
+  });
+
+  describe.skip('rapidapi', () => {
+    test('amadeus flight availabilities', async () => {
+      const departureDate = new Date().toISOString().split('T')[0];
+      const arrivalDate = new Date(new Date().getTime() + 7 * 24 * 60 * 60 * 1000).toISOString().split('T')[0];
+
+      const url = `https://amadeus-api2.p.rapidapi.com/serpapi-flight-search?departure_id=PEK&arrival_id=AUS&outbound_date=${departureDate}&return_date=${arrivalDate}&currency=USD&hl=en`;
+      const options = {
+        method: 'GET',
+        headers: {
+          'x-rapidapi-key': RAPID_API_CREDENTIALS.key,
+          'x-rapidapi-host': 'amadeus-api2.p.rapidapi.com',
+        },
+      };
+
+      const response = await fetch(url, options);
+      log.info('response', { status: response.status, body: await response.json() });
+      expect(response.status).toBe(200);
+    });
+  });
+});
+
+const FLIGHT_SEARCH_API =
+  'https://api.apis.guru/v2/specs/amadeus.com/amadeus-flight-availabilities-search/1.0.2/swagger.json';
+const HOTEL_SEARCH_API = 'https://api.apis.guru/v2/specs/amadeus.com/amadeus-hotel-search/3.0.8/swagger.json';
+const HOTEL_NAME_AUTOCOMPLETE_API =
+  'https://api.apis.guru/v2/specs/amadeus.com/amadeus-hotel-name-autocomplete/1.0.3/swagger.json';
+const WEATHER_API = 'https://api.apis.guru/v2/specs/visualcrossing.com/weather/4.6/openapi.json';
+
+const WEATHER_INSTRUCTIONS = `
+  If the user doesn't provide a date, use today's date.
+  Make sure to provide the start and end dates when possible to reduce the amount of data returned.
+  Use the tool that accepts the date parameters.
+`;
+
+const AMADEUS_AUTH: ApiAuthorization = {
+  type: 'oauth',
+  clientId: 'BOEnpLd1sMyKjAPGKYeAPFFy60u53QEG',
+  clientSecret: 'n4qldSN7usvD57gm',
+  tokenUrl: 'https://test.api.amadeus.com/v1/security/oauth2/token',
+  grantType: 'client_credentials',
+};
+
+const VISUAL_CROSSING_CREDENTIALS: ApiAuthorization = {
+  type: 'api-key',
+  key: 'FDPRVS953KB4GQQLD25GRT975',
+  placement: {
+    type: 'query',
+    name: 'key',
+  },
+};
+
+const RAPID_API_CREDENTIALS = {
+  key: '92271b6740msh32fd821d70f050dp16665bjsna69195c9e838',
+};