@dxos/assistant-toolkit 0.8.4-main.ae835ea → 0.8.4-main.c85a9c8dae

package/src/functions/entity-extraction/entity-extraction.ts

@@ -2,53 +2,62 @@
 // Copyright 2025 DXOS.org
 //
 
-import * as Toolkit from '@effect/ai/Toolkit';
 import * as Effect from 'effect/Effect';
 import * as Layer from 'effect/Layer';
 import * as Predicate from 'effect/Predicate';
 import * as Schema from 'effect/Schema';
 
 import { AiService } from '@dxos/ai';
-import { AiSession, makeToolExecutionServiceFromFunctions, makeToolResolverFromFunctions } from '@dxos/assistant';
-import { Filter, Obj, Ref } from '@dxos/echo';
-import { DatabaseService, FunctionInvocationService, defineFunction } from '@dxos/functions';
+import { GenericToolkit } from '@dxos/ai';
+import { AiSession, ToolExecutionServices } from '@dxos/assistant';
+import { Database, Filter, Obj, Ref } from '@dxos/echo';
+import { defineFunction } from '@dxos/functions';
+import { FunctionInvocationServiceLayerTest } from '@dxos/functions-runtime/testing';
 import { type DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
+import { type Actor, LegacyOrganization, Message, Organization, Person } from '@dxos/types';
 import { trim } from '@dxos/util';
 
-import { contextQueueLayerFromResearchGraph, makeGraphWriterHandler, makeGraphWriterToolkit } from '../research';
+import { ResearchGraph } from '../../blueprints';
+import { makeGraphWriterHandler, makeGraphWriterToolkit } from '../../crud';
 
 export default defineFunction({
   key: 'dxos.org/functions/entity-extraction',
   name: 'Entity Extraction',
   description: 'Extracts entities from emails and transcripts.',
   inputSchema: Schema.Struct({
-    source: DataType.Message.annotations({ description: 'Email or transcript to extract entities from.' }),
+    source: Message.Message.annotations({
+      description: 'Email or transcript to extract entities from.',
+    }),
 
     // TODO(dmaretskyi): Consider making this an array of blueprints instead.
-    instructions: Schema.optional(Schema.String).annotations({ description: 'Instructions extraction process.' }),
+    instructions: Schema.optional(Schema.String).annotations({
+      description: 'Instructions extraction process.',
+    }),
   }),
   outputSchema: Schema.Struct({
     entities: Schema.optional(
-      Schema.Array(Obj.Any).annotations({
+      Schema.Array(Obj.Unknown).annotations({
         description: 'Extracted entities.',
       }),
     ),
   }),
   handler: Effect.fnUntraced(
-    function* ({ data: { source, instructions } }) {
-      const contact = yield* extractContact(source);
-      let organization: DataType.Organization | null = null;
+    function* ({ data: { source: message, instructions } }) {
+      const tags = Obj.getMeta(message)?.tags;
+      const contact = yield* extractContact(message.sender, tags);
+      let organization: Organization.Organization | null = null;
 
       if (contact && !contact.organization) {
         const created: DXN[] = [];
-        const GraphWriterToolkit = makeGraphWriterToolkit({ schema: [DataType.LegacyOrganization] }).pipe();
+        const GraphWriterToolkit = makeGraphWriterToolkit({
+          schema: [LegacyOrganization],
+        }).pipe();
         const GraphWriterHandler = makeGraphWriterHandler(GraphWriterToolkit, {
           onAppend: (dxns) => created.push(...dxns),
         });
         const toolkit = yield* GraphWriterToolkit.pipe(
-          Effect.provide(GraphWriterHandler.pipe(Layer.provide(contextQueueLayerFromResearchGraph))),
+          Effect.provide(GraphWriterHandler.pipe(Layer.provide(ResearchGraph.contextQueueLayer))),
         );
 
         yield* new AiSession().run({
@@ -57,17 +66,22 @@ export default defineFunction({
             The extracted organization URL must match the sender's email domain.
             ${instructions ? '<user_intructions>' + instructions + '</user_intructions>' : ''},
           `,
-          prompt: JSON.stringify({ source, contact }),
+          prompt: JSON.stringify({ source: message, contact }),
           toolkit,
         });
 
         if (created.length > 1) {
           throw new Error('Multiple organizations created');
         } else if (created.length === 1) {
-          organization = yield* DatabaseService.resolve(created[0], DataType.Organization);
-          Obj.getMeta(organization).tags ??= [];
-          Obj.getMeta(organization).tags!.push(...(Obj.getMeta(source)?.tags ?? []));
-          contact.organization = Ref.make(organization);
+          organization = yield* Database.resolve(created[0], Organization.Organization);
+          Obj.change(organization, (org) => {
+            const meta = Obj.getMeta(org);
+            meta.tags ??= [];
+            meta.tags.push(...(tags ?? []));
+          });
+          Obj.change(contact, (c) => {
+            c.organization = Ref.make(organization!);
+          });
         }
       }
 
@@ -78,27 +92,26 @@ export default defineFunction({
     Effect.provide(
       Layer.mergeAll(
         AiService.model('@anthropic/claude-sonnet-4-0'), // TODO(dmaretskyi): Extract.
-        makeToolResolverFromFunctions([], Toolkit.make()),
-        makeToolExecutionServiceFromFunctions(Toolkit.make() as any, Layer.empty as any),
+        ToolExecutionServices,
       ).pipe(
         Layer.provide(
           // TODO(dmaretskyi): This should be provided by environment.
-          Layer.mergeAll(FunctionInvocationService.layerTest()),
+          Layer.mergeAll(GenericToolkit.providerEmpty, FunctionInvocationServiceLayerTest()),
         ),
       ),
     ),
   ),
 });
 
-const extractContact = Effect.fn('extractContact')(function* (message: DataType.Message) {
-  const name = message.sender.name;
-  const email = message.sender.email;
+const extractContact = Effect.fn('extractContact')(function* (actor: Actor.Actor, tags?: readonly string[]) {
+  const name = actor.name;
+  const email = actor.email;
   if (!email) {
-    log.warn('email is required for contact extraction', { sender: message.sender });
+    log.warn('email is required for contact extraction', { actor });
     return undefined;
   }
 
-  const { objects: existingContacts } = yield* DatabaseService.runQuery(Filter.type(DataType.Person));
+  const existingContacts = yield* Database.runQuery(Filter.type(Person.Person));
 
   // Check for existing contact
   // TODO(dmaretskyi): Query filter DSL - https://linear.app/dxos/issue/DX-541/filtercontains-should-work-with-partial-objects
@@ -111,16 +124,16 @@ const extractContact = Effect.fn('extractContact')(function* (message: DataType.
     return existingContact;
   }
 
-  const newContact = Obj.make(DataType.Person, {
-    [Obj.Meta]: {
-      tags: Obj.getMeta(message)?.tags,
-    },
+  const newContact = Obj.make(Person.Person, {
+    ...(tags ? { [Obj.Meta]: { tags: [...tags] } } : {}),
     emails: [{ value: email }],
   });
-  yield* DatabaseService.add(newContact);
+  yield* Database.add(newContact);
 
   if (name) {
-    newContact.fullName = name;
+    Obj.change(newContact, (c) => {
+      c.fullName = name;
+    });
   }
 
   const emailDomain = email.split('@')[1]?.toLowerCase();
@@ -131,7 +144,7 @@ const extractContact = Effect.fn('extractContact')(function* (message: DataType.
 
   log.info('extracted email domain', { emailDomain });
 
-  const { objects: existingOrganisations } = yield* DatabaseService.runQuery(Filter.type(DataType.Organization));
+  const existingOrganisations = yield* Database.runQuery(Filter.type(Organization.Organization));
   const matchingOrg = existingOrganisations.find((org) => {
     if (org.website) {
       try {
@@ -147,7 +160,10 @@ const extractContact = Effect.fn('extractContact')(function* (message: DataType.
           emailDomain.endsWith(`.${websiteDomain}`)
         );
       } catch (e) {
-        log.warn('Error parsing website URL', { website: org.website, error: e });
+        log.warn('Error parsing website URL', {
+          website: org.website,
+          error: e,
+        });
         return false;
       }
     }
@@ -156,7 +172,9 @@ const extractContact = Effect.fn('extractContact')(function* (message: DataType.
 
   if (matchingOrg) {
     log.info('found matching organization', { organization: matchingOrg });
-    newContact.organization = Ref.make(matchingOrg);
+    Obj.change(newContact, (c) => {
+      c.organization = Ref.make(matchingOrg);
+    });
   }
 
   return newContact;

package/src/functions/entity-extraction/index.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { default as extract$ } from './entity-extraction';
+import Extract from './entity-extraction';
 
-export namespace EntityExtraction {
-  export const extract = extract$;
-}
+export const EntityExtractionFunctions = {
+  Extract,
+};

package/src/functions/exa/exa.ts

@@ -9,8 +9,8 @@ import Exa from 'exa-js';
 import { CredentialsService, defineFunction } from '@dxos/functions';
 
 export default defineFunction({
-  key: 'dxos.org/function/exa',
-  name: 'Exa',
+  key: 'dxos.org/function/exa-search',
+  name: 'Exa Search',
   description: 'Search the web for information',
   inputSchema: Schema.Struct({
     query: Schema.String.annotations({

package/src/functions/exa/index.ts

@@ -2,5 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-export { default as exaFunction } from './exa';
-export { default as exaMockFunction } from './mock';
+import Search from './exa';
+import Mock from './mock';
+
+export const ExaFunctions = {
+  Search,
+  Mock,
+};

package/src/functions/exa/mock.ts

@@ -7,7 +7,7 @@ import * as Schema from 'effect/Schema';
 
 import { defineFunction } from '@dxos/functions';
 
-import { SEARCH_RESULTS } from '../../testing';
+import { SEARCH_RESULTS } from './data';
 
 export default defineFunction({
   key: 'dxos.org/function/exa-mock',

package/src/functions/index.ts

@@ -3,9 +3,5 @@
 //
 
 export * from './agent';
-export * from './discord';
-export * from './document';
 export * from './entity-extraction';
-export * from './linear';
-export * from './research';
-export * from './tasks';
+export * from './exa';

package/src/index.ts

@@ -3,6 +3,8 @@
 //
 
 export * from './blueprints';
+export * from './crud';
 export * from './functions';
-export * from './plugins';
 export * from './sync';
+export * from './toolkits';
+export * from './types';

package/src/sync/sync.ts

@@ -5,7 +5,7 @@
 import * as Effect from 'effect/Effect';
 
 import { Filter, Obj, Query, Ref } from '@dxos/echo';
-import { DatabaseService } from '@dxos/functions';
+import { Database } from '@dxos/echo';
 import { failedInvariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
@@ -18,9 +18,12 @@ import { log } from '@dxos/log';
  * @param opts.foreignKeyId - The key to use for matching objects.
  */
 export const syncObjects: (
-  objs: Obj.Any[],
+  objs: Obj.Unknown[],
   opts: { foreignKeyId: string },
-) => Effect.Effect<Obj.Any[], never, DatabaseService> = Effect.fn('syncObjects')(function* (objs, { foreignKeyId }) {
+) => Effect.Effect<Obj.Unknown[], never, Database.Service> = Effect.fn('syncObjects')(function* (
+  objs,
+  { foreignKeyId },
+) {
   return yield* Effect.forEach(
     objs,
     Effect.fnUntraced(function* (obj) {
@@ -28,7 +31,7 @@ export const syncObjects: (
       for (const key of Object.keys(obj)) {
         if (typeof key !== 'string' || key === 'id') continue;
         if (!Ref.isRef((obj as any)[key])) continue;
-        const ref: Ref.Any = (obj as any)[key];
+        const ref: Ref.Unknown = (obj as any)[key];
         if (!ref.target) continue;
         if (Obj.getDXN(ref.target).isLocalObjectId()) {
           // obj not persisted to db.
@@ -39,9 +42,7 @@ export const syncObjects: (
 
       const schema = Obj.getSchema(obj) ?? failedInvariant('No schema.');
       const foreignId = Obj.getKeys(obj, foreignKeyId)[0]?.id ?? failedInvariant('No foreign key.');
-      const {
-        objects: [existing],
-      } = yield* DatabaseService.runQuery(
+      const [existing] = yield* Database.runQuery(
         Query.select(Filter.foreignKeys(schema, [{ source: foreignKeyId, id: foreignId }])),
       );
       log('sync object', {
@@ -50,7 +51,7 @@ export const syncObjects: (
         existing: existing ? Obj.getDXN(existing) : undefined,
       });
       if (!existing) {
-        yield* DatabaseService.add(obj);
+        yield* Database.add(obj);
         return obj;
       } else {
         copyObjectData(existing, obj);
@@ -61,27 +62,34 @@ export const syncObjects: (
   );
 });
 
-const copyObjectData = (existing: Obj.Any, newObj: Obj.Any) => {
-  for (const key of Object.keys(newObj)) {
-    if (typeof key !== 'string' || key === 'id') continue;
-    if (
-      typeof (newObj as any)[key] !== 'string' &&
-      typeof (newObj as any)[key] !== 'number' &&
-      typeof (newObj as any)[key] !== 'boolean' &&
-      !Ref.isRef((newObj as any)[key])
-    )
-      continue;
-    (existing as any)[key] = (newObj as any)[key];
-  }
-  for (const key of Object.keys(existing)) {
-    if (typeof key !== 'string' || key === 'id') continue;
-    if (!(key in newObj)) {
-      delete (existing as any)[key];
+const copyObjectData = (existing: Obj.Unknown, newObj: Obj.Unknown) => {
+  Obj.change(existing, (obj) => {
+    // Copy properties from newObj to existing.
+    for (const key of Object.keys(newObj)) {
+      if (typeof key !== 'string' || key === 'id') continue;
+      if (
+        typeof (newObj as any)[key] !== 'string' &&
+        typeof (newObj as any)[key] !== 'number' &&
+        typeof (newObj as any)[key] !== 'boolean' &&
+        !Ref.isRef((newObj as any)[key])
+      )
+        continue;
+      (obj as any)[key] = (newObj as any)[key];
     }
-  }
-  for (const foreignKey of Obj.getMeta(newObj).keys) {
-    Obj.deleteKeys(existing, foreignKey.source);
-    // TODO(dmaretskyi): Doesn't work: `Obj.getMeta(existing).keys.push(foreignKey);`
-    Obj.getMeta(existing).keys.push({ ...foreignKey });
-  }
+
+    // Delete properties that don't exist in newObj.
+    for (const key of Object.keys(obj)) {
+      if (typeof key !== 'string' || key === 'id') continue;
+      if (!(key in newObj)) {
+        delete (obj as any)[key];
+      }
+    }
+
+    // Update foreign keys.
+    for (const foreignKey of Obj.getMeta(newObj).keys) {
+      Obj.deleteKeys(obj, foreignKey.source);
+      // TODO(dmaretskyi): Doesn't work: `Obj.getMeta(existing).keys.push(foreignKey);`
+      Obj.getMeta(obj).keys.push({ ...foreignKey });
+    }
+  });
 };

package/src/testing/index.ts

@@ -2,4 +2,5 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './data';
+export * from '../blueprints/testing';
+export * from './plugins';

package/src/testing/plugins.tsx

@@ -0,0 +1,72 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import React from 'react';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { Surface } from '@dxos/app-framework/ui';
+import { Format, type Obj, Type } from '@dxos/echo';
+import { Card } from '@dxos/react-ui';
+import { JsonFilter } from '@dxos/react-ui-syntax-highlighter';
+
+export const MapSchema = Schema.Struct({
+  coordinates: Format.GeoPoint,
+}).pipe(
+  Type.object({
+    typename: 'example.com/type/Map',
+    version: '0.1.0',
+  }),
+);
+
+export type MapSchema = Schema.Schema.Type<typeof MapSchema>;
+
+// TODO(burdon): Move to ECHO def.
+export type ArtifactsContext = {
+  items: Obj.Unknown[];
+  getArtifacts: () => Obj.Unknown[];
+  addArtifact: (artifact: Obj.Unknown) => void;
+};
+
+declare global {
+  interface ToolContextExtensions {
+    artifacts?: ArtifactsContext;
+  }
+}
+
+// TODO(dmaretskyi): Removed images from conductor GPT implementation.
+const isImage = (data: any): data is any => false;
+
+export const capabilities: Capability.Any[] = [
+  Capability.contributes(
+    Capabilities.ReactSurface,
+    Surface.create({
+      id: 'plugin-image',
+      role: 'card--content',
+      filter: (data: any): data is any => isImage(data.value),
+      component: ({ data }) => (
+        <Card.Content>
+          <img
+            className='grow object-cover'
+            src={`data:image/jpeg;base64,${data.value.source.data}`}
+            alt={data.value.prompt ?? `Generated image [id=${data.value.id}]`}
+          />
+        </Card.Content>
+      ),
+    }),
+  ),
+  Capability.contributes(
+    Capabilities.ReactSurface,
+    Surface.create({
+      id: 'plugin-default',
+      role: 'card--content',
+      position: 'fallback',
+      component: ({ data }) => (
+        <Card.Content>
+          <JsonFilter data={data} />
+        </Card.Content>
+      ),
+    }),
+  ),
+];

package/src/toolkits/WebToolkit.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Tool from '@effect/ai/Tool';
+import * as Toolkit from '@effect/ai/Toolkit';
+import * as AnthropicTool from '@effect/ai-anthropic/AnthropicTool';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+// TODO(dmaretskyi): Testing only.
+export const WebToolkit = Toolkit.make(
+  AnthropicTool.WebSearch_20250305({}).pipe(
+    // TODO(dmaretskyi): Effect bug -- provider-defined tools don't support annotations.
+    // ToolFormatter.assign({
+    //   debugFormatResult: (result) =>
+    //     Array.isArray(result) ? result.map(Struct.pick('title', 'type', 'url', 'page_age')) : result,
+    // }),
+  ),
+  Tool.make('WebFetch', {
+    parameters: {
+      url: Schema.String,
+    },
+    success: Schema.String,
+  }),
+);
+
+export const layer = WebToolkit.toLayer({
+  WebFetch: Effect.fnUntraced(function* ({ url }) {
+    const response = yield* Effect.promise(() => fetch(url).then((response) => response.text()));
+    return response;
+  }),
+});

package/src/toolkits/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * as WebToolkit from './WebToolkit';

package/src/types/Chat.ts

@@ -0,0 +1,49 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { Annotation, Obj, Ref, Type } from '@dxos/echo';
+import { FormInputAnnotation, LabelAnnotation } from '@dxos/echo/internal';
+import { Queue } from '@dxos/echo-db';
+
+/**
+ * AI chat.
+ */
+export const Chat = Schema.Struct({
+  name: Schema.String.pipe(Schema.optional),
+  queue: Ref.Ref(Queue).pipe(FormInputAnnotation.set(false)),
+  // TODO(dmaretskyi): Eventually this and the message queue will be the same.
+  traceQueue: Ref.Ref(Queue).pipe(FormInputAnnotation.set(false), Schema.optional),
+}).pipe(
+  Type.object({
+    typename: 'dxos.org/type/assistant/Chat',
+    version: '0.2.0',
+  }),
+  LabelAnnotation.set(['name']),
+  Annotation.IconAnnotation.set({
+    icon: 'ph--atom--regular',
+    hue: 'blue',
+  }),
+);
+
+export interface Chat extends Schema.Schema.Type<typeof Chat> {}
+
+export const make = (props: Obj.MakeProps<typeof Chat>) => Obj.make(Chat, props);
+
+/**
+ * Relation between a Chat and companion objects (e.g., artifacts).
+ */
+export const CompanionTo = Schema.Struct({
+  id: Obj.ID,
+}).pipe(
+  Type.relation({
+    typename: 'dxos.org/relation/assistant/CompanionTo',
+    version: '0.1.0',
+    source: Chat,
+    target: Obj.Unknown,
+  }),
+);
+
+export interface CompanionTo extends Schema.Schema.Type<typeof CompanionTo> {}

package/src/types/Plan.ts

@@ -0,0 +1,107 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { Obj, Ref, Type } from '@dxos/echo';
+
+import * as Chat from './Chat';
+
+export const TaskId = Schema.String.pipe(Schema.brand('@dxos/assistant-toolkit/TaskId'));
+export type TaskId = Schema.Schema.Type<typeof TaskId>;
+
+export const Task = Schema.Struct({
+  /**
+   * Short task ID unique within the plan.
+   */
+  id: TaskId,
+
+  title: Schema.String.annotations({
+    description: 'Task title and description.',
+  }),
+
+  status: Schema.Literal('todo', 'in-progress', 'done'),
+
+  /**
+   * Parent task ID.
+   */
+  parent: Schema.optional(TaskId).annotations({
+    description: 'Parent task ID.',
+  }),
+
+  /**
+   * Chat object that this task is associated with.
+   */
+  chat: Schema.optional(Ref.Ref(Chat.Chat)),
+});
+export interface Task extends Schema.Schema.Type<typeof Task> {}
+
+/**
+ * Hierarchical collection of tasks for humans and agents to track progress.
+ */
+export const Plan = Schema.Struct({
+  tasks: Schema.Array(Task),
+}).pipe(
+  Type.object({
+    typename: 'dxos.org/type/Plan',
+    version: '0.1.0',
+  }),
+);
+export interface Plan extends Schema.Schema.Type<typeof Plan> {}
+
+export const generateTaskId = (plan: Plan): TaskId => {
+  const existingIds = plan.tasks
+    .map((task) => {
+      const [num, letter] = task.id.split('-');
+      return parseInt(num);
+    })
+    .filter((_) => !isNaN(_));
+  let newId;
+  if (existingIds.length === 0) {
+    newId = 1;
+  } else {
+    newId = Math.max(...existingIds) + 1;
+  }
+  // Add random suffix to avoid collisions.
+  return TaskId.make(`${newId}-${crypto.randomUUID().slice(0, 2)}`);
+};
+
+/**
+ * Add new tasks to a plan, generating IDs for new tasks.
+ * Use inside an `Obj.change` callback.
+ */
+export const addTasks = (
+  plan: Obj.Mutable<Plan>,
+  tasks: (Pick<Task, 'title'> & Partial<Pick<Task, 'status' | 'parent' | 'chat'>>)[],
+) => {
+  for (const task of tasks) {
+    const taskId = generateTaskId(plan);
+    plan.tasks.push({ id: taskId, ...task, status: task.status ?? 'todo' });
+  }
+  return plan;
+};
+
+interface MakePlanProps {
+  tasks: {
+    title: string;
+    status?: 'todo' | 'in-progress' | 'done';
+  }[];
+}
+
+export const makePlan = (props: MakePlanProps): Plan => {
+  const plan = Obj.make(Plan, { tasks: [] });
+  Obj.change(plan, (plan) => {
+    addTasks(plan, props.tasks);
+  });
+  return plan;
+};
+
+/**
+ * Formats plan to markdown format.
+ */
+export const formatPlan = (plan: Plan): string => {
+  return plan.tasks
+    .map((task) => `- **${task.status?.toLocaleUpperCase()}**: ${task.title ?? 'No title'} <!-- id=${task.id} -->`)
+    .join('\n');
+};