@dxos/assistant 0.8.0 → 0.8.1-main.013e445

package/src/conversation/conversation.ts

@@ -147,10 +147,10 @@ export const runLLM = async (params: CreateLLMConversationParams) => {
   };
 };
 
-export const isToolUse = (message: Message) => {
+export const isToolUse = (message: Message, { onlyToolNames }: { onlyToolNames?: string[] } = {}) => {
   const block = message.content.at(-1);
   invariant(block);
-  return block.type === 'tool_use';
+  return block.type === 'tool_use' && (!onlyToolNames || onlyToolNames.includes(block.name));
 };
 
 export type RunToolsOptions = {
@@ -220,6 +220,7 @@ export const runTools = async ({ message, tools, extensions }: RunToolsOptions):
             content:
               typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result) ?? '',
           },
+          ...(toolResult.extractContentBlocks ?? []),
         ],
       });
 

package/src/conversation/types.ts

@@ -2,7 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
-import { Schema as S } from '@effect/schema';
+import { Schema as S } from 'effect';
 
 import { type Message, type MessageContentBlock } from '@dxos/artifact';
 import { toJsonSchema, JsonSchemaType, ObjectId } from '@dxos/echo-schema';

package/src/index.ts

@@ -5,3 +5,4 @@
 export * from './ai-service';
 export * from './cypher';
 export * from './conversation';
+export * from './session';

package/src/session/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './session';

package/src/session/session.test.ts

@@ -0,0 +1,256 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema as S } from 'effect';
+import { inspect } from 'node:util';
+import { describe, test } from 'vitest';
+
+import {
+  ArtifactId,
+  defineArtifact,
+  defineTool,
+  ToolResult,
+  type Message,
+  type MessageContentBlock,
+} from '@dxos/artifact';
+import { AIServiceEdgeClient } from '@dxos/assistant';
+import { createStatic, EchoObject, ObjectId } from '@dxos/echo-schema';
+import { DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import { AISession } from './session';
+import { AI_SERVICE_ENDPOINT } from '../testing';
+
+// Define a calendar event artifact schema
+const CalendarEventSchema = S.Struct({
+  title: S.String,
+  startTime: S.String,
+  endTime: S.String,
+  description: S.String,
+}).pipe(EchoObject('example.com/type/CalendarEvent', '0.1.0'));
+
+type CalendarEvent = S.Schema.Type<typeof CalendarEventSchema>;
+
+describe('AISession with Ollama', () => {
+  test('create calendar itinerary', { timeout: 60_000 }, async () => {
+    const aiClient = new AIServiceEdgeClient({ endpoint: AI_SERVICE_ENDPOINT.REMOTE });
+    // const aiClient = new OllamaClient({
+    //   overrides: { model: 'llama3.1:8b' },
+    // });
+    const session = new AISession({ operationModel: 'immediate' });
+
+    const objects = new Set<string>();
+
+    // Define calendar artifact
+    const calendarArtifact = defineArtifact({
+      id: 'artifact:dxos.org/plugin/calendar',
+      name: 'Calendar',
+      instructions: 'Use this to create and query calendar events.',
+      schema: CalendarEventSchema,
+      tools: [
+        defineTool('calendar', {
+          name: 'query',
+          description: 'Query the calendar for events',
+          schema: S.Struct({}),
+          execute: async () => {
+            return ToolResult.Success(CALENDAR_EVENTS);
+          },
+        }),
+      ],
+    });
+
+    const tableArtifact = defineArtifact({
+      id: 'artifact:dxos.org/plugin/table',
+      name: 'Table',
+      instructions: 'Use this to create and manage tables. Each table has a unique id.',
+      schema: S.Struct({}),
+      tools: [
+        defineTool('table', {
+          name: 'create',
+          description: 'Create a table',
+          schema: S.Struct({
+            data: S.Array(S.Any).annotations({ description: 'Array of data payloads to add as rows' }),
+          }),
+          execute: async ({ data }) => {
+            log.info('create table', { data });
+            const id = DXN.fromLocalObjectId(ObjectId.random()).toString();
+            objects.add(id);
+            // TODO(dmaretskyi): consider xml for refs instead of @dxn:echo:@:XXXXX
+            return ToolResult.Success(`table @${id}`);
+          },
+        }),
+      ],
+    });
+
+    const mapArtifact = defineArtifact({
+      id: 'artifact:dxos.org/plugin/map',
+      name: 'Map',
+      instructions:
+        'Use this to create and manage maps. Maps source data from tables. Table id is required to create a map.',
+      schema: S.Struct({}),
+      tools: [
+        defineTool('map', {
+          name: 'create',
+          description: 'Create a map',
+          schema: S.Struct({
+            source: ArtifactId.annotations({
+              description: 'The table that will be used as the source of the map',
+            }),
+          }),
+          execute: async ({ source }) => {
+            // TODO(dmaretskyi): Use effect-schema decode instead of manual parsing.
+            const sourceId = ArtifactId.toDXN(source);
+            if (!objects.has(sourceId.toString())) {
+              return ToolResult.Error(`table id=${source} not found`);
+            }
+            log.info('create map', { sourceId });
+            const id = DXN.fromLocalObjectId(ObjectId.random()).toString();
+            objects.add(id);
+            // TODO(dmaretskyi): consider xml for refs instead of @dxn:echo:@:XXXXX
+            return ToolResult.Success(`map @${id}`);
+          },
+        }),
+      ],
+    });
+
+    const scriptArtifact = defineArtifact({
+      id: 'artifact:dxos.org/plugin/script',
+      name: 'Script',
+      instructions: 'Use this to create and manage scripts',
+      schema: S.Struct({}),
+      tools: [],
+    });
+
+    // session.streamEvent.on((event) => {
+    //   printStreamEvent(event);
+    // });
+
+    session.message.on(printMessage);
+    session.userMessage.on(printMessage);
+    session.block.on(printContentBlock);
+
+    // session.update.on((update) => {
+    //   log.info('update', { update });
+    // });
+
+    // Test creating an itinerary
+    const response = await session.run({
+      client: aiClient,
+      tools: [],
+      artifacts: [calendarArtifact, tableArtifact, mapArtifact, scriptArtifact],
+      history: [],
+      generationOptions: {
+        model: '@anthropic/claude-3-5-haiku-20241022',
+      },
+      prompt: 'create a table and map for a travel itinerary based on events in my calendar',
+    });
+
+    log.info('result', {
+      objects,
+      finalMessage: response.at(-1),
+    });
+  });
+});
+
+// Travel to rome, florence, livorno, siena, madrid for conferences
+const CALENDAR_EVENTS: CalendarEvent[] = [
+  createStatic(CalendarEventSchema, {
+    title: 'Exploring Ancient Ruins in Rome',
+    startTime: '2024-01-01T10:00:00Z',
+    endTime: '2024-01-01T11:00:00Z',
+    description: 'Tech conference at the historic Colosseum with networking opportunities',
+  }),
+  createStatic(CalendarEventSchema, {
+    title: 'Renaissance Tech Summit in Florence',
+    startTime: '2024-01-01T11:00:00Z',
+    endTime: '2024-01-01T12:00:00Z',
+    description: 'Discussing AI innovations surrounded by Renaissance art',
+  }),
+  createStatic(CalendarEventSchema, {
+    title: 'Travel to Livorno',
+    startTime: '2024-01-01T12:00:00Z',
+    endTime: '2024-01-01T13:00:00Z',
+    description: 'Travel to Livorno',
+  }),
+  createStatic(CalendarEventSchema, {
+    title: 'Travel to Siena',
+    startTime: '2024-01-01T13:00:00Z',
+    endTime: '2024-01-01T14:00:00Z',
+    description: 'Travel to Siena',
+  }),
+  createStatic(CalendarEventSchema, {
+    title: 'Travel to Madrid',
+    startTime: '2024-01-01T14:00:00Z',
+    endTime: '2024-01-01T15:00:00Z',
+    description: 'Travel to Madrid',
+  }),
+];
+
+const printMessage = (message: Message) => {
+  console.log(`${message.role.toUpperCase()}\n`);
+  for (const content of message.content) {
+    printContentBlock(content);
+  }
+};
+
+const printContentBlock = (content: MessageContentBlock) => {
+  switch (content.type) {
+    case 'text':
+      console.log(content.text);
+      break;
+    case 'tool_use':
+      console.log(`⚙️ [Tool Use] ${content.name} ${inspect(content.input, { depth: null, colors: true })}`);
+      break;
+    case 'tool_result': {
+      let data: any;
+      try {
+        data = JSON.parse(content.content);
+      } catch {
+        data = content.content;
+      }
+      console.log(`⚙️ [Tool Result] ${content.toolUseId} ${inspect(data, { depth: null, colors: true })}`);
+      break;
+    }
+  }
+};
+
+// const printStreamEvent = (event: GenerationStreamEvent) => {
+//   switch (event.type) {
+//     case 'message_start': {
+//       process.stdout.write(`${event.message.role.toUpperCase()}\n\n`);
+//       for (const content of event.message.content) {
+//         printContentBlock(content);
+//       }
+//       break;
+//     }
+//     case 'content_block_start': {
+//       printContentBlock(event.content);
+//       break;
+//     }
+//     case 'content_block_delta': {
+//       switch (event.delta.type) {
+//         case 'text_delta': {
+//           process.stdout.write(event.delta.text);
+//           break;
+//         }
+//         case 'input_json_delta': {
+//           process.stdout.write(event.delta.partial_json);
+//           break;
+//         }
+//       }
+//       break;
+//     }
+//     case 'content_block_stop': {
+//       process.stdout.write('\n');
+//       break;
+//     }
+//     case 'message_delta': {
+//       break;
+//     }
+//     case 'message_stop': {
+//       process.stdout.write('\n\n');
+//       break;
+//     }
+//   }
+// };

package/src/session/session.ts

@@ -0,0 +1,369 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema as S } from 'effect';
+
+import { defineTool, Message, ToolResult, type ArtifactDefinition, type Tool } from '@dxos/artifact';
+import { Event, synchronized } from '@dxos/async';
+import { createStatic } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+import { MixedStreamParser, type AIServiceClient, type GenerateRequest, type GenerationStream } from '../ai-service';
+import { isToolUse, runTools } from '../conversation';
+/**
+ * Contains message history, tools, current context.
+ * Current context means the state of the app, time of day, and other contextual information.
+ * It makes requests to the model, its a state machine.
+ * It keeps track of the current goal.
+ * It manages the context window.
+ * Tracks the success criteria of reaching the goal, exposing metrics (stretch)
+ * Could be run locally in the app or remotely.
+ * Could be personal or shared.
+ */
+
+export type SessionRunOptions = {
+  client: AIServiceClient;
+
+  artifacts: ArtifactDefinition[];
+
+  /**
+   * Non-artifact specific tools.
+   */
+  tools: Tool[];
+
+  history: Message[];
+
+  prompt: string;
+
+  generationOptions?: Pick<GenerateRequest, 'model' | 'systemPrompt'>;
+
+  extensions?: ToolContextExtensions;
+
+  /**
+   * Pre-require artifacts.
+   */
+  requiredArtifactIds?: string[];
+};
+
+type OperationModel = 'planning' | 'immediate';
+
+export type AiSessionOptions = {
+  /**
+   * In planning mode, the model will create a plan before executing the user's request.
+   * The plan will determine the artifacts that will be used to perform the actions.
+   * In require mode, the model will select the artifacts that will be used to perform the actions.
+   */
+  operationModel: OperationModel;
+};
+
+export class AISession {
+  /** SSE stream parser. */
+  private readonly _parser = new MixedStreamParser();
+
+  /** Pending messages (incl. the current user request). */
+  private _pending: Message[] = [];
+
+  /** Current streaming response. */
+  private _stream: GenerationStream | undefined;
+
+  /** Prior history from queue. */
+  private _history: Message[] = [];
+
+  /**
+   * New message.
+   */
+  public readonly message = this._parser.message;
+
+  /**
+   * Complete block added to Message.
+   */
+  public readonly block = this._parser.block;
+
+  /**
+   * Update partial block (while streaming).
+   */
+  public readonly update = this._parser.update;
+
+  public readonly streamEvent = this._parser.streamEvent;
+
+  /**
+   * User prompt or tool result.
+   */
+  public readonly userMessage = new Event<Message>();
+
+  constructor(private readonly _options: AiSessionOptions) {
+    // Message complete.
+    this._parser.message.on((message) => {
+      this._pending.push(message);
+    });
+  }
+
+  @synchronized
+  async run(options: SessionRunOptions): Promise<Message[]> {
+    const systemTools: Tool[] = [
+      defineTool('system', {
+        name: 'query_artifacts',
+        description: 'Query the available artifacts',
+        schema: S.Struct({}),
+        execute: async () => {
+          return ToolResult.Success({
+            artifacts: options.artifacts.map((artifact) => ({
+              id: artifact.id,
+              name: artifact.name,
+              description: artifact.description,
+            })),
+          });
+        },
+      }),
+    ];
+    switch (this._options.operationModel) {
+      case 'planning':
+        systemTools.push(
+          defineTool('system', {
+            name: 'create_plan',
+            description:
+              'Create a plan. Make sure that each step is independent and can be executed solely on the data returned by the previous step. The steps only share the data that is specified in the plan.',
+            schema: S.Struct({
+              goal: S.String.annotations({ description: 'The goal that the plan will achieve.' }),
+              steps: S.Array(
+                S.Struct({
+                  action: S.String.annotations({
+                    description: 'Complete, detailed action to perform.',
+                  }),
+                  input: S.String.annotations({
+                    description: 'The required input data for this step. First step must not require any data.',
+                  }),
+                  output: S.String.annotations({
+                    description: 'The output data from this step. This is passed to the next step as input.',
+                  }),
+                  requiredArtifactIds: S.Array(S.String).annotations({
+                    description: 'The ids of the artifacts required to perform this step.',
+                    examples: [['artifact:dxos.org/example/Test']],
+                  }),
+                }),
+              ).annotations({ description: 'Steps' }),
+            }),
+            execute: async ({ goal, steps }) => {
+              const missingArtifactIds = steps
+                .flatMap((step) => step.requiredArtifactIds)
+                .filter((artifactId) => !options.artifacts.some((artifact) => artifact.id === artifactId));
+              if (missingArtifactIds.length > 0) {
+                return ToolResult.Error(`One or more artifact ids are invalid: ${missingArtifactIds.join(', ')}`);
+              }
+
+              const stepResults: any[] = [];
+              for (const step of steps) {
+                log.info('executing step', { action: step.action });
+                const session = new AISession({ operationModel: 'immediate' });
+                session.message.on((ev) => this.message.emit(ev));
+                session.block.on((ev) => this.block.emit(ev));
+                session.update.on((ev) => this.update.emit(ev));
+                session.streamEvent.on((ev) => this.streamEvent.emit(ev));
+                session.userMessage.on((ev) => this.userMessage.emit(ev));
+
+                const messages = await session.run({
+                  client: options.client,
+                  artifacts: options.artifacts,
+                  tools: options.tools,
+                  history: options.history,
+                  extensions: options.extensions,
+                  generationOptions: options.generationOptions,
+                  requiredArtifactIds: step.requiredArtifactIds.slice(),
+                  prompt: `
+                    You are an agent that executes the subtask in a plan.
+                    
+                    While you know the plan's goal only focus on your subtask
+                    Use the data from the previous step to complete your task.
+                    Return all of the data from "output" in your last message.
+                    
+                    Current step (that you need to complete):
+                    ${JSON.stringify(
+                      {
+                        goal,
+                        input: step.input,
+                        output: step.output,
+                        yourTask: step.action,
+                      },
+                      null,
+                      2,
+                    )}
+                    
+                    Previous step results:
+                    ${JSON.stringify(stepResults.at(-1))}.
+                  `,
+                });
+                const result = messages.at(-1);
+                stepResults.push(result);
+              }
+
+              return ToolResult.Success({ stepResults });
+            },
+          }),
+        );
+        break;
+      case 'immediate':
+        systemTools.push(
+          defineTool('system', {
+            name: 'require_artifacts',
+            description:
+              'Require the use of specific artifacts. This will allow the model to interact with artifacts and use their tools.',
+            schema: S.Struct({
+              artifactIds: S.Array(S.String).annotations({
+                description: 'The ids of the artifacts to require',
+                examples: [['artifact:dxos.org/example/Test']],
+              }),
+            }),
+            execute: async ({ artifactIds }) => {
+              const missingArtifactIds = artifactIds.filter(
+                (artifactId) => !options.artifacts.some((artifact) => artifact.id === artifactId),
+              );
+              if (missingArtifactIds.length > 0) {
+                return ToolResult.Error(`One or more artifact ids are invalid: ${missingArtifactIds.join(', ')}`);
+              }
+
+              for (const artifactId of artifactIds) {
+                requiredArtifactIds.add(artifactId);
+              }
+
+              return ToolResult.Success({});
+            },
+          }),
+        );
+        break;
+    }
+
+    this._history = [...options.history];
+    this._pending = [
+      createStatic(Message, {
+        role: 'user',
+        content: [{ type: 'text', text: options.prompt }],
+      }),
+    ];
+    this.userMessage.emit(this._pending.at(-1)!);
+    this._stream = undefined;
+    let error: Error | undefined;
+
+    const requiredArtifactIds = new Set<string>(options.requiredArtifactIds ?? []);
+    try {
+      let more = false;
+      do {
+        const tools = [
+          ...systemTools,
+          ...options.tools,
+          ...options.artifacts
+            .filter((artifact) => requiredArtifactIds.has(artifact.id))
+            .flatMap((artifact) => artifact.tools),
+        ];
+
+        log('request', {
+          pending: this._pending.length,
+          history: this._history.length,
+          tools: tools.map((tool) => tool.name),
+        });
+
+        // Open request stream.
+        this._stream = await options.client.exec({
+          ...(options.generationOptions ?? {}),
+          // TODO(burdon): Rename messages or separate history/message.
+          history: [...this._history, ...this._pending],
+          tools,
+          systemPrompt: createBaseInstructions({
+            availableArtifacts: Array.from(requiredArtifactIds),
+            operationModel: this._options.operationModel,
+          }),
+        });
+
+        // Wait until complete.
+        await this._parser.parse(this._stream);
+        await this._stream.complete();
+
+        // Add messages.
+        log('response', { pending: this._pending });
+
+        // Resolve tool use locally.
+        more = false;
+        const message = this._pending.at(-1);
+        invariant(message);
+        if (isToolUse(message)) {
+          log('tool request...');
+          const response = await runTools({
+            message: this._pending.at(-1)!,
+            tools,
+            extensions: options.extensions ?? {},
+          });
+
+          log('tool response', { response });
+          switch (response.type) {
+            case 'continue': {
+              this._pending = [...this._pending, response.message];
+              this.userMessage.emit(response.message);
+              more = true;
+              break;
+            }
+          }
+        }
+      } while (more);
+    } catch (err) {
+      log.catch(err);
+      if (err instanceof Error && err.message.includes('Overloaded')) {
+        error = new AIServiecOverloadedError('AI service overloaded', { cause: err });
+      } else {
+        error = new Error('AI service error', { cause: err });
+      }
+    } finally {
+      this._stream = undefined;
+    }
+
+    if (error) {
+      throw error;
+    }
+
+    return this._pending;
+  }
+
+  abort() {
+    this._stream?.abort();
+  }
+}
+
+// TODO(burdon): Use handlebars template with effect-schema input.
+const createBaseInstructions = ({
+  availableArtifacts,
+  operationModel,
+}: {
+  availableArtifacts: string[];
+  operationModel: OperationModel;
+}) => `
+  You are a friendly, advanced AI assistant capable of creating and managing artifacts from available data and tools. 
+  Your task is to process user commands and questions and decide how best to respond.
+  In some cases, you will need to create or reference data objects called artifacts.
+
+  Follow these guidelines carefully:
+
+  Decision-making:
+
+  - Analyze the structure and type of the content in the user's message.
+  - Can you complete the task using the available artifacts?
+  - If you can't complete the task using the available artifacts, query the list of available artifacts using the appropriate tool.
+  - Identify which artifacts are relevant to the user's request.
+  ${
+    operationModel === 'planning'
+      ? `
+        - Break down the user's request into a step-by-step plan that you will use to execute the user's request, the plan items should include artifacts which will be used to perform the actions.
+      `
+      : `
+      - Are the required artifacts already available?
+      - If not, select which artifact(s) will be the most relevant and require them using the require_artifacts tool.
+      - The require'd artifact tools will be available for use after require.
+    `
+  }
+
+  Artifacts already required: ${availableArtifacts.join('\n')}
+`;
+
+export class AIServiecOverloadedError extends S.TaggedError<AIServiecOverloadedError>()(
+  'AIServiecOverloadedError',
+  {},
+) {}

package/src/testing/artifacts.test.ts

@@ -11,7 +11,7 @@ import { invariant } from '@dxos/invariant';
 import { SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
 
-import { AIServiceClientImpl, Tool } from '../ai-service';
+import { AIServiceEdgeClient, Tool } from '../ai-service';
 import { defineTool, ToolResult } from '../conversation';
 
 // TODO(dmaretskyi): Effect schema.
@@ -108,7 +108,7 @@ If there's
 
 describe('Artifacts', () => {
   test('shopping list', async () => {
-    const client = new AIServiceClientImpl({
+    const client = new AIServiceEdgeClient({
       endpoint: ENDPOINT,
     });
 
@@ -136,7 +136,7 @@ describe('Artifacts', () => {
     ]);
 
     const stream = await client.generate({
-      model: DEFAULT_LLM_MODEL,
+      model: DEFAULT_EDGE_MODEL,
       spaceId,
       threadId,
       systemPrompt: 'You are a helpful assistant.',
@@ -164,7 +164,7 @@ describe('Artifacts', () => {
     ]);
 
     const stream2 = await client.generate({
-      model: DEFAULT_LLM_MODEL,
+      model: DEFAULT_EDGE_MODEL,
       spaceId,
       threadId,
       systemPrompt: 'You are a helpful assistant.',