@dxos/assistant 0.8.3 → 0.8.4-main.03d5cd7b56

package/src/blueprint/machine.ts

@@ -1,251 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Match, Schema } from 'effect';
-
-import {
-  createTool,
-  type AiServiceClient,
-  Message,
-  type MessageContentBlock,
-  ToolResult,
-  type ToolUseContentBlock,
-  type ToolRegistry,
-} from '@dxos/ai';
-import { Event } from '@dxos/async';
-import { Key, Obj } from '@dxos/echo';
-import { type ObjectId } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { isNonNullable } from '@dxos/util';
-
-import type { Blueprint, BlueprintStep } from './blueprint';
-import { AISession } from '../session';
-
-const SYSTEM_PROMPT = `
-You are a smart Rule-Following Agent.
-Rule-Following Agent executes the command that the user sent in the last message.
-After doing the work, the Rule-Following Agent calls report tool.
-If the Rule-Following agent believes that no action is needed, it calls the report tool with "skipped" status.
-If the Rule-Following Agent is unable to perform the task, it calls the report tool with "bail" status.
-Rule-Following Agent explains the reason it is unable to perform the task before bailing.
-The Rule-Following Agent can express creativity and imagination in the way it performs the task.
-The Rule-Following Agent precisely follows the instructions.
-`;
-
-export type BlueprintTraceStep = {
-  status: 'done' | 'bailed' | 'skipped';
-  stepId: ObjectId;
-  comment: string;
-};
-
-export type BlueprintMachineState = {
-  history: Message[];
-  trace: BlueprintTraceStep[];
-  state: 'working' | 'bail' | 'done';
-};
-
-const INITIAL_STATE: BlueprintMachineState = {
-  state: 'working',
-  history: [],
-  trace: [],
-};
-
-type ExecutionOptions = {
-  aiClient: AiServiceClient;
-
-  /**
-   * Input to the blueprint.
-   */
-  input?: unknown;
-};
-
-export type BlueprintEvent =
-  | { type: 'begin'; invocationId: string }
-  | { type: 'end'; invocationId: string }
-  | { type: 'step-start'; invocationId: string; step: BlueprintStep }
-  | { type: 'step-complete'; invocationId: string; step: BlueprintStep }
-  | { type: 'message'; invocationId: string; message: Message }
-  | { type: 'block'; invocationId: string; block: MessageContentBlock };
-
-export interface BlueprintLogger {
-  log(event: BlueprintEvent): void;
-}
-
-/**
- * Blueprint state machine.
- */
-export class BlueprintMachine {
-  public readonly begin = new Event<void>();
-  public readonly end = new Event<void>();
-  public readonly stepStart = new Event<BlueprintStep>();
-  public readonly stepComplete = new Event<BlueprintStep>();
-  public readonly message = new Event<Message>();
-  public readonly block = new Event<MessageContentBlock>();
-
-  // TODO(burdon): Replace events with logger?
-  private logger?: BlueprintLogger;
-
-  private _state: BlueprintMachineState = structuredClone(INITIAL_STATE);
-
-  constructor(
-    readonly registry: ToolRegistry,
-    readonly blueprint: Blueprint,
-  ) {}
-
-  get state(): BlueprintMachineState {
-    return this._state;
-  }
-
-  setLogger(logger: BlueprintLogger): this {
-    this.logger = logger;
-    return this;
-  }
-
-  async runToCompletion(options: ExecutionOptions): Promise<void> {
-    const invocationId = Key.ObjectId.random();
-    log.info('runToCompletion', { invocationId, options });
-
-    this.begin.emit();
-    this.logger?.log({ type: 'begin', invocationId });
-
-    let firstStep = true;
-    while (this._state.state !== 'done') {
-      const input = firstStep ? options.input : undefined;
-      firstStep = false;
-
-      this._state = await this._execStep(invocationId, this._state, { input, aiClient: options.aiClient });
-
-      const step = this.blueprint.steps.find((step) => step.id === this._state.trace.at(-1)?.stepId)!;
-      this.stepComplete.emit(step);
-      this.logger?.log({ type: 'step-complete', invocationId, step });
-
-      if (this._state.state === 'bail') {
-        throw new Error('Agent unable to follow the blueprint');
-      }
-    }
-
-    this.end.emit();
-    this.logger?.log({ type: 'end', invocationId });
-  }
-
-  private async _execStep(
-    invocationId: string,
-    state: BlueprintMachineState,
-    options: ExecutionOptions,
-  ): Promise<BlueprintMachineState> {
-    const prevStep = this.blueprint.steps.findIndex((step) => step.id === this._state.trace.at(-1)?.stepId);
-    if (prevStep === this.blueprint.steps.length - 1) {
-      throw new Error('Done execution blueprint');
-    }
-
-    const nextStep = this.blueprint.steps[prevStep + 1];
-    const onLastStep = prevStep === this.blueprint.steps.length - 2;
-
-    this.stepStart.emit(nextStep);
-    this.logger?.log({ type: 'step-start', invocationId, step: nextStep });
-
-    const ReportSchema = Schema.Struct({
-      status: Schema.Literal('done', 'bailed', 'skipped').annotations({
-        description: 'The status of the task completion',
-      }),
-      comment: Schema.String.annotations({
-        description:
-          'A comment about the task completion. If you bailed, you must explain why in detail (min couple of sentences).',
-      }),
-    });
-
-    const report = createTool('system', {
-      name: 'report',
-      description: 'This tool reports that the agent has completed the task or is unable to do so.',
-      schema: ReportSchema,
-      execute: async (input) => ToolResult.Break(input),
-    });
-
-    const session = new AISession({
-      operationModel: 'configured',
-    });
-
-    session.userMessage.pipeInto(this.message);
-    session.message.pipeInto(this.message);
-    session.block.pipeInto(this.block);
-
-    const inputMessages = options.input
-      ? [
-          Obj.make(Message, {
-            role: 'user',
-            content: [taggedDataBlock('input', options.input)],
-          }),
-        ]
-      : [];
-    inputMessages.forEach((message) => {
-      this.message.emit(message);
-      this.logger?.log({ type: 'message', invocationId, message });
-    });
-
-    // TODO(wittjosiah): Warn if tool is not found.
-    const tools = nextStep.tools?.map((tool) => this.registry.get(tool)).filter(isNonNullable) ?? [];
-    const messages = await session.run({
-      systemPrompt: SYSTEM_PROMPT,
-      history: [...state.history, ...inputMessages],
-      tools: [...tools, report],
-      artifacts: [],
-      client: options.aiClient,
-      prompt: nextStep.instructions,
-    });
-
-    const { messages: trimmedHistory, call: lastBlock } = popLastToolCall(messages);
-    if (!lastBlock) {
-      // TODO(dmaretskyi): Handle this with grace.
-      throw new Error('Agent did not call the report tool');
-    }
-
-    const { status, comment } = ReportSchema.pipe(Schema.decodeUnknownSync)(lastBlock.input);
-
-    return {
-      history: [...state.history, ...inputMessages, ...trimmedHistory],
-      trace: [
-        ...state.trace,
-        {
-          stepId: nextStep.id,
-          status,
-          comment,
-        },
-      ],
-      state: Match.value(status).pipe(
-        Match.withReturnType<BlueprintMachineState['state']>(),
-        Match.when('done', () => (onLastStep ? 'done' : 'working')),
-        Match.when('skipped', () => (onLastStep ? 'done' : 'working')),
-        Match.when('bailed', () => 'bail'),
-        Match.exhaustive,
-      ),
-    };
-  }
-}
-
-const taggedDataBlock = (tag: string, data: unknown): MessageContentBlock => {
-  return {
-    type: 'text',
-    text: `<${tag}>\n${JSON.stringify(data, null, 2)}\n</${tag}>`,
-  } satisfies MessageContentBlock;
-};
-
-const popLastToolCall = (messages: Message[]): { messages: Message[]; call?: ToolUseContentBlock } => {
-  const lastBlock = messages.at(-1)?.content.at(-1);
-  if (lastBlock?.type !== 'tool_use') {
-    return { messages };
-  }
-
-  const trimmedHistory = [
-    ...messages.slice(0, -1),
-    {
-      ...messages.at(-1)!,
-      content: [...messages.at(-1)!.content.slice(0, -1)],
-    } satisfies Message,
-  ];
-
-  return {
-    messages: trimmedHistory,
-    call: lastBlock,
-  };
-};

package/src/blueprint/test-data.ts

@@ -1,55 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Obj } from '@dxos/echo';
-import { DataType } from '@dxos/schema';
-
-// TODO(burdon): Convert to queue.
-export const TEST_EMAILS: DataType.Message[] = [
-  Obj.make(DataType.Message, {
-    properties: {
-      subject: 'Introduction email',
-    },
-    sender: {
-      email: 'alice@example.com',
-    },
-    blocks: [
-      {
-        type: 'text',
-        text: "Hi there! I came across your work and would love to connect. I'm working on some interesting projects in the decentralized space and think there could be good collaboration opportunities.",
-      },
-    ],
-    created: new Date().toISOString(),
-  }),
-  Obj.make(DataType.Message, {
-    properties: {
-      subject: 'Question about integration',
-    },
-    sender: {
-      email: 'bob@example.com',
-    },
-    blocks: [
-      {
-        type: 'text',
-        text: "Hello, I was wondering if your system supports integration with external APIs? We have some custom services we'd like to connect. Could you explain the process?",
-      },
-    ],
-    created: new Date().toISOString(),
-  }),
-  Obj.make(DataType.Message, {
-    properties: {
-      subject: 'MAKE MONEY FAST!!!',
-    },
-    sender: {
-      email: 'spammer123@sketchy.com',
-    },
-    blocks: [
-      {
-        type: 'text',
-        text: 'CONGRATULATIONS!!! You have been selected to receive $10 MILLION dollars! Just send us your bank details and social security number to claim your prize NOW!!!',
-      },
-    ],
-    created: new Date().toISOString(),
-  }),
-];

package/src/extraction/extraction-llm-function/extraction-llm-function.test.ts

@@ -1,95 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { describe, test, beforeAll } from 'vitest';
-
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { log } from '@dxos/log';
-import { createTestData } from '@dxos/schema/testing';
-import { range } from '@dxos/util';
-
-import { extractionAnthropicFn } from './extraction-llm-function';
-import { processTranscriptMessage } from '../extraction';
-
-describe.skip('LLM EntityExtraction', () => {
-  let executor: FunctionExecutor;
-
-  beforeAll(async () => {
-    executor = new FunctionExecutor(
-      new ServiceContainer().setServices({
-        ai: {
-          client: new EdgeAiServiceClient({
-            endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-            defaultGenerationOptions: {
-              // model: '@anthropic/claude-sonnet-4-20250514',
-              model: '@anthropic/claude-3-5-haiku-20241022',
-            },
-          }),
-        },
-      }),
-    );
-  });
-
-  test('should process a transcript block', async () => {
-    const { transcriptMessages, documents, contacts } = createTestData();
-    log.info('context', { documents, contacts });
-
-    for (const message of transcriptMessages.slice(0, 1)) {
-      log.info('input', message);
-      const { message: enhancedMessage } = await processTranscriptMessage({
-        input: {
-          message,
-          objects: [...documents, ...Object.values(contacts)],
-        },
-        function: extractionAnthropicFn,
-        executor,
-      });
-      log.info('output', enhancedMessage);
-    }
-  });
-
-  test('computational irreducibility', async () => {
-    const { transcriptWoflram, documents, contacts } = createTestData();
-
-    log.info('context', { documents, contacts });
-    const message = transcriptWoflram[0];
-    log.info('input', message);
-
-    await Promise.all(
-      range(10).map(async () => {
-        const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
-          input: {
-            message,
-            objects: [...documents, ...Object.values(contacts)],
-          },
-          function: extractionAnthropicFn,
-          executor,
-        });
-        log.info('output', { message: enhancedMessage.blocks[0], timeElapsed });
-      }),
-    );
-  });
-
-  test('org and document linking', async () => {
-    const { transcriptJosiah, documents, contacts, organizations } = createTestData();
-
-    log.info('context', { contacts, organizations, documents });
-
-    for (const message of transcriptJosiah) {
-      log.info('input', message);
-
-      const { message: enhancedMessage } = await processTranscriptMessage({
-        input: {
-          message,
-          objects: [...documents, ...Object.values(contacts), ...Object.values(organizations)],
-        },
-        function: extractionAnthropicFn,
-        executor,
-      });
-      log.info('output', enhancedMessage);
-    }
-  });
-});

package/src/extraction/extraction-llm-function/extraction-llm-function.ts

@@ -1,65 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Message } from '@dxos/ai';
-import { Obj } from '@dxos/echo';
-import { create } from '@dxos/echo-schema';
-import { AiService, defineFunction, type FunctionDefinition } from '@dxos/functions';
-import { DataType } from '@dxos/schema';
-
-import PROMPT from './instructions.tpl?raw';
-import { AISession } from '../../session';
-import { ExtractionInput, ExtractionOutput } from '../extraction';
-import { insertReferences, ReferencedQuotes } from '../quotes';
-
-export const extractionAnthropicFn: FunctionDefinition<ExtractionInput, ExtractionOutput> = defineFunction({
-  description: 'Extract entities from the transcript message and add them to the message.',
-  inputSchema: ExtractionInput,
-  outputSchema: ExtractionOutput,
-  handler: async ({ data: { message, objects }, context }) => {
-    const startTime = performance.now();
-    const ai = context.getService(AiService);
-    const session = new AISession({ operationModel: 'configured' });
-    const result = await session.runStructured(ReferencedQuotes, {
-      generationOptions: {
-        model: '@anthropic/claude-3-5-haiku-20241022',
-      },
-      client: ai.client,
-      systemPrompt: PROMPT,
-      history: [
-        Obj.make(Message, {
-          role: 'user',
-          content: [
-            {
-              type: 'text',
-              text: `<context>${JSON.stringify(objects)}</context>`,
-            },
-            {
-              type: 'text',
-              text: `<transcript>${JSON.stringify(message.blocks)}</transcript>`,
-            },
-          ],
-        }),
-      ],
-      artifacts: [],
-      prompt: '',
-      tools: [],
-    });
-
-    return {
-      message: create(DataType.Message, {
-        ...message,
-        blocks: message.blocks.map((block, i) =>
-          block.type !== 'transcription'
-            ? block
-            : {
-                ...block,
-                text: insertReferences(block.text, result),
-              },
-        ),
-      }),
-      timeElapsed: performance.now() - startTime,
-    };
-  },
-});

package/src/extraction/extraction-llm-function/index.ts

@@ -1,5 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-export * from './extraction-llm-function';

package/src/extraction/extraction-ner-function/extraction-ner-function.test.ts

@@ -1,115 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { describe, test, beforeAll } from 'vitest';
-
-import { type EchoDatabase } from '@dxos/echo-db';
-import { EchoTestBuilder } from '@dxos/echo-db/testing';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
-import { createTestData, Testing } from '@dxos/schema/testing';
-import { range } from '@dxos/util';
-
-import { extractionNerFn } from './extraction-ner-function';
-import { processTranscriptMessage } from '../extraction';
-
-describe.skip('NER EntityExtraction', () => {
-  let builder: EchoTestBuilder;
-  let db: EchoDatabase;
-  let executor: FunctionExecutor;
-  let testData: {
-    transcriptJosiah: DataType.Message[];
-    transcriptWoflram: DataType.Message[];
-    transcriptMessages: DataType.Message[];
-    documents: Testing.DocumentType[];
-    contacts: Record<string, DataType.Person>;
-    organizations: Record<string, DataType.Organization>;
-  };
-  const TYPES = [DataType.Organization, DataType.Person, Testing.Contact, Testing.DocumentType];
-
-  beforeAll(async () => {
-    // TODO(dmaretskyi): Helper to scaffold this from a config.
-    builder = await new EchoTestBuilder().open();
-    const { db: db1 } = await builder.createDatabase({ indexing: { vector: true } });
-    db = db1;
-    db.graph.schemaRegistry.addSchema(TYPES);
-    const data = createTestData();
-    testData = {
-      transcriptJosiah: data.transcriptJosiah,
-      transcriptWoflram: data.transcriptWoflram,
-      transcriptMessages: data.transcriptMessages,
-      documents: data.documents.map((document) => db.add(document)),
-      contacts: Object.fromEntries(Object.entries(data.contacts).map(([key, value]) => [key, db.add(value)])),
-      organizations: Object.fromEntries(Object.entries(data.organizations).map(([key, value]) => [key, db.add(value)])),
-    };
-    await db.flush();
-
-    executor = new FunctionExecutor(
-      new ServiceContainer().setServices({
-        database: { db },
-      }),
-    );
-  }, 30_000);
-
-  test('should process a transcript block', async () => {
-    const { transcriptMessages, documents, contacts } = testData;
-    log.info('context', { documents, contacts });
-
-    for (const message of transcriptMessages) {
-      log.info('input', message);
-      const { message: enhancedMessage } = await processTranscriptMessage({
-        input: {
-          message,
-          objects: [...documents, ...Object.values(contacts)],
-        },
-        function: extractionNerFn,
-        executor,
-      });
-      log.info('output', enhancedMessage);
-    }
-  });
-
-  test('computational irreducibility', async () => {
-    const { transcriptWoflram, documents, contacts } = testData;
-
-    log.info('context', { documents, contacts });
-    const message = transcriptWoflram[0];
-    log.info('input', message);
-
-    await Promise.all(
-      range(10).map(async () => {
-        const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
-          input: {
-            message,
-            objects: [...documents, ...Object.values(contacts)],
-          },
-          function: extractionNerFn,
-          executor,
-        });
-        log.info('output', { message: enhancedMessage.blocks[0], timeElapsed });
-      }),
-    );
-  });
-
-  test('org and document linking', async () => {
-    const { transcriptJosiah, documents, contacts, organizations } = testData;
-
-    log.info('context', { contacts, organizations, documents });
-
-    for (const message of transcriptJosiah) {
-      log.info('input', message);
-
-      const { message: enhancedMessage } = await processTranscriptMessage({
-        input: {
-          message,
-          objects: [...documents, ...Object.values(contacts), ...Object.values(organizations)],
-        },
-        function: extractionNerFn,
-        executor,
-      });
-      log.info('output', enhancedMessage);
-    }
-  });
-});

package/src/extraction/extraction-ner-function/extraction-ner-function.ts

@@ -1,54 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { asyncTimeout } from '@dxos/async';
-import { create } from '@dxos/echo-schema';
-import { DatabaseService, defineFunction } from '@dxos/functions';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
-
-import { extractFullEntities } from './named-entity-recognition';
-import { ExtractionInput, ExtractionOutput } from '../extraction';
-import { findQuotes, insertReferences } from '../quotes';
-
-// TODO(mykola): Make it use vector index to query objects and not pass objects as input.
-export const extractionNerFn = defineFunction({
-  description: 'Extracts named entities from a text',
-  inputSchema: ExtractionInput,
-  outputSchema: ExtractionOutput,
-  handler: async ({ data: { message, options }, context }) => {
-    log.info('input', { message, options });
-    const startTime = performance.now();
-    const { db } = context.getService(DatabaseService);
-
-    const entitiesPromise = Promise.all(
-      message.blocks.map(async (block) => {
-        invariant(block.type === 'transcription' || block.type === 'text', 'Block must have text');
-        return extractFullEntities(block.text);
-      }),
-    ).then((entities) => entities.flat());
-    const entities = options?.timeout ? await asyncTimeout(entitiesPromise, options.timeout) : await entitiesPromise;
-    log.info('entities', { entities });
-    const quoteReferences = await findQuotes(
-      entities.map((entity) => entity.word),
-      db,
-    );
-    const blocksWithReferences = message.blocks.map((block, i) => {
-      invariant(block.type === 'transcription' || block.type === 'text', 'Block must have text');
-      return { ...block, text: insertReferences(block.text, quoteReferences) };
-    });
-    const messageWithReferences = create(DataType.Message, {
-      ...message,
-      blocks: blocksWithReferences,
-    });
-
-    log.info('output', { messageWithReferences });
-
-    return {
-      message: messageWithReferences,
-      timeElapsed: performance.now() - startTime,
-    };
-  },
-});