@dxos/assistant-toolkit 0.8.4-main.3eb6e50203 → 0.8.4-main.3fbcb4aa9b

package/src/functions/research/research.test.ts

@@ -1,144 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { inspect } from 'node:util';
-
-import { describe, it } from '@effect/vitest';
-import * as Effect from 'effect/Effect';
-
-import { ConsolePrinter } from '@dxos/ai';
-import { MemoizedAiService } from '@dxos/ai/testing';
-import { AiConversation, type ContextBinding, GenerationObserver } from '@dxos/assistant';
-import { AssistantTestLayer } from '@dxos/assistant/testing';
-import { Blueprint } from '@dxos/blueprints';
-import { Database, Filter, Obj, Query, Ref } from '@dxos/echo';
-import { acquireReleaseResource } from '@dxos/effect';
-import { TestHelpers } from '@dxos/effect/testing';
-import { FunctionInvocationService, QueueService } from '@dxos/functions';
-import { invariant } from '@dxos/invariant';
-import { ObjectId } from '@dxos/keys';
-import { MarkdownBlueprint } from '@dxos/plugin-markdown/blueprints';
-import { Markdown } from '@dxos/plugin-markdown/types';
-import { HasSubject, type Message, Organization } from '@dxos/types';
-
-import { ResearchBlueprint } from '../../blueprints';
-
-import { default as createDocument } from './document-create';
-import { default as research } from './research';
-import { ResearchGraph, queryResearchGraph } from './research-graph';
-import { ResearchDataTypes } from './types';
-
-ObjectId.dangerouslyDisableRandomness();
-
-const TestLayer = AssistantTestLayer({
-  functions: [research, createDocument, ...MarkdownBlueprint.functions],
-  types: [...ResearchDataTypes, ResearchGraph, Blueprint.Blueprint, Markdown.Document, HasSubject.HasSubject],
-});
-
-describe('Research', () => {
-  it.effect(
-    'call a function to generate a research report',
-    Effect.fnUntraced(
-      function* (_) {
-        yield* Database.add(
-          Obj.make(Organization.Organization, {
-            name: 'BlueYard',
-            website: 'https://blueyard.com',
-          }),
-        );
-        yield* Database.flush({ indexes: true });
-        const result = yield* FunctionInvocationService.invokeFunction(research, {
-          query: 'Founders and portfolio of BlueYard.',
-        });
-
-        console.log(inspect(result, { depth: null, colors: true }));
-        console.log(JSON.stringify(result, null, 2));
-
-        yield* Database.flush({ indexes: true });
-        const researchGraph = yield* queryResearchGraph();
-        if (researchGraph) {
-          const data = yield* Database.load(researchGraph.queue).pipe(
-            Effect.flatMap((queue) => Effect.promise(() => queue.queryObjects())),
-          );
-          console.log(inspect(data, { depth: null, colors: true }));
-        }
-      },
-      Effect.provide(TestLayer),
-      TestHelpers.provideTestContext,
-    ),
-    MemoizedAiService.isGenerationEnabled() ? 240_000 : 30_000,
-  );
-
-  it.scoped(
-    'create and update research report',
-    Effect.fnUntraced(
-      function* (_) {
-        const organization = yield* Database.add(
-          Obj.make(Organization.Organization, {
-            name: 'BlueYard',
-            website: 'https://blueyard.com',
-          }),
-        );
-
-        const queue = yield* QueueService.createQueue<Message.Message | ContextBinding>();
-        const conversation = yield* acquireReleaseResource(() => new AiConversation({ queue }));
-
-        yield* Database.flush({ indexes: true });
-        const researchBlueprint = yield* Database.add(Obj.clone(ResearchBlueprint));
-        const markdownBlueprint = yield* Database.add(Obj.clone(MarkdownBlueprint.make()));
-        yield* Effect.promise(() =>
-          conversation.context.bind({
-            blueprints: [Ref.make(researchBlueprint), Ref.make(markdownBlueprint)],
-            objects: [Ref.make(organization)],
-          }),
-        );
-
-        const observer = GenerationObserver.fromPrinter(new ConsolePrinter());
-
-        yield* conversation.createRequest({
-          observer,
-          prompt: `Create a research summary about ${organization.name}.`,
-        });
-        {
-          const docs = yield* Database.runQuery(
-            Query.select(Filter.id(organization.id)).targetOf(HasSubject.HasSubject).source(),
-          );
-          if (docs.length !== 1) {
-            throw new Error(`Expected 1 research document; got ${docs.length}: ${docs.map((_) => _.name)}`);
-          }
-
-          const doc = docs[0];
-          invariant(Obj.instanceOf(Markdown.Document, doc));
-          console.log({
-            name: doc.name,
-            content: yield* Database.load(doc.content).pipe(Effect.map((_) => _.content)),
-          });
-        }
-
-        yield* conversation.createRequest({
-          observer,
-          prompt: 'Add a section about their portfolio.',
-        });
-        {
-          const docs = yield* Database.runQuery(
-            Query.select(Filter.id(organization.id)).targetOf(HasSubject.HasSubject).source(),
-          );
-          if (docs.length !== 1) {
-            throw new Error(`Expected 1 research document; got ${docs.length}: ${docs.map((_) => _.name)}`);
-          }
-
-          const doc = docs[0];
-          invariant(Obj.instanceOf(Markdown.Document, doc));
-          console.log({
-            name: doc.name,
-            content: yield* Database.load(doc.content).pipe(Effect.map((_) => _.content)),
-          });
-        }
-      },
-      Effect.provide(TestLayer),
-      TestHelpers.provideTestContext,
-    ),
-    MemoizedAiService.isGenerationEnabled() ? 240_000 : 30_000,
-  );
-});

package/src/functions/research/research.ts

@@ -1,182 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Toolkit from '@effect/ai/Toolkit';
-import * as AnthropicTool from '@effect/ai-anthropic/AnthropicTool';
-import * as Array from 'effect/Array';
-import * as Effect from 'effect/Effect';
-import * as Function from 'effect/Function';
-import * as Layer from 'effect/Layer';
-import * as Option from 'effect/Option';
-import * as Schema from 'effect/Schema';
-import * as String from 'effect/String';
-
-import { AiService, ConsolePrinter } from '@dxos/ai';
-import { AiSession, GenerationObserver, GenericToolkit, ToolExecutionServices, createToolkit } from '@dxos/assistant';
-import { Template } from '@dxos/blueprints';
-import { type DXN, Entity, Obj } from '@dxos/echo';
-import { Database } from '@dxos/echo';
-import { TracingService, defineFunction } from '@dxos/functions';
-import { FunctionInvocationServiceLayerTestMocked } from '@dxos/functions-runtime/testing';
-import { type Message, Person } from '@dxos/types';
-import { trim } from '@dxos/util';
-
-import { LocalSearchHandler, LocalSearchToolkit, makeGraphWriterHandler, makeGraphWriterToolkit } from '../../crud';
-import { exaFunction, exaMockFunction } from '../exa';
-
-import { contextQueueLayerFromResearchGraph } from './research-graph';
-import PROMPT from './research-instructions.tpl?raw';
-import { ResearchDataTypes } from './types';
-
-/**
- * Exec external service and return the results as a Subgraph.
- */
-export default defineFunction({
-  key: 'dxos.org/function/research',
-  name: 'Research',
-  description: trim`
-    Search the web to research information about the given subject.
-    Inserts structured data into the research graph. 
-    Creates a research summary and returns the objects created.
-  `,
-  inputSchema: Schema.Struct({
-    query: Schema.String.annotations({
-      description: trim`
-        The search query.
-        If doing research on an object then load it first and pass it as JSON.
-      `,
-    }),
-
-    instructions: Schema.optional(Schema.String).annotations({
-      description: trim`
-        The instructions for the research agent. 
-      `,
-    }),
-
-    // TOOD(burdon): Move to context.
-    mockSearch: Schema.optional(Schema.Boolean).annotations({
-      description: 'Whether to use the mock search tool.',
-      default: false,
-    }),
-
-    entityExtraction: Schema.optional(Schema.Boolean).annotations({
-      description: trim`
-        Whether to extract structured entities from the research. 
-        Experimental feature only enable if user explicitly requests it.
-      `,
-      default: false,
-    }),
-  }),
-  outputSchema: Schema.Struct({
-    document: Schema.optional(Schema.String).annotations({
-      description: 'The generated research document.',
-    }),
-    objects: Schema.Array(Schema.Unknown).annotations({
-      description: 'Structured objects created during the research process.',
-    }),
-  }),
-  handler: Effect.fnUntraced(
-    function* ({ data: { query, instructions, mockSearch = false, entityExtraction = false } }) {
-      if (mockSearch) {
-        const mockPerson = yield* Database.add(
-          Obj.make(Person.Person, {
-            preferredName: 'John Doe',
-            emails: [{ value: 'john.doe@example.com' }],
-            phoneNumbers: [{ value: '123-456-7890' }],
-          }),
-        );
-
-        return {
-          document: trim`
-            The research ran in test-mode and was mocked. Proceed as usual.
-            We reference John Doe to test reference: ${Obj.getDXN(mockPerson)}
-          `,
-          objects: [Obj.toJSON(mockPerson)],
-        };
-      }
-
-      yield* Database.flush({ indexes: true });
-      yield* TracingService.emitStatus({ message: 'Starting research...' });
-
-      const NativeWebSearch = Toolkit.make(AnthropicTool.WebSearch_20250305({}));
-
-      let toolkit: Toolkit.Any = NativeWebSearch;
-      let handlers: Layer.Layer<any, any> = Layer.empty as any;
-
-      const objectDXNs: DXN[] = [];
-      if (entityExtraction) {
-        const GraphWriterToolkit = makeGraphWriterToolkit({ schema: ResearchDataTypes });
-        const GraphWriterHandler = makeGraphWriterHandler(GraphWriterToolkit, {
-          onAppend: (dxns) => objectDXNs.push(...dxns),
-        });
-
-        toolkit = Toolkit.merge(toolkit, LocalSearchToolkit, GraphWriterToolkit);
-        handlers = Layer.mergeAll(handlers, LocalSearchHandler, GraphWriterHandler).pipe(
-          Layer.provide(contextQueueLayerFromResearchGraph),
-        ) as any;
-      }
-
-      const finishedToolkit = yield* createToolkit({ toolkit }).pipe(Effect.provide(handlers));
-
-      const session = new AiSession();
-      const result = yield* session.run({
-        prompt: query,
-        system: join(
-          Template.process(PROMPT, { entityExtraction }),
-          instructions && `<instructions>${instructions}</instructions>`,
-        ),
-        toolkit: finishedToolkit,
-        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'research' })),
-      });
-
-      const objects = yield* Effect.forEach(objectDXNs, (dxn) => Database.resolve(dxn)).pipe(
-        Effect.map(Array.map((obj) => Entity.toJSON(obj))),
-      );
-
-      return {
-        document: extractLastTextBlock(result),
-        objects,
-      };
-    },
-    Effect.provide(
-      Layer.mergeAll(AiService.model('@anthropic/claude-sonnet-4-0'), ToolExecutionServices).pipe(
-        Layer.provide(
-          // TODO(dmaretskyi): This should be provided by environment.
-
-          Layer.mergeAll(
-            GenericToolkit.providerEmpty,
-            FunctionInvocationServiceLayerTestMocked({ functions: [exaFunction, exaMockFunction] }),
-          ),
-        ),
-      ),
-    ),
-  ),
-});
-
-// TODO(burdon): Factor out.
-const join = (...strings: (string | undefined)[]) => strings.filter(Boolean).join('\n\n');
-
-/**
- * Extracts the last text block from the result.
- * Skips citations.
- */
-// TODO(burdon): Factor out.
-const extractLastTextBlock = (result: Message.Message[]) => {
-  return Function.pipe(
-    result,
-    Array.last,
-    Option.map(
-      Function.flow(
-        (_: Message.Message) => _.blocks,
-        Array.reverse,
-        Array.dropWhile((_: any) => _._tag === 'summary'),
-        Array.takeWhile((_: any) => _._tag === 'text'),
-        Array.reverse,
-        Array.map((_: any) => _.text),
-        Array.reduce('', String.concat),
-      ),
-    ),
-    Option.getOrElse(() => ''),
-  );
-};

package/src/functions/research/types.ts

@@ -1,26 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Type } from '@dxos/echo';
-import { Text } from '@dxos/schema';
-import { Event, HasConnection, HasRelationship, LegacyOrganization, LegacyPerson, Pipeline, Task } from '@dxos/types';
-
-/**
- * Data types for research.
- */
-export const ResearchDataTypes: Type.Entity.Any[] = [
-  // Objects
-  Event.Event,
-  LegacyOrganization,
-  Task.Task,
-  Text.Text,
-
-  // Relations
-  // TODO(wittjosiah): Until views (e.g., Table) support relations this needs to be expressed via organization ref.
-  // Employer.Employer,
-  LegacyPerson,
-  Pipeline.Pipeline,
-  HasRelationship.HasRelationship,
-  HasConnection.HasConnection,
-];

package/src/functions/tasks/index.ts

@@ -1,11 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { default as read$ } from './read';
-import { default as update$ } from './update';
-
-export namespace Tasks {
-  export const read = read$;
-  export const update = update$;
-}

package/src/functions/tasks/read.ts

@@ -1,34 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import * as Schema from 'effect/Schema';
-
-import { Database, Type } from '@dxos/echo';
-import { defineFunction } from '@dxos/functions';
-import { Markdown } from '@dxos/plugin-markdown/types';
-
-export default defineFunction({
-  key: 'dxos.org/function/markdown/read-tasks',
-  name: 'Read',
-  description: 'Read markdown tasks.',
-  inputSchema: Schema.Struct({
-    doc: Type.Ref(Markdown.Document).annotations({
-      description: 'The ID of the document to read.',
-    }),
-  }),
-  outputSchema: Schema.Struct({
-    content: Schema.String,
-  }),
-  handler: Effect.fn(function* ({ data: { doc } }) {
-    const document = yield* Database.load(doc);
-
-    // Return content with line numbers prefixed.
-    const { content } = yield* Database.load(document.content);
-    const lines = content.split('\n');
-    const len = String(lines.length).length;
-    const numbered = lines.map((line, i) => `${String(i + 1).padStart(len, ' ')}. ${line}`).join('\n');
-    return { content: numbered };
-  }),
-});

package/src/functions/tasks/task-list.test.ts

@@ -1,99 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { describe, expect, it } from 'vitest';
-
-import { MarkdownTasks, type TaskOperation } from './task-list';
-
-describe('MarkdownTasks', () => {
-  it('should initialize with content', () => {
-    const content = '- [ ] Task 1\n- [x] Task 2';
-    const manager = new MarkdownTasks(content);
-    expect(manager.getRawContent()).toBe(content);
-  });
-
-  it('should provide numbered content', () => {
-    const manager = new MarkdownTasks('- [ ] Task 1\n  - [ ] Subtask');
-    const numbered = manager.getNumberedContent();
-    expect(numbered).toBe('  1→- [ ] Task 1\n  2→  - [ ] Subtask');
-  });
-
-  it('should insert tasks with indentation', () => {
-    const manager = new MarkdownTasks('- [ ] Main task');
-    manager.insertTask(2, 'Subtask', false, 2);
-    expect(manager.getRawContent()).toBe('- [ ] Main task\n  - [ ] Subtask');
-  });
-
-  it('should insert completed tasks', () => {
-    const manager = new MarkdownTasks('- [ ] Task 1');
-    manager.insertTask(2, 'Completed task', true, 0);
-    expect(manager.getRawContent()).toBe('- [ ] Task 1\n- [x] Completed task');
-  });
-
-  it('should delete tasks', () => {
-    const manager = new MarkdownTasks('- [ ] Task 1\n- [ ] Task 2');
-    const success = manager.deleteTask(2);
-    expect(success).toBe(true);
-    expect(manager.getRawContent()).toBe('- [ ] Task 1');
-  });
-
-  it('should update task text', () => {
-    const manager = new MarkdownTasks('- [ ] Old task');
-    const success = manager.updateTaskText(1, 'New task description');
-    expect(success).toBe(true);
-    expect(manager.getRawContent()).toBe('- [ ] New task description');
-  });
-
-  it('should toggle task completion', () => {
-    const manager = new MarkdownTasks('- [ ] Task 1\n- [x] Task 2');
-
-    // Complete first task
-    manager.toggleTaskCompletion(1, true);
-    expect(manager.getRawContent()).toBe('- [x] Task 1\n- [x] Task 2');
-
-    // Uncomplete second task
-    manager.toggleTaskCompletion(2, false);
-    expect(manager.getRawContent()).toBe('- [x] Task 1\n- [ ] Task 2');
-  });
-
-  it('should change task indentation', () => {
-    const manager = new MarkdownTasks('- [ ] Task 1');
-    const success = manager.setTaskIndent(1, 4);
-    expect(success).toBe(true);
-    expect(manager.getRawContent()).toBe('    - [ ] Task 1');
-  });
-
-  it('should apply multiple operations', () => {
-    const manager = new MarkdownTasks('- [ ] Task 1');
-
-    const operations: TaskOperation[] = [
-      { type: 'insertTask', lineNumber: 2, text: 'New task', indent: 2 },
-      { type: 'toggleTaskCompletion', lineNumber: 1, completed: true },
-      { type: 'insertTask', lineNumber: 3, text: 'Another task', indent: 0 },
-    ];
-
-    manager.applyOperations(operations);
-    const result = manager.getRawContent();
-    expect(result).toBe('- [x] Task 1\n  - [ ] New task\n- [ ] Another task');
-  });
-
-  it('should handle hierarchical task structure', () => {
-    const manager = new MarkdownTasks();
-
-    // Build a hierarchical structure
-    manager.insertTask(1, 'Main task', false, 0);
-    manager.insertTask(2, 'Subtask 1', false, 2);
-    manager.insertTask(3, 'Sub-subtask', false, 4);
-    manager.insertTask(4, 'Subtask 2', false, 2);
-
-    const expected = '- [ ] Main task\n  - [ ] Subtask 1\n    - [ ] Sub-subtask\n  - [ ] Subtask 2';
-    expect(manager.getRawContent()).toBe(expected);
-  });
-
-  it('should handle line insertions at the end', () => {
-    const manager = new MarkdownTasks('- [ ] Task 1');
-    manager.insertTask(999, 'End task', false, 0);
-    expect(manager.getRawContent()).toBe('- [ ] Task 1\n- [ ] End task');
-  });
-});

package/src/functions/tasks/task-list.ts

@@ -1,165 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-/**
- * Hierarchical Task List Manager.
- *
- * A framework for managing hierarchical task lists where each line is a task.
- * Designed to work with language model agents that receive documents with line numbers.
- */
-export class MarkdownTasks {
-  private _lineEndings: string;
-  private _content: string[];
-
-  constructor(initialContent: string = '') {
-    // Detect line endings.
-    this._lineEndings = initialContent.includes('\r\n') ? '\r\n' : '\n';
-    this._content = initialContent ? initialContent.split(this._lineEndings) : [];
-  }
-
-  /**
-   * Get the current document content with line numbers prefixed.
-   */
-  getNumberedContent(): string {
-    return this._content
-      .map((line, index) => `${(index + 1).toString().padStart(3, ' ')}→${line}`)
-      .join(this._lineEndings);
-  }
-
-  /**
-   * Get the raw document content without line numbers.
-   */
-  getRawContent(): string {
-    return this._content.join(this._lineEndings);
-  }
-
-  /**
-   * Insert a new task at the specified line number (1-based).
-   * Indentation level determines hierarchy (0, 2, 4, 6 spaces etc.)
-   */
-  insertTask(lineNumber: number, taskText: string, completed: boolean = false, indent: number = 0): void {
-    const indentStr = ' '.repeat(indent);
-    const taskLine = completed ? `${indentStr}- [x] ${taskText}` : `${indentStr}- [ ] ${taskText}`;
-    const insertIndex = Math.max(0, Math.min(lineNumber - 1, this._content.length));
-    this._content.splice(insertIndex, 0, taskLine);
-  }
-
-  /**
-   * Delete a task at the specified line number (1-based).
-   */
-  deleteTask(lineNumber: number): boolean {
-    if (lineNumber < 1 || lineNumber > this._content.length) {
-      return false;
-    }
-    this._content.splice(lineNumber - 1, 1);
-    return true;
-  }
-
-  /**
-   * Update the text of a task at the specified line number (1-based).
-   */
-  updateTaskText(lineNumber: number, newText: string): boolean {
-    if (lineNumber < 1 || lineNumber > this._content.length) {
-      return false;
-    }
-
-    const currentLine = this._content[lineNumber - 1];
-    const taskMatch = currentLine.match(/^(\s*- \[[x ]\] )(.*)$/);
-
-    if (taskMatch) {
-      this._content[lineNumber - 1] = `${taskMatch[1]}${newText}`;
-      return true;
-    }
-    return false;
-  }
-
-  /**
-   * Mark a task as complete or incomplete at the specified line number (1-based).
-   */
-  toggleTaskCompletion(lineNumber: number, completed?: boolean): boolean {
-    if (lineNumber < 1 || lineNumber > this._content.length) {
-      return false;
-    }
-
-    const currentLine = this._content[lineNumber - 1];
-    const taskMatch = currentLine.match(/^(\s*- \[)([x ])(.*)$/);
-    if (taskMatch) {
-      const isCurrentlyComplete = taskMatch[2] === 'x';
-      const newStatus = completed !== undefined ? completed : !isCurrentlyComplete;
-      const statusChar = newStatus ? 'x' : ' ';
-      this._content[lineNumber - 1] = `${taskMatch[1]}${statusChar}${taskMatch[3]}`;
-      return true;
-    }
-
-    return false;
-  }
-
-  /**
-   * Change the indentation level of a task (for hierarchy).
-   */
-  setTaskIndent(lineNumber: number, indent: number): boolean {
-    if (lineNumber < 1 || lineNumber > this._content.length) {
-      return false;
-    }
-
-    const currentLine = this._content[lineNumber - 1];
-    const taskMatch = currentLine.match(/^\s*- (\[[x ]\] .*)$/);
-    if (taskMatch) {
-      const indentStr = ' '.repeat(indent);
-      this._content[lineNumber - 1] = `${indentStr}- ${taskMatch[1]}`;
-      return true;
-    }
-
-    return false;
-  }
-
-  /**
-   * Get the total number of lines in the document.
-   */
-  getLineCount(): number {
-    return this._content.length;
-  }
-
-  /**
-   * Apply multiple operations atomically.
-   */
-  applyOperations(operations: TaskOperation[]): void {
-    // Sort operations by line number in descending order to avoid index shifts.
-    const sortedOps = [...operations].sort((a, b) => {
-      const aLine = 'lineNumber' in a ? a.lineNumber : 0;
-      const bLine = 'lineNumber' in b ? b.lineNumber : 0;
-      return bLine - aLine;
-    });
-
-    for (const op of sortedOps) {
-      switch (op.type) {
-        case 'insertTask':
-          this.insertTask(op.lineNumber, op.text, op.completed, op.indent);
-          break;
-        case 'deleteTask':
-          this.deleteTask(op.lineNumber);
-          break;
-        case 'updateTaskText':
-          this.updateTaskText(op.lineNumber, op.text);
-          break;
-        case 'toggleTaskCompletion':
-          this.toggleTaskCompletion(op.lineNumber, op.completed);
-          break;
-        case 'setTaskIndent':
-          this.setTaskIndent(op.lineNumber, op.indent);
-          break;
-      }
-    }
-  }
-}
-
-/**
- * Operation types for batch task updates.
- */
-export type TaskOperation =
-  | { type: 'insertTask'; lineNumber: number; text: string; completed?: boolean; indent?: number }
-  | { type: 'deleteTask'; lineNumber: number }
-  | { type: 'updateTaskText'; lineNumber: number; text: string }
-  | { type: 'toggleTaskCompletion'; lineNumber: number; completed?: boolean }
-  | { type: 'setTaskIndent'; lineNumber: number; indent: number };