@dxos/assistant 0.8.2-main.2f9c567 → 0.8.2-main.30e4dbb

package/src/experimental/research/exa.ts

@@ -0,0 +1,38 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema } from 'effect';
+import Exa from 'exa-js';
+
+import { defineTool, ToolResult } from '@dxos/artifact';
+import { log } from '@dxos/log';
+
+type CreateExaToolOptions = {
+  apiKey: string;
+};
+
+export const createExaTool = ({ apiKey }: CreateExaToolOptions) => {
+  const exa = new Exa(apiKey);
+
+  return defineTool('search', {
+    name: 'web_search',
+    description: 'Search the web for information',
+    schema: Schema.Struct({
+      query: Schema.String.annotations({ description: 'The query to search for' }),
+    }),
+    execute: async ({ query }) => {
+      const context = await exa.searchAndContents(query, {
+        type: 'auto',
+        text: {
+          maxCharacters: 3_000,
+        },
+        livecrawl: 'always',
+      });
+
+      log.info('exa search', { query, costDollars: context.costDollars });
+
+      return ToolResult.Success(context);
+    },
+  });
+};

package/src/experimental/research/instructions.tpl

@@ -0,0 +1,64 @@
+You are an expert research agent. Your job is to assist a user by conducting in-depth research using real-time web search.
+You output the results in a structured format matching the schema provided.
+
+You are equipped with the ability to:
+
+- Generate precise and effective search queries.
+- Request web pages by query (through a `web_search` tool).
+- Read the full content of retrieved pages.
+- Synthesize accurate, clear, and structured answers using reliable information from the retrieved content.
+
+Always follow these principles:
+
+- Relevance First: Only return facts that are supported by content in the retrieved web pages. Never fabricate or guess information.
+- Summarize, Don't Copy: Synthesize and rephrase content in your own words. Quote only when necessary.
+- Multiple Sources: Cross-reference at least 2 sources before drawing conclusions, unless the information is directly stated and non-controversial.
+- Transparency: Mention which sources were used and explain how you arrived at your conclusions.
+- Accuracy Over Brevity: Prefer detailed, technically accurate explanations over shallow summaries.
+- If uncertain, say so. It's better to admit uncertainty than to mislead.
+- Pick the most concrete schema types for the extracted information.
+- Fill the schema fields as completely as possible with the information you are confident about, do not fill the fields that you are not confident about.
+- When outputting results, you can add extra data that isn't directly related to the user's question but fits the schema.
+
+The user may ask for:
+
+- Technical explanations
+- Literature reviews
+- Comparisons
+- Emerging trends
+- Implementation strategies
+
+Begin by interpreting the user's request, then:
+
+Break it into sub-questions (if applicable).
+
+For each sub-question, generate a clear, concise web search query.
+
+Use `web_search`(query) to retrieve information.
+
+Extract and synthesize relevant answers.
+
+Output should include:
+
+- A clear, structured answer to the user’s question.
+- A citation list or link list of the sources used.
+
+Optionally, follow-up suggestions or questions for deeper inquiry.
+
+Here's your operational instructions:
+
+1. Analyze user's request and identify key topics to search for (3 or more). Print them out.
+2. Perform a web search for each topic.
+3. Read and analyze the results, cross reference information from multiple sources, in case of conflict, represent the information as a range of possible values.
+4. Create a list of follow-up topics to research. Print them out.
+5. Perform a web search for each follow-up topic.
+6. Read and analyze the results, cross reference information from multiple sources, in case of conflict, represent the information as a range of possible values.
+9. Create a clear, structured answer to the user's question.
+10. Create a citation list or link list of the sources used.
+11. Select the most concrete schema types for the extracted information. You can use multiple different schema types. Print your decision and reasoning.
+12. Submit the results using the specific schema.
+
+
+
+
+

package/src/experimental/research/research.test.ts

@@ -0,0 +1,263 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Option, Schema, SchemaAST } from 'effect';
+import { describe, test } from 'vitest';
+
+import { structuredOutputParser } from '@dxos/artifact';
+import { isEncodedReference } from '@dxos/echo-protocol';
+import { create, getSchemaDXN, ObjectId, ReferenceAnnotationId } from '@dxos/echo-schema';
+import { mapAst } from '@dxos/effect';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
+import { deepMapValues } from '@dxos/util';
+
+import { createExaTool } from './exa';
+import INSTRUCTIONS from './instructions.tpl?raw';
+import { AIServiceEdgeClient, OllamaClient } from '../../ai-service';
+import { AISession } from '../../session';
+import { AI_SERVICE_ENDPOINT, ConsolePrinter } from '../../testing';
+
+const EXA_API_KEY = '9c7e17ff-0c85-4cd5-827a-8b489f139e03';
+const REMOTE_AI = true;
+
+const aiService = REMOTE_AI
+  ? new AIServiceEdgeClient({
+      endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+    })
+  : new OllamaClient({
+      overrides: {
+        model: 'llama3.1:8b',
+      },
+    });
+
+const TYPES = [DataType.Event, DataType.Organization, DataType.Person, DataType.Project, DataType.Task, DataType.Text];
+
+describe.skip('Research', () => {
+  test('should generate a research report', { timeout: 1000000 }, async () => {
+    const searchTool = createExaTool({ apiKey: EXA_API_KEY });
+
+    const session = new AISession({ operationModel: 'configured' });
+
+    const printer = new ConsolePrinter();
+    session.message.on((message) => printer.printMessage(message));
+    session.userMessage.on((message) => printer.printMessage(message));
+    session.block.on((block) => printer.printContentBlock(block));
+
+    // TODO(dmaretskyi): Consider adding this pattern as the "Graph" output mode for the session.
+    const outputSchema = createExtractionSchema(TYPES);
+    const result = await session.runStructured(outputSchema, {
+      client: aiService,
+      systemPrompt: INSTRUCTIONS,
+      artifacts: [],
+      tools: [searchTool],
+      generationOptions: {
+        model: '@anthropic/claude-3-5-sonnet-20241022',
+      },
+      history: [],
+      prompt: 'Find projects that are in the space of AI and personal knowledge management',
+    });
+    const data = sanitizeObjects(TYPES, result as any);
+
+    log.info('result', { data });
+  });
+});
+
+describe.skip('misc', () => {
+  test('createExtractionSchema', () => {
+    const schema = createExtractionSchema(TYPES);
+    log.info('schema', { schema });
+  });
+
+  test('extract schema json schema', () => {
+    const schema = createExtractionSchema(TYPES);
+    const parser = structuredOutputParser(schema);
+    log.info('schema', { json: parser.tool.parameters });
+  });
+
+  test('getSanitizedSchemaName', () => {
+    const names = TYPES.map(getSanitizedSchemaName);
+    log.info('names', { names });
+  });
+
+  test('getTypeAnnotation', () => {
+    for (const schema of TYPES) {
+      const dxn = getSchemaDXN(schema);
+      log.info('dxn', { schema, dxn });
+    }
+  });
+
+  test('sanitizeObjects', () => {
+    const TEST_DATA = {
+      objects_dxos_org_type_Project: [
+        {
+          name: 'Reor',
+          description:
+            'A private & local AI personal knowledge management app focused on high entropy thinkers. Features include Q&A chat with full context of notes, automatic connection of ideas, semantic search, and local-first architecture with LLMs and vector databases.',
+          image: 'https://reorhomepage-2-cwy0zagzg-reor-team.vercel.app/opengraph-image.jpg',
+          id: 'reor-project-01',
+        },
+        {
+          name: 'Personal AI',
+          description:
+            'A platform for creating personal AI models with unlimited memory capabilities. Features include memory stacking, data uploads, and personalized language models that can be trained on individual knowledge bases.',
+          image: 'https://cdn.prod.website-files.com/5ff65c460ce39f5ec5681c6a/663d12aab1b425e1ad40d3a6_Memory-min.jpg',
+          id: 'personal-ai-project-01',
+        },
+        {
+          name: 'IKI AI',
+          description:
+            'An AI-native workspace for research, strategy, and creative work. Offers features like AI summarization, content analysis, and team knowledge sharing capabilities.',
+          image: 'https://framerusercontent.com/assets/cI6Uo7x4q0W3uxzOt2preXjv6aE.jpg',
+          id: 'iki-ai-project-01',
+        },
+        {
+          name: 'Forethink',
+          description:
+            'A privacy-first, on-device AI system that proactively links current activities to relevant knowledge. Features real-time learning across teams and local data processing.',
+          image: 'https://framerusercontent.com/assets/XDhN6tICnlElB134JWcWXOuXc.jpeg',
+          id: 'forethink-project-01',
+        },
+        {
+          name: 'Amurex',
+          description:
+            'An invisible AI companion that handles tasks, remembers important information, and supports workflow. Includes features for meeting automation, email categorization, and cross-platform search.',
+          image: 'https://amurex.ai/ogimages/og_amurex.jpg',
+          id: 'amurex-project-01',
+        },
+      ],
+      objects_dxos_org_type_Organization: [
+        {
+          name: 'Reor',
+          description:
+            'A technology company focused on developing local-first AI solutions for personal knowledge management.',
+          status: 'active',
+          website: 'https://www.reorproject.org',
+          id: 'reor-org-01',
+        },
+        {
+          name: 'Personal.ai',
+          description: 'A company specializing in personal AI development and memory management solutions.',
+          status: 'active',
+          website: 'https://www.personal.ai',
+          id: 'personal-ai-org-01',
+        },
+        {
+          name: 'IKI AI',
+          description: 'A startup backed by 500 Global, developing AI-native workspaces for knowledge workers.',
+          status: 'active',
+          website: 'https://iki.ai',
+          id: 'iki-ai-org-01',
+        },
+        {
+          name: 'Forethink.ai',
+          description:
+            'A company focused on privacy-first AI solutions for knowledge management and team collaboration.',
+          status: 'active',
+          website: 'https://www.forethink.ai',
+          id: 'forethink-org-01',
+        },
+      ],
+      objects_dxos_org_type_Text: [
+        {
+          content:
+            'Current trends in AI-powered Personal Knowledge Management (PKM) systems include: 1) Shift from document-based to graph-based systems for better knowledge connection, 2) Integration of large language models for enhanced content understanding and retrieval, 3) Focus on privacy-first and local-first architectures, 4) Emphasis on automatic knowledge connection and discovery, 5) Implementation of semantic search capabilities.',
+          id: 'pkm-trends-text-01',
+        },
+        {
+          content:
+            'Key features emerging across modern PKM tools include: automated content categorization, semantic analysis and linking, personalized AI models that learn from user interaction, integration with existing workflows, and robust privacy protection. These tools are increasingly focusing on proactive knowledge surfacing rather than passive storage.',
+          id: 'pkm-features-text-01',
+        },
+      ],
+    };
+
+    const data = sanitizeObjects(TYPES, TEST_DATA);
+    log.info('data', { data });
+  });
+});
+
+const createExtractionSchema = (types: Schema.Schema.AnyNoContext[]) => {
+  return Schema.Struct({
+    ...Object.fromEntries(
+      types.map(preprocessSchema).map((schema, index) => [
+        `objects_${getSanitizedSchemaName(types[index])}`,
+        Schema.optional(Schema.Array(schema)).annotations({
+          description: `The objects to answer the query of type: ${getSchemaDXN(types[index])?.asTypeDXN()!.type}`,
+        }),
+      ]),
+    ),
+  });
+};
+
+const getSanitizedSchemaName = (schema: Schema.Schema.AnyNoContext) => {
+  return getSchemaDXN(schema)!
+    .asTypeDXN()!
+    .type.replaceAll(/[^a-zA-Z0-9]+/g, '_');
+};
+
+const sanitizeObjects = (types: Schema.Schema.AnyNoContext[], data: Record<string, readonly unknown[]>) => {
+  const entries = types
+    .map(
+      (type) =>
+        data[`objects_${getSanitizedSchemaName(type)}`]?.map((object: any) => ({
+          data: object,
+          schema: type,
+        })) ?? [],
+    )
+    .flat();
+
+  const idMap = new Map<string, string>();
+
+  return entries
+    .map((entry) => {
+      idMap.set(entry.data.id, ObjectId.random());
+      entry.data.id = idMap.get(entry.data.id);
+      return entry;
+    })
+    .map((entry) => {
+      const data = deepMapValues(entry.data, (value, recurse) => {
+        if (isEncodedReference(value)) {
+          const ref = value['/'];
+          if (idMap.has(ref)) {
+            // TODO(dmaretskyi): Whats the best way to represent a local url.
+            return { '/': `dxn:echo:@:${idMap.get(ref)}` };
+          } else {
+            // Search URIs?
+            return { '/': `search:?q=${encodeURIComponent(ref)}` };
+          }
+        }
+
+        return recurse(value);
+      });
+
+      return create(entry.schema, data);
+    });
+};
+
+const SoftRef = Schema.Struct({
+  '/': Schema.String,
+}).annotations({
+  description: 'Reference to another object.',
+});
+
+const preprocessSchema = (schema: Schema.Schema.AnyNoContext) => {
+  const go = (ast: SchemaAST.AST): SchemaAST.AST => {
+    if (SchemaAST.getAnnotation(ast, ReferenceAnnotationId).pipe(Option.isSome)) {
+      return SoftRef.ast;
+    }
+
+    return mapAst(ast, go);
+  };
+
+  const processed = Schema.make<any, any, never>(mapAst(schema.ast, go));
+  return Schema.extend(
+    processed.pipe(Schema.omit('id')),
+    Schema.Struct({
+      id: Schema.String.annotations({
+        description: 'The id of this object. Come up with a unique id based on your judgement.',
+      }),
+    }),
+  );
+};

package/src/index.ts

@@ -3,6 +3,7 @@
 //
 
 export * from './ai-service';
-export * from './cypher';
-export * from './conversation';
 export * from './session';
+
+// TODO(dmaretskyi): Separate entrypoint for experimental features.
+export * from './experimental';

package/src/session/index.ts

@@ -4,3 +4,4 @@
 
 export * from './session';
 export * from './version-pin';
+export * from './tools';

package/src/session/session.test.ts

@@ -3,17 +3,9 @@
 //
 
 import { Schema } from 'effect';
-import { inspect } from 'node:util';
 import { describe, test } from 'vitest';
 
-import {
-  ArtifactId,
-  defineArtifact,
-  defineTool,
-  ToolResult,
-  type Message,
-  type MessageContentBlock,
-} from '@dxos/artifact';
+import { ArtifactId, defineArtifact, defineTool, ToolResult } from '@dxos/artifact';
 import { AIServiceEdgeClient } from '@dxos/assistant';
 import { Type } from '@dxos/echo';
 import { create, ObjectId } from '@dxos/echo-schema';
@@ -21,7 +13,7 @@ import { DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
 
 import { AISession } from './session';
-import { AI_SERVICE_ENDPOINT } from '../testing';
+import { AI_SERVICE_ENDPOINT, ConsolePrinter } from '../testing';
 
 // Define a calendar event artifact schema.
 const CalendarEventSchema = Schema.Struct({
@@ -133,9 +125,10 @@ describe.skip('AISession with Ollama', () => {
     //   printStreamEvent(event);
     // });
 
-    session.message.on(printMessage);
-    session.userMessage.on(printMessage);
-    session.block.on(printContentBlock);
+    const printer = new ConsolePrinter();
+    session.message.on((message) => printer.printMessage(message));
+    session.userMessage.on((message) => printer.printMessage(message));
+    session.block.on((block) => printer.printContentBlock(block));
 
     // session.update.on((update) => {
     //   log('update', { update });
@@ -195,34 +188,6 @@ const CALENDAR_EVENTS: CalendarEvent[] = [
   }),
 ];
 
-const printMessage = (message: Message) => {
-  console.log(`${message.role.toUpperCase()}\n`);
-  for (const content of message.content) {
-    printContentBlock(content);
-  }
-};
-
-const printContentBlock = (content: MessageContentBlock) => {
-  switch (content.type) {
-    case 'text':
-      console.log(content.text);
-      break;
-    case 'tool_use':
-      console.log(`⚙️ [Tool Use] ${content.name} ${inspect(content.input, { depth: null, colors: true })}`);
-      break;
-    case 'tool_result': {
-      let data: any;
-      try {
-        data = JSON.parse(content.content);
-      } catch {
-        data = content.content;
-      }
-      console.log(`⚙️ [Tool Result] ${content.toolUseId} ${inspect(data, { depth: null, colors: true })}`);
-      break;
-    }
-  }
-};
-
 // const printStreamEvent = (event: GenerationStreamEvent) => {
 //   switch (event.type) {
 //     case 'message_start': {

package/src/session/session.ts

@@ -7,6 +7,7 @@ import { Option, Schema } from 'effect';
 import {
   defineTool,
   Message,
+  structuredOutputParser,
   ToolResult,
   type ArtifactDefinition,
   type MessageContentBlock,
@@ -18,9 +19,9 @@ import { create, type ObjectId } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
+import { isToolUse, runTools } from './tools';
 import { VersionPin } from './version-pin';
 import { MixedStreamParser, type AIServiceClient, type GenerateRequest, type GenerationStream } from '../ai-service';
-import { isToolUse, runTools } from '../conversation';
 
 /**
  * Contains message history, tools, current context.
@@ -245,6 +246,15 @@ export class AISession {
     return this._pending;
   }
 
+  async runStructured<S extends Schema.Schema.AnyNoContext>(schema: S, options: SessionRunOptions) {
+    const parser = structuredOutputParser(schema);
+    const result = await this.run({
+      ...options,
+      tools: [...options.tools, parser.tool],
+    });
+    return parser.getResult(result);
+  }
+
   private async _formatUserPrompt(
     artifactDiffResolver: ArtifactDiffResolver | undefined,
     prompt: string,

package/src/session/tools.ts

@@ -0,0 +1,117 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Message, type Tool, ToolResult } from '@dxos/artifact';
+import { create } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+export const isToolUse = (message: Message, { onlyToolNames }: { onlyToolNames?: string[] } = {}) => {
+  const block = message.content.at(-1);
+  invariant(block);
+  return block.type === 'tool_use' && (!onlyToolNames || onlyToolNames.includes(block.name));
+};
+
+export type RunToolsOptions = {
+  message: Message;
+  tools: Tool[];
+  extensions?: ToolContextExtensions;
+};
+
+export type RunToolsResult =
+  | {
+      type: 'continue';
+      message: Message;
+    }
+  | {
+      type: 'break';
+      result: unknown;
+    };
+
+export const runTools = async ({ message, tools, extensions }: RunToolsOptions): Promise<RunToolsResult> => {
+  const toolCalls = message.content.filter((block) => block.type === 'tool_use');
+  invariant(toolCalls.length === 1);
+  const toolCall = toolCalls[0];
+  const tool = tools.find((tool) => tool.name === toolCall.name);
+  if (!tool) {
+    const resultMessage = create(Message, {
+      role: 'user',
+      content: [
+        {
+          type: 'tool_result',
+          toolUseId: toolCall.id,
+          content: `Tool not found: ${toolCall.name}`,
+          isError: true,
+        },
+      ],
+    });
+
+    // TODO(burdon): Retry count?
+    return {
+      type: 'continue',
+      message: resultMessage,
+    };
+  }
+
+  let toolResult: ToolResult;
+  try {
+    invariant(tool.execute);
+    toolResult = await tool.execute(toolCall.input, { extensions });
+  } catch (error: any) {
+    log('tool error', { error });
+    toolResult = ToolResult.Error(error.message);
+  }
+
+  switch (toolResult.kind) {
+    case 'error': {
+      log('tool error', { message: toolResult.message });
+      const resultMessage = create(Message, {
+        role: 'user',
+        content: [
+          {
+            type: 'tool_result',
+            toolUseId: toolCall.id,
+            content: toolResult.message,
+            isError: true,
+          },
+        ],
+      });
+
+      // TODO(burdon): Retry count?
+      return {
+        type: 'continue',
+        message: resultMessage,
+      };
+    }
+
+    case 'success': {
+      log('tool success', { result: toolResult.result });
+      const resultMessage = create(Message, {
+        role: 'user',
+        content: [
+          {
+            type: 'tool_result',
+            toolUseId: toolCall.id,
+            content:
+              typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result) ?? '',
+          },
+          ...(toolResult.extractContentBlocks ?? []),
+        ],
+      });
+
+      return {
+        type: 'continue',
+        message: resultMessage,
+      };
+    }
+
+    case 'break': {
+      log('tool break', { result: toolResult.result });
+      return {
+        type: 'break',
+        result: toolResult.result,
+      };
+    }
+  }
+};

package/src/session/version-pin.ts

@@ -18,6 +18,12 @@ const VersionPinSchema = Schema.Struct({
   // TODO(dmaretskyi): Could be opaque
   version: ObjectVersionSchema,
 });
+
+/**
+ * Used to pin a specific version of an object during the course of a conversation.
+ *
+ * This allows us to detect when the object changes during the course of the conversation.
+ */
 export interface VersionPin extends Schema.Schema.Type<typeof VersionPinSchema> {}
 
 export const VersionPin: typeof VersionPinSchema & {

package/src/testing/index.ts

@@ -8,3 +8,4 @@ export * from './ollama';
 export * from './query-promts';
 export * from './test-data';
 export * from './test-schema';
+export * from './printer';

package/src/testing/logger.ts

@@ -4,7 +4,7 @@
 
 import type { ImageSource, MessageContentBlock } from '@dxos/artifact';
 
-import type { ConversationEvent } from '../conversation';
+import type { ConversationEvent } from '../experimental';
 
 export const createLogger = ({
   stream,

package/src/testing/printer.ts

@@ -0,0 +1,41 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { inspect } from 'node:util';
+
+import type { Message, MessageContentBlock } from '@dxos/artifact';
+
+export class ConsolePrinter {
+  printMessage = (message: Message) => {
+    // eslint-disable-next-line no-console
+    console.log(`${message.role.toUpperCase()}\n`);
+    for (const content of message.content) {
+      this.printContentBlock(content);
+    }
+  };
+
+  printContentBlock = (content: MessageContentBlock) => {
+    switch (content.type) {
+      case 'text':
+        // eslint-disable-next-line no-console
+        console.log(content.text);
+        break;
+      case 'tool_use':
+        // eslint-disable-next-line no-console
+        console.log(`⚙️ [Tool Use] ${content.name} ${inspect(content.input, { depth: null, colors: true })}`);
+        break;
+      case 'tool_result': {
+        let data: any;
+        try {
+          data = JSON.parse(content.content);
+        } catch {
+          data = content.content;
+        }
+        // eslint-disable-next-line no-console
+        console.log(`⚙️ [Tool Result] ${content.toolUseId} ${inspect(data, { depth: null, colors: true })}`);
+        break;
+      }
+    }
+  };
+}