@dxos/assistant 0.8.2-main.12df754 → 0.8.2-main.30e4dbb

package/src/{conversation → experimental/conversation}/conversation.ts

@@ -2,13 +2,13 @@
 // Copyright 2024 DXOS.org
 //
 
-import { type Tool, ToolResult, Message } from '@dxos/artifact';
-import { create, ObjectId } from '@dxos/echo-schema';
+import { type Message, type Tool } from '@dxos/artifact';
+import { ObjectId } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';
 import { type SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
 
-import { type AIServiceClient, type GenerationStreamEvent, type LLMModel } from '../ai-service';
+import { type AIServiceClient, type GenerationStreamEvent, type LLMModel } from '../../ai-service';
 
 export type CreateLLMConversationParams = {
   model: LLMModel;
@@ -140,112 +140,3 @@ export const runLLM = async (params: CreateLLMConversationParams) => {
     history,
   };
 };
-
-export const isToolUse = (message: Message, { onlyToolNames }: { onlyToolNames?: string[] } = {}) => {
-  const block = message.content.at(-1);
-  invariant(block);
-  return block.type === 'tool_use' && (!onlyToolNames || onlyToolNames.includes(block.name));
-};
-
-export type RunToolsOptions = {
-  message: Message;
-  tools: Tool[];
-  extensions?: ToolContextExtensions;
-};
-
-export type RunToolsResult =
-  | {
-      type: 'continue';
-      message: Message;
-    }
-  | {
-      type: 'break';
-      result: unknown;
-    };
-
-export const runTools = async ({ message, tools, extensions }: RunToolsOptions): Promise<RunToolsResult> => {
-  const toolCalls = message.content.filter((block) => block.type === 'tool_use');
-  invariant(toolCalls.length === 1);
-  const toolCall = toolCalls[0];
-  const tool = tools.find((tool) => tool.name === toolCall.name);
-  if (!tool) {
-    const resultMessage = create(Message, {
-      role: 'user',
-      content: [
-        {
-          type: 'tool_result',
-          toolUseId: toolCall.id,
-          content: `Tool not found: ${toolCall.name}`,
-          isError: true,
-        },
-      ],
-    });
-
-    // TODO(burdon): Retry count?
-    return {
-      type: 'continue',
-      message: resultMessage,
-    };
-  }
-
-  let toolResult: ToolResult;
-  try {
-    invariant(tool.execute);
-    toolResult = await tool.execute(toolCall.input, { extensions });
-  } catch (error: any) {
-    log('tool error', { error });
-    toolResult = ToolResult.Error(error.message);
-  }
-
-  switch (toolResult.kind) {
-    case 'error': {
-      log('tool error', { message: toolResult.message });
-      const resultMessage = create(Message, {
-        role: 'user',
-        content: [
-          {
-            type: 'tool_result',
-            toolUseId: toolCall.id,
-            content: toolResult.message,
-            isError: true,
-          },
-        ],
-      });
-
-      // TODO(burdon): Retry count?
-      return {
-        type: 'continue',
-        message: resultMessage,
-      };
-    }
-
-    case 'success': {
-      log('tool success', { result: toolResult.result });
-      const resultMessage = create(Message, {
-        role: 'user',
-        content: [
-          {
-            type: 'tool_result',
-            toolUseId: toolCall.id,
-            content:
-              typeof toolResult.result === 'string' ? toolResult.result : JSON.stringify(toolResult.result) ?? '',
-          },
-          ...(toolResult.extractContentBlocks ?? []),
-        ],
-      });
-
-      return {
-        type: 'continue',
-        message: resultMessage,
-      };
-    }
-
-    case 'break': {
-      log('tool break', { result: toolResult.result });
-      return {
-        type: 'break',
-        result: toolResult.result,
-      };
-    }
-  }
-};

package/src/{conversation → experimental/conversation}/types.ts

@@ -2,7 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
-import { Schema as S } from 'effect';
+import { Schema } from 'effect';
 
 import { type Message, type MessageContentBlock } from '@dxos/artifact';
 import { toJsonSchema, JsonSchemaType, ObjectId } from '@dxos/echo-schema';
@@ -17,11 +17,11 @@ export const createUserMessage = (text: string): Message => ({
 /**
  * @deprecated Use {@link LLMTool} instead.
  */
-export const ToolDefinition = S.Struct({
-  name: S.String,
-  description: S.String,
+export const ToolDefinition = Schema.Struct({
+  name: Schema.String,
+  description: Schema.String,
   parameters: JsonSchemaType,
-  execute: S.Any,
+  execute: Schema.Any,
 });
 
 /**
@@ -83,14 +83,14 @@ export const ToolResult = Object.freeze({
   Break: (result: unknown): ToolResult => ({ kind: 'break', result }),
 });
 
-export type DefineToolParams<Params extends S.Schema.AnyNoContext> = {
+export type DefineToolParams<Params extends Schema.Schema.AnyNoContext> = {
   name: string;
   description: string;
   schema: Params;
-  execute: (params: S.Schema.Type<Params>, context: ToolExecutionContext) => Promise<ToolResult>;
+  execute: (params: Schema.Schema.Type<Params>, context: ToolExecutionContext) => Promise<ToolResult>;
 };
 
-export const defineTool = <Params extends S.Schema.AnyNoContext>({
+export const defineTool = <Params extends Schema.Schema.AnyNoContext>({
   name,
   description,
   schema,
@@ -101,7 +101,7 @@ export const defineTool = <Params extends S.Schema.AnyNoContext>({
     description,
     parameters: toFunctionParameterSchema(schema),
     execute: (params, context) => {
-      const sanitized = S.decodeSync(schema)(params);
+      const sanitized = Schema.decodeSync(schema)(params);
       return execute(sanitized, context);
     },
   };
@@ -110,7 +110,7 @@ export const defineTool = <Params extends S.Schema.AnyNoContext>({
 /**
  * Adapts schems to be able to pass to AI providers.
  */
-const toFunctionParameterSchema = (schema: S.Schema.All) => {
+const toFunctionParameterSchema = (schema: Schema.Schema.All) => {
   const jsonSchema = toJsonSchema(schema);
   log('tool schema', { jsonSchema });
   delete jsonSchema.anyOf;

package/src/{cypher → experimental/cypher}/combinators.test.ts

@@ -77,7 +77,7 @@ describe('Cypher Combinators', () => {
   test('match clause', ({ expect }) => {
     testParser(matchClause, 'MATCH (n:Person)');
     testParser(matchClause, 'MATCH (n:Person)-[r:KNOWS]->(m:Person)');
-    testParser(matchClause, 'MATCH (o:Org)-[:HAS_EMPLOYEE]->(e:Employee)');
+    testParser(matchClause, 'MATCH (o:Organization)-[:HAS_EMPLOYEE]->(e:Employee)');
   });
 
   test('where clause', ({ expect }) => {
@@ -116,15 +116,15 @@ describe('Cypher Combinators', () => {
     );
     testParser(
       cypherQuery,
-      'MATCH (o:Org)-[:HAS_EMPLOYEE]->(e:Employee)-[:WORKS_ON]->(p:Project) WHERE o.name = "DXOS" AND p.name = "Composer" RETURN e.name, e.id, p.name, p.id',
+      'MATCH (o:Organization)-[:HAS_EMPLOYEE]->(e:Employee)-[:WORKS_ON]->(p:Project) WHERE o.name = "DXOS" AND p.name = "Composer" RETURN e.name, e.id, p.name, p.id',
     );
     testParser(
       cypherQuery,
-      'MATCH (n:Org {name: "DXOS"})-[:HAS_EMPLOYEE]->(m:Employee)<-[:WORKS_ON]-(p:Project {name: "Composer"}) RETURN m.name AS employee',
+      'MATCH (n:Organization {name: "DXOS"})-[:HAS_EMPLOYEE]->(m:Employee)<-[:WORKS_ON]-(p:Project {name: "Composer"}) RETURN m.name AS employee',
     );
     testParser(
       cypherQuery,
-      "MATCH (org:Org {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(t:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'}) RETURN c.name",
+      "MATCH (org:Organization {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(t:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'}) RETURN c.name",
     );
   });
 });

package/src/{cypher → experimental/cypher}/echo-data-source.ts

@@ -2,9 +2,11 @@
 // Copyright 2025 DXOS.org
 //
 
+import { type Schema } from 'effect';
+
 import { raise } from '@dxos/debug';
-import { type EchoDatabase, Filter, type ReactiveEchoObject } from '@dxos/echo-db';
-import { getSchema, getSchemaTypename, type S, StoredSchema, toJsonSchema } from '@dxos/echo-schema';
+import { type EchoDatabase, Filter, type AnyLiveObject } from '@dxos/echo-db';
+import { getSchema, getSchemaTypename, StoredSchema, toJsonSchema } from '@dxos/echo-schema';
 import { log } from '@dxos/log';
 
 import type { DataSource, Node, Relationship } from './query-executor';
@@ -62,7 +64,7 @@ export class EchoDataSource implements DataSource {
     ).flat();
   }
 
-  private async _getAllSchema(): Promise<S.Schema.AnyNoContext[]> {
+  private async _getAllSchema(): Promise<Schema.Schema.AnyNoContext[]> {
     return [
       ...(await this._db.schemaRegistry.query().run()),
       // TODO(dmaretskyi): Remove once we can serialize recursive schema.
@@ -70,7 +72,7 @@ export class EchoDataSource implements DataSource {
     ].filter((schema) => getSchemaTypename(schema) !== StoredSchema.typename);
   }
 
-  private _objectToNode(object: ReactiveEchoObject<any>): Node {
+  private _objectToNode(object: AnyLiveObject<any>): Node {
     const { id, ...properties } = object;
     return {
       id,
@@ -80,7 +82,7 @@ export class EchoDataSource implements DataSource {
     };
   }
 
-  private async _projectRefRelationship(object: ReactiveEchoObject<any>, prop: string): Promise<Relationship[]> {
+  private async _projectRefRelationship(object: AnyLiveObject<any>, prop: string): Promise<Relationship[]> {
     if (!object[prop]) {
       return [];
     }

package/src/{cypher → experimental/cypher}/execution-plan.test.ts

@@ -21,7 +21,7 @@ describe('Execution Plan', () => {
   test('complex query', ({ expect }) => {
     const plan = createExecutionPlan(
       parseCypherQuery(
-        "MATCH (org:Org {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'}) RETURN c.name",
+        "MATCH (org:Organization {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'}) RETURN c.name",
       ),
     );
     log('', { plan });

package/src/{cypher → experimental/cypher}/query-executor.test.ts

@@ -7,7 +7,7 @@ import { test } from 'vitest';
 import { log } from '@dxos/log';
 
 import { executeQuery } from './query-executor';
-import { createTestData } from '../testing';
+import { createTestData } from '../../testing';
 
 test.skip('run', async ({ expect }) => {
   const dataSource = createTestData();
@@ -25,7 +25,7 @@ test.skip('run', async ({ expect }) => {
   const result = await executeQuery(
     dataSource,
     `
-      MATCH (org:Org {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(t:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'})
+      MATCH (org:Organization {name: 'DXOS'})-[:ORG_EMPLOYEES]->(c:Contact)<-[:TASK_ASSIGNEE]-(t:Task)-[:TASK_PROJECT]->(p:Project {name: 'Composer'})
       RETURN c.name, t.name
     `,
   );

package/src/{cypher → experimental/cypher}/schema.test.ts

@@ -7,10 +7,10 @@ import { test } from 'vitest';
 import { toJsonSchema } from '@dxos/echo-schema';
 
 import { formatJsonSchemaForLLM } from './schema';
-import { Contact, Org, Project, Task } from '../testing';
+import { Contact, Organization, Project, Task } from '../../testing';
 
-test('Org-Project-Task-Contact schema', async ({ expect }) => {
-  const jsonSchema = [Org, Project, Task, Contact].map((schema) => toJsonSchema(schema));
+test('Organization-Project-Task-Contact schema', async ({ expect }) => {
+  const jsonSchema = [Organization, Project, Task, Contact].map((schema) => toJsonSchema(schema));
   const result = formatJsonSchemaForLLM(jsonSchema);
   // log('result', { result });
   expect(result).toHaveLength(8);

package/src/{cypher → experimental/cypher}/schema.ts

@@ -5,7 +5,7 @@
 import { raise } from '@dxos/debug';
 import { JSON_SCHEMA_ECHO_REF_ID, type JsonSchemaType } from '@dxos/echo-schema';
 
-import { trim } from '../util';
+import { trim } from '../../util';
 
 /**
  * Formats ECHO schema to be consumed by LLM.

package/src/experimental/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './conversation';
+export * from './cypher';

package/src/experimental/ner/ner.test.ts

@@ -0,0 +1,163 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { pipeline } from '@xenova/transformers';
+import { beforeAll, describe, expect, test } from 'vitest';
+
+import { log } from '@dxos/log';
+import { createTestData } from '@dxos/schema/testing';
+
+type NerToken = {
+  entity: string;
+  score: number;
+  index: number;
+  word: string;
+  start: number | null;
+  end: number | null;
+};
+
+const createTokenGroups = (tokens: NerToken[]): NerToken[][] => {
+  const tokenGroups: NerToken[][] = [];
+
+  const canBeCombined = (prevToken: NerToken, currentToken: NerToken) => {
+    const [currentState, currentEntity] = currentToken.entity.split('-'); // B, I, O
+    const [prevState, prevEntity] = prevToken.entity.split('-'); // B, I, O
+
+    if (prevEntity !== currentEntity) {
+      return false;
+    }
+
+    if (prevState === 'O') {
+      return false;
+    }
+
+    if (currentState === 'I') {
+      return true;
+    }
+
+    if ((prevState === 'B' || prevState === 'I') && currentState === 'B') {
+      return currentToken.word.startsWith('#');
+    }
+
+    return false;
+  };
+
+  for (const token of tokens) {
+    if (tokenGroups.length === 0) {
+      tokenGroups.push([token]);
+      continue;
+    }
+
+    const lastTokenGroup = tokenGroups.at(-1)!;
+    if (canBeCombined(lastTokenGroup.at(-1)!, token)) {
+      lastTokenGroup.push(token);
+      continue;
+    }
+
+    tokenGroups.push([token]);
+  }
+
+  return tokenGroups;
+};
+
+const combineNerTokens = (group: NerToken[]): NerToken => {
+  const combinedToken: NerToken = {
+    entity: group.at(0)!.entity.split('-')[1],
+    score: group.reduce((acc, token) => acc + token.score, 0) / group.length,
+    index: group.at(0)!.index,
+    word: group.at(0)!.word,
+    start: group.at(0)!.start,
+    end: group.at(-1)!.end,
+  };
+
+  for (const token of group.slice(1)) {
+    if (token.word.startsWith('#')) {
+      combinedToken.word += token.word.replace(/^#+/, '');
+    } else {
+      combinedToken.word += ' ' + token.word;
+    }
+  }
+
+  return combinedToken;
+};
+
+describe.skip('Named Entity Recognition', () => {
+  let ner: any;
+
+  beforeAll(async () => {
+    // Initialize the NER pipeline
+    ner = await pipeline('ner', 'Xenova/bert-base-NER');
+  });
+
+  test.skip('should identify named entities in text', async () => {
+    const text = 'Elon Musk is the CEO of SpaceX and Tesla.';
+    const result = await ner(text);
+
+    // Verify the structure of the result
+    expect(Array.isArray(result)).toBe(true);
+    expect(result.length).toBeGreaterThan(0);
+
+    log.info('ner', { result });
+
+    // Check for specific entities
+    const entities = result.map((item: any) => ({
+      text: item.word,
+      entity: item.entity,
+      score: item.score,
+    }));
+
+    // Verify Elon Musk is recognized as a person
+    const elonMusk = entities.find((e: any) => e.text === 'Elon' || e.text === 'Musk');
+    expect(elonMusk).toBeDefined();
+    expect(elonMusk?.entity).toMatch(/PER/);
+
+    // Verify SpaceX and Tesla are recognized as organizations
+    const companies = entities.filter((e: any) => e.text === 'SpaceX' || e.text === 'Tesla');
+    expect(companies.length).toBe(2);
+    companies.forEach((company: any) => {
+      expect(company.entity).toMatch(/ORG/);
+    });
+  });
+
+  test('own data', async () => {
+    const { transcriptWoflram, transcriptJosiah } = await createTestData();
+
+    const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message) =>
+      message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
+    );
+
+    for (const block of blocks) {
+      const result = await ner(block);
+      const combinedTokens = createTokenGroups(result).map(combineNerTokens);
+      log.info('ner', { text: block, result: combinedTokens });
+    }
+  });
+
+  test('more own data', async () => {
+    const { transcriptMessages } = await createTestData();
+
+    const blocks = transcriptMessages.flatMap((message) =>
+      message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
+    );
+
+    for (const block of blocks) {
+      const result = await ner(block);
+      const combinedTokens = createTokenGroups(result).map(combineNerTokens);
+      log.info('ner', { text: block, result: combinedTokens });
+    }
+  });
+
+  test('should handle empty text', async () => {
+    const result = await ner('');
+    expect(Array.isArray(result)).toBe(true);
+    expect(result.length).toBe(0);
+  });
+
+  test('should handle text with no named entities', async () => {
+    const text = 'The weather is nice today.';
+    const result = await ner(text);
+    expect(Array.isArray(result)).toBe(true);
+    expect(result.length).toBe(0);
+  });
+});

package/src/experimental/research/exa.ts

@@ -0,0 +1,38 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema } from 'effect';
+import Exa from 'exa-js';
+
+import { defineTool, ToolResult } from '@dxos/artifact';
+import { log } from '@dxos/log';
+
+type CreateExaToolOptions = {
+  apiKey: string;
+};
+
+export const createExaTool = ({ apiKey }: CreateExaToolOptions) => {
+  const exa = new Exa(apiKey);
+
+  return defineTool('search', {
+    name: 'web_search',
+    description: 'Search the web for information',
+    schema: Schema.Struct({
+      query: Schema.String.annotations({ description: 'The query to search for' }),
+    }),
+    execute: async ({ query }) => {
+      const context = await exa.searchAndContents(query, {
+        type: 'auto',
+        text: {
+          maxCharacters: 3_000,
+        },
+        livecrawl: 'always',
+      });
+
+      log.info('exa search', { query, costDollars: context.costDollars });
+
+      return ToolResult.Success(context);
+    },
+  });
+};

package/src/experimental/research/instructions.tpl

@@ -0,0 +1,64 @@
+You are an expert research agent. Your job is to assist a user by conducting in-depth research using real-time web search.
+You output the results in a structured format matching the schema provided.
+
+You are equipped with the ability to:
+
+- Generate precise and effective search queries.
+- Request web pages by query (through a `web_search` tool).
+- Read the full content of retrieved pages.
+- Synthesize accurate, clear, and structured answers using reliable information from the retrieved content.
+
+Always follow these principles:
+
+- Relevance First: Only return facts that are supported by content in the retrieved web pages. Never fabricate or guess information.
+- Summarize, Don't Copy: Synthesize and rephrase content in your own words. Quote only when necessary.
+- Multiple Sources: Cross-reference at least 2 sources before drawing conclusions, unless the information is directly stated and non-controversial.
+- Transparency: Mention which sources were used and explain how you arrived at your conclusions.
+- Accuracy Over Brevity: Prefer detailed, technically accurate explanations over shallow summaries.
+- If uncertain, say so. It's better to admit uncertainty than to mislead.
+- Pick the most concrete schema types for the extracted information.
+- Fill the schema fields as completely as possible with the information you are confident about, do not fill the fields that you are not confident about.
+- When outputting results, you can add extra data that isn't directly related to the user's question but fits the schema.
+
+The user may ask for:
+
+- Technical explanations
+- Literature reviews
+- Comparisons
+- Emerging trends
+- Implementation strategies
+
+Begin by interpreting the user's request, then:
+
+Break it into sub-questions (if applicable).
+
+For each sub-question, generate a clear, concise web search query.
+
+Use `web_search`(query) to retrieve information.
+
+Extract and synthesize relevant answers.
+
+Output should include:
+
+- A clear, structured answer to the user’s question.
+- A citation list or link list of the sources used.
+
+Optionally, follow-up suggestions or questions for deeper inquiry.
+
+Here's your operational instructions:
+
+1. Analyze user's request and identify key topics to search for (3 or more). Print them out.
+2. Perform a web search for each topic.
+3. Read and analyze the results, cross reference information from multiple sources, in case of conflict, represent the information as a range of possible values.
+4. Create a list of follow-up topics to research. Print them out.
+5. Perform a web search for each follow-up topic.
+6. Read and analyze the results, cross reference information from multiple sources, in case of conflict, represent the information as a range of possible values.
+9. Create a clear, structured answer to the user's question.
+10. Create a citation list or link list of the sources used.
+11. Select the most concrete schema types for the extracted information. You can use multiple different schema types. Print your decision and reasoning.
+12. Submit the results using the specific schema.
+
+
+
+
+