@dxos/assistant 0.8.4-main.84f28bd → 0.8.4-main.9735255

package/src/extraction/extraction-llm-function/extraction-llm-function.ts

@@ -2,65 +2,61 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Message, ToolRegistry } from '@dxos/ai';
-import { Obj } from '@dxos/echo';
-import { create } from '@dxos/echo-schema';
-import { AiService, defineFunction, type FunctionDefinition } from '@dxos/functions';
-import { DataType } from '@dxos/schema';
+import { type FunctionDefinition, defineFunction } from '@dxos/functions';
 
-import PROMPT from './instructions.tpl?raw';
-import { AISession } from '../../session';
 import { ExtractionInput, ExtractionOutput } from '../extraction';
-import { insertReferences, ReferencedQuotes } from '../quotes';
 
-export const extractionAnthropicFn: FunctionDefinition<ExtractionInput, ExtractionOutput> = defineFunction({
+export const extractionAnthropicFunction: FunctionDefinition<ExtractionInput, ExtractionOutput> = defineFunction({
+  key: 'dxos.org/function/extraction/extract-entities',
+  name: 'Extract Entities',
   description: 'Extract entities from the transcript message and add them to the message.',
   inputSchema: ExtractionInput,
   outputSchema: ExtractionOutput,
   handler: async ({ data: { message, objects }, context }) => {
-    const startTime = performance.now();
-    const ai = context.getService(AiService);
-    const session = new AISession({ operationModel: 'configured' });
-    const result = await session.runStructured(ReferencedQuotes, {
-      generationOptions: {
-        model: '@anthropic/claude-3-5-haiku-20241022',
-      },
-      client: ai.client,
-      systemPrompt: PROMPT,
-      history: [
-        Obj.make(Message, {
-          role: 'user',
-          content: [
-            {
-              type: 'text',
-              text: `<context>${JSON.stringify(objects)}</context>`,
-            },
-            {
-              type: 'text',
-              text: `<transcript>${JSON.stringify(message.blocks)}</transcript>`,
-            },
-          ],
-        }),
-      ],
-      artifacts: [],
-      prompt: '',
-      tools: [],
-      toolResolver: new ToolRegistry([]),
-    });
+    throw new Error('not implemented');
+    // const startTime = performance.now();
+    // const ai = context.getService(AiService.AiService);
+    // const session = new AiSession({ operationModel: 'configured' });
+    // const result = await session.runStructured(ReferencedQuotes, {
+    //   generationOptions: {
+    //     model: '@anthropic/claude-3-5-haiku-20241022',
+    //   },
+    //   client: ai.client,
+    //   systemPrompt: PROMPT,
+    //   history: [
+    //     Obj.make(DataType.Message.Message, {
+    //       created: new Date().toISOString(),
+    //       sender: { role: 'user' },
+    //       blocks: [
+    //         {
+    //           _tag: 'text',
+    //           text: `<context>${JSON.stringify(objects)}</context>`,
+    //         } satisfies DataType.ContentBlock.Text,
+    //         {
+    //           _tag: 'text',
+    //           text: `<transcript>${JSON.stringify(message.blocks)}</transcript>`,
+    //         } satisfies DataType.ContentBlock.Text,
+    //       ],
+    //     }),
+    //   ],
+    //   artifacts: [],
+    //   prompt: '',
+    //   tools: [],
+    // } as any); // TODO(burdon): Rewrite test.
 
-    return {
-      message: create(DataType.Message, {
-        ...message,
-        blocks: message.blocks.map((block, i) =>
-          block.type !== 'transcription'
-            ? block
-            : {
-                ...block,
-                text: insertReferences(block.text, result),
-              },
-        ),
-      }),
-      timeElapsed: performance.now() - startTime,
-    };
+    // return {
+    //   timeElapsed: performance.now() - startTime,
+    //   message: create(DataType.Message.Message, {
+    //     ...message,
+    //     blocks: message.blocks.map((block) =>
+    //       block._tag !== 'transcript'
+    //         ? block
+    //         : {
+    //             ...block,
+    //             text: insertReferences(block.text, result),
+    //           },
+    //     ),
+    //   }),
+    // };
   },
 });

package/src/extraction/extraction-ner-function/extraction-ner-function.test.ts

@@ -2,39 +2,42 @@
 // Copyright 2025 DXOS.org
 //
 
-import { describe, test, beforeAll } from 'vitest';
+import { beforeAll, describe, test } from 'vitest';
 
-import { type EchoDatabase } from '@dxos/echo-db';
+import { type Database } from '@dxos/echo';
 import { EchoTestBuilder } from '@dxos/echo-db/testing';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
-import { createTestData, Testing } from '@dxos/schema/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { type Message, Organization, Person } from '@dxos/types';
+import { createTestData } from '@dxos/types/testing';
 import { range } from '@dxos/util';
 
-import { extractionNerFn } from './extraction-ner-function';
 import { processTranscriptMessage } from '../extraction';
 
+import { extractionNerFunction } from './extraction-ner-function';
+
 describe.skip('NER EntityExtraction', () => {
   let builder: EchoTestBuilder;
-  let db: EchoDatabase;
+  let db: Database.Database;
   let executor: FunctionExecutor;
   let testData: {
-    transcriptJosiah: DataType.Message[];
-    transcriptWoflram: DataType.Message[];
-    transcriptMessages: DataType.Message[];
-    documents: Testing.DocumentType[];
-    contacts: Record<string, DataType.Person>;
-    organizations: Record<string, DataType.Organization>;
+    transcriptJosiah: Message.Message[];
+    transcriptWoflram: Message.Message[];
+    transcriptMessages: Message.Message[];
+    documents: TestSchema.DocumentType[];
+    contacts: Record<string, Person.Person>;
+    organizations: Record<string, Organization.Organization>;
   };
-  const TYPES = [DataType.Organization, DataType.Person, Testing.Contact, Testing.DocumentType];
+
+  const TYPES = [Organization.Organization, Person.Person, TestSchema.Person, TestSchema.DocumentType];
 
   beforeAll(async () => {
     // TODO(dmaretskyi): Helper to scaffold this from a config.
     builder = await new EchoTestBuilder().open();
     const { db: db1 } = await builder.createDatabase({ indexing: { vector: true } });
     db = db1;
-    db.graph.schemaRegistry.addSchema(TYPES);
+    await db.graph.schemaRegistry.register(TYPES);
     const data = createTestData();
     testData = {
       transcriptJosiah: data.transcriptJosiah,
@@ -44,8 +47,8 @@ describe.skip('NER EntityExtraction', () => {
       contacts: Object.fromEntries(Object.entries(data.contacts).map(([key, value]) => [key, db.add(value)])),
       organizations: Object.fromEntries(Object.entries(data.organizations).map(([key, value]) => [key, db.add(value)])),
     };
-    await db.flush();
 
+    await db.flush();
     executor = new FunctionExecutor(
       new ServiceContainer().setServices({
         database: { db },
@@ -64,7 +67,7 @@ describe.skip('NER EntityExtraction', () => {
           message,
           objects: [...documents, ...Object.values(contacts)],
         },
-        function: extractionNerFn,
+        function: extractionNerFunction,
         executor,
       });
       log.info('output', enhancedMessage);
@@ -73,7 +76,6 @@ describe.skip('NER EntityExtraction', () => {
 
   test('computational irreducibility', async () => {
     const { transcriptWoflram, documents, contacts } = testData;
-
     log.info('context', { documents, contacts });
     const message = transcriptWoflram[0];
     log.info('input', message);
@@ -85,7 +87,7 @@ describe.skip('NER EntityExtraction', () => {
             message,
             objects: [...documents, ...Object.values(contacts)],
           },
-          function: extractionNerFn,
+          function: extractionNerFunction,
           executor,
         });
         log.info('output', { message: enhancedMessage.blocks[0], timeElapsed });
@@ -95,7 +97,6 @@ describe.skip('NER EntityExtraction', () => {
 
   test('org and document linking', async () => {
     const { transcriptJosiah, documents, contacts, organizations } = testData;
-
     log.info('context', { contacts, organizations, documents });
 
     for (const message of transcriptJosiah) {
@@ -106,7 +107,7 @@ describe.skip('NER EntityExtraction', () => {
           message,
           objects: [...documents, ...Object.values(contacts), ...Object.values(organizations)],
         },
-        function: extractionNerFn,
+        function: extractionNerFunction,
         executor,
       });
       log.info('output', enhancedMessage);

package/src/extraction/extraction-ner-function/extraction-ner-function.ts

@@ -2,53 +2,55 @@
 // Copyright 2025 DXOS.org
 //
 
-import { asyncTimeout } from '@dxos/async';
-import { create } from '@dxos/echo-schema';
-import { DatabaseService, defineFunction } from '@dxos/functions';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
-
-import { extractFullEntities } from './named-entity-recognition';
+import { defineFunction } from '@dxos/functions';
+
 import { ExtractionInput, ExtractionOutput } from '../extraction';
-import { findQuotes, insertReferences } from '../quotes';
 
+/**
+ * Named Entity Recognition model
+ */
 // TODO(mykola): Make it use vector index to query objects and not pass objects as input.
-export const extractionNerFn = defineFunction({
+export const extractionNerFunction = defineFunction({
+  key: 'dxos.org/function/extraction/extract-entities',
+  name: 'Extract Entities',
   description: 'Extracts named entities from a text',
   inputSchema: ExtractionInput,
   outputSchema: ExtractionOutput,
   handler: async ({ data: { message, options }, context }) => {
-    log.info('input', { message, options });
-    const startTime = performance.now();
-    const { db } = context.getService(DatabaseService);
-
-    const entitiesPromise = Promise.all(
-      message.blocks.map(async (block) => {
-        invariant(block.type === 'transcription' || block.type === 'text', 'Block must have text');
-        return extractFullEntities(block.text);
-      }),
-    ).then((entities) => entities.flat());
-    const entities = options?.timeout ? await asyncTimeout(entitiesPromise, options.timeout) : await entitiesPromise;
-    log.info('entities', { entities });
-    const quoteReferences = await findQuotes(
-      entities.map((entity) => entity.word),
-      db,
-    );
-    const blocksWithReferences = message.blocks.map((block, i) => {
-      invariant(block.type === 'transcription' || block.type === 'text', 'Block must have text');
-      return { ...block, text: insertReferences(block.text, quoteReferences) };
-    });
-    const messageWithReferences = create(DataType.Message, {
-      ...message,
-      blocks: blocksWithReferences,
-    });
-
-    log.info('output', { messageWithReferences });
-
-    return {
-      message: messageWithReferences,
-      timeElapsed: performance.now() - startTime,
-    };
+    throw new Error('not implemented');
+    // log.info('input', { message, options });
+    // const startTime = performance.now();
+    // const { db } = context.getService(Database.Service);
+    // const { extractFullEntities } = await import('./named-entity-recognition');
+
+    // const entitiesPromise = Promise.all(
+    //   message.blocks.map(async (block) => {
+    //     invariant(block._tag === 'transcript' || block._tag === 'text');
+    //     return extractFullEntities(block.text);
+    //   }),
+    // ).then((entities) => entities.flat());
+    // const entities = options?.timeout ? await asyncTimeout(entitiesPromise, options.timeout) : await entitiesPromise;
+
+    // log.info('entities', { entities });
+    // const quoteReferences = await findQuotes(
+    //   entities.map((entity) => entity.word),
+    //   db,
+    // );
+
+    // const blocksWithReferences = message.blocks.map((block) => {
+    //   invariant(block._tag === 'transcript' || block._tag === 'text');
+    //   return { ...block, text: insertReferences(block._tag, quoteReferences) };
+    // });
+
+    // const messageWithReferences = create(Message.Message, {
+    //   ...message,
+    //   blocks: blocksWithReferences,
+    // });
+
+    // log.info('output', { messageWithReferences });
+    // return {
+    //   message: messageWithReferences,
+    //   timeElapsed: performance.now() - startTime,
+    // };
   },
 });

package/src/extraction/extraction-ner-function/named-entity-recognition.test.ts

@@ -6,11 +6,13 @@ import { type TokenClassificationOutput, type TokenClassificationPipelineType }
 import { beforeAll, describe, expect, test } from 'vitest';
 
 import { log } from '@dxos/log';
-import { createTestData } from '@dxos/schema/testing';
+import { type ContentBlock, type Message } from '@dxos/types';
+import { createTestData } from '@dxos/types/testing';
 
 import { combineNerTokens, createTokenGroups, extractFullEntities, getNer } from './named-entity-recognition';
 
-describe('NamedEntityRecognition', () => {
+// TODO(mykola): This should not run on CI.
+describe.skip('NamedEntityRecognition', () => {
   let ner: TokenClassificationPipelineType;
 
   beforeAll(async () => {
@@ -56,8 +58,10 @@ describe('NamedEntityRecognition', () => {
   test('own data', async () => {
     const { transcriptWoflram, transcriptJosiah } = await createTestData();
 
-    const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message) =>
-      message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
+    const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message: Message.Message) =>
+      message.blocks.flatMap((block: ContentBlock.Any) =>
+        block._tag === 'transcript' ? [(block as ContentBlock.Transcript).text] : [],
+      ),
     );
 
     for (const block of blocks) {
@@ -69,8 +73,10 @@ describe('NamedEntityRecognition', () => {
   test('more own data', async () => {
     const { transcriptMessages } = await createTestData();
 
-    const blocks = transcriptMessages.flatMap((message) =>
-      message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
+    const blocks = transcriptMessages.flatMap((message: Message.Message) =>
+      message.blocks.flatMap((block: ContentBlock.Any) =>
+        block._tag === 'transcript' ? [(block as ContentBlock.Transcript).text] : [],
+      ),
     );
 
     for (const block of blocks) {

package/src/extraction/extraction-ner-function/named-entity-recognition.ts

@@ -4,14 +4,15 @@
 
 import {
   type TokenClassificationOutput,
+  type TokenClassificationPipelineType,
   type TokenClassificationSingle,
   pipeline,
-  type TokenClassificationPipelineType,
 } from '@xenova/transformers';
 
 import { log } from '@dxos/log';
 
 let _ner: Promise<TokenClassificationPipelineType>;
+
 /**
  * Named Entity Recognition pipeline.
  * Initializes the pipeline on first call.
@@ -24,6 +25,7 @@ export const getNer = () => {
       return ner;
     });
   }
+
   return _ner;
 };
 

package/src/extraction/extraction.ts

@@ -2,17 +2,18 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
 import { asyncTimeout } from '@dxos/async';
-import { Expando } from '@dxos/echo-schema';
-import { type FunctionDefinition, type FunctionExecutor } from '@dxos/functions';
+import { Type } from '@dxos/echo';
+import { type FunctionDefinition } from '@dxos/functions';
+import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
 export const ExtractionInput = Schema.Struct({
-  message: DataType.Message,
-  objects: Schema.optional(Schema.Array(Expando)),
+  message: Message.Message,
+  objects: Schema.optional(Schema.Array(Type.Obj)),
   options: Schema.optional(
     Schema.Struct({
       timeout: Schema.optional(Schema.Number),
@@ -23,12 +24,12 @@ export const ExtractionInput = Schema.Struct({
 export interface ExtractionInput extends Schema.Schema.Type<typeof ExtractionInput> {}
 
 export const ExtractionOutput = Schema.Struct({
-  message: DataType.Message,
+  message: Message.Message,
   timeElapsed: Schema.Number,
 });
 export interface ExtractionOutput extends Schema.Schema.Type<typeof ExtractionOutput> {}
 
-export type ProcessTranscriptMessageParams = {
+export type ProcessTranscriptMessageProps = {
   input: ExtractionInput;
   function: FunctionDefinition<ExtractionInput, ExtractionOutput>;
   executor: FunctionExecutor;
@@ -54,7 +55,7 @@ export type ExtractionFunction = FunctionDefinition<ExtractionInput, ExtractionO
  * Extract entities from the transcript message and add them to the message.
  */
 // TODO(dmaretskyi): Move context to a vector search index.
-export const processTranscriptMessage = async (params: ProcessTranscriptMessageParams): Promise<ExtractionOutput> => {
+export const processTranscriptMessage = async (params: ProcessTranscriptMessageProps): Promise<ExtractionOutput> => {
   try {
     if (params.options?.timeout && params.options.timeout > 0) {
       return await asyncTimeout(params.executor.invoke(params.function, params.input), params.options.timeout);

package/src/extraction/quotes.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import { describe, test, expect } from 'vitest';
+import { describe, expect, test } from 'vitest';
 
 import { insertReferences } from './quotes';
 
@@ -13,7 +13,7 @@ describe('insertReferences', () => {
     };
 
     expect(insertReferences('This is a computational irreducibility test.', quotes)).toBe(
-      'This is a [computational irreducibility][dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1] test.',
+      'This is a [computational irreducibility](dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1) test.',
     );
     expect(
       insertReferences(
@@ -21,7 +21,7 @@ describe('insertReferences', () => {
         quotes,
       ),
     ).toBe(
-      "And what I'd like to talk today about is Steven Wolfram's concept of a [computational irreducibility][dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1].",
+      "And what I'd like to talk today about is Steven Wolfram's concept of a [computational irreducibility](dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1).",
     );
   });
 });

package/src/extraction/quotes.ts

@@ -2,11 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
-import { Query, Filter } from '@dxos/echo';
-import { type EchoDatabase } from '@dxos/echo-db';
-import { ObjectId } from '@dxos/echo-schema';
+import { type Database, Filter, Query } from '@dxos/echo';
+import { ObjectId } from '@dxos/keys';
 import { DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
 
@@ -28,10 +27,10 @@ export const ReferencedQuotes = Schema.Struct({
 });
 export type ReferencedQuotes = Schema.Schema.Type<typeof ReferencedQuotes>;
 
-export const findQuotes = async (quotes: string[], db: EchoDatabase): Promise<ReferencedQuotes> => {
+export const findQuotes = async (quotes: string[], db: Database.Database): Promise<ReferencedQuotes> => {
   const references = await Promise.all(
     quotes.map(async (quote) => {
-      const { objects } = await db.query(Query.select(Filter.text(quote, { type: 'vector' }))).run();
+      const objects = await db.query(Query.select(Filter.text(quote, { type: 'vector' }))).run();
       log.info('found quotes in vector index', { objects, quote });
       return objects.length > 0 ? [{ id: objects[0].id.toString(), quote }] : [];
     }),
@@ -51,7 +50,7 @@ export const insertReferences = (text: string, quotes: ReferencedQuotes) => {
 
     // Use a case-insensitive regular expression to replace the quote.
     const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
-    text = text.replace(regex, `[${quote.quote}][${DXN.fromLocalObjectId(quote.id).toString()}]`);
+    text = text.replace(regex, `[${quote.quote}](${DXN.fromLocalObjectId(quote.id).toString()})`);
   }
 
   return text;

package/src/functions/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './services';