@dxos/assistant 0.8.4-main.84f28bd → 0.8.4-main.8baae0fced

package/package.json

@@ -1,18 +1,27 @@
 {
   "name": "@dxos/assistant",
-  "version": "0.8.4-main.84f28bd",
+  "version": "0.8.4-main.8baae0fced",
   "description": "Natural language interface to DXOS",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
-  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "FSL-1.1-Apache-2.0",
   "author": "info@dxos.org",
   "sideEffects": true,
   "type": "module",
   "exports": {
     ".": {
+      "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "browser": "./dist/lib/browser/index.mjs",
-      "node": "./dist/lib/node-esm/index.mjs"
+      "default": "./dist/lib/neutral/index.mjs"
+    },
+    "./extraction": {
+      "source": "./src/extraction/index.ts",
+      "types": "./dist/types/src/extraction/index.d.ts",
+      "default": "./dist/lib/neutral/extraction/index.mjs"
     }
   },
   "types": "dist/types/src/index.d.ts",
@@ -21,42 +30,47 @@
     "src"
   ],
   "dependencies": {
-    "@preact/signals-core": "^1.9.0",
+    "@automerge/automerge": "3.2.6",
+    "@effect-atom/atom": "^0.5.1",
+    "@effect-atom/atom-react": "^0.5.0",
+    "@effect/ai": "0.33.2",
+    "@effect/experimental": "0.58.0",
+    "@effect/platform": "0.94.4",
     "@xenova/transformers": "^2.17.2",
-    "chalk": "^4.1.0",
-    "exa-js": "^1.5.12",
-    "@dxos/ai": "0.8.4-main.84f28bd",
-    "@dxos/artifact": "0.8.4-main.84f28bd",
-    "@dxos/async": "0.8.4-main.84f28bd",
-    "@dxos/debug": "0.8.4-main.84f28bd",
-    "@dxos/echo": "0.8.4-main.84f28bd",
-    "@dxos/context": "0.8.4-main.84f28bd",
-    "@dxos/echo-db": "0.8.4-main.84f28bd",
-    "@dxos/echo-protocol": "0.8.4-main.84f28bd",
-    "@dxos/echo-schema": "0.8.4-main.84f28bd",
-    "@dxos/effect": "0.8.4-main.84f28bd",
-    "@dxos/functions": "0.8.4-main.84f28bd",
-    "@dxos/graph": "0.8.4-main.84f28bd",
-    "@dxos/invariant": "0.8.4-main.84f28bd",
-    "@dxos/keys": "0.8.4-main.84f28bd",
-    "@dxos/live-object": "0.8.4-main.84f28bd",
-    "@dxos/log": "0.8.4-main.84f28bd",
-    "@dxos/node-std": "0.8.4-main.84f28bd",
-    "@dxos/schema": "0.8.4-main.84f28bd",
-    "@dxos/util": "0.8.4-main.84f28bd"
+    "lodash.defaultsdeep": "^4.6.1",
+    "@dxos/ai": "0.8.4-main.8baae0fced",
+    "@dxos/compute": "0.8.4-main.8baae0fced",
+    "@dxos/context": "0.8.4-main.8baae0fced",
+    "@dxos/async": "0.8.4-main.8baae0fced",
+    "@dxos/debug": "0.8.4-main.8baae0fced",
+    "@dxos/echo-db": "0.8.4-main.8baae0fced",
+    "@dxos/echo": "0.8.4-main.8baae0fced",
+    "@dxos/echo-protocol": "0.8.4-main.8baae0fced",
+    "@dxos/errors": "0.8.4-main.8baae0fced",
+    "@dxos/invariant": "0.8.4-main.8baae0fced",
+    "@dxos/graph": "0.8.4-main.8baae0fced",
+    "@dxos/keys": "0.8.4-main.8baae0fced",
+    "@dxos/effect": "0.8.4-main.8baae0fced",
+    "@dxos/mcp-client": "0.8.4-main.8baae0fced",
+    "@dxos/log": "0.8.4-main.8baae0fced",
+    "@dxos/node-std": "0.8.4-main.8baae0fced",
+    "@dxos/types": "0.8.4-main.8baae0fced",
+    "@dxos/schema": "0.8.4-main.8baae0fced",
+    "@dxos/util": "0.8.4-main.8baae0fced",
+    "@dxos/functions": "0.8.4-main.8baae0fced"
   },
   "devDependencies": {
-    "effect": "3.16.13",
-    "inquirer": "^12.0.0"
+    "@types/lodash.defaultsdeep": "^4.6.6",
+    "effect": "3.20.0"
   },
   "peerDependencies": {
-    "effect": "^3.13.3"
+    "effect": "3.20.0"
   },
   "publishConfig": {
     "access": "public"
   },
   "beast": {},
   "scripts": {
-    "run:chat": "tsx ./bin/chat.ts"
+    "run:chat": "vite-node ./bin/chat.ts"
   }
 }

package/src/extraction/extraction-ner-function/index.ts

@@ -2,5 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './extraction-ner-function';
 export * from './named-entity-recognition';

package/src/extraction/extraction-ner-function/named-entity-recognition.test.ts

@@ -6,11 +6,13 @@ import { type TokenClassificationOutput, type TokenClassificationPipelineType }
 import { beforeAll, describe, expect, test } from 'vitest';
 
 import { log } from '@dxos/log';
-import { createTestData } from '@dxos/schema/testing';
+import { type ContentBlock, type Message } from '@dxos/types';
+import { createTestData } from '@dxos/types/testing';
 
 import { combineNerTokens, createTokenGroups, extractFullEntities, getNer } from './named-entity-recognition';
 
-describe('NamedEntityRecognition', () => {
+// TODO(mykola): This should not run on CI.
+describe.skip('NamedEntityRecognition', () => {
   let ner: TokenClassificationPipelineType;
 
   beforeAll(async () => {
@@ -56,8 +58,10 @@ describe('NamedEntityRecognition', () => {
   test('own data', async () => {
     const { transcriptWoflram, transcriptJosiah } = await createTestData();
 
-    const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message) =>
-      message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
+    const blocks = [...transcriptWoflram, ...transcriptJosiah].flatMap((message: Message.Message) =>
+      message.blocks.flatMap((block: ContentBlock.Any) =>
+        block._tag === 'transcript' ? [(block as ContentBlock.Transcript).text] : [],
+      ),
     );
 
     for (const block of blocks) {
@@ -69,8 +73,10 @@ describe('NamedEntityRecognition', () => {
   test('more own data', async () => {
     const { transcriptMessages } = await createTestData();
 
-    const blocks = transcriptMessages.flatMap((message) =>
-      message.blocks.flatMap((block) => (block.type === 'transcription' ? [block.text] : [])),
+    const blocks = transcriptMessages.flatMap((message: Message.Message) =>
+      message.blocks.flatMap((block: ContentBlock.Any) =>
+        block._tag === 'transcript' ? [(block as ContentBlock.Transcript).text] : [],
+      ),
     );
 
     for (const block of blocks) {

package/src/extraction/extraction-ner-function/named-entity-recognition.ts

@@ -4,14 +4,15 @@
 
 import {
   type TokenClassificationOutput,
+  type TokenClassificationPipelineType,
   type TokenClassificationSingle,
   pipeline,
-  type TokenClassificationPipelineType,
 } from '@xenova/transformers';
 
 import { log } from '@dxos/log';
 
 let _ner: Promise<TokenClassificationPipelineType>;
+
 /**
  * Named Entity Recognition pipeline.
  * Initializes the pipeline on first call.
@@ -24,6 +25,7 @@ export const getNer = () => {
       return ner;
     });
   }
+
   return _ner;
 };
 

package/src/extraction/extraction.ts

@@ -2,17 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
-import { asyncTimeout } from '@dxos/async';
-import { Expando } from '@dxos/echo-schema';
-import { type FunctionDefinition, type FunctionExecutor } from '@dxos/functions';
+import { Operation } from '@dxos/compute';
+import { Obj } from '@dxos/echo';
 import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
 export const ExtractionInput = Schema.Struct({
-  message: DataType.Message,
-  objects: Schema.optional(Schema.Array(Expando)),
+  message: Message.Message,
+  objects: Schema.optional(Schema.Array(Obj.Unknown)),
   options: Schema.optional(
     Schema.Struct({
       timeout: Schema.optional(Schema.Number),
@@ -23,15 +22,14 @@ export const ExtractionInput = Schema.Struct({
 export interface ExtractionInput extends Schema.Schema.Type<typeof ExtractionInput> {}
 
 export const ExtractionOutput = Schema.Struct({
-  message: DataType.Message,
+  message: Message.Message,
   timeElapsed: Schema.Number,
 });
 export interface ExtractionOutput extends Schema.Schema.Type<typeof ExtractionOutput> {}
 
-export type ProcessTranscriptMessageParams = {
+export type ProcessTranscriptMessageProps = {
   input: ExtractionInput;
-  function: FunctionDefinition<ExtractionInput, ExtractionOutput>;
-  executor: FunctionExecutor;
+  function: Operation.Definition<ExtractionInput, ExtractionOutput>;
 
   options?: {
     /**
@@ -48,18 +46,46 @@ export type ProcessTranscriptMessageParams = {
   };
 };
 
-export type ExtractionFunction = FunctionDefinition<ExtractionInput, ExtractionOutput>;
+export type ExtractionFunction = Operation.Definition<ExtractionInput, ExtractionOutput>;
+
+/**
+ * Placeholder for Anthropic-based entity extraction.
+ * Implementation is currently disabled pending restoration of extraction logic.
+ */
+export const extractionAnthropicFunction = Operation.make({
+  meta: {
+    key: 'org.dxos.function.extraction.anthropic',
+    name: 'Entity Extraction (Anthropic)',
+    description: 'Extract entities from transcript using Anthropic LLM.',
+  },
+  input: ExtractionInput,
+  output: ExtractionOutput,
+});
+
+/**
+ * Placeholder for NER-based entity extraction.
+ * Implementation is currently disabled pending restoration of extraction logic.
+ */
+export const extractionNerFunction = Operation.make({
+  meta: {
+    key: 'org.dxos.function.extraction.ner',
+    name: 'Entity Extraction (NER)',
+    description: 'Extract entities from transcript using Named Entity Recognition.',
+  },
+  input: ExtractionInput,
+  output: ExtractionOutput,
+});
 
 /**
  * Extract entities from the transcript message and add them to the message.
  */
 // TODO(dmaretskyi): Move context to a vector search index.
-export const processTranscriptMessage = async (params: ProcessTranscriptMessageParams): Promise<ExtractionOutput> => {
+export const processTranscriptMessage = async (params: ProcessTranscriptMessageProps): Promise<ExtractionOutput> => {
   try {
     if (params.options?.timeout && params.options.timeout > 0) {
-      return await asyncTimeout(params.executor.invoke(params.function, params.input), params.options.timeout);
+      // return await asyncTimeout(params.executor.invoke(params.function, params.input), params.options.timeout);
     } else {
-      return await params.executor.invoke(params.function, params.input);
+      // return await params.executor.invoke(params.function, params.input);
     }
   } catch (error) {
     if (params.options?.fallbackToRaw) {
@@ -72,4 +98,7 @@ export const processTranscriptMessage = async (params: ProcessTranscriptMessageP
       throw error;
     }
   }
+
+  // TODO(dmaretskyi): Restore extraction logic.
+  return { message: params.input.message, timeElapsed: 0 };
 };

package/src/extraction/index.ts

@@ -3,6 +3,5 @@
 //
 
 export * from './extraction';
-export * from './extraction-llm-function';
 export * from './extraction-ner-function';
 export * from './quotes';

package/src/extraction/quotes.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import { describe, test, expect } from 'vitest';
+import { describe, expect, test } from 'vitest';
 
 import { insertReferences } from './quotes';
 
@@ -13,7 +13,7 @@ describe('insertReferences', () => {
     };
 
     expect(insertReferences('This is a computational irreducibility test.', quotes)).toBe(
-      'This is a [computational irreducibility][dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1] test.',
+      'This is a [computational irreducibility](dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1) test.',
     );
     expect(
       insertReferences(
@@ -21,7 +21,7 @@ describe('insertReferences', () => {
         quotes,
       ),
     ).toBe(
-      "And what I'd like to talk today about is Steven Wolfram's concept of a [computational irreducibility][dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1].",
+      "And what I'd like to talk today about is Steven Wolfram's concept of a [computational irreducibility](dxn:echo:@:01JTG9JW11XGWJZ32AW8ET93D1).",
     );
   });
 });

package/src/extraction/quotes.ts

@@ -2,11 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
-import { Query, Filter } from '@dxos/echo';
-import { type EchoDatabase } from '@dxos/echo-db';
-import { ObjectId } from '@dxos/echo-schema';
+import { type Database, Filter, Query } from '@dxos/echo';
+import { ObjectId } from '@dxos/keys';
 import { DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
 
@@ -28,10 +27,10 @@ export const ReferencedQuotes = Schema.Struct({
 });
 export type ReferencedQuotes = Schema.Schema.Type<typeof ReferencedQuotes>;
 
-export const findQuotes = async (quotes: string[], db: EchoDatabase): Promise<ReferencedQuotes> => {
+export const findQuotes = async (quotes: string[], db: Database.Database): Promise<ReferencedQuotes> => {
   const references = await Promise.all(
     quotes.map(async (quote) => {
-      const { objects } = await db.query(Query.select(Filter.text(quote, { type: 'vector' }))).run();
+      const objects = await db.query(Query.select(Filter.text(quote, { type: 'vector' }))).run();
       log.info('found quotes in vector index', { objects, quote });
       return objects.length > 0 ? [{ id: objects[0].id.toString(), quote }] : [];
     }),
@@ -51,7 +50,7 @@ export const insertReferences = (text: string, quotes: ReferencedQuotes) => {
 
     // Use a case-insensitive regular expression to replace the quote.
     const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
-    text = text.replace(regex, `[${quote.quote}][${DXN.fromLocalObjectId(quote.id).toString()}]`);
+    text = text.replace(regex, `[${quote.quote}](${DXN.fromLocalObjectId(quote.id).toString()})`);
   }
 
   return text;

package/src/{conversation → functions}/index.ts

@@ -2,4 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './conversation';
+export * from './services';

package/src/functions/services.ts

@@ -0,0 +1,214 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Tool from '@effect/ai/Tool';
+import type * as Toolkit from '@effect/ai/Toolkit';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Record from 'effect/Record';
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { AiToolNotFoundError, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { OpaqueToolkit } from '@dxos/ai';
+import { Operation, OperationRegistry } from '@dxos/compute';
+import { todo } from '@dxos/debug';
+import { Ref } from '@dxos/echo';
+import { invariant } from '@dxos/invariant';
+
+import { RefFromLLM } from '../util';
+
+export const makeToolResolverFromOperations = <R = never>({
+  toolkit: extraToolkit = OpaqueToolkit.empty,
+}: { toolkit?: OpaqueToolkit.OpaqueToolkit<never, never, R> } = {}): Layer.Layer<
+  ToolResolverService,
+  never,
+  OpaqueToolkit.OpaqueToolkitProvider | OperationRegistry.Service | R
+> => {
+  return Layer.effect(
+    ToolResolverService,
+    Effect.gen(function* () {
+      const toolkitProvider = yield* OpaqueToolkit.OpaqueToolkitProvider;
+      const operationRegistry = yield* OperationRegistry.Service;
+      return {
+        resolve: (id): Effect.Effect<Tool.Any, AiToolNotFoundError> =>
+          Effect.gen(function* () {
+            const toolkit = OpaqueToolkit.merge(extraToolkit, toolkitProvider.getToolkit());
+
+            const tool = toolkit.toolkit.tools[id];
+            if (tool) {
+              return tool;
+            }
+
+            return yield* operationRegistry.resolve(id).pipe(
+              Effect.flatMap(
+                Option.match({
+                  onSome: (_) => Effect.succeed(projectFunctionToTool(_)),
+                  onNone: () => Effect.fail(new AiToolNotFoundError(id)),
+                }),
+              ),
+            );
+          }),
+      } satisfies Context.Tag.Service<ToolResolverService>;
+    }),
+  );
+};
+
+export const makeToolExecutionService = <E, R>(opts: {
+  invoke: (tool: Tool.Any, input: unknown) => Effect.Effect<unknown>;
+}): Layer.Layer<ToolExecutionService, never, OpaqueToolkit.OpaqueToolkitProvider> =>
+  Layer.effect(
+    ToolExecutionService,
+    Effect.gen(function* () {
+      const toolkitProvider = yield* OpaqueToolkit.OpaqueToolkitProvider;
+      const toolkit = toolkitProvider.getToolkit();
+
+      const toolkitHandler = yield* toolkit.toolkit.pipe(Effect.provide(toolkit.layer));
+      invariant(isHandlerLike(toolkitHandler));
+
+      return {
+        handlersFor: (toolkit) => {
+          const makeHandler = (tool: Tool.Any): ((params: unknown) => Effect.Effect<unknown, any, any>) => {
+            return Effect.fn(`toolFunctionHandler ${tool.name}`)(function* (input: any) {
+              if (toolkitHandler.tools[tool.name]) {
+                if (Tool.isProviderDefined(tool)) {
+                  throw new Error('Attempted to call a provider-defined tool');
+                }
+
+                // TODO(wittjosiah): Everything is `never` here.
+                const { result } = yield* (toolkitHandler.handle as any)(tool.name, input);
+                return result;
+              }
+
+              return yield* opts.invoke(tool, input).pipe(Effect.catchAllDefect((defect) => Effect.fail(defect)));
+            });
+          };
+
+          return toolkit.of(
+            Record.map(toolkit.tools, (tool, _name) => (Tool.isUserDefined(tool) ? makeHandler(tool) : null)) as any,
+          ) as any;
+        },
+      };
+    }),
+  );
+
+export const makeToolExecutionServiceFromOperationInvoker = (): Layer.Layer<
+  ToolExecutionService,
+  never,
+  Operation.Service | OpaqueToolkit.OpaqueToolkitProvider
+> => {
+  return Layer.unwrapEffect(
+    Effect.gen(function* () {
+      const operationInvoker = yield* Operation.Service;
+
+      return makeToolExecutionService({
+        invoke: (tool, input) =>
+          Effect.gen(function* () {
+            const operationDef = getOperationFromTool(tool).pipe(Option.getOrThrow);
+
+            return yield* operationInvoker.invoke(operationDef, input).pipe(Effect.orDie);
+          }),
+      });
+    }),
+  );
+};
+
+export const ToolExecutionServices = Layer.mergeAll(
+  makeToolResolverFromOperations(),
+  makeToolExecutionServiceFromOperationInvoker(),
+);
+
+class FunctionToolAnnotation extends Context.Tag('@dxos/assistant/FunctionToolAnnotation')<
+  FunctionToolAnnotation,
+  { definition: Operation.Definition.Any }
+>() {}
+
+export const getOperationFromTool = (tool: Tool.Any): Option.Option<Operation.Definition.Any> => {
+  return Context.getOption(FunctionToolAnnotation)(tool.annotations).pipe(Option.map(({ definition }) => definition));
+};
+
+const toolCache = new WeakMap<Operation.Definition.Any, Tool.Any>();
+
+/**
+ * Projects an `Operation.Definition` into an `AiTool`.
+ */
+const projectFunctionToTool = (fn: Operation.Definition.Any): Tool.Any => {
+  if (toolCache.has(fn)) {
+    return toolCache.get(fn)!;
+  }
+
+  const tool = Tool.make(makeToolName(fn.meta.name ?? fn.meta.key), {
+    description: fn.meta.description,
+    parameters: createStructFieldsFromSchema(fn.input),
+    failure: Schema.Any,
+  }).annotate(FunctionToolAnnotation, { definition: fn });
+  toolCache.set(fn, tool);
+  return tool;
+};
+
+/**
+ * @returns Tool name produced from function name by escaping invalid characters.
+ */
+const makeToolName = (name: string) => {
+  const toolName = name.toLowerCase().replace(/[^a-zA-Z0-9]/g, '-');
+  invariant(toolName.match(/^[a-z_][a-z0-9-_]*$/));
+  return toolName;
+};
+
+// TODO(dmaretskyi): Factor out.
+const createStructFieldsFromSchema = (schema: Schema.Schema<any, any>): Record<string, Schema.Schema<any, any>> => {
+  switch (schema.ast._tag) {
+    case 'TypeLiteral':
+      return Object.fromEntries(
+        schema.ast.propertySignatures.map((prop) => [prop.name, Schema.make(mapSchemaTypeForLLM(prop.type))]),
+      );
+    case 'VoidKeyword':
+      return {};
+    default:
+      return todo(`Unsupported schema AST: ${schema.ast._tag}`);
+  }
+};
+
+/**
+ * Picks an LLM-friendly schema type for the given schema AST.
+ * The picked schema type decodes to the original schema type.
+ */
+const mapSchemaTypeForLLM = (ast: SchemaAST.AST): SchemaAST.AST => {
+  if (Ref.isRefType(ast)) {
+    const description = ast.annotations.description
+      ? ast.annotations.description + '\n' + RefFromLLM.ast.annotations.description
+      : (RefFromLLM.ast.annotations.description as string);
+    return RefFromLLM.annotations({ description }).ast;
+  } else if (SchemaAST.isTupleType(ast)) {
+    return new SchemaAST.TupleType(
+      ast.elements.map((t) => new SchemaAST.OptionalType(mapSchemaTypeForLLM(t.type), t.isOptional, t.annotations)),
+      ast.rest.map((t) => new SchemaAST.Type(mapSchemaTypeForLLM(t.type), t.annotations)),
+      ast.isReadonly,
+      ast.annotations,
+    );
+  } else if (SchemaAST.isTypeLiteral(ast)) {
+    return new SchemaAST.TypeLiteral(
+      ast.propertySignatures.map(
+        (p) =>
+          new SchemaAST.PropertySignature(
+            p.name,
+            mapSchemaTypeForLLM(p.type),
+            p.isOptional,
+            p.isReadonly,
+            p.annotations,
+          ),
+      ),
+      ast.indexSignatures,
+      ast.annotations,
+    );
+  }
+
+  return ast;
+};
+
+const isHandlerLike = (value: unknown): value is Toolkit.WithHandler<Record<string, Tool.Any>> => {
+  return typeof (value as any).tools === 'object' && typeof (value as any).handle === 'function';
+};

package/src/index.ts

@@ -2,11 +2,8 @@
 // Copyright 2024 DXOS.org
 //
 
-export * from './blueprint';
-export * from './context';
-export * from './conversation';
-export * from './extraction';
-export * from './research';
+export * from './functions';
+export * from './request';
 export * from './session';
-export * from './types';
-export * from './conversation';
+export * from './templates';
+export * from './util';