@dxos/functions 0.8.4-main.84f28bd → 0.8.4-main.ae835ea

package/src/services/service-registry.test.ts

@@ -0,0 +1,45 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, it } from '@effect/vitest';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+
+import { ServiceRegistry } from './service-registry';
+
+class MyTag extends Context.Tag('MyTag')<MyTag, { value: string }>() {}
+
+const mockRegistry = Layer.succeed(ServiceRegistry, {
+  resolve: (tag) => ((tag as any) === MyTag ? Option.some({ value: 'test' } as any) : Option.none()),
+});
+
+describe('ServiceRegistry', () => {
+  it.effect(
+    'provide',
+    Effect.fn(function* ({ expect }) {
+      const body = Effect.gen(function* () {
+        const { value } = yield* MyTag;
+        expect(value).toEqual('test');
+      });
+
+      const eff = body.pipe(ServiceRegistry.provide(MyTag));
+      yield* eff;
+    }, Effect.provide(mockRegistry)),
+  );
+
+  it.effect(
+    'provide or die',
+    Effect.fn(function* ({ expect }) {
+      const body = Effect.gen(function* () {
+        const { value } = yield* MyTag;
+        expect(value).toEqual('test');
+      });
+
+      const eff = body.pipe(ServiceRegistry.provideOrDie(MyTag));
+      yield* eff;
+    }, Effect.provide(mockRegistry)),
+  );
+});

package/src/services/service-registry.ts

@@ -0,0 +1,63 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import type * as Option from 'effect/Option';
+
+import { ServiceNotAvailableError } from '../errors';
+
+export namespace ServiceRegistry {
+  export interface Service {
+    resolve: <T extends Context.Tag<any, any>>(tag: T) => Option.Option<Context.Tag.Service<T>>;
+  }
+}
+
+export class ServiceRegistry extends Context.Tag('@dxos/functions/ServiceRegistry')<
+  ServiceRegistry,
+  ServiceRegistry.Service
+>() {
+  /**
+   * Resolves the service from the registry.
+   * @param tag Service tag to resolve.
+   * @throws {@link ServiceNotAvailableError} if the service is not found.
+   * @returns Effect that resolve to the service.
+   */
+  static resolve: <T extends Context.Tag<any, any>>(
+    tag: T,
+  ) => Effect.Effect<T, ServiceNotAvailableError, ServiceRegistry> = (tag) =>
+    ServiceRegistry.pipe(
+      Effect.flatMap((_) => _.resolve(tag)),
+      Effect.mapError(() => new ServiceNotAvailableError(tag.key)),
+    );
+
+  static provide: {
+    <Tags extends [Context.Tag<any, any>, ...Context.Tag<any, any>[]]>(
+      ...tags: Tags
+    ): <A, E, R>(
+      effect: Effect.Effect<A, E, R>,
+    ) => Effect.Effect<
+      A,
+      E | ServiceNotAvailableError,
+      Exclude<R, { [K in keyof Tags]: Context.Tag.Identifier<Tags[K]> }[number]> | ServiceRegistry
+    >;
+  } = (...tags) =>
+    (Function.flow as any)(...tags.map((tag) => Effect.provideServiceEffect(tag, ServiceRegistry.resolve(tag))));
+
+  static provideOrDie: {
+    <Tags extends [Context.Tag<any, any>, ...Context.Tag<any, any>[]]>(
+      ...tags: Tags
+    ): <A, E, R>(
+      effect: Effect.Effect<A, E, R>,
+    ) => Effect.Effect<
+      A,
+      E,
+      Exclude<R, { [K in keyof Tags]: Context.Tag.Identifier<Tags[K]> }[number]> | ServiceRegistry
+    >;
+  } = (...tags) =>
+    (Function.flow as any)(
+      ...tags.map((tag) => Effect.provideServiceEffect(tag, ServiceRegistry.resolve(tag).pipe(Effect.orDie))),
+    );
+}

package/src/services/tracing.ts

@@ -2,26 +2,142 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context } from 'effect';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
 
-import type { AnyEchoObject } from '@dxos/echo-schema';
+import { AgentStatus } from '@dxos/ai';
+import { Obj } from '@dxos/echo';
+import type { ObjectId } from '@dxos/echo/internal';
+import type { Queue } from '@dxos/echo-db';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
 
-export class TracingService extends Context.Tag('TracingService')<
+/**
+ * Provides a way for compute primitives (functions, workflows, tools)
+ * to emit an execution trace as a series of structured ECHO objects.
+ */
+export class TracingService extends Context.Tag('@dxos/functions/TracingService')<
   TracingService,
   {
+    /**
+     * Gets the parent message ID.
+     */
+    getTraceContext: () => TracingService.TraceContext;
+
     /**
      * Write an event to the tracing queue.
      * @param event - The event to write. Must be an a typed object.
      */
-    write(event: AnyEchoObject): void;
+    write: (event: Obj.Any) => void;
   }
 >() {
-  static noop: Context.Tag.Service<TracingService> = { write: () => {} };
+  static noop: Context.Tag.Service<TracingService> = { write: () => {}, getTraceContext: () => ({}) };
+
+  static layerNoop = Layer.succeed(TracingService, TracingService.noop);
 
   static console: Context.Tag.Service<TracingService> = {
     write: (event) => {
-      // eslint-disable-next-line no-console
       console.log(event);
     },
+    getTraceContext: () => ({}),
   };
+
+  static layerConsole = Layer.succeed(TracingService, TracingService.console);
+
+  static layerLogInfo = () =>
+    Layer.succeed(TracingService, {
+      write: (event) => {
+        if (Obj.instanceOf(AgentStatus, event)) {
+          log.info('status', { message: event.message });
+        }
+      },
+      getTraceContext: () => ({}),
+    });
+
+  /**
+   * Creates a TracingService layer that emits events to the parent tracing service.
+   */
+  static layerSubframe = (mapContext: (currentContext: TracingService.TraceContext) => TracingService.TraceContext) =>
+    Layer.effect(
+      TracingService,
+      Effect.gen(function* () {
+        const tracing = yield* TracingService;
+        const context = mapContext(tracing.getTraceContext());
+        return {
+          write: (event) => tracing.write(event),
+          getTraceContext: () => context,
+        };
+      }),
+    );
+
+  static layerQueue = (queue: Queue) =>
+    Layer.effect(
+      TracingService,
+      Effect.gen(function* () {
+        // TODO(dmaretskyi): Batching.
+        return {
+          write: (event) => queue.append([event]),
+          getTraceContext: () => ({
+            debugInfo: {
+              queue: queue.dxn.toString(),
+            },
+          }),
+        };
+      }),
+    );
+
+  /**
+   * Emit the current human-readable execution status.
+   */
+  static emitStatus: (
+    data: Omit<Obj.MakeProps<typeof AgentStatus>, 'created'>,
+  ) => Effect.Effect<void, never, TracingService> = Effect.fnUntraced(function* (data) {
+    const tracing = yield* TracingService;
+    tracing.write(
+      Obj.make(AgentStatus, {
+        parentMessage: tracing.getTraceContext().parentMessage,
+        toolCallId: tracing.getTraceContext().toolCallId,
+        created: new Date().toISOString(),
+        ...data,
+      }),
+    );
+  });
+
+  static emitConverationMessage: (
+    data: Obj.MakeProps<typeof DataType.Message>,
+  ) => Effect.Effect<void, never, TracingService> = Effect.fnUntraced(function* (data) {
+    const tracing = yield* TracingService;
+    tracing.write(
+      Obj.make(DataType.Message, {
+        parentMessage: tracing.getTraceContext().parentMessage,
+        ...data,
+        properties: {
+          [MESSAGE_PROPERTY_TOOL_CALL_ID]: tracing.getTraceContext().toolCallId,
+          ...data.properties,
+        },
+      }),
+    );
+  });
 }
+
+export namespace TracingService {
+  export interface TraceContext {
+    /**
+     * If this thread sprung from a tool call, this is the ID of the message containing the tool call.
+     */
+    parentMessage?: ObjectId;
+
+    /**
+     * If the current thread is a byproduct of a tool call, this is the ID of the tool call.
+     */
+    toolCallId?: string;
+
+    debugInfo?: unknown;
+  }
+}
+
+/**
+ * Goes into {@link DataType.Message['properties']}
+ */
+export const MESSAGE_PROPERTY_TOOL_CALL_ID = 'toolCallId' as const;

package/src/testing/index.ts

@@ -3,3 +3,4 @@
 //
 
 export * from './services';
+export * from './layer';

package/src/testing/layer.ts

@@ -0,0 +1,114 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import type * as Schema from 'effect/Schema';
+
+import type { EchoDatabaseImpl, QueueFactory } from '@dxos/echo-db';
+import { EchoTestBuilder } from '@dxos/echo-db/testing';
+import type { EchoHostIndexingConfig } from '@dxos/echo-pipeline';
+import { acquireReleaseResource } from '@dxos/effect';
+import { PublicKey } from '@dxos/keys';
+import type { LevelDB } from '@dxos/kv-store';
+import { createTestLevel } from '@dxos/kv-store/testing';
+import { log } from '@dxos/log';
+
+import { DatabaseService, QueueService } from '../services';
+
+const testBuilder = acquireReleaseResource(() => new EchoTestBuilder());
+
+export const testStoragePath = ({ name = PublicKey.random().toHex() }: { name?: string }) => {
+  return `/tmp/dxos-${name}`;
+};
+
+export type TestDatabaseOptions = {
+  indexing?: Partial<EchoHostIndexingConfig>;
+  types?: Schema.Schema.AnyNoContext[];
+  spaceKey?: PublicKey;
+  storagePath?: string;
+  onInit?: () => Effect.Effect<void, never, DatabaseService | QueueService>;
+};
+
+export const TestDatabaseLayer = ({ indexing, types, spaceKey, storagePath, onInit }: TestDatabaseOptions = {}) =>
+  Layer.scopedContext(
+    Effect.gen(function* () {
+      const builder = yield* testBuilder;
+
+      let kv: LevelDB | undefined;
+      if (storagePath) {
+        kv = createTestLevel(storagePath);
+        yield* Effect.promise(() => kv!.open());
+        // const keyCount = yield* Effect.promise(async () => (await kv!.iterator({ values: false }).all()).length);
+        // log.info('opened test db', { storagePath, keyCount });
+      }
+      const peer = yield* Effect.promise(() => builder.createPeer({ indexing, types, kv }));
+
+      let db: EchoDatabaseImpl | undefined;
+      let queues: QueueFactory | undefined;
+
+      if (storagePath) {
+        const testMetadata = yield* Effect.promise(async () => {
+          try {
+            return await kv!.get('test-metadata', { valueEncoding: 'json' });
+          } catch (e) {
+            if ((e as any).code === 'LEVEL_NOT_FOUND') {
+              return undefined;
+            }
+            throw e;
+          }
+        });
+        log('starting persistant test db', { storagePath, testMetadata });
+        if (!testMetadata) {
+          const key = spaceKey ?? PublicKey.random();
+          db = yield* Effect.promise(() => peer.createDatabase(key));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+
+          yield* Effect.promise(() =>
+            kv!.put('test-metadata', { key: key.toHex(), rootUrl: db!.rootUrl }, { valueEncoding: 'json' }),
+          );
+
+          if (onInit) {
+            yield* onInit().pipe(
+              Effect.provideService(DatabaseService, DatabaseService.make(db)),
+              Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+            );
+          }
+        } else {
+          const key = PublicKey.from((testMetadata as any).key);
+          const rootUrl = (testMetadata as any).rootUrl;
+          db = yield* Effect.promise(() => peer.openDatabase(key, rootUrl));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+        }
+      } else {
+        db = yield* Effect.promise(() => peer.createDatabase(spaceKey));
+        queues = peer.client.constructQueueFactory(db.spaceId);
+        if (onInit) {
+          yield* onInit().pipe(
+            Effect.provideService(DatabaseService, DatabaseService.make(db)),
+            Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+          );
+        }
+      }
+
+      yield* Effect.addFinalizer(() =>
+        Effect.promise(async () => {
+          if (kv) {
+            // {
+            //   const keyCount = (await kv.iterator({ values: false }).all()).length;
+            //   log.info('closing persistant test db', { storagePath, keyCount });
+            // }
+
+            await kv.close();
+          }
+        }),
+      );
+
+      return Context.mergeAll(
+        Context.make(DatabaseService, DatabaseService.make(db)),
+        Context.make(QueueService, QueueService.make(queues, undefined)),
+      );
+    }),
+  );

package/src/testing/logger.ts

@@ -2,15 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Effect, type Context } from 'effect';
+import type * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
 
 import { LogLevel } from '@dxos/log';
 
-import { type EventLogger, createEventLogger } from '../services';
+import { type ComputeEventLogger, createEventLogger } from '../services';
 
-export const noopLogger: Context.Tag.Service<EventLogger> = {
+export const noopLogger: Context.Tag.Service<ComputeEventLogger> = {
   log: () => Effect.succeed(undefined),
   nodeId: undefined,
 };
 
-export const consoleLogger: Context.Tag.Service<EventLogger> = createEventLogger(LogLevel.INFO);
+export const consoleLogger: Context.Tag.Service<ComputeEventLogger> = createEventLogger(LogLevel.INFO);

package/src/testing/persist-database.test.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+
+import { Filter, Obj, Query, Type } from '@dxos/echo';
+import { DataType } from '@dxos/schema';
+
+import { DatabaseService } from '../services';
+
+import { TestDatabaseLayer, testStoragePath } from './layer';
+
+describe('TestDatabaseLayer', { timeout: 600_000 }, () => {
+  it.effect(
+    'persist database to disk',
+    Effect.fnUntraced(function* (_) {
+      const DbLayer = TestDatabaseLayer({
+        storagePath: testStoragePath({ name: `feed-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        yield* DatabaseService.add(Obj.make(Type.Expando, { label: 'test' }));
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.everything()));
+        expect(objects[0]?.label).toEqual('test');
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect(
+    'reload database -- save index before restart',
+    Effect.fnUntraced(function* (_) {
+      const NUM_OBJECTS = 500;
+      const DbLayer = TestDatabaseLayer({
+        types: [DataType.Person],
+        storagePath: testStoragePath({ name: `reload-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+        expect(objects.length).toEqual(NUM_OBJECTS);
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect.skip(
+    'reload database -- save index before restart [manual]',
+    Effect.fnUntraced(
+      function* (_) {
+        const NUM_OBJECTS = 500;
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+      },
+      Effect.provide(
+        TestDatabaseLayer({
+          types: [DataType.Person],
+          storagePath: testStoragePath({ name: `reload-test` }),
+        }),
+      ),
+    ),
+  );
+});

package/src/testing/services.ts

@@ -2,28 +2,25 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type Context } from 'effect';
+import type * as Context from 'effect/Context';
 
-import { type AiServiceEdgeClientOptions, EdgeAiServiceClient, type AiServiceClient, ToolRegistry } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT, createTestAiServiceClient } from '@dxos/ai/testing';
 import type { Space } from '@dxos/client/echo';
 import type { EchoDatabase, QueueFactory } from '@dxos/echo-db';
 import { assertArgument } from '@dxos/invariant';
 
-import { consoleLogger, noopLogger } from './logger';
 import {
-  AiService,
+  type ComputeEventLogger,
   ConfiguredCredentialsService,
   type CredentialsService,
   DatabaseService,
-  type EventLogger,
   QueueService,
   ServiceContainer,
   type ServiceCredential,
-  ToolResolverService,
   type TracingService,
 } from '../services';
 
+import { consoleLogger, noopLogger } from './logger';
+
 // TODO(burdon): Factor out.
 export type OneOf<T> = {
   [K in keyof T]: { [P in K]: T[P] } & { [P in Exclude<keyof T, K>]?: never };
@@ -35,23 +32,7 @@ export type TestServiceOptions = {
   /**
    * AI service configuration.
    */
-  ai?: OneOf<{
-    /**
-     * Custom AI service client.
-     */
-    client?: AiServiceClient;
-
-    /**
-     * Edge AI service at specified endpoint.
-     */
-    endpoint?: AiServiceEdgeClientOptions['endpoint'];
-
-    /**
-     * Predefined AI service configuration.
-     */
-    // TODO(burdon): 'dev' and 'edge' are redundant with providing an endpoint.
-    provider?: AiServiceProvider;
-  }>;
+  ai?: any;
 
   /**
    * Credentials service configuration.
@@ -84,7 +65,7 @@ export type TestServiceOptions = {
    */
   logging?: {
     enabled?: boolean;
-    logger?: Context.Tag.Service<EventLogger>;
+    logger?: Context.Tag.Service<ComputeEventLogger>;
   };
 
   /**
@@ -95,10 +76,11 @@ export type TestServiceOptions = {
   tracing?: {
     service?: Context.Tag.Service<TracingService>;
   };
-
-  toolResolver?: Context.Tag.Service<ToolResolverService>;
 };
 
+/**
+ * @deprecated
+ */
 export const createTestServices = ({
   ai,
   credentials,
@@ -107,60 +89,19 @@ export const createTestServices = ({
   queues,
   space,
   tracing,
-  toolResolver,
 }: TestServiceOptions = {}): ServiceContainer => {
-  assertArgument(!(!!space && (!!db || !!queues)), 'space can be provided only if db and queues are not');
+  assertArgument(!(!!space && (!!db || !!queues)), 'space', 'space can be provided only if db and queues are not');
 
   return new ServiceContainer().setServices({
-    ai: createAiService(ai),
+    // ai: createAiService(ai),
     credentials: createCredentialsService(credentials),
     database: space || db ? DatabaseService.make(space?.db || db!) : undefined,
-    eventLogger: logging?.logger ?? logging?.enabled ? consoleLogger : noopLogger,
+    eventLogger: (logging?.logger ?? logging?.enabled) ? consoleLogger : noopLogger,
     queues: space || queues ? QueueService.make(space?.queues || queues!, undefined) : undefined,
     tracing: tracing?.service,
-    toolResolver: toolResolver ?? ToolResolverService.make(new ToolRegistry([])),
   });
 };
 
-// TODO(burdon): Enable model configuration.
-const createAiService = (ai: TestServiceOptions['ai']): Context.Tag.Service<AiService> | undefined => {
-  if (ai?.client != null) {
-    return AiService.make(ai.client);
-  }
-
-  if (ai?.endpoint != null) {
-    return AiService.make(new EdgeAiServiceClient({ endpoint: ai.endpoint }));
-  }
-
-  switch (ai?.provider) {
-    case 'dev':
-      return AiService.make(
-        new EdgeAiServiceClient({
-          endpoint: AI_SERVICE_ENDPOINT.LOCAL,
-          defaultGenerationOptions: {
-            model: '@anthropic/claude-3-5-sonnet-20241022',
-          },
-        }),
-      );
-
-    case 'edge':
-      return AiService.make(
-        new EdgeAiServiceClient({
-          endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-          defaultGenerationOptions: {
-            model: '@anthropic/claude-3-5-sonnet-20241022',
-          },
-        }),
-      );
-
-    case 'ollama':
-      return AiService.make(createTestAiServiceClient());
-
-    case 'lmstudio':
-      throw new Error('LMStudio is not supported');
-  }
-};
-
 const createCredentialsService = (
   credentials: TestServiceOptions['credentials'] | undefined,
 ): Context.Tag.Service<CredentialsService> | undefined => {