@dxos/functions 0.8.4-main.c1de068 → 0.8.4-main.dedc0f3

package/src/services/service-container.ts

@@ -9,9 +9,9 @@ import { entries } from '@dxos/util';
 
 import { ConfiguredCredentialsService, CredentialsService } from './credentials';
 import { DatabaseService } from './database';
-import { EventLogger } from './event-logger';
-import { FunctionCallService } from './function-call-service';
+import { ComputeEventLogger } from './event-logger';
 import { QueueService } from './queues';
+import { RemoteFunctionExecutionService } from './remote-function-execution-service';
 import { TracingService } from './tracing';
 
 // TODO(dmaretskyi): Refactor this module to only rely on tags and not the human-assigned names.
@@ -20,11 +20,11 @@ import { TracingService } from './tracing';
  * List of all services.
  */
 const SERVICES = {
-  ai: AiService,
+  ai: AiService.AiService,
   credentials: CredentialsService,
   database: DatabaseService,
-  eventLogger: EventLogger,
-  functionCallService: FunctionCallService,
+  eventLogger: ComputeEventLogger,
+  functionCallService: RemoteFunctionExecutionService,
   queues: QueueService,
   tracing: TracingService,
 } as const satisfies Record<string, Context.TagClass<any, string, any>>;
@@ -88,9 +88,10 @@ export class ServiceContainer {
     return new ServiceContainer().setServices({ ...this._services });
   }
 
-  // TODO(dmaretskyi): `getService` is designed to error at runtime if the service is not available, but layer forces us to provide all services and makes stubs for the ones that are not available.
+  // TODO(dmaretskyi): `getService` is designed to error at runtime if the service is not available, but Layer forces us to provide all services and makes stubs for the ones that are not available.
   createLayer(): Layer.Layer<Services> {
-    const ai = this._services.ai != null ? Layer.succeed(AiService, this._services.ai) : AiService.notAvailable;
+    const ai =
+      this._services.ai != null ? Layer.succeed(AiService.AiService, this._services.ai) : AiService.notAvailable;
     const credentials = Layer.succeed(
       CredentialsService,
       this._services.credentials ?? new ConfiguredCredentialsService(),
@@ -102,10 +103,10 @@ export class ServiceContainer {
     const queues =
       this._services.queues != null ? Layer.succeed(QueueService, this._services.queues) : QueueService.notAvailable;
     const tracing = Layer.succeed(TracingService, this._services.tracing ?? TracingService.noop);
-    const eventLogger = Layer.succeed(EventLogger, this._services.eventLogger ?? EventLogger.noop);
+    const eventLogger = Layer.succeed(ComputeEventLogger, this._services.eventLogger ?? ComputeEventLogger.noop);
     const functionCallService = Layer.succeed(
-      FunctionCallService,
-      this._services.functionCallService ?? FunctionCallService.mock(),
+      RemoteFunctionExecutionService,
+      this._services.functionCallService ?? RemoteFunctionExecutionService.mock(),
     );
 
     return Layer.mergeAll(ai, credentials, database, queues, tracing, eventLogger, functionCallService);

package/src/services/service-registry.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect, flow, type Option } from 'effect';
+import { Context, Effect, type Option, flow } from 'effect';
 
 import { ServiceNotAvailableError } from '../errors';
 
@@ -12,7 +12,10 @@ export namespace ServiceRegistry {
   }
 }
 
-export class ServiceRegistry extends Context.Tag('ServiceRegistry')<ServiceRegistry, ServiceRegistry.Service>() {
+export class ServiceRegistry extends Context.Tag('@dxos/functions/ServiceRegistry')<
+  ServiceRegistry,
+  ServiceRegistry.Service
+>() {
   /**
    * Resolves the service from the registry.
    * @param tag Service tag to resolve.

package/src/services/tracing.ts

@@ -2,41 +2,139 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect } from 'effect';
+import { Context, Effect, Layer } from 'effect';
 
 import { AgentStatus } from '@dxos/ai';
 import { Obj } from '@dxos/echo';
-import type { AnyEchoObject } from '@dxos/echo-schema';
+import type { Queue } from '@dxos/echo-db';
+import type { ObjectId } from '@dxos/echo-schema';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
 
 /**
  * Provides a way for compute primitives (functions, workflows, tools)
  * to emit an execution trace as a series of structured ECHO objects.
  */
-export class TracingService extends Context.Tag('TracingService')<
+export class TracingService extends Context.Tag('@dxos/functions/TracingService')<
   TracingService,
   {
+    /**
+     * Gets the parent message ID.
+     */
+    getTraceContext: () => TracingService.TraceContext;
+
     /**
      * Write an event to the tracing queue.
      * @param event - The event to write. Must be an a typed object.
      */
-    write(event: AnyEchoObject): void;
+    write: (event: Obj.Any) => void;
   }
 >() {
-  static noop: Context.Tag.Service<TracingService> = { write: () => {} };
+  static noop: Context.Tag.Service<TracingService> = { write: () => {}, getTraceContext: () => ({}) };
+
+  static layerNoop = Layer.succeed(TracingService, TracingService.noop);
 
   static console: Context.Tag.Service<TracingService> = {
     write: (event) => {
       // eslint-disable-next-line no-console
       console.log(event);
     },
+    getTraceContext: () => ({}),
   };
 
+  static layerConsole = Layer.succeed(TracingService, TracingService.console);
+
+  static layerLogInfo = () =>
+    Layer.succeed(TracingService, {
+      write: (event) => {
+        if (Obj.instanceOf(AgentStatus, event)) {
+          log.info('status', { message: event.message });
+        }
+      },
+      getTraceContext: () => ({}),
+    });
+
+  /**
+   * Creates a TracingService layer that emits events to the parent tracing service.
+   */
+  static layerSubframe = (mapContext: (currentContext: TracingService.TraceContext) => TracingService.TraceContext) =>
+    Layer.effect(
+      TracingService,
+      Effect.gen(function* () {
+        const tracing = yield* TracingService;
+        const context = mapContext(tracing.getTraceContext());
+        return {
+          write: (event) => tracing.write(event),
+          getTraceContext: () => context,
+        };
+      }),
+    );
+
+  static layerQueue = (queue: Queue) =>
+    Layer.effect(
+      TracingService,
+      Effect.gen(function* () {
+        // TODO(dmaretskyi): Batching.
+        return {
+          write: (event) => queue.append([event]),
+          getTraceContext: () => ({
+            debugInfo: {
+              queue: queue.dxn.toString(),
+            },
+          }),
+        };
+      }),
+    );
+
   /**
    * Emit the current human-readable execution status.
    */
-  static emitStatus: (data: Obj.MakeProps<typeof AgentStatus>) => Effect.Effect<unknown, never, void> =
+  static emitStatus: (data: Obj.MakeProps<typeof AgentStatus>) => Effect.Effect<void, never, TracingService> =
     Effect.fnUntraced(function* (data) {
       const tracing = yield* TracingService;
-      tracing.write(Obj.make(AgentStatus, data));
+      tracing.write(
+        Obj.make(AgentStatus, {
+          parentMessage: tracing.getTraceContext().parentMessage,
+          toolCallId: tracing.getTraceContext().toolCallId,
+          ...data,
+        }),
+      );
     });
+
+  static emitConverationMessage: (
+    data: Obj.MakeProps<typeof DataType.Message>,
+  ) => Effect.Effect<void, never, TracingService> = Effect.fnUntraced(function* (data) {
+    const tracing = yield* TracingService;
+    tracing.write(
+      Obj.make(DataType.Message, {
+        parentMessage: tracing.getTraceContext().parentMessage,
+        ...data,
+        properties: {
+          [MESSAGE_PROPERTY_TOOL_CALL_ID]: tracing.getTraceContext().toolCallId,
+          ...data.properties,
+        },
+      }),
+    );
+  });
 }
+
+export namespace TracingService {
+  export interface TraceContext {
+    /**
+     * If this thread sprung from a tool call, this is the ID of the message containing the tool call.
+     */
+    parentMessage?: ObjectId;
+
+    /**
+     * If the current thread is a byproduct of a tool call, this is the ID of the tool call.
+     */
+    toolCallId?: string;
+
+    debugInfo?: unknown;
+  }
+}
+
+/**
+ * Goes into {@link DataType.Message['properties']}
+ */
+export const MESSAGE_PROPERTY_TOOL_CALL_ID = 'toolCallId' as const;

package/src/testing/layer.ts

@@ -2,27 +2,107 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect, Layer } from 'effect';
 import type { Schema } from 'effect';
+import { Context, Effect, Layer } from 'effect';
 
+import type { EchoDatabaseImpl, QueueFactory } from '@dxos/echo-db';
 import { EchoTestBuilder } from '@dxos/echo-db/testing';
 import type { EchoHostIndexingConfig } from '@dxos/echo-pipeline';
 import { accuireReleaseResource } from '@dxos/effect';
+import { PublicKey } from '@dxos/keys';
+import type { LevelDB } from '@dxos/kv-store';
+import { createTestLevel } from '@dxos/kv-store/testing';
+import { log } from '@dxos/log';
 
 import { DatabaseService, QueueService } from '../services';
 
 const testBuilder = accuireReleaseResource(() => new EchoTestBuilder());
 
+export const testStoragePath = ({ name = PublicKey.random().toHex() }: { name?: string }) => {
+  return `/tmp/dxos-${name}`;
+};
+
 export type TestDatabaseOptions = {
   indexing?: Partial<EchoHostIndexingConfig>;
   types?: Schema.Schema.AnyNoContext[];
+  storagePath?: string;
+  onInit?: () => Effect.Effect<void, never, DatabaseService | QueueService>;
 };
 
-export const TestDatabaseLayer = ({ indexing, types }: TestDatabaseOptions = {}) =>
+export const TestDatabaseLayer = ({ indexing, types, storagePath, onInit }: TestDatabaseOptions = {}) =>
   Layer.scopedContext(
     Effect.gen(function* () {
       const builder = yield* testBuilder;
-      const { db, queues } = yield* Effect.promise(() => builder.createDatabase({ indexing, types }));
+
+      let kv: LevelDB | undefined;
+      if (storagePath) {
+        kv = createTestLevel(storagePath);
+        yield* Effect.promise(() => kv!.open());
+        // const keyCount = yield* Effect.promise(async () => (await kv!.iterator({ values: false }).all()).length);
+        // log.info('opened test db', { storagePath, keyCount });
+      }
+      const peer = yield* Effect.promise(() => builder.createPeer({ indexing, types, kv }));
+
+      let db: EchoDatabaseImpl | undefined;
+      let queues: QueueFactory | undefined;
+
+      if (storagePath) {
+        const testMetadata = yield* Effect.promise(async () => {
+          try {
+            return await kv!.get('test-metadata', { valueEncoding: 'json' });
+          } catch (e) {
+            if ((e as any).code === 'LEVEL_NOT_FOUND') {
+              return undefined;
+            }
+            throw e;
+          }
+        });
+        log('starting persistant test db', { storagePath, testMetadata });
+        if (!testMetadata) {
+          const key = PublicKey.random();
+          db = yield* Effect.promise(() => peer.createDatabase(key));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+
+          yield* Effect.promise(() =>
+            kv!.put('test-metadata', { key: key.toHex(), rootUrl: db!.rootUrl }, { valueEncoding: 'json' }),
+          );
+
+          if (onInit) {
+            yield* onInit().pipe(
+              Effect.provideService(DatabaseService, DatabaseService.make(db)),
+              Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+            );
+          }
+        } else {
+          const key = PublicKey.from((testMetadata as any).key);
+          const rootUrl = (testMetadata as any).rootUrl;
+          db = yield* Effect.promise(() => peer.openDatabase(key, rootUrl));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+        }
+      } else {
+        db = yield* Effect.promise(() => peer.createDatabase());
+        queues = peer.client.constructQueueFactory(db.spaceId);
+        if (onInit) {
+          yield* onInit().pipe(
+            Effect.provideService(DatabaseService, DatabaseService.make(db)),
+            Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+          );
+        }
+      }
+
+      yield* Effect.addFinalizer(() =>
+        Effect.promise(async () => {
+          if (kv) {
+            // {
+            //   const keyCount = (await kv.iterator({ values: false }).all()).length;
+            //   log.info('closing persistant test db', { storagePath, keyCount });
+            // }
+
+            await kv.close();
+          }
+        }),
+      );
+
       return Context.mergeAll(
         Context.make(DatabaseService, DatabaseService.make(db)),
         Context.make(QueueService, QueueService.make(queues, undefined)),

package/src/testing/logger.ts

@@ -2,15 +2,15 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Effect, type Context } from 'effect';
+import { type Context, Effect } from 'effect';
 
 import { LogLevel } from '@dxos/log';
 
-import { type EventLogger, createEventLogger } from '../services';
+import { type ComputeEventLogger, createEventLogger } from '../services';
 
-export const noopLogger: Context.Tag.Service<EventLogger> = {
+export const noopLogger: Context.Tag.Service<ComputeEventLogger> = {
   log: () => Effect.succeed(undefined),
   nodeId: undefined,
 };
 
-export const consoleLogger: Context.Tag.Service<EventLogger> = createEventLogger(LogLevel.INFO);
+export const consoleLogger: Context.Tag.Service<ComputeEventLogger> = createEventLogger(LogLevel.INFO);

package/src/testing/persist-database.test.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, it } from '@effect/vitest';
+import { Effect } from 'effect';
+
+import { Filter, Obj, Query, Type } from '@dxos/echo';
+import { DataType } from '@dxos/schema';
+
+import { DatabaseService } from '../services';
+
+import { TestDatabaseLayer, testStoragePath } from './layer';
+
+describe('TestDatabaseLayer', { timeout: 600_000 }, () => {
+  it.effect(
+    'persist database to disk',
+    Effect.fnUntraced(function* ({ expect: _ }) {
+      const DbLayer = TestDatabaseLayer({
+        storagePath: testStoragePath({ name: `feed-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        yield* DatabaseService.add(Obj.make(Type.Expando, { label: 'test' }));
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.everything()));
+        expect(objects[0]?.label).toEqual('test');
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect(
+    'reload database -- save index before restart',
+    Effect.fnUntraced(function* ({ expect: _ }) {
+      const NUM_OBJECTS = 500;
+      const DbLayer = TestDatabaseLayer({
+        types: [DataType.Person],
+        storagePath: testStoragePath({ name: `reload-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+        expect(objects.length).toEqual(NUM_OBJECTS);
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect.skip(
+    'reload database -- save index before restart [manual]',
+    Effect.fnUntraced(
+      function* ({ expect: _ }) {
+        const NUM_OBJECTS = 500;
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+      },
+      Effect.provide(
+        TestDatabaseLayer({
+          types: [DataType.Person],
+          storagePath: testStoragePath({ name: `reload-test` }),
+        }),
+      ),
+    ),
+  );
+});

package/src/testing/services.ts

@@ -9,15 +9,16 @@ import type { EchoDatabase, QueueFactory } from '@dxos/echo-db';
 import { assertArgument } from '@dxos/invariant';
 
 import {
+  type ComputeEventLogger,
   ConfiguredCredentialsService,
   type CredentialsService,
   DatabaseService,
-  type EventLogger,
   QueueService,
   ServiceContainer,
   type ServiceCredential,
   type TracingService,
 } from '../services';
+
 import { consoleLogger, noopLogger } from './logger';
 
 // TODO(burdon): Factor out.
@@ -64,7 +65,7 @@ export type TestServiceOptions = {
    */
   logging?: {
     enabled?: boolean;
-    logger?: Context.Tag.Service<EventLogger>;
+    logger?: Context.Tag.Service<ComputeEventLogger>;
   };
 
   /**
@@ -89,7 +90,7 @@ export const createTestServices = ({
   space,
   tracing,
 }: TestServiceOptions = {}): ServiceContainer => {
-  assertArgument(!(!!space && (!!db || !!queues)), 'space can be provided only if db and queues are not');
+  assertArgument(!(!!space && (!!db || !!queues)), 'space', 'space can be provided only if db and queues are not');
 
   return new ServiceContainer().setServices({
     // ai: createAiService(ai),

package/src/trace.ts

@@ -4,12 +4,12 @@
 
 import { Schema } from 'effect';
 
-import { Type, type Ref } from '@dxos/echo';
+import { type Ref, Type } from '@dxos/echo';
 import { Queue } from '@dxos/echo-db';
 import { ObjectId } from '@dxos/echo-schema';
 import { log } from '@dxos/log';
 
-import { FunctionTrigger, type FunctionTriggerType } from './types';
+import { FunctionTrigger } from './types';
 
 export enum InvocationOutcome {
   SUCCESS = 'success',
@@ -24,7 +24,7 @@ export enum InvocationTraceEventType {
 }
 
 export const TraceEventException = Schema.Struct({
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   message: Schema.String,
   name: Schema.String,
   stack: Schema.optional(Schema.String),
@@ -44,7 +44,7 @@ export const InvocationTraceStartEvent = Schema.Struct({
   /**
    * Event generation time.
    */
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   /**
    * Data passed to function / workflow as an argument.
    */
@@ -53,11 +53,11 @@ export const InvocationTraceStartEvent = Schema.Struct({
   /**
    * Queue  for function/workflow invocation events.
    */
-  invocationTraceQueue: Type.Ref(Queue),
+  invocationTraceQueue: Schema.optional(Type.Ref(Queue)),
   /**
    * DXN of the invoked function/workflow.
    */
-  invocationTarget: Type.Ref(Type.Expando),
+  invocationTarget: Schema.optional(Type.Ref(Type.Expando)),
   /**
    * Present for automatic invocations.
    */
@@ -80,7 +80,7 @@ export const InvocationTraceEndEvent = Schema.Struct({
    * Event generation time.
    */
   // TODO(burdon): Remove ms suffix.
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   outcome: Schema.Enums(InvocationOutcome),
   exception: Schema.optional(TraceEventException),
 }).pipe(Type.Obj({ typename: 'dxos.org/type/InvocationTraceEnd', version: '0.1.0' }));
@@ -90,7 +90,7 @@ export type InvocationTraceEndEvent = Schema.Schema.Type<typeof InvocationTraceE
 export type InvocationTraceEvent = InvocationTraceStartEvent | InvocationTraceEndEvent;
 
 export const TraceEventLog = Schema.Struct({
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   level: Schema.String,
   message: Schema.String,
   context: Schema.optional(Schema.Object),
@@ -101,10 +101,8 @@ export const TraceEvent = Schema.Struct({
   // TODO(burdon): Need enum/numeric result (not string).
   outcome: Schema.String,
   truncated: Schema.Boolean,
-  /**
-   * Time when the event was persisted.
-   */
-  ingestionTimestampMs: Schema.Number,
+  /** Time when the event was persisted. */
+  ingestionTimestamp: Schema.Number,
   logs: Schema.Array(TraceEventLog),
   exceptions: Schema.Array(TraceEventException),
 }).pipe(Type.Obj({ typename: 'dxos.org/type/TraceEvent', version: '0.1.0' }));
@@ -117,13 +115,13 @@ export type TraceEvent = Schema.Schema.Type<typeof TraceEvent>;
  */
 export type InvocationSpan = {
   id: string;
-  timestampMs: number;
+  timestamp: number;
+  duration: number;
   outcome: InvocationOutcome;
   input: object;
-  durationMs: number;
-  invocationTraceQueue: Ref.Ref<Queue>;
-  invocationTarget: Ref.Ref<Type.Expando>;
-  trigger?: Ref.Ref<FunctionTriggerType>;
+  invocationTraceQueue?: Ref.Ref<Queue>;
+  invocationTarget?: Ref.Ref<Type.Expando>;
+  trigger?: Ref.Ref<FunctionTrigger>;
   exception?: TraceEventException;
 };
 
@@ -165,8 +163,8 @@ export const createInvocationSpans = (items?: InvocationTraceEvent[]): Invocatio
 
     result.push({
       id: invocationId,
-      timestampMs: start.timestampMs,
-      durationMs: isInProgress ? now - start.timestampMs : end!.timestampMs - start.timestampMs,
+      timestamp: start.timestamp,
+      duration: isInProgress ? now - start.timestamp : end!.timestamp - start.timestamp,
       outcome: end?.outcome ?? InvocationOutcome.PENDING,
       exception: end?.exception,
       input: start.input,

package/src/triggers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './invocation-tracer';
+export * from './trigger-dispatcher';
+export * from './trigger-state-store';

package/src/triggers/input-builder.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type EventType, type FunctionTrigger } from '@dxos/functions';
+
+export const createInvocationPayload = (trigger: FunctionTrigger, event: EventType): any => {
+  if (!trigger.input) {
+    return event;
+  }
+
+  const payload: any = {};
+  for (const [key, value] of Object.entries(trigger.input)) {
+    if (typeof value !== 'string' || !(value.startsWith('{{') && value.endsWith('}}'))) {
+      payload[key] = value;
+      continue;
+    }
+
+    const propertyPath = value.slice(2, -2);
+    let valueSubstitution: any = propertyPath.startsWith('trigger.')
+      ? trigger
+      : propertyPath.startsWith('event.')
+        ? event
+        : undefined;
+
+    for (const pathSegment of propertyPath.split('.').slice(1)) {
+      if (valueSubstitution && typeof valueSubstitution === 'object') {
+        valueSubstitution = valueSubstitution[pathSegment];
+      }
+    }
+
+    payload[key] = valueSubstitution;
+  }
+  return payload;
+};