@dxos/functions 0.8.4-main.b97322e → 0.8.4-main.c4373fc

package/src/services/tracing.ts

@@ -2,41 +2,142 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect } from 'effect';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
 
 import { AgentStatus } from '@dxos/ai';
 import { Obj } from '@dxos/echo';
-import type { AnyEchoObject } from '@dxos/echo-schema';
+import type { ObjectId } from '@dxos/echo/internal';
+import type { Queue } from '@dxos/echo-db';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
 
 /**
  * Provides a way for compute primitives (functions, workflows, tools)
  * to emit an execution trace as a series of structured ECHO objects.
  */
-export class TracingService extends Context.Tag('TracingService')<
+export class TracingService extends Context.Tag('@dxos/functions/TracingService')<
   TracingService,
   {
+    /**
+     * Gets the parent message ID.
+     */
+    getTraceContext: () => TracingService.TraceContext;
+
     /**
      * Write an event to the tracing queue.
      * @param event - The event to write. Must be an a typed object.
      */
-    write(event: AnyEchoObject): void;
+    write: (event: Obj.Any) => void;
   }
 >() {
-  static noop: Context.Tag.Service<TracingService> = { write: () => {} };
+  static noop: Context.Tag.Service<TracingService> = { write: () => {}, getTraceContext: () => ({}) };
+
+  static layerNoop = Layer.succeed(TracingService, TracingService.noop);
 
   static console: Context.Tag.Service<TracingService> = {
     write: (event) => {
-      // eslint-disable-next-line no-console
       console.log(event);
     },
+    getTraceContext: () => ({}),
   };
 
+  static layerConsole = Layer.succeed(TracingService, TracingService.console);
+
+  static layerLogInfo = () =>
+    Layer.succeed(TracingService, {
+      write: (event) => {
+        if (Obj.instanceOf(AgentStatus, event)) {
+          log.info('status', { message: event.message });
+        }
+      },
+      getTraceContext: () => ({}),
+    });
+
+  /**
+   * Creates a TracingService layer that emits events to the parent tracing service.
+   */
+  static layerSubframe = (mapContext: (currentContext: TracingService.TraceContext) => TracingService.TraceContext) =>
+    Layer.effect(
+      TracingService,
+      Effect.gen(function* () {
+        const tracing = yield* TracingService;
+        const context = mapContext(tracing.getTraceContext());
+        return {
+          write: (event) => tracing.write(event),
+          getTraceContext: () => context,
+        };
+      }),
+    );
+
+  static layerQueue = (queue: Queue) =>
+    Layer.effect(
+      TracingService,
+      Effect.gen(function* () {
+        // TODO(dmaretskyi): Batching.
+        return {
+          write: (event) => queue.append([event]),
+          getTraceContext: () => ({
+            debugInfo: {
+              queue: queue.dxn.toString(),
+            },
+          }),
+        };
+      }),
+    );
+
   /**
    * Emit the current human-readable execution status.
    */
-  static emitStatus: (data: Obj.MakeProps<typeof AgentStatus>) => Effect.Effect<unknown, never, void> =
-    Effect.fnUntraced(function* (data) {
-      const tracing = yield* TracingService;
-      tracing.write(Obj.make(AgentStatus, data));
-    });
+  static emitStatus: (
+    data: Omit<Obj.MakeProps<typeof AgentStatus>, 'created'>,
+  ) => Effect.Effect<void, never, TracingService> = Effect.fnUntraced(function* (data) {
+    const tracing = yield* TracingService;
+    tracing.write(
+      Obj.make(AgentStatus, {
+        parentMessage: tracing.getTraceContext().parentMessage,
+        toolCallId: tracing.getTraceContext().toolCallId,
+        created: new Date().toISOString(),
+        ...data,
+      }),
+    );
+  });
+
+  static emitConverationMessage: (
+    data: Obj.MakeProps<typeof DataType.Message>,
+  ) => Effect.Effect<void, never, TracingService> = Effect.fnUntraced(function* (data) {
+    const tracing = yield* TracingService;
+    tracing.write(
+      Obj.make(DataType.Message, {
+        parentMessage: tracing.getTraceContext().parentMessage,
+        ...data,
+        properties: {
+          [MESSAGE_PROPERTY_TOOL_CALL_ID]: tracing.getTraceContext().toolCallId,
+          ...data.properties,
+        },
+      }),
+    );
+  });
 }
+
+export namespace TracingService {
+  export interface TraceContext {
+    /**
+     * If this thread sprung from a tool call, this is the ID of the message containing the tool call.
+     */
+    parentMessage?: ObjectId;
+
+    /**
+     * If the current thread is a byproduct of a tool call, this is the ID of the tool call.
+     */
+    toolCallId?: string;
+
+    debugInfo?: unknown;
+  }
+}
+
+/**
+ * Goes into {@link DataType.Message['properties']}
+ */
+export const MESSAGE_PROPERTY_TOOL_CALL_ID = 'toolCallId' as const;

package/src/testing/layer.ts

@@ -2,27 +2,110 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect, Layer } from 'effect';
-import type { Schema } from 'effect';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import type * as Schema from 'effect/Schema';
 
+import type { EchoDatabaseImpl, QueueFactory } from '@dxos/echo-db';
 import { EchoTestBuilder } from '@dxos/echo-db/testing';
 import type { EchoHostIndexingConfig } from '@dxos/echo-pipeline';
 import { accuireReleaseResource } from '@dxos/effect';
+import { PublicKey } from '@dxos/keys';
+import type { LevelDB } from '@dxos/kv-store';
+import { createTestLevel } from '@dxos/kv-store/testing';
+import { log } from '@dxos/log';
 
 import { DatabaseService, QueueService } from '../services';
 
 const testBuilder = accuireReleaseResource(() => new EchoTestBuilder());
 
+export const testStoragePath = ({ name = PublicKey.random().toHex() }: { name?: string }) => {
+  return `/tmp/dxos-${name}`;
+};
+
 export type TestDatabaseOptions = {
   indexing?: Partial<EchoHostIndexingConfig>;
   types?: Schema.Schema.AnyNoContext[];
+  spaceKey?: PublicKey;
+  storagePath?: string;
+  onInit?: () => Effect.Effect<void, never, DatabaseService | QueueService>;
 };
 
-export const TestDatabaseLayer = ({ indexing, types }: TestDatabaseOptions = {}) =>
+export const TestDatabaseLayer = ({ indexing, types, spaceKey, storagePath, onInit }: TestDatabaseOptions = {}) =>
   Layer.scopedContext(
     Effect.gen(function* () {
       const builder = yield* testBuilder;
-      const { db, queues } = yield* Effect.promise(() => builder.createDatabase({ indexing, types }));
+
+      let kv: LevelDB | undefined;
+      if (storagePath) {
+        kv = createTestLevel(storagePath);
+        yield* Effect.promise(() => kv!.open());
+        // const keyCount = yield* Effect.promise(async () => (await kv!.iterator({ values: false }).all()).length);
+        // log.info('opened test db', { storagePath, keyCount });
+      }
+      const peer = yield* Effect.promise(() => builder.createPeer({ indexing, types, kv }));
+
+      let db: EchoDatabaseImpl | undefined;
+      let queues: QueueFactory | undefined;
+
+      if (storagePath) {
+        const testMetadata = yield* Effect.promise(async () => {
+          try {
+            return await kv!.get('test-metadata', { valueEncoding: 'json' });
+          } catch (e) {
+            if ((e as any).code === 'LEVEL_NOT_FOUND') {
+              return undefined;
+            }
+            throw e;
+          }
+        });
+        log('starting persistant test db', { storagePath, testMetadata });
+        if (!testMetadata) {
+          const key = spaceKey ?? PublicKey.random();
+          db = yield* Effect.promise(() => peer.createDatabase(key));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+
+          yield* Effect.promise(() =>
+            kv!.put('test-metadata', { key: key.toHex(), rootUrl: db!.rootUrl }, { valueEncoding: 'json' }),
+          );
+
+          if (onInit) {
+            yield* onInit().pipe(
+              Effect.provideService(DatabaseService, DatabaseService.make(db)),
+              Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+            );
+          }
+        } else {
+          const key = PublicKey.from((testMetadata as any).key);
+          const rootUrl = (testMetadata as any).rootUrl;
+          db = yield* Effect.promise(() => peer.openDatabase(key, rootUrl));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+        }
+      } else {
+        db = yield* Effect.promise(() => peer.createDatabase(spaceKey));
+        queues = peer.client.constructQueueFactory(db.spaceId);
+        if (onInit) {
+          yield* onInit().pipe(
+            Effect.provideService(DatabaseService, DatabaseService.make(db)),
+            Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+          );
+        }
+      }
+
+      yield* Effect.addFinalizer(() =>
+        Effect.promise(async () => {
+          if (kv) {
+            // {
+            //   const keyCount = (await kv.iterator({ values: false }).all()).length;
+            //   log.info('closing persistant test db', { storagePath, keyCount });
+            // }
+
+            await kv.close();
+          }
+        }),
+      );
+
       return Context.mergeAll(
         Context.make(DatabaseService, DatabaseService.make(db)),
         Context.make(QueueService, QueueService.make(queues, undefined)),

package/src/testing/logger.ts

@@ -2,15 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Effect, type Context } from 'effect';
+import type * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
 
 import { LogLevel } from '@dxos/log';
 
-import { type EventLogger, createEventLogger } from '../services';
+import { type ComputeEventLogger, createEventLogger } from '../services';
 
-export const noopLogger: Context.Tag.Service<EventLogger> = {
+export const noopLogger: Context.Tag.Service<ComputeEventLogger> = {
   log: () => Effect.succeed(undefined),
   nodeId: undefined,
 };
 
-export const consoleLogger: Context.Tag.Service<EventLogger> = createEventLogger(LogLevel.INFO);
+export const consoleLogger: Context.Tag.Service<ComputeEventLogger> = createEventLogger(LogLevel.INFO);

package/src/testing/persist-database.test.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, it } from '@effect/vitest';
+import * as Effect from 'effect/Effect';
+
+import { Filter, Obj, Query, Type } from '@dxos/echo';
+import { DataType } from '@dxos/schema';
+
+import { DatabaseService } from '../services';
+
+import { TestDatabaseLayer, testStoragePath } from './layer';
+
+describe('TestDatabaseLayer', { timeout: 600_000 }, () => {
+  it.effect(
+    'persist database to disk',
+    Effect.fnUntraced(function* (_) {
+      const DbLayer = TestDatabaseLayer({
+        storagePath: testStoragePath({ name: `feed-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        yield* DatabaseService.add(Obj.make(Type.Expando, { label: 'test' }));
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.everything()));
+        expect(objects[0]?.label).toEqual('test');
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect(
+    'reload database -- save index before restart',
+    Effect.fnUntraced(function* (_) {
+      const NUM_OBJECTS = 500;
+      const DbLayer = TestDatabaseLayer({
+        types: [DataType.Person],
+        storagePath: testStoragePath({ name: `reload-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+        expect(objects.length).toEqual(NUM_OBJECTS);
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect.skip(
+    'reload database -- save index before restart [manual]',
+    Effect.fnUntraced(
+      function* (_) {
+        const NUM_OBJECTS = 500;
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+      },
+      Effect.provide(
+        TestDatabaseLayer({
+          types: [DataType.Person],
+          storagePath: testStoragePath({ name: `reload-test` }),
+        }),
+      ),
+    ),
+  );
+});

package/src/testing/services.ts

@@ -2,26 +2,25 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type Context } from 'effect';
+import type * as Context from 'effect/Context';
 
-import { AiService, type AiServiceClient, type AiServiceEdgeClientOptions, EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT, createTestAiServiceClient } from '@dxos/ai/testing';
 import type { Space } from '@dxos/client/echo';
 import type { EchoDatabase, QueueFactory } from '@dxos/echo-db';
 import { assertArgument } from '@dxos/invariant';
 
-import { consoleLogger, noopLogger } from './logger';
 import {
+  type ComputeEventLogger,
   ConfiguredCredentialsService,
   type CredentialsService,
   DatabaseService,
-  type EventLogger,
   QueueService,
   ServiceContainer,
   type ServiceCredential,
   type TracingService,
 } from '../services';
 
+import { consoleLogger, noopLogger } from './logger';
+
 // TODO(burdon): Factor out.
 export type OneOf<T> = {
   [K in keyof T]: { [P in K]: T[P] } & { [P in Exclude<keyof T, K>]?: never };
@@ -33,23 +32,7 @@ export type TestServiceOptions = {
   /**
    * AI service configuration.
    */
-  ai?: OneOf<{
-    /**
-     * Custom AI service client.
-     */
-    client?: AiServiceClient;
-
-    /**
-     * Edge AI service at specified endpoint.
-     */
-    endpoint?: AiServiceEdgeClientOptions['endpoint'];
-
-    /**
-     * Predefined AI service configuration.
-     */
-    // TODO(burdon): 'dev' and 'edge' are redundant with providing an endpoint.
-    provider?: AiServiceProvider;
-  }>;
+  ai?: any;
 
   /**
    * Credentials service configuration.
@@ -82,7 +65,7 @@ export type TestServiceOptions = {
    */
   logging?: {
     enabled?: boolean;
-    logger?: Context.Tag.Service<EventLogger>;
+    logger?: Context.Tag.Service<ComputeEventLogger>;
   };
 
   /**
@@ -95,6 +78,9 @@ export type TestServiceOptions = {
   };
 };
 
+/**
+ * @deprecated
+ */
 export const createTestServices = ({
   ai,
   credentials,
@@ -104,10 +90,10 @@ export const createTestServices = ({
   space,
   tracing,
 }: TestServiceOptions = {}): ServiceContainer => {
-  assertArgument(!(!!space && (!!db || !!queues)), 'space can be provided only if db and queues are not');
+  assertArgument(!(!!space && (!!db || !!queues)), 'space', 'space can be provided only if db and queues are not');
 
   return new ServiceContainer().setServices({
-    ai: createAiService(ai),
+    // ai: createAiService(ai),
     credentials: createCredentialsService(credentials),
     database: space || db ? DatabaseService.make(space?.db || db!) : undefined,
     eventLogger: (logging?.logger ?? logging?.enabled) ? consoleLogger : noopLogger,
@@ -116,45 +102,6 @@ export const createTestServices = ({
   });
 };
 
-// TODO(burdon): Enable model configuration.
-const createAiService = (ai: TestServiceOptions['ai']): Context.Tag.Service<AiService> | undefined => {
-  if (ai?.client != null) {
-    return AiService.make(ai.client);
-  }
-
-  if (ai?.endpoint != null) {
-    return AiService.make(new EdgeAiServiceClient({ endpoint: ai.endpoint }));
-  }
-
-  switch (ai?.provider) {
-    case 'dev':
-      return AiService.make(
-        new EdgeAiServiceClient({
-          endpoint: AI_SERVICE_ENDPOINT.LOCAL,
-          defaultGenerationOptions: {
-            model: '@anthropic/claude-3-5-sonnet-20241022',
-          },
-        }),
-      );
-
-    case 'edge':
-      return AiService.make(
-        new EdgeAiServiceClient({
-          endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-          defaultGenerationOptions: {
-            model: '@anthropic/claude-3-5-sonnet-20241022',
-          },
-        }),
-      );
-
-    case 'ollama':
-      return AiService.make(createTestAiServiceClient());
-
-    case 'lmstudio':
-      throw new Error('LMStudio is not supported');
-  }
-};
-
 const createCredentialsService = (
   credentials: TestServiceOptions['credentials'] | undefined,
 ): Context.Tag.Service<CredentialsService> | undefined => {

package/src/trace.ts

@@ -2,14 +2,14 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
-import { Type, type Ref } from '@dxos/echo';
+import { type Ref, Type } from '@dxos/echo';
+import { ObjectId } from '@dxos/echo/internal';
 import { Queue } from '@dxos/echo-db';
-import { ObjectId } from '@dxos/echo-schema';
 import { log } from '@dxos/log';
 
-import { FunctionTrigger, type FunctionTriggerType } from './types';
+import { FunctionTrigger } from './types';
 
 export enum InvocationOutcome {
   SUCCESS = 'success',
@@ -24,7 +24,7 @@ export enum InvocationTraceEventType {
 }
 
 export const TraceEventException = Schema.Struct({
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   message: Schema.String,
   name: Schema.String,
   stack: Schema.optional(Schema.String),
@@ -44,7 +44,7 @@ export const InvocationTraceStartEvent = Schema.Struct({
   /**
    * Event generation time.
    */
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   /**
    * Data passed to function / workflow as an argument.
    */
@@ -53,11 +53,11 @@ export const InvocationTraceStartEvent = Schema.Struct({
   /**
    * Queue  for function/workflow invocation events.
    */
-  invocationTraceQueue: Type.Ref(Queue),
+  invocationTraceQueue: Schema.optional(Type.Ref(Queue)),
   /**
    * DXN of the invoked function/workflow.
    */
-  invocationTarget: Type.Ref(Type.Expando),
+  invocationTarget: Schema.optional(Type.Ref(Type.Expando)),
   /**
    * Present for automatic invocations.
    */
@@ -80,7 +80,7 @@ export const InvocationTraceEndEvent = Schema.Struct({
    * Event generation time.
    */
   // TODO(burdon): Remove ms suffix.
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   outcome: Schema.Enums(InvocationOutcome),
   exception: Schema.optional(TraceEventException),
 }).pipe(Type.Obj({ typename: 'dxos.org/type/InvocationTraceEnd', version: '0.1.0' }));
@@ -90,7 +90,7 @@ export type InvocationTraceEndEvent = Schema.Schema.Type<typeof InvocationTraceE
 export type InvocationTraceEvent = InvocationTraceStartEvent | InvocationTraceEndEvent;
 
 export const TraceEventLog = Schema.Struct({
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   level: Schema.String,
   message: Schema.String,
   context: Schema.optional(Schema.Object),
@@ -101,10 +101,8 @@ export const TraceEvent = Schema.Struct({
   // TODO(burdon): Need enum/numeric result (not string).
   outcome: Schema.String,
   truncated: Schema.Boolean,
-  /**
-   * Time when the event was persisted.
-   */
-  ingestionTimestampMs: Schema.Number,
+  /** Time when the event was persisted. */
+  ingestionTimestamp: Schema.Number,
   logs: Schema.Array(TraceEventLog),
   exceptions: Schema.Array(TraceEventException),
 }).pipe(Type.Obj({ typename: 'dxos.org/type/TraceEvent', version: '0.1.0' }));
@@ -117,13 +115,13 @@ export type TraceEvent = Schema.Schema.Type<typeof TraceEvent>;
  */
 export type InvocationSpan = {
   id: string;
-  timestampMs: number;
+  timestamp: number;
+  duration: number;
   outcome: InvocationOutcome;
   input: object;
-  durationMs: number;
-  invocationTraceQueue: Ref.Ref<Queue>;
-  invocationTarget: Ref.Ref<Type.Expando>;
-  trigger?: Ref.Ref<FunctionTriggerType>;
+  invocationTraceQueue?: Ref.Ref<Queue>;
+  invocationTarget?: Ref.Ref<Type.Expando>;
+  trigger?: Ref.Ref<FunctionTrigger>;
   exception?: TraceEventException;
 };
 
@@ -165,8 +163,8 @@ export const createInvocationSpans = (items?: InvocationTraceEvent[]): Invocatio
 
     result.push({
       id: invocationId,
-      timestampMs: start.timestampMs,
-      durationMs: isInProgress ? now - start.timestampMs : end!.timestampMs - start.timestampMs,
+      timestamp: start.timestamp,
+      duration: isInProgress ? now - start.timestamp : end!.timestamp - start.timestamp,
       outcome: end?.outcome ?? InvocationOutcome.PENDING,
       exception: end?.exception,
       input: start.input,

package/src/triggers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './invocation-tracer';
+export * from './trigger-dispatcher';
+export * from './trigger-state-store';

package/src/triggers/input-builder.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type EventType, type FunctionTrigger } from '@dxos/functions';
+
+export const createInvocationPayload = (trigger: FunctionTrigger, event: EventType): any => {
+  if (!trigger.input) {
+    return event;
+  }
+
+  const payload: any = {};
+  for (const [key, value] of Object.entries(trigger.input)) {
+    if (typeof value !== 'string' || !(value.startsWith('{{') && value.endsWith('}}'))) {
+      payload[key] = value;
+      continue;
+    }
+
+    const propertyPath = value.slice(2, -2);
+    let valueSubstitution: any = propertyPath.startsWith('trigger.')
+      ? trigger
+      : propertyPath.startsWith('event.')
+        ? event
+        : undefined;
+
+    for (const pathSegment of propertyPath.split('.').slice(1)) {
+      if (valueSubstitution && typeof valueSubstitution === 'object') {
+        valueSubstitution = valueSubstitution[pathSegment];
+      }
+    }
+
+    payload[key] = valueSubstitution;
+  }
+  return payload;
+};