@dxos/functions 0.8.3 → 0.8.4-main.1da679c

package/src/testing/layer.ts

@@ -0,0 +1,111 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import type { Schema } from 'effect';
+import { Context, Effect, Layer } from 'effect';
+
+import type { EchoDatabaseImpl, QueueFactory } from '@dxos/echo-db';
+import { EchoTestBuilder } from '@dxos/echo-db/testing';
+import type { EchoHostIndexingConfig } from '@dxos/echo-pipeline';
+import { accuireReleaseResource } from '@dxos/effect';
+import { PublicKey } from '@dxos/keys';
+import type { LevelDB } from '@dxos/kv-store';
+import { createTestLevel } from '@dxos/kv-store/testing';
+import { log } from '@dxos/log';
+
+import { DatabaseService, QueueService } from '../services';
+
+const testBuilder = accuireReleaseResource(() => new EchoTestBuilder());
+
+export const testStoragePath = ({ name = PublicKey.random().toHex() }: { name?: string }) => {
+  return `/tmp/dxos-${name}`;
+};
+
+export type TestDatabaseOptions = {
+  indexing?: Partial<EchoHostIndexingConfig>;
+  types?: Schema.Schema.AnyNoContext[];
+  storagePath?: string;
+  onInit?: () => Effect.Effect<void, never, DatabaseService | QueueService>;
+};
+
+export const TestDatabaseLayer = ({ indexing, types, storagePath, onInit }: TestDatabaseOptions = {}) =>
+  Layer.scopedContext(
+    Effect.gen(function* () {
+      const builder = yield* testBuilder;
+
+      let kv: LevelDB | undefined;
+      if (storagePath) {
+        kv = createTestLevel(storagePath);
+        yield* Effect.promise(() => kv!.open());
+        // const keyCount = yield* Effect.promise(async () => (await kv!.iterator({ values: false }).all()).length);
+        // log.info('opened test db', { storagePath, keyCount });
+      }
+      const peer = yield* Effect.promise(() => builder.createPeer({ indexing, types, kv }));
+
+      let db: EchoDatabaseImpl | undefined;
+      let queues: QueueFactory | undefined;
+
+      if (storagePath) {
+        const testMetadata = yield* Effect.promise(async () => {
+          try {
+            return await kv!.get('test-metadata', { valueEncoding: 'json' });
+          } catch (e) {
+            if ((e as any).code === 'LEVEL_NOT_FOUND') {
+              return undefined;
+            }
+            throw e;
+          }
+        });
+        log('starting persistant test db', { storagePath, testMetadata });
+        if (!testMetadata) {
+          const key = PublicKey.random();
+          db = yield* Effect.promise(() => peer.createDatabase(key));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+
+          yield* Effect.promise(() =>
+            kv!.put('test-metadata', { key: key.toHex(), rootUrl: db!.rootUrl }, { valueEncoding: 'json' }),
+          );
+
+          if (onInit) {
+            yield* onInit().pipe(
+              Effect.provideService(DatabaseService, DatabaseService.make(db)),
+              Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+            );
+          }
+        } else {
+          const key = PublicKey.from((testMetadata as any).key);
+          const rootUrl = (testMetadata as any).rootUrl;
+          db = yield* Effect.promise(() => peer.openDatabase(key, rootUrl));
+          queues = peer.client.constructQueueFactory(db.spaceId);
+        }
+      } else {
+        db = yield* Effect.promise(() => peer.createDatabase());
+        queues = peer.client.constructQueueFactory(db.spaceId);
+        if (onInit) {
+          yield* onInit().pipe(
+            Effect.provideService(DatabaseService, DatabaseService.make(db)),
+            Effect.provideService(QueueService, QueueService.make(queues, undefined)),
+          );
+        }
+      }
+
+      yield* Effect.addFinalizer(() =>
+        Effect.promise(async () => {
+          if (kv) {
+            // {
+            //   const keyCount = (await kv.iterator({ values: false }).all()).length;
+            //   log.info('closing persistant test db', { storagePath, keyCount });
+            // }
+
+            await kv.close();
+          }
+        }),
+      );
+
+      return Context.mergeAll(
+        Context.make(DatabaseService, DatabaseService.make(db)),
+        Context.make(QueueService, QueueService.make(queues, undefined)),
+      );
+    }),
+  );

package/src/testing/logger.ts

@@ -2,15 +2,15 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Effect, type Context } from 'effect';
+import { type Context, Effect } from 'effect';
 
 import { LogLevel } from '@dxos/log';
 
-import { type EventLogger, createEventLogger } from '../services';
+import { type ComputeEventLogger, createEventLogger } from '../services';
 
-export const noopLogger: Context.Tag.Service<EventLogger> = {
+export const noopLogger: Context.Tag.Service<ComputeEventLogger> = {
   log: () => Effect.succeed(undefined),
   nodeId: undefined,
 };
 
-export const consoleLogger: Context.Tag.Service<EventLogger> = createEventLogger(LogLevel.INFO);
+export const consoleLogger: Context.Tag.Service<ComputeEventLogger> = createEventLogger(LogLevel.INFO);

package/src/testing/persist-database.test.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, it } from '@effect/vitest';
+import { Effect } from 'effect';
+
+import { Filter, Obj, Query, Type } from '@dxos/echo';
+import { DataType } from '@dxos/schema';
+
+import { DatabaseService } from '../services';
+
+import { TestDatabaseLayer, testStoragePath } from './layer';
+
+describe('TestDatabaseLayer', { timeout: 600_000 }, () => {
+  it.effect(
+    'persist database to disk',
+    Effect.fnUntraced(function* ({ expect: _ }) {
+      const DbLayer = TestDatabaseLayer({
+        storagePath: testStoragePath({ name: `feed-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        yield* DatabaseService.add(Obj.make(Type.Expando, { label: 'test' }));
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.everything()));
+        expect(objects[0]?.label).toEqual('test');
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect(
+    'reload database -- save index before restart',
+    Effect.fnUntraced(function* ({ expect: _ }) {
+      const NUM_OBJECTS = 500;
+      const DbLayer = TestDatabaseLayer({
+        types: [DataType.Person],
+        storagePath: testStoragePath({ name: `reload-test-${Date.now()}` }),
+      });
+
+      yield* Effect.gen(function* () {
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+      }).pipe(Effect.provide(DbLayer));
+
+      yield* Effect.gen(function* () {
+        const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+        expect(objects.length).toEqual(NUM_OBJECTS);
+      }).pipe(Effect.provide(DbLayer));
+    }),
+  );
+
+  it.effect.skip(
+    'reload database -- save index before restart [manual]',
+    Effect.fnUntraced(
+      function* ({ expect: _ }) {
+        const NUM_OBJECTS = 500;
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+
+        for (let i = 0; i < NUM_OBJECTS; i++) {
+          yield* DatabaseService.add(Obj.make(DataType.Person, { nickname: `Person ${i}` }));
+        }
+        yield* DatabaseService.flush({ indexes: true });
+
+        {
+          const { objects } = yield* DatabaseService.runQuery(Query.select(Filter.type(DataType.Person)));
+          console.log({ count: objects.length });
+        }
+      },
+      Effect.provide(
+        TestDatabaseLayer({
+          types: [DataType.Person],
+          storagePath: testStoragePath({ name: `reload-test` }),
+        }),
+      ),
+    ),
+  );
+});

package/src/testing/services.ts

@@ -4,29 +4,112 @@
 
 import { type Context } from 'effect';
 
-import type { AiServiceClient } from '@dxos/ai';
-import type { QueueFactory } from '@dxos/echo-db';
+import type { Space } from '@dxos/client/echo';
+import type { EchoDatabase, QueueFactory } from '@dxos/echo-db';
+import { assertArgument } from '@dxos/invariant';
+
+import {
+  type ComputeEventLogger,
+  ConfiguredCredentialsService,
+  type CredentialsService,
+  DatabaseService,
+  QueueService,
+  ServiceContainer,
+  type ServiceCredential,
+  type TracingService,
+} from '../services';
 
 import { consoleLogger, noopLogger } from './logger';
-import { AiService, QueueService, ServiceContainer } from '../services';
-import type { EventLogger } from '../services/event-logger';
+
+// TODO(burdon): Factor out.
+export type OneOf<T> = {
+  [K in keyof T]: { [P in K]: T[P] } & { [P in Exclude<keyof T, K>]?: never };
+}[keyof T];
+
+export type AiServiceProvider = 'dev' | 'edge' | 'ollama' | 'lmstudio';
 
 export type TestServiceOptions = {
-  ai?: AiServiceClient;
-  queueFactory?: QueueFactory;
-  enableLogging?: boolean;
-  logger?: Context.Tag.Service<EventLogger>;
+  /**
+   * AI service configuration.
+   */
+  ai?: any;
+
+  /**
+   * Credentials service configuration.
+   */
+  credentials?: OneOf<{
+    /**
+     * Predefined credentials list.
+     */
+    services?: ServiceCredential[];
+
+    /**
+     * Custom credentials service.
+     */
+    service?: Context.Tag.Service<CredentialsService>;
+  }>;
+
+  /**
+   * Database configuration.
+   */
+  db?: EchoDatabase;
+
+  /**
+   * Gets database and queue services from the space.
+   * Exclusive with: `db`, `queues`
+   */
+  space?: Space;
+
+  /**
+   * Logging configuration.
+   */
+  logging?: {
+    enabled?: boolean;
+    logger?: Context.Tag.Service<ComputeEventLogger>;
+  };
+
+  /**
+   * Queue service configuration.
+   */
+  queues?: QueueFactory;
+
+  tracing?: {
+    service?: Context.Tag.Service<TracingService>;
+  };
 };
 
+/**
+ * @deprecated
+ */
 export const createTestServices = ({
   ai,
-  queueFactory,
-  enableLogging = false,
-  logger = enableLogging ? consoleLogger : noopLogger,
+  credentials,
+  db,
+  logging,
+  queues,
+  space,
+  tracing,
 }: TestServiceOptions = {}): ServiceContainer => {
+  assertArgument(!(!!space && (!!db || !!queues)), 'space', 'space can be provided only if db and queues are not');
+
   return new ServiceContainer().setServices({
-    ai: ai != null ? AiService.make(ai) : undefined,
-    eventLogger: logger,
-    queues: queueFactory != null ? QueueService.make(queueFactory, undefined) : undefined,
+    // ai: createAiService(ai),
+    credentials: createCredentialsService(credentials),
+    database: space || db ? DatabaseService.make(space?.db || db!) : undefined,
+    eventLogger: (logging?.logger ?? logging?.enabled) ? consoleLogger : noopLogger,
+    queues: space || queues ? QueueService.make(space?.queues || queues!, undefined) : undefined,
+    tracing: tracing?.service,
   });
 };
+
+const createCredentialsService = (
+  credentials: TestServiceOptions['credentials'] | undefined,
+): Context.Tag.Service<CredentialsService> | undefined => {
+  if (credentials?.services) {
+    return new ConfiguredCredentialsService(credentials.services);
+  }
+
+  if (credentials?.service) {
+    return credentials.service;
+  }
+};

package/src/trace.ts

@@ -4,12 +4,12 @@
 
 import { Schema } from 'effect';
 
-import { Type, type Ref } from '@dxos/echo';
+import { type Ref, Type } from '@dxos/echo';
 import { Queue } from '@dxos/echo-db';
 import { ObjectId } from '@dxos/echo-schema';
 import { log } from '@dxos/log';
 
-import { FunctionTrigger, type FunctionTriggerType } from './types';
+import { FunctionTrigger } from './types';
 
 export enum InvocationOutcome {
   SUCCESS = 'success',
@@ -24,7 +24,7 @@ export enum InvocationTraceEventType {
 }
 
 export const TraceEventException = Schema.Struct({
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   message: Schema.String,
   name: Schema.String,
   stack: Schema.optional(Schema.String),
@@ -44,7 +44,7 @@ export const InvocationTraceStartEvent = Schema.Struct({
   /**
    * Event generation time.
    */
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   /**
    * Data passed to function / workflow as an argument.
    */
@@ -53,11 +53,11 @@ export const InvocationTraceStartEvent = Schema.Struct({
   /**
    * Queue  for function/workflow invocation events.
    */
-  invocationTraceQueue: Type.Ref(Queue),
+  invocationTraceQueue: Schema.optional(Type.Ref(Queue)),
   /**
    * DXN of the invoked function/workflow.
    */
-  invocationTarget: Type.Ref(Type.Expando),
+  invocationTarget: Schema.optional(Type.Ref(Type.Expando)),
   /**
    * Present for automatic invocations.
    */
@@ -80,7 +80,7 @@ export const InvocationTraceEndEvent = Schema.Struct({
    * Event generation time.
    */
   // TODO(burdon): Remove ms suffix.
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   outcome: Schema.Enums(InvocationOutcome),
   exception: Schema.optional(TraceEventException),
 }).pipe(Type.Obj({ typename: 'dxos.org/type/InvocationTraceEnd', version: '0.1.0' }));
@@ -90,7 +90,7 @@ export type InvocationTraceEndEvent = Schema.Schema.Type<typeof InvocationTraceE
 export type InvocationTraceEvent = InvocationTraceStartEvent | InvocationTraceEndEvent;
 
 export const TraceEventLog = Schema.Struct({
-  timestampMs: Schema.Number,
+  timestamp: Schema.Number,
   level: Schema.String,
   message: Schema.String,
   context: Schema.optional(Schema.Object),
@@ -101,10 +101,8 @@ export const TraceEvent = Schema.Struct({
   // TODO(burdon): Need enum/numeric result (not string).
   outcome: Schema.String,
   truncated: Schema.Boolean,
-  /**
-   * Time when the event was persisted.
-   */
-  ingestionTimestampMs: Schema.Number,
+  /** Time when the event was persisted. */
+  ingestionTimestamp: Schema.Number,
   logs: Schema.Array(TraceEventLog),
   exceptions: Schema.Array(TraceEventException),
 }).pipe(Type.Obj({ typename: 'dxos.org/type/TraceEvent', version: '0.1.0' }));
@@ -117,13 +115,13 @@ export type TraceEvent = Schema.Schema.Type<typeof TraceEvent>;
  */
 export type InvocationSpan = {
   id: string;
-  timestampMs: number;
+  timestamp: number;
+  duration: number;
   outcome: InvocationOutcome;
   input: object;
-  durationMs: number;
-  invocationTraceQueue: Ref.Ref<Queue>;
-  invocationTarget: Ref.Ref<Type.Expando>;
-  trigger?: Ref.Ref<FunctionTriggerType>;
+  invocationTraceQueue?: Ref.Ref<Queue>;
+  invocationTarget?: Ref.Ref<Type.Expando>;
+  trigger?: Ref.Ref<FunctionTrigger>;
   exception?: TraceEventException;
 };
 
@@ -165,8 +163,8 @@ export const createInvocationSpans = (items?: InvocationTraceEvent[]): Invocatio
 
     result.push({
       id: invocationId,
-      timestampMs: start.timestampMs,
-      durationMs: isInProgress ? now - start.timestampMs : end!.timestampMs - start.timestampMs,
+      timestamp: start.timestamp,
+      duration: isInProgress ? now - start.timestamp : end!.timestamp - start.timestamp,
       outcome: end?.outcome ?? InvocationOutcome.PENDING,
       exception: end?.exception,
       input: start.input,

package/src/translations.ts

@@ -2,14 +2,14 @@
 // Copyright 2023 DXOS.org
 //
 
-import { Type } from '@dxos/echo';
+import { type Resource } from 'i18next';
 
 import { ScriptType } from './schema';
 
-export default [
+export const translations = [
   {
     'en-US': {
-      [Type.getTypename(ScriptType)]: {
+      [ScriptType.typename]: {
         'typename label': 'Script',
         'typename label_zero': 'Scripts',
         'typename label_one': 'Script',
@@ -17,4 +17,4 @@ export default [
       },
     },
   },
-];
+] as const satisfies Resource[];

package/src/triggers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './invocation-tracer';
+export * from './trigger-dispatcher';
+export * from './trigger-state-store';

package/src/triggers/input-builder.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type EventType, type FunctionTrigger } from '@dxos/functions';
+
+export const createInvocationPayload = (trigger: FunctionTrigger, event: EventType): any => {
+  if (!trigger.input) {
+    return event;
+  }
+
+  const payload: any = {};
+  for (const [key, value] of Object.entries(trigger.input)) {
+    if (typeof value !== 'string' || !(value.startsWith('{{') && value.endsWith('}}'))) {
+      payload[key] = value;
+      continue;
+    }
+
+    const propertyPath = value.slice(2, -2);
+    let valueSubstitution: any = propertyPath.startsWith('trigger.')
+      ? trigger
+      : propertyPath.startsWith('event.')
+        ? event
+        : undefined;
+
+    for (const pathSegment of propertyPath.split('.').slice(1)) {
+      if (valueSubstitution && typeof valueSubstitution === 'object') {
+        valueSubstitution = valueSubstitution[pathSegment];
+      }
+    }
+
+    payload[key] = valueSubstitution;
+  }
+  return payload;
+};

package/src/triggers/invocation-tracer.ts

@@ -0,0 +1,99 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Context, Effect, Layer } from 'effect';
+
+import { Obj, Ref } from '@dxos/echo';
+import { type Queue } from '@dxos/echo-db';
+import { DXN, ObjectId } from '@dxos/keys';
+
+import { QueueService } from '../services';
+import {
+  InvocationOutcome,
+  InvocationTraceEndEvent,
+  InvocationTraceEventType,
+  InvocationTraceStartEvent,
+} from '../trace';
+import type { TriggerKind } from '../types';
+
+export type FunctionInvocationPayload = {
+  data?: any;
+  inputNodeId?: string;
+  trigger?: {
+    id: string;
+    kind: TriggerKind;
+  };
+};
+
+export type TraceData = {
+  invocationId: ObjectId;
+  invocationTraceQueue: Queue;
+};
+
+export class InvocationTracer extends Context.Tag('@dxos/functions/InvocationTracer')<
+  InvocationTracer,
+  {
+    traceInvocationStart({
+      payload,
+      target,
+    }: {
+      payload: FunctionInvocationPayload;
+      target?: DXN;
+    }): Effect.Effect<TraceData, never, QueueService>;
+
+    traceInvocationEnd({ trace, exception }: { trace: TraceData; exception?: any }): Effect.Effect<void>;
+  }
+>() {
+  static layerLive = (opts: { invocationTraceQueue: Queue }) =>
+    Layer.effect(
+      InvocationTracer,
+      Effect.gen(function* () {
+        return {
+          traceInvocationStart: Effect.fn('traceInvocationStart')(function* ({ payload, target }) {
+            const invocationId = ObjectId.random();
+            const invocationTraceQueue = yield* QueueService.createQueue({ subspaceTag: 'trace' });
+            const now = Date.now();
+            const traceEvent = Obj.make(InvocationTraceStartEvent, {
+              type: InvocationTraceEventType.START,
+              invocationId,
+              timestamp: now,
+              // TODO(dmaretskyi): Not json-stringifying this makes ECHO fail when one ECHO object becomes embedded in another ECHO object.
+              input: JSON.parse(JSON.stringify(payload.data ?? {})),
+              invocationTraceQueue: Ref.fromDXN(invocationTraceQueue.dxn),
+              invocationTarget: target ? Ref.fromDXN(target) : undefined,
+              trigger: payload.trigger ? Ref.fromDXN(DXN.fromLocalObjectId(payload.trigger.id)) : undefined,
+            });
+            yield* QueueService.append(opts.invocationTraceQueue, [traceEvent]);
+
+            return { invocationId, invocationTraceQueue };
+          }),
+          traceInvocationEnd: Effect.fn('traceInvocationEnd')(function* ({ trace, exception }) {
+            const now = Date.now();
+            const traceEvent = Obj.make(InvocationTraceEndEvent, {
+              type: InvocationTraceEventType.END,
+              invocationId: trace.invocationId,
+              timestamp: now,
+              outcome: exception ? InvocationOutcome.FAILURE : InvocationOutcome.SUCCESS,
+              exception: exception
+                ? {
+                    name: exception.constructor.name,
+                    timestamp: now,
+                    message: exception?.message ?? 'Unknown error',
+                    stack: exception?.stack,
+                  }
+                : undefined,
+            });
+            yield* QueueService.append(opts.invocationTraceQueue, [traceEvent]);
+          }),
+        };
+      }),
+    );
+
+  static layerTest = Layer.unwrapEffect(
+    Effect.gen(function* () {
+      const queue = yield* QueueService.createQueue({ subspaceTag: 'trace' });
+      return InvocationTracer.layerLive({ invocationTraceQueue: queue });
+    }),
+  );
+}