@dxos/functions 0.8.4-main.fd6878d → 0.8.4-main.fffef41

package/src/sdk.ts

@@ -0,0 +1,226 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import type * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { type AiService } from '@dxos/ai';
+import { Obj, Type } from '@dxos/echo';
+import { type DatabaseService } from '@dxos/echo-db';
+import { assertArgument, failedInvariant } from '@dxos/invariant';
+
+import {
+  type CredentialsService,
+  type FunctionInvocationService,
+  type QueueService,
+  type TracingService,
+} from './services';
+import { Function } from './types';
+import { getUserFunctionIdInMetadata, setUserFunctionIdInMetadata } from './types';
+
+// TODO(burdon): Model after http request. Ref Lambda/OpenFaaS.
+// https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
+// https://www.serverless.com/framework/docs/providers/aws/guide/serverless.yml/#functions
+// https://www.npmjs.com/package/aws-lambda
+
+/**
+ * Services that are provided at the function call site by the caller.
+ */
+export type InvocationServices = TracingService;
+
+/**
+ * Services that are available to invoked functions.
+ */
+export type FunctionServices =
+  | InvocationServices
+  | AiService.AiService
+  | CredentialsService
+  | DatabaseService
+  | QueueService
+  | FunctionInvocationService;
+
+/**
+ * Function handler.
+ */
+export type FunctionHandler<TData = {}, TOutput = any, S extends FunctionServices = FunctionServices> = (params: {
+  /**
+   * Context available to the function.
+   */
+  context: FunctionContext;
+
+  /**
+   * Data passed as the input to the function.
+   * Must match the function's input schema.
+   * This will be the payload from the trigger or other data passed into the function in a workflow.
+   */
+  data: TData;
+}) => TOutput | Promise<TOutput> | Effect.Effect<TOutput, any, S>;
+
+/**
+ * Function context.
+ */
+export interface FunctionContext {
+  // TODO(dmaretskyi): Consider what we should put into context.
+}
+
+const typeId = Symbol.for('@dxos/functions/FunctionDefinition');
+
+export type FunctionDefinition<T = any, O = any, S extends FunctionServices = FunctionServices> = {
+  [typeId]: true;
+  key: string;
+  name: string;
+  description?: string;
+  inputSchema: Schema.Schema<T, any>;
+  outputSchema?: Schema.Schema<O, any>;
+
+  /**
+   * Keys of the required services.
+   */
+  services: readonly string[];
+
+  handler: FunctionHandler<T, O, S>;
+  meta?: {
+    /**
+     * Tools that are projected from functions have this annotation.
+     *
+     * deployedFunctionId:
+     * - Backend deployment ID assigned by the EDGE function service (typically a UUID).
+     * - Used for remote invocation via `FunctionInvocationService` → `RemoteFunctionExecutionService`.
+     * - Persisted on the corresponding ECHO `Function.Function` object's metadata under the
+     *   `FUNCTIONS_META_KEY` and retrieved with `getUserFunctionIdInMetadata`.
+     */
+    deployedFunctionId?: string;
+  };
+};
+
+export declare namespace FunctionDefinition {
+  export type Any = FunctionDefinition<any, any, any>;
+  export type Input<T extends Any> = T extends FunctionDefinition<infer I, infer _O, infer _S> ? I : never;
+  export type Output<T extends Any> = T extends FunctionDefinition<infer _I, infer O, infer _S> ? O : never;
+  export type Services<T extends Any> = T extends FunctionDefinition<infer _I, infer _O, infer S> ? S : never;
+}
+
+export type FunctionProps<T, O> = {
+  key: string;
+  name: string;
+  description?: string;
+  inputSchema: Schema.Schema<T, any>;
+  outputSchema?: Schema.Schema<O, any>;
+  // TODO(dmaretskyi): This currently doesn't cause a compile-time error if the handler requests a service that is not specified
+  services?: readonly Context.Tag<any, any>[];
+
+  handler: FunctionHandler<T, O, FunctionServices>;
+};
+
+// TODO(dmaretskyi): Output type doesn't get typechecked.
+export const defineFunction: {
+  <I, O>(params: FunctionProps<I, O>): FunctionDefinition<I, O, FunctionServices>;
+} = ({ key, name, description, inputSchema, outputSchema = Schema.Any, handler, services }) => {
+  if (!Schema.isSchema(inputSchema)) {
+    throw new Error('Input schema must be a valid schema');
+  }
+  if (typeof handler !== 'function') {
+    throw new Error('Handler must be a function');
+  }
+
+  // Captures the function definition location.
+  const limit = Error.stackTraceLimit;
+  Error.stackTraceLimit = 2;
+  const traceError = new Error();
+  Error.stackTraceLimit = limit;
+  let cache: false | string = false;
+  const captureStackTrace = () => {
+    if (cache !== false) {
+      return cache;
+    }
+    if (traceError.stack !== undefined) {
+      const stack = traceError.stack.split('\n');
+      if (stack[2] !== undefined) {
+        cache = stack[2].trim();
+        return cache;
+      }
+    }
+  };
+
+  const handlerWithSpan = (...args: any[]) => {
+    const result = (handler as any)(...args);
+    if (Effect.isEffect(result)) {
+      return Effect.withSpan(result, `${key ?? name}`, {
+        captureStackTrace,
+      });
+    }
+    return result;
+  };
+
+  return {
+    [typeId]: true,
+    key,
+    name,
+    description,
+    inputSchema,
+    outputSchema,
+    handler: handlerWithSpan,
+    services: !services ? [] : getServiceKeys(services),
+  } satisfies FunctionDefinition.Any;
+};
+
+const getServiceKeys = (services: readonly Context.Tag<any, any>[]) => {
+  return services.map((tag: any) => {
+    if (typeof tag.key === 'string') {
+      return tag.key;
+    }
+    console.log(tag);
+    failedInvariant();
+  });
+};
+
+export const FunctionDefinition = {
+  make: defineFunction,
+  isFunction: (value: unknown): value is FunctionDefinition.Any => {
+    return typeof value === 'object' && value !== null && Symbol.for('@dxos/functions/FunctionDefinition') in value;
+  },
+  serialize: (functionDef: FunctionDefinition.Any): Function.Function => {
+    assertArgument(FunctionDefinition.isFunction(functionDef), 'functionDef');
+    return serializeFunction(functionDef);
+  },
+  deserialize: (functionObj: Function.Function): FunctionDefinition.Any => {
+    assertArgument(Obj.instanceOf(Function.Function, functionObj), 'functionObj');
+    return deserializeFunction(functionObj);
+  },
+};
+
+export const serializeFunction = (functionDef: FunctionDefinition.Any): Function.Function => {
+  const fn = Function.make({
+    key: functionDef.key,
+    name: functionDef.name,
+    version: '0.1.0',
+    description: functionDef.description,
+    inputSchema: Type.toJsonSchema(functionDef.inputSchema),
+    outputSchema: !functionDef.outputSchema ? undefined : Type.toJsonSchema(functionDef.outputSchema),
+    services: functionDef.services,
+  });
+  if (functionDef.meta?.deployedFunctionId) {
+    setUserFunctionIdInMetadata(Obj.getMeta(fn), functionDef.meta.deployedFunctionId);
+  }
+  return fn;
+};
+
+export const deserializeFunction = (functionObj: Function.Function): FunctionDefinition<unknown, unknown> => {
+  return {
+    [typeId]: true,
+    // TODO(dmaretskyi): Fix key.
+    key: functionObj.key ?? functionObj.name,
+    name: functionObj.name,
+    description: functionObj.description,
+    inputSchema: !functionObj.inputSchema ? Schema.Unknown : Type.toEffectSchema(functionObj.inputSchema),
+    outputSchema: !functionObj.outputSchema ? undefined : Type.toEffectSchema(functionObj.outputSchema),
+    // TODO(dmaretskyi): This should throw error.
+    handler: () => {},
+    services: functionObj.services ?? [],
+    meta: {
+      deployedFunctionId: getUserFunctionIdInMetadata(Obj.getMeta(functionObj)),
+    },
+  };
+};

package/src/services/credentials.ts

@@ -2,14 +2,19 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect, Layer } from 'effect';
+import * as HttpClient from '@effect/platform/HttpClient';
+import * as HttpClientRequest from '@effect/platform/HttpClientRequest';
+import type * as Config from 'effect/Config';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Redacted from 'effect/Redacted';
 
 import { Query } from '@dxos/echo';
-import { DataType } from '@dxos/schema';
+import { DatabaseService } from '@dxos/echo-db';
+import { AccessToken } from '@dxos/types';
 
-import { DatabaseService } from './database';
-
-type CredentialQuery = {
+export type CredentialQuery = {
   service?: string;
 };
 
@@ -43,16 +48,42 @@ export class CredentialsService extends Context.Tag('@dxos/functions/Credentials
       return yield* Effect.promise(() => credentials.getCredential(query));
     });
 
+  static getApiKey = (query: CredentialQuery): Effect.Effect<Redacted.Redacted<string>, never, CredentialsService> =>
+    Effect.gen(function* () {
+      const credential = yield* CredentialsService.getCredential(query);
+      if (!credential.apiKey) {
+        throw new Error(`API key not found for service: ${query.service}`);
+      }
+      return Redacted.make(credential.apiKey);
+    });
+
   static configuredLayer = (credentials: ServiceCredential[]) =>
     Layer.succeed(CredentialsService, new ConfiguredCredentialsService(credentials));
 
+  static layerConfig = (credentials: { service: string; apiKey: Config.Config<Redacted.Redacted<string>> }[]) =>
+    Layer.effect(
+      CredentialsService,
+      Effect.gen(function* () {
+        const serviceCredentials = yield* Effect.forEach(credentials, ({ service, apiKey }) =>
+          Effect.gen(function* () {
+            return {
+              service,
+              apiKey: Redacted.value(yield* apiKey),
+            };
+          }),
+        );
+
+        return new ConfiguredCredentialsService(serviceCredentials);
+      }),
+    );
+
   static layerFromDatabase = () =>
     Layer.effect(
       CredentialsService,
       Effect.gen(function* () {
         const dbService = yield* DatabaseService;
         const queryCredentials = async (query: CredentialQuery): Promise<ServiceCredential[]> => {
-          const { objects: accessTokens } = await dbService.db.query(Query.type(DataType.AccessToken)).run();
+          const { objects: accessTokens } = await dbService.db.query(Query.type(AccessToken.AccessToken)).run();
           return accessTokens
             .filter((accessToken) => accessToken.source === query.service)
             .map((accessToken) => ({
@@ -66,6 +97,7 @@ export class CredentialsService extends Context.Tag('@dxos/functions/Credentials
             if (credentials.length === 0) {
               throw new Error(`Credential not found for service: ${query.service}`);
             }
+
             return credentials[0];
           },
           queryCredentials: async (query) => {
@@ -93,6 +125,19 @@ export class ConfiguredCredentialsService implements Context.Tag.Service<Credent
     if (!credential) {
       throw new Error(`Credential not found for service: ${query.service}`);
     }
+
     return credential;
   }
 }
+
+/**
+ * Maps the request to include the API key from the credential.
+ */
+export const withAuthorization = (query: CredentialQuery, kind?: 'Bearer' | 'Basic') =>
+  HttpClient.mapRequestEffect(
+    Effect.fnUntraced(function* (request) {
+      const key = yield* CredentialsService.getApiKey(query).pipe(Effect.map(Redacted.value));
+      const authorization = kind ? `${kind} ${key}` : key;
+      return HttpClientRequest.setHeader(request, 'Authorization', authorization);
+    }),
+  );

package/src/services/event-logger.ts

@@ -2,7 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect, Layer, Schema } from 'effect';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+import * as Schema from 'effect/Schema';
 
 import { Obj, Type } from '@dxos/echo';
 import { invariant } from '@dxos/invariant';
@@ -14,12 +17,18 @@ export const ComputeEventPayload = Schema.Union(
   Schema.Struct({
     type: Schema.Literal('begin-compute'),
     nodeId: Schema.String,
-    inputs: Schema.Record({ key: Schema.String, value: Schema.Any }),
+    /**
+     * Names of the inputs begin computed.
+     */
+    inputs: Schema.Array(Schema.String),
   }),
   Schema.Struct({
     type: Schema.Literal('end-compute'),
     nodeId: Schema.String,
-    outputs: Schema.Record({ key: Schema.String, value: Schema.Any }),
+    /**
+     * Names of the outputs computed.
+     */
+    outputs: Schema.Array(Schema.String),
   }),
   Schema.Struct({
     type: Schema.Literal('compute-input'),

package/src/services/function-invocation-service.ts

@@ -0,0 +1,23 @@
+//
+// Copyright 2025 DXOS.org
+//
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+
+import { type FunctionDefinition, type InvocationServices } from '../sdk';
+
+export class FunctionInvocationService extends Context.Tag('@dxos/functions/FunctionInvocationService')<
+  FunctionInvocationService,
+  {
+    invokeFunction<I, O>(
+      functionDef: FunctionDefinition<I, O, any>,
+      input: I,
+    ): Effect.Effect<O, never, InvocationServices>;
+  }
+>() {
+  static invokeFunction = <I, O>(
+    functionDef: FunctionDefinition<I, O, any>,
+    input: I,
+  ): Effect.Effect<O, never, FunctionInvocationService | InvocationServices> =>
+    Effect.serviceFunctionEffect(FunctionInvocationService, (service) => service.invokeFunction)(functionDef, input);
+}

package/src/services/index.ts

@@ -2,11 +2,12 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './database';
-export * from './queues';
-export * from './service-container';
+export { DatabaseService } from '@dxos/echo-db';
+
 export * from './credentials';
-export * from './tracing';
+export { ConfiguredCredentialsService, type ServiceCredential } from './credentials';
 export * from './event-logger';
-export * from './remote-function-execution-service';
-export * from './local-function-execution';
+export { createEventLogger, createDefectLogger } from './event-logger';
+export * from './function-invocation-service';
+export * from './queues';
+export * from './tracing';

package/src/services/queues.ts

@@ -2,7 +2,9 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect, Layer } from 'effect';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
 
 import type { Obj, Relation } from '@dxos/echo';
 import type { Queue, QueueAPI, QueueFactory } from '@dxos/echo-db';
@@ -62,6 +64,11 @@ export class QueueService extends Context.Tag('@dxos/functions/QueueService')<
     subspaceTag?: QueueSubspaceTag;
   }): Effect.Effect<Queue<T>, never, QueueService> =>
     QueueService.pipe(Effect.map(({ queues }) => queues.create<T>(options)));
+
+  static append = <T extends Obj.Any | Relation.Any = Obj.Any | Relation.Any>(
+    queue: Queue<T>,
+    objects: T[],
+  ): Effect.Effect<void> => Effect.promise(() => queue.append(objects));
 }
 
 /**

package/src/services/tracing.ts

@@ -2,11 +2,14 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Context, Effect, Layer } from 'effect';
+import * as Context from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
 
 import { AgentStatus } from '@dxos/ai';
 import { Obj } from '@dxos/echo';
-import type { AnyEchoObject } from '@dxos/echo-schema';
+import type { ObjectId } from '@dxos/echo/internal';
+import { Message } from '@dxos/types';
 
 /**
  * Provides a way for compute primitives (functions, workflows, tools)
@@ -15,30 +18,91 @@ import type { AnyEchoObject } from '@dxos/echo-schema';
 export class TracingService extends Context.Tag('@dxos/functions/TracingService')<
   TracingService,
   {
+    /**
+     * Gets the parent message ID.
+     */
+    getTraceContext: () => TracingService.TraceContext;
+
     /**
      * Write an event to the tracing queue.
      * @param event - The event to write. Must be an a typed object.
      */
-    write(event: AnyEchoObject): void;
+    write: (event: Obj.Any) => void;
   }
 >() {
-  static noop: Context.Tag.Service<TracingService> = { write: () => {} };
-
-  static layerNoop = Layer.succeed(TracingService, TracingService.noop);
-
-  static console: Context.Tag.Service<TracingService> = {
-    write: (event) => {
-      // eslint-disable-next-line no-console
-      console.log(event);
-    },
+  static noop: Context.Tag.Service<TracingService> = {
+    getTraceContext: () => ({}),
+    write: () => {},
   };
 
+  static layerNoop: Layer.Layer<TracingService> = Layer.succeed(TracingService, TracingService.noop);
+  /**
+   * Creates a TracingService layer that emits events to the parent tracing service.
+   */
+  static layerSubframe = (mapContext: (currentContext: TracingService.TraceContext) => TracingService.TraceContext) =>
+    Layer.effect(
+      TracingService,
+      Effect.gen(function* () {
+        const tracing = yield* TracingService;
+        const context = mapContext(tracing.getTraceContext());
+        return {
+          write: (event) => tracing.write(event),
+          getTraceContext: () => context,
+        };
+      }),
+    );
+
   /**
    * Emit the current human-readable execution status.
    */
-  static emitStatus: (data: Obj.MakeProps<typeof AgentStatus>) => Effect.Effect<void, never, TracingService> =
-    Effect.fnUntraced(function* (data) {
-      const tracing = yield* TracingService;
-      tracing.write(Obj.make(AgentStatus, data));
-    });
+  static emitStatus: (
+    data: Omit<Obj.MakeProps<typeof AgentStatus>, 'created'>,
+  ) => Effect.Effect<void, never, TracingService> = Effect.fnUntraced(function* (data) {
+    const tracing = yield* TracingService;
+    tracing.write(
+      Obj.make(AgentStatus, {
+        parentMessage: tracing.getTraceContext().parentMessage,
+        toolCallId: tracing.getTraceContext().toolCallId,
+        created: new Date().toISOString(),
+        ...data,
+      }),
+    );
+  });
+
+  static emitConverationMessage: (
+    data: Obj.MakeProps<typeof Message.Message>,
+  ) => Effect.Effect<void, never, TracingService> = Effect.fnUntraced(function* (data) {
+    const tracing = yield* TracingService;
+    tracing.write(
+      Obj.make(Message.Message, {
+        parentMessage: tracing.getTraceContext().parentMessage,
+        ...data,
+        properties: {
+          [MESSAGE_PROPERTY_TOOL_CALL_ID]: tracing.getTraceContext().toolCallId,
+          ...data.properties,
+        },
+      }),
+    );
+  });
 }
+
+export namespace TracingService {
+  export interface TraceContext {
+    /**
+     * If this thread sprung from a tool call, this is the ID of the message containing the tool call.
+     */
+    parentMessage?: ObjectId;
+
+    /**
+     * If the current thread is a byproduct of a tool call, this is the ID of the tool call.
+     */
+    toolCallId?: string;
+
+    debugInfo?: unknown;
+  }
+}
+
+/**
+ * Goes into {@link Message['properties']}
+ */
+export const MESSAGE_PROPERTY_TOOL_CALL_ID = 'toolCallId' as const;

package/src/types/Function.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { Obj, Type } from '@dxos/echo';
+import { JsonSchemaType, LabelAnnotation, Ref } from '@dxos/echo/internal';
+
+import { Script } from './Script';
+
+/**
+ * Function deployment.
+ */
+export const Function = Schema.Struct({
+  /**
+   * Global registry ID.
+   * NOTE: The `key` property refers to the original registry entry.
+   */
+  // TODO(burdon): Create Format type for DXN-like ids, such as this and schema type.
+  // TODO(dmaretskyi): Consider making it part of ECHO meta.
+  // TODO(dmaretskyi): Make required.
+  key: Schema.optional(Schema.String).annotations({
+    description: 'Unique registration key for the blueprint',
+  }),
+
+  // TODO(burdon): Rename to id/uri?
+  name: Schema.NonEmptyString,
+  version: Schema.String,
+
+  description: Schema.optional(Schema.String),
+
+  /**
+   * ISO date string of the last deployment.
+   */
+  updated: Schema.optional(Schema.String),
+
+  // Reference to a source script if it exists within ECHO.
+  // TODO(burdon): Don't ref ScriptType directly (core).
+  source: Schema.optional(Ref(Script)),
+
+  inputSchema: Schema.optional(JsonSchemaType),
+  outputSchema: Schema.optional(JsonSchemaType),
+
+  /**
+   * List of required services.
+   * Match the Context.Tag keys of the FunctionServices variants.
+   */
+  services: Schema.optional(Schema.Array(Schema.String)),
+
+  // Local binding to a function name.
+  binding: Schema.optional(Schema.String),
+}).pipe(
+  Type.Obj({
+    typename: 'dxos.org/type/Function',
+    version: '0.1.0',
+  }),
+  LabelAnnotation.set(['name']),
+);
+export interface Function extends Schema.Schema.Type<typeof Function> {}
+
+export const make = (props: Obj.MakeProps<typeof Function>) => Obj.make(Function, props);

package/src/types/Script.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { Obj, Ref, Type } from '@dxos/echo';
+import { FormAnnotation, LabelAnnotation } from '@dxos/echo/internal';
+import { Text } from '@dxos/schema';
+
+/**
+ * Source script.
+ */
+export const Script = Schema.Struct({
+  name: Schema.String.pipe(Schema.optional),
+  description: Schema.String.pipe(Schema.optional),
+  // TODO(burdon): Change to hash of deployed content.
+  // Whether source has changed since last deploy.
+  changed: Schema.Boolean.pipe(FormAnnotation.set(false), Schema.optional),
+  source: Type.Ref(Text.Text).pipe(FormAnnotation.set(false)),
+}).pipe(
+  Type.Obj({
+    typename: 'dxos.org/type/Script',
+    version: '0.1.0',
+  }),
+  LabelAnnotation.set(['name']),
+);
+export interface Script extends Schema.Schema.Type<typeof Script> {}
+
+type Props = Omit<Obj.MakeProps<typeof Script>, 'source'> & { source?: string };
+
+export const make = ({ source = '', ...props }: Props = {}) =>
+  Obj.make(Script, { ...props, source: Ref.make(Text.make(source)) });