@dxos/functions 0.8.4-main.c85a9c8dae → 0.8.4-main.cb12b3f963

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/functions",
-  "version": "0.8.4-main.c85a9c8dae",
+  "version": "0.8.4-main.cb12b3f963",
   "description": "Functions API.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -26,23 +26,24 @@
     "src"
   ],
   "dependencies": {
+    "@effect-atom/atom": "^0.5.1",
     "@effect/ai-anthropic": "0.23.0",
     "@effect/platform": "0.94.4",
-    "effect": "3.19.16",
-    "@dxos/echo-db": "0.8.4-main.c85a9c8dae",
-    "@dxos/echo": "0.8.4-main.c85a9c8dae",
-    "@dxos/ai": "0.8.4-main.c85a9c8dae",
-    "@dxos/effect": "0.8.4-main.c85a9c8dae",
-    "@dxos/errors": "0.8.4-main.c85a9c8dae",
-    "@dxos/keys": "0.8.4-main.c85a9c8dae",
-    "@dxos/log": "0.8.4-main.c85a9c8dae",
-    "@dxos/node-std": "0.8.4-main.c85a9c8dae",
-    "@dxos/invariant": "0.8.4-main.c85a9c8dae",
-    "@dxos/operation": "0.8.4-main.c85a9c8dae",
-    "@dxos/types": "0.8.4-main.c85a9c8dae",
-    "@dxos/schema": "0.8.4-main.c85a9c8dae",
-    "@dxos/context": "0.8.4-main.c85a9c8dae",
-    "@dxos/protocols": "0.8.4-main.c85a9c8dae"
+    "effect": "3.20.0",
+    "@dxos/ai": "0.8.4-main.cb12b3f963",
+    "@dxos/echo": "0.8.4-main.cb12b3f963",
+    "@dxos/context": "0.8.4-main.cb12b3f963",
+    "@dxos/effect": "0.8.4-main.cb12b3f963",
+    "@dxos/compute": "0.8.4-main.cb12b3f963",
+    "@dxos/echo-db": "0.8.4-main.cb12b3f963",
+    "@dxos/errors": "0.8.4-main.cb12b3f963",
+    "@dxos/invariant": "0.8.4-main.cb12b3f963",
+    "@dxos/log": "0.8.4-main.cb12b3f963",
+    "@dxos/keys": "0.8.4-main.cb12b3f963",
+    "@dxos/types": "0.8.4-main.cb12b3f963",
+    "@dxos/protocols": "0.8.4-main.cb12b3f963",
+    "@dxos/schema": "0.8.4-main.cb12b3f963",
+    "@dxos/node-std": "0.8.4-main.cb12b3f963"
   },
   "publishConfig": {
     "access": "public"

package/src/index.ts

@@ -2,8 +2,6 @@
 // Copyright 2024 DXOS.org
 //
 
-export * from './errors';
-export * from './example';
 export * from './sdk';
 export * from './services';
 export * from './types';

package/src/protocol/protocol.test.ts

@@ -4,17 +4,16 @@
 
 import { describe, test } from 'vitest';
 
-import { FunctionError } from '../errors';
-import fibFunc from '../example/fib';
-import replyFunc from '../example/reply';
+import { InvalidOperationInputError } from '@dxos/compute';
+import { FibonacciHandler, ReplyHandler } from '@dxos/compute/testing';
 
 import { wrapFunctionHandler } from './protocol';
 
 describe('wrapFunctionHandler', () => {
   test('wraps reply function and executes handler', async ({ expect }) => {
-    const wrapped = wrapFunctionHandler(replyFunc);
+    const wrapped = wrapFunctionHandler(ReplyHandler);
 
-    expect(wrapped.meta.key).toBe('example.org/function/reply');
+    expect(wrapped.meta.key).toBe('org.example.function.reply');
     expect(wrapped.meta.name).toBe('Reply');
 
     const testData = { message: 'hello' };
@@ -29,9 +28,9 @@ describe('wrapFunctionHandler', () => {
   });
 
   test('wraps fibonacci function with valid input', async ({ expect }) => {
-    const wrapped = wrapFunctionHandler(fibFunc);
+    const wrapped = wrapFunctionHandler(FibonacciHandler);
 
-    expect(wrapped.meta.key).toBe('example.org/function/fib');
+    expect(wrapped.meta.key).toBe('org.example.function.fib');
     expect(wrapped.meta.name).toBe('Fibonacci');
 
     const result = await wrapped.handler({
@@ -44,8 +43,8 @@ describe('wrapFunctionHandler', () => {
     expect(result).toEqual({ result: '55' });
   });
 
-  test('throws FunctionError on invalid input schema for fibonacci', async ({ expect }) => {
-    const wrapped = wrapFunctionHandler(fibFunc);
+  test('throws InvalidOperationInputError on invalid input schema for fibonacci', async ({ expect }) => {
+    const wrapped = wrapFunctionHandler(FibonacciHandler);
 
     await expect(
       wrapped.handler({
@@ -54,6 +53,6 @@ describe('wrapFunctionHandler', () => {
           services: {},
         },
       }),
-    ).rejects.toThrow(FunctionError);
+    ).rejects.toThrow(InvalidOperationInputError);
   });
 });

package/src/protocol/protocol.ts

@@ -10,43 +10,55 @@ import * as SchemaAST from 'effect/SchemaAST';
 
 import { AiModelResolver, AiService } from '@dxos/ai';
 import { AnthropicResolver } from '@dxos/ai/resolvers';
+import {
+  FunctionError,
+  InvalidOperationInputError,
+  InvalidOperationOutputError,
+  Operation,
+  Trace,
+} from '@dxos/compute';
 import { LifecycleState, Resource } from '@dxos/context';
 import { Database, Feed, JsonSchema, Ref, type Type } from '@dxos/echo';
-import { refFromEncodedReference } from '@dxos/echo/internal';
 import { EchoClient, type EchoDatabaseImpl, type QueueFactory, createFeedServiceLayer } from '@dxos/echo-db';
+import { refFromEncodedReference } from '@dxos/echo/internal';
 import { runAndForwardErrors } from '@dxos/effect';
 import { assertState, failedInvariant, invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { type FunctionProtocol } from '@dxos/protocols';
 
-import { FunctionError } from '../errors';
-import { FunctionDefinition, type FunctionServices } from '../sdk';
-import { CredentialsService, FunctionInvocationService, QueueService, TracingService } from '../services';
-
+import { type FunctionServices } from '../sdk';
+import {
+  configuredCredentialsLayer,
+  credentialsLayerFromDatabase,
+  FunctionInvocationService,
+  QueueService,
+} from '../services';
 import { FunctionsAiHttpClient } from './functions-ai-http-client';
 
 /**
  * Wraps a function handler made with `defineFunction` to a protocol that the functions-runtime expects.
  */
-export const wrapFunctionHandler = (func: FunctionDefinition): FunctionProtocol.Func => {
-  if (!FunctionDefinition.isFunction(func)) {
-    throw new TypeError('Invalid function definition');
+export const wrapFunctionHandler = (func: Operation.WithHandler<Operation.Definition.Any>): FunctionProtocol.Func => {
+  if (!Operation.isOperationWithHandler(func)) {
+    throw new TypeError('Expected operation with handler');
   }
 
+  const serviceTags = func.services.map((service) => service.key);
+
   return {
     meta: {
-      key: func.key,
-      name: func.name,
-      description: func.description,
-      inputSchema: JsonSchema.toJsonSchema(func.inputSchema),
-      outputSchema: func.outputSchema === undefined ? undefined : JsonSchema.toJsonSchema(func.outputSchema),
-      services: func.services,
+      key: func.meta.key,
+      name: func.meta.name,
+      description: func.meta.description,
+      inputSchema: JsonSchema.toJsonSchema(func.input),
+      outputSchema: func.output === undefined ? undefined : JsonSchema.toJsonSchema(func.output),
+      services: func.services.map((service) => service.key),
     },
     handler: async ({ data, context }) => {
       if (
-        (func.services.includes(Database.Service.key) ||
-          func.services.includes(QueueService.key) ||
-          func.services.includes(Feed.Service.key)) &&
+        (serviceTags.includes(Database.Service.key) ||
+          serviceTags.includes(QueueService.key) ||
+          serviceTags.includes(Feed.FeedService.key)) &&
         (!context.services.dataService || !context.services.queryService)
       ) {
         throw new FunctionError({
@@ -56,11 +68,14 @@ export const wrapFunctionHandler = (func: FunctionDefinition): FunctionProtocol.
 
       // eslint-disable-next-line no-useless-catch
       try {
-        if (!SchemaAST.isAnyKeyword(func.inputSchema.ast)) {
+        if (!SchemaAST.isAnyKeyword(func.input.ast)) {
           try {
-            Schema.validateSync(func.inputSchema)(data);
-          } catch (error) {
-            throw new FunctionError({ message: 'Invalid input schema', cause: error });
+            Schema.validateSync(func.input, { onExcessProperty: 'error' })(data);
+          } catch (error: any) {
+            throw new InvalidOperationInputError({
+              message: `Operation input did not match schema (${func.meta.key}): ${error.message}`,
+              cause: error,
+            });
           }
         }
 
@@ -72,15 +87,11 @@ export const wrapFunctionHandler = (func: FunctionDefinition): FunctionProtocol.
         }
 
         const dataWithDecodedRefs =
-          funcContext.db && !SchemaAST.isAnyKeyword(func.inputSchema.ast)
-            ? decodeRefsFromSchema(func.inputSchema.ast, data, funcContext.db)
+          funcContext.db && !SchemaAST.isAnyKeyword(func.input.ast)
+            ? decodeRefsFromSchema(func.input.ast, data, funcContext.db)
             : data;
 
-        let result = await func.handler({
-          // TODO(dmaretskyi): Fix the types.
-          context: context as any,
-          data: dataWithDecodedRefs,
-        });
+        let result: any = await func.handler(dataWithDecodedRefs);
 
         if (Effect.isEffect(result)) {
           result = await runAndForwardErrors(
@@ -91,8 +102,15 @@ export const wrapFunctionHandler = (func: FunctionDefinition): FunctionProtocol.
           );
         }
 
-        if (func.outputSchema && !SchemaAST.isAnyKeyword(func.outputSchema.ast)) {
-          Schema.validateSync(func.outputSchema)(result);
+        if (func.output && !SchemaAST.isAnyKeyword(func.output.ast)) {
+          try {
+            Schema.validateSync(func.output, { onExcessProperty: 'error' })(result);
+          } catch (error: any) {
+            throw new InvalidOperationOutputError({
+              message: `Operation output did not match schema (${func.meta.key}): ${error.message}`,
+              cause: error,
+            });
+          }
         }
 
         return result;
@@ -155,10 +173,10 @@ class FunctionContext extends Resource {
     const queuesLayer = this.queues ? QueueService.layer(this.queues) : QueueService.notAvailable;
     const feedLayer = this.queues ? createFeedServiceLayer(this.queues) : Feed.notAvailable;
     const credentials = dbLayer
-      ? CredentialsService.layerFromDatabase({ caching: true }).pipe(Layer.provide(dbLayer))
-      : CredentialsService.configuredLayer([]);
+      ? credentialsLayerFromDatabase({ caching: true }).pipe(Layer.provide(dbLayer))
+      : configuredCredentialsLayer([]);
     const functionInvocationService = MockedFunctionInvocationService;
-    const tracing = TracingService.layerNoop;
+    const operationServiceLayer = MockedOperationServiceLayer;
 
     const aiLayer = this.context.services.functionsAiService
       ? AiModelResolver.AiModelResolver.buildAiService.pipe(
@@ -175,7 +193,17 @@ class FunctionContext extends Resource {
         )
       : AiService.notAvailable;
 
-    return Layer.mergeAll(dbLayer, queuesLayer, feedLayer, credentials, functionInvocationService, aiLayer, tracing);
+    return Layer.mergeAll(
+      dbLayer,
+      queuesLayer,
+      feedLayer,
+      credentials,
+      functionInvocationService,
+      operationServiceLayer,
+      aiLayer,
+      // TODO(dmaretskyi): Forward trace events.
+      Trace.writerLayerNoop,
+    );
   }
 }
 
@@ -184,6 +212,12 @@ const MockedFunctionInvocationService = Layer.succeed(FunctionInvocationService,
   resolveFunction: () => Effect.die('Not implemented.'),
 });
 
+const MockedOperationServiceLayer = Layer.succeed(Operation.Service, {
+  invoke: () => Effect.die('Calling operations from functions is not implemented yet.'),
+  schedule: () => Effect.die('Not implemented.'),
+  invokePromise: async () => ({ error: new Error('Not implemented') }),
+} as any);
+
 const decodeRefsFromSchema = (ast: SchemaAST.AST, value: unknown, db: EchoDatabaseImpl): unknown => {
   if (value == null) {
     return value;

package/src/sdk.ts

@@ -2,296 +2,28 @@
 // Copyright 2023 DXOS.org
 //
 
-import type * as Context from 'effect/Context';
-import * as Effect from 'effect/Effect';
-import * as Schema from 'effect/Schema';
-
 import { type AiService } from '@dxos/ai';
-import { type Feed, JsonSchema, Obj, type Type } from '@dxos/echo';
-import { type Database } from '@dxos/echo';
-import { assertArgument, failedInvariant } from '@dxos/invariant';
-import { Operation } from '@dxos/operation';
+import { type Credential, type Operation, type Trace } from '@dxos/compute';
+import { type Database, type Feed } from '@dxos/echo';
 
-import {
-  type CredentialsService,
-  type FunctionInvocationService,
-  type QueueService,
-  type TracingService,
-} from './services';
-import { Function } from './types';
-import { getUserFunctionIdInMetadata, setUserFunctionIdInMetadata } from './types';
+import { type FunctionInvocationService, type QueueService } from './services';
 
 // TODO(burdon): Model after http request. Ref Lambda/OpenFaaS.
 // https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
 // https://www.serverless.com/framework/docs/providers/aws/guide/serverless.yml/#functions
 // https://www.npmjs.com/package/aws-lambda
 
-/**
- * Services that are provided at the function call site by the caller.
- */
-export type InvocationServices = TracingService;
-
 /**
  * Services that are available to invoked functions.
+ * @deprecated
  */
 export type FunctionServices =
-  | InvocationServices
   | AiService.AiService
-  | CredentialsService
+  | Credential.CredentialsService
   | Database.Service
-  // TODO(wittjosiah): Remove QueueService — use Feed.Service instead.
+  // TODO(wittjosiah): Remove QueueService — use Feed.FeedService instead.
   | QueueService
-  | Feed.Service
-  | FunctionInvocationService;
-
-/**
- * Function handler.
- */
-export type FunctionHandler<TData = {}, TOutput = any, S extends FunctionServices = FunctionServices> = (params: {
-  /**
-   * Context available to the function.
-   */
-  context: FunctionContext;
-
-  /**
-   * Data passed as the input to the function.
-   * Must match the function's input schema.
-   * This will be the payload from the trigger or other data passed into the function in a workflow.
-   */
-  data: TData;
-}) => TOutput | Promise<TOutput> | Effect.Effect<TOutput, any, S>;
-
-/**
- * Function context.
- */
-export interface FunctionContext {
-  // TODO(dmaretskyi): Consider what we should put into context.
-}
-
-const typeId = Symbol.for('@dxos/functions/FunctionDefinition');
-
-/**
- * Deployable function definition.
- */
-export type FunctionDefinition<TInput = any, TOutput = any, S extends FunctionServices = FunctionServices> = {
-  [typeId]: true;
-
-  key: string;
-  name: string;
-  description?: string;
-  inputSchema: Schema.Schema<TInput, any>;
-  outputSchema?: Schema.Schema<TOutput, any>;
-
-  /**
-   * List of types the function uses.
-   * This is used to ensure that the types are available when the function is executed.
-   */
-  types: readonly Type.AnyEntity[];
-
-  /**
-   * Keys of the required services.
-   */
-  services: readonly string[];
-
-  meta?: {
-    /**
-     * Tools that are projected from functions have this annotation.
-     *
-     * deployedFunctionId:
-     * - Backend deployment ID assigned by the EDGE function service (typically a UUID).
-     * - Used for remote invocation via `FunctionInvocationService` → `RemoteFunctionExecutionService`.
-     * - Persisted on the corresponding ECHO `Function.Function` object's metadata under the
-     *   `FUNCTIONS_META_KEY` and retrieved with `getUserFunctionIdInMetadata`.
-     */
-    deployedFunctionId?: string;
-  };
-
-  handler: FunctionHandler<TInput, TOutput, S>;
-};
-
-export declare namespace FunctionDefinition {
-  export type Any = FunctionDefinition<any, any, any>;
-  export type Input<T extends Any> = T extends FunctionDefinition<infer I, infer _O, infer _S> ? I : never;
-  export type Output<T extends Any> = T extends FunctionDefinition<infer _I, infer O, infer _S> ? O : never;
-  export type Services<T extends Any> = T extends FunctionDefinition<infer _I, infer _O, infer S> ? S : never;
-}
-
-export type FunctionProps<T, O> = {
-  key: string;
-  name: string;
-  description?: string;
-  inputSchema: Schema.Schema<T, any>;
-  outputSchema?: Schema.Schema<O, any>;
-
-  /**
-   * List of types the function uses.
-   * This is used to ensure that the types are available when the function is executed.
-   */
-  types?: readonly Type.AnyEntity[];
-
-  // TODO(dmaretskyi): This currently doesn't cause a compile-time error if the handler requests a service that is not specified
-  services?: readonly Context.Tag<any, any>[];
-
-  handler: FunctionHandler<T, O, FunctionServices>;
-};
-
-// TODO(dmaretskyi): Output type doesn't get typechecked.
-export const defineFunction: {
-  <I, O>(params: FunctionProps<I, O>): FunctionDefinition<I, O, FunctionServices>;
-} = ({ key, name, description, inputSchema, outputSchema = Schema.Any, handler, types, services }) => {
-  if (!Schema.isSchema(inputSchema)) {
-    throw new Error('Input schema must be a valid schema');
-  }
-  if (typeof handler !== 'function') {
-    throw new Error('Handler must be a function');
-  }
-
-  // Captures the function definition location.
-  const limit = Error.stackTraceLimit;
-  Error.stackTraceLimit = 2;
-  const traceError = new Error();
-  Error.stackTraceLimit = limit;
-  let cache: false | string = false;
-  const captureStackTrace = () => {
-    if (cache !== false) {
-      return cache;
-    }
-    if (traceError.stack !== undefined) {
-      const stack = traceError.stack.split('\n');
-      if (stack[2] !== undefined) {
-        cache = stack[2].trim();
-        return cache;
-      }
-    }
-  };
-
-  const handlerWithSpan = (...args: any[]) => {
-    const result = (handler as any)(...args);
-    if (Effect.isEffect(result)) {
-      return Effect.withSpan(result, `${key ?? name}`, {
-        captureStackTrace,
-      });
-    }
-    return result;
-  };
-
-  return {
-    [typeId]: true,
-    key,
-    name,
-    description,
-    inputSchema,
-    outputSchema,
-    handler: handlerWithSpan,
-    types: types ?? [],
-    services: !services ? [] : getServiceKeys(services),
-  };
-};
-
-const getServiceKeys = (services: readonly Context.Tag<any, any>[]) => {
-  return services.map((tag: any) => {
-    if (typeof tag.key === 'string') {
-      return tag.key;
-    }
-    failedInvariant();
-  });
-};
-
-/**
- * Converts a FunctionDefinition to an OperationDefinition with handler.
- * The function handler is adapted to the OperationHandler format.
- *
- * Note: FunctionDefinition stores service keys as strings, not Tag types,
- * so we can't use Operation.withHandler's type inference here.
- */
-export const toOperation = <T, O, S extends FunctionServices = FunctionServices>(
-  functionDef: FunctionDefinition<T, O, S>,
-): Operation.Definition<T, O> & { handler: Operation.Handler<T, O, any, S> } => {
-  const op = Operation.make({
-    schema: {
-      input: functionDef.inputSchema,
-      output: functionDef.outputSchema ?? Schema.Any,
-    },
-    meta: {
-      key: functionDef.key,
-      name: functionDef.name,
-      description: functionDef.description,
-    },
-  });
-
-  // Adapt FunctionHandler signature to OperationHandler format.
-  // FunctionHandler expects { context, data }, OperationHandler expects just input.
-  const operationHandler: Operation.Handler<T, O, any, S> = (input: T) => {
-    const result = functionDef.handler({
-      context: {} as FunctionContext,
-      data: input,
-    });
-
-    // Convert Promise or plain value to Effect.
-    if (Effect.isEffect(result)) {
-      return result;
-    }
-    if (result instanceof Promise) {
-      return Effect.tryPromise(() => result);
-    }
-    return Effect.succeed(result as O);
-  };
-
-  // Manually attach handler since FunctionDefinition stores service keys as strings,
-  // not Tag types, so withHandler's type inference doesn't apply.
-  return {
-    ...op,
-    handler: operationHandler,
-  };
-};
-
-export const FunctionDefinition = {
-  make: defineFunction,
-  isFunction: (value: unknown): value is FunctionDefinition.Any => {
-    return typeof value === 'object' && value !== null && Symbol.for('@dxos/functions/FunctionDefinition') in value;
-  },
-  serialize: (functionDef: FunctionDefinition.Any): Function.Function => {
-    assertArgument(FunctionDefinition.isFunction(functionDef), 'functionDef');
-    return serializeFunction(functionDef);
-  },
-  deserialize: (functionObj: Function.Function): FunctionDefinition.Any => {
-    assertArgument(Obj.instanceOf(Function.Function, functionObj), 'functionObj');
-    return deserializeFunction(functionObj);
-  },
-  toOperation,
-};
-
-export const serializeFunction = (functionDef: FunctionDefinition.Any): Function.Function => {
-  const fn = Function.make({
-    key: functionDef.key,
-    name: functionDef.name,
-    version: '0.1.0',
-    description: functionDef.description,
-    inputSchema: JsonSchema.toJsonSchema(functionDef.inputSchema),
-    outputSchema: !functionDef.outputSchema ? undefined : JsonSchema.toJsonSchema(functionDef.outputSchema),
-    services: functionDef.services,
-  });
-  if (functionDef.meta?.deployedFunctionId) {
-    Obj.change(fn, (fn) => setUserFunctionIdInMetadata(Obj.getMeta(fn), functionDef.meta!.deployedFunctionId!));
-  }
-  return fn;
-};
-
-export const deserializeFunction = (functionObj: Function.Function): FunctionDefinition<unknown, unknown> => {
-  return {
-    [typeId]: true,
-    // TODO(dmaretskyi): Fix key.
-    key: functionObj.key ?? functionObj.name,
-    name: functionObj.name,
-    description: functionObj.description,
-    inputSchema: !functionObj.inputSchema ? Schema.Unknown : JsonSchema.toEffectSchema(functionObj.inputSchema),
-    outputSchema: !functionObj.outputSchema ? undefined : JsonSchema.toEffectSchema(functionObj.outputSchema),
-    // TODO(dmaretskyi): This should throw error.
-    handler: () => {},
-    services: functionObj.services ?? [],
-    types: [],
-    meta: {
-      deployedFunctionId: getUserFunctionIdInMetadata(Obj.getMeta(functionObj)),
-    },
-  };
-};
+  | Feed.FeedService
+  | Trace.TraceService
+  | FunctionInvocationService
+  | Operation.Service;