@dxos/functions 0.8.2-main.fbd8ed0 → 0.8.2-staging.7ac8446

package/src/handler.ts

@@ -2,15 +2,17 @@
 // Copyright 2023 DXOS.org
 //
 
-import { Schema } from 'effect';
+import { Schema as S } from 'effect';
 import { type Effect } from 'effect';
 
-import { type AIServiceClient } from '@dxos/assistant';
-// import { type Space } from '@dxos/client/echo';
-import type { CoreDatabase, EchoDatabase } from '@dxos/echo-db';
+import { type Client, PublicKey } from '@dxos/client';
+import { type Space, type SpaceId } from '@dxos/client/echo';
+import type { CoreDatabase, EchoDatabase, ReactiveEchoObject } from '@dxos/echo-db';
 import { type HasId } from '@dxos/echo-schema';
-import { type SpaceId, type DXN } from '@dxos/keys';
+import { type DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
 import { type QueryResult } from '@dxos/protocols';
+import { isNonNullable } from '@dxos/util';
 
 // TODO(burdon): Model after http request. Ref Lambda/OpenFaaS.
 // https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
@@ -20,18 +22,13 @@ import { type QueryResult } from '@dxos/protocols';
 /**
  * Function handler.
  */
-export type FunctionHandler<TData = {}, TOutput = any> = (params: {
-  /**
-   * Services and context available to the function.
-   */
+export type FunctionHandler<TData = {}, TMeta = {}, TOutput = any> = (params: {
   context: FunctionContext;
-
+  event: FunctionEvent<TData, TMeta>;
   /**
-   * Data passed as the input to the function.
-   * Must match the function's input schema.
-   * This will be the payload from the trigger or other data passed into the function in a workflow.
+   * @deprecated
    */
-  data: TData;
+  response: FunctionResponse;
 }) => TOutput | Promise<TOutput> | Effect.Effect<TOutput, any>;
 
 /**
@@ -45,7 +42,18 @@ export interface FunctionContext {
    */
   space: SpaceAPI | undefined;
 
-  ai: AIServiceClient;
+  ai: FunctionContextAi;
+
+  /**
+   * @deprecated
+   */
+  // TODO(burdon): Limit access to individual space.
+  client: Client;
+  /**
+   * @deprecated
+   */
+  // TODO(burdon): Replace with storage service abstraction.
+  dataDir?: string;
 }
 
 export interface FunctionContextAi {
@@ -53,6 +61,28 @@ export interface FunctionContextAi {
   run(model: string, inputs: any, options?: any): Promise<any>;
 }
 
+/**
+ * Event payload.
+ */
+// TODO(dmaretskyi): Update type definitions to match the actual payload.
+export type FunctionEvent<TData = {}, TMeta = {}> = {
+  data: FunctionEventMeta<TMeta> & TData;
+};
+
+/**
+ * Metadata from trigger.
+ */
+export type FunctionEventMeta<TMeta = {}> = {
+  meta: TMeta;
+};
+
+/**
+ * Function response.
+ */
+export type FunctionResponse = {
+  status(code: number): FunctionResponse;
+};
+
 //
 // API.
 //
@@ -78,21 +108,28 @@ export interface SpaceAPI {
   get queues(): QueuesAPI;
 }
 
-// TODO(wittjosiah): Queues are incompatible.
+// TODO(wittjosiah): Fix this.
 const __assertFunctionSpaceIsCompatibleWithTheClientSpace = () => {
   // const _: SpaceAPI = {} as Space;
 };
 
-export type FunctionDefinition<T = {}, O = any> = {
+export type FunctionDefinition = {
+  description?: string;
+  inputSchema: S.Schema.AnyNoContext;
+  outputSchema?: S.Schema.AnyNoContext;
+  handler: FunctionHandler<any>;
+};
+
+export type DefineFunctionParams<T, O = any> = {
   description?: string;
-  inputSchema: Schema.Schema<T, any>;
-  outputSchema?: Schema.Schema<O, any>;
-  handler: FunctionHandler<T, O>;
+  inputSchema: S.Schema<T, any>;
+  outputSchema?: S.Schema<O, any>;
+  handler: FunctionHandler<T, any, O>;
 };
 
 // TODO(dmaretskyi): Bind input type to function handler.
-export const defineFunction = <T, O>(params: FunctionDefinition<T, O>): FunctionDefinition<T, O> => {
-  if (!Schema.isSchema(params.inputSchema)) {
+export const defineFunction = <T, O>(params: DefineFunctionParams<T, O>): FunctionDefinition => {
+  if (!S.isSchema(params.inputSchema)) {
     throw new Error('Input schema must be a valid schema');
   }
   if (typeof params.handler !== 'function') {
@@ -102,7 +139,71 @@ export const defineFunction = <T, O>(params: FunctionDefinition<T, O>): Function
   return {
     description: params.description,
     inputSchema: params.inputSchema,
-    outputSchema: params.outputSchema ?? Schema.Any,
+    outputSchema: params.outputSchema ?? S.Any,
     handler: params.handler,
   };
 };
+
+//
+// Subscription utils.
+//
+
+export type RawSubscriptionData = {
+  spaceKey?: string;
+  objects?: string[];
+};
+
+export type SubscriptionData = {
+  space?: Space;
+  objects?: ReactiveEchoObject<any>[];
+};
+
+/**
+ * Handler wrapper for subscription events; extracts space and objects.
+ *
+ * To test:
+ * ```
+ * curl -s -X POST -H "Content-Type: application/json" --data '{"space": "0446...1cbb"}' http://localhost:7100/dev/email-extractor
+ * ```
+ *
+ * NOTE: Get space key from devtools or `dx space list --json`
+ */
+// TODO(burdon): Evolve into plugin definition like Composer.
+export const subscriptionHandler = <TMeta>(
+  handler: FunctionHandler<SubscriptionData, TMeta>,
+  types?: S.Schema.AnyNoContext[],
+): FunctionHandler<RawSubscriptionData, TMeta> => {
+  return async ({ event: { data }, context, response, ...rest }) => {
+    const { client } = context;
+    const space = data.spaceKey ? client.spaces.get(PublicKey.from(data.spaceKey)) : undefined;
+    if (!space) {
+      log.error('Invalid space');
+      return response.status(500);
+    }
+
+    registerTypes(space, types);
+    const objects = space
+      ? data.objects
+          ?.map<ReactiveEchoObject<any> | undefined>((id) => space!.db.getObjectById(id))
+          .filter(isNonNullable)
+      : [];
+
+    if (!!data.spaceKey && !space) {
+      log.warn('invalid space', { data });
+    } else {
+      log.info('handler', { space: space?.key.truncate(), objects: objects?.length });
+    }
+
+    return handler({ event: { data: { ...data, space, objects } }, context, response, ...rest });
+  };
+};
+
+// TODO(burdon): Evolve types as part of function metadata.
+const registerTypes = (space: Space, types: S.Schema.AnyNoContext[] = []) => {
+  const registry = space.db.graph.schemaRegistry;
+  for (const type of types) {
+    if (!registry.hasSchema(type)) {
+      registry.addSchema([type]);
+    }
+  }
+};

package/src/index.ts

@@ -1,9 +1,10 @@
 //
-// Copyright 2024 DXOS.org
+// Copyright 2023 DXOS.org
 //
 
+export * from './edge';
+export * from './function';
 export * from './handler';
-export * from './schema';
-export * from './trace';
+// export * from './runtime';
+export * from './trigger';
 export * from './types';
-export * from './url';

package/src/runtime/dev-server.test.ts

@@ -0,0 +1,79 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { getRandomPort } from 'get-port-please';
+import path from 'node:path';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+
+import { sleep, waitForCondition } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { TestBuilder } from '@dxos/client/testing';
+
+import { DevServer } from './dev-server';
+import { FunctionRegistry } from '../function';
+import { createFunctionRuntime, testFunctionManifest } from '../testing';
+import { initFunctionsPlugin } from '../testing/plugin-init';
+
+// TODO(wittjosiah): Doesn't work in vitest.
+describe.skip('dev server', () => {
+  let client: Client;
+  let testBuilder: TestBuilder;
+
+  beforeAll(async () => {
+    testBuilder = new TestBuilder();
+    client = await createFunctionRuntime(testBuilder, initFunctionsPlugin);
+    expect(client.services.services.FunctionRegistryService).to.exist;
+  });
+
+  afterAll(async () => {
+    await testBuilder.destroy();
+  });
+
+  test('function registry open after dev server started', async () => {
+    const { space, registry, server } = await setupTest();
+    await registry.register(space, testFunctionManifest.functions);
+    await server.start();
+    await registry.open();
+    await waitForCondition({ condition: () => server.functions.length > 0 });
+    await expectTestFunctionInvocable(server);
+  });
+
+  test('function registry open before dev server started', async () => {
+    const { space, registry, server } = await setupTest();
+    await registry.register(space, testFunctionManifest.functions);
+    await registry.open();
+    await server.start();
+    await waitForCondition({ condition: () => server.functions.length > 0 });
+    await expectTestFunctionInvocable(server);
+  });
+
+  test('unsubscribes from functions after stopped', async () => {
+    const { space, registry, server } = await setupTest();
+    await registry.register(space, testFunctionManifest.functions);
+    await server.start();
+    await server.stop();
+    await registry.open();
+    await sleep(20);
+    expect(server.functions.length).to.eq(0);
+  });
+
+  const expectTestFunctionInvocable = async (server: DevServer) => {
+    const seq = server.stats.seq;
+    await server.invoke('test', {});
+    expect(server.stats.seq).to.eq(seq + 1);
+  };
+
+  const setupTest = async () => {
+    const registry = new FunctionRegistry(client);
+    const server = new DevServer(client, registry, {
+      baseDir: path.join(__dirname, '../testing'),
+      port: await getRandomPort('127.0.0.1'),
+    });
+    const space = await client.spaces.create();
+    // TODO(burdon): Doesn't shut down cleanly.
+    //  Error: invariant violation [this._client.services.services.FunctionRegistryService]
+    testBuilder.ctx.onDispose(() => server.stop());
+    return { registry, server, space };
+  };
+});

package/src/runtime/dev-server.ts

@@ -0,0 +1,240 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import express from 'express';
+import { getPort } from 'get-port-please';
+import type http from 'http';
+import { join } from 'node:path';
+
+import { asyncTimeout, Event, Trigger } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { Context } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+import { type FunctionRegistry } from '../function';
+import { type FunctionContext, type FunctionEvent, type FunctionHandler, type FunctionResponse } from '../handler';
+import { type FunctionDef } from '../types';
+
+const FN_TIMEOUT = 20_000;
+
+export type DevServerOptions = {
+  baseDir: string;
+  port?: number;
+  reload?: boolean;
+  dataDir?: string;
+};
+
+/**
+ * Functions dev server provides a local HTTP server for loading and invoking functions.
+ * Functions are executed in the context of an authenticated client.
+ */
+export class DevServer {
+  private _ctx = createContext();
+
+  // Function handlers indexed by name (URL path).
+  private readonly _handlers: Record<string, { def: FunctionDef; handler: FunctionHandler<any> }> = {};
+
+  private _server?: http.Server;
+  private _port?: number;
+  private _functionServiceRegistration?: string;
+  private _proxy?: string;
+  private _seq = 0;
+
+  public readonly update = new Event<number>();
+
+  constructor(
+    private readonly _client: Client,
+    private readonly _functionsRegistry: FunctionRegistry,
+    private readonly _options: DevServerOptions,
+  ) {}
+
+  get stats() {
+    return {
+      seq: this._seq,
+    };
+  }
+
+  get endpoint() {
+    invariant(this._port);
+    return `http://localhost:${this._port}`;
+  }
+
+  get proxy() {
+    return this._proxy;
+  }
+
+  get functions() {
+    return Object.values(this._handlers);
+  }
+
+  async start() {
+    invariant(!this._server);
+    log.info('starting...');
+    this._ctx = createContext();
+
+    // TODO(burdon): Change to hono.
+    const app = express();
+    app.use(express.json());
+
+    app.post('/:path', async (req, res) => {
+      const { path } = req.params;
+      try {
+        log.info('calling', { path });
+        if (this._options.reload) {
+          const { def } = this._handlers['/' + path];
+          await this._load(def, true);
+        }
+
+        // TODO(burdon): Get function context.
+        res.statusCode = await asyncTimeout(this.invoke('/' + path, req.body), FN_TIMEOUT);
+        res.end();
+      } catch (err: any) {
+        log.catch(err);
+        res.statusCode = 500;
+        res.end();
+      }
+    });
+
+    this._port = this._options.port ?? (await getPort({ host: 'localhost', port: 7200, portRange: [7200, 7299] }));
+    this._server = app.listen(this._port);
+
+    try {
+      // Register functions.
+      const { registrationId, endpoint } = await this._client.services.services.FunctionRegistryService!.register({
+        endpoint: this.endpoint,
+      });
+
+      log.info('registered', { endpoint });
+      this._proxy = endpoint;
+      this._functionServiceRegistration = registrationId;
+
+      // Open after registration, so that it can be updated with the list of function definitions.
+      await this._handleNewFunctions(this._functionsRegistry.getUniqueByUri());
+      this._ctx.onDispose(this._functionsRegistry.registered.on(({ added }) => this._handleNewFunctions(added)));
+    } catch (err: any) {
+      await this.stop();
+      throw new Error('FunctionRegistryService not available (check plugin is configured).');
+    }
+
+    log.info('started', { port: this._port });
+  }
+
+  async stop() {
+    if (!this._server) {
+      return;
+    }
+
+    log.info('stopping...');
+    await this._ctx.dispose();
+
+    const trigger = new Trigger();
+    this._server.close(async () => {
+      log.info('server stopped');
+      try {
+        if (this._functionServiceRegistration) {
+          invariant(this._client.services.services.FunctionRegistryService);
+          await this._client.services.services.FunctionRegistryService.unregister({
+            registrationId: this._functionServiceRegistration,
+          });
+
+          log.info('unregistered', { registrationId: this._functionServiceRegistration });
+          this._functionServiceRegistration = undefined;
+          this._proxy = undefined;
+        }
+
+        trigger.wake();
+      } catch (err) {
+        trigger.throw(err as Error);
+      }
+    });
+
+    await trigger.wait();
+    this._port = undefined;
+    this._server = undefined;
+    log.info('stopped');
+  }
+
+  private async _handleNewFunctions(newFunctions: FunctionDef[]) {
+    newFunctions.forEach((def) => this._load(def));
+    await this._safeUpdateRegistration();
+    log('new functions loaded', { newFunctions });
+  }
+
+  /**
+   * Load function.
+   */
+  private async _load(def: FunctionDef, force?: boolean | undefined) {
+    const { uri, route, handler } = def;
+    const filePath = join(this._options.baseDir, handler);
+    log.info('loading', { uri, force });
+
+    // Remove from cache.
+    if (force) {
+      Object.keys(require.cache)
+        .filter((key) => key.startsWith(filePath))
+        .forEach((key) => {
+          delete require.cache[key];
+        });
+    }
+
+    // TODO(burdon): Import types.
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const module = require(filePath);
+    if (typeof module.default !== 'function') {
+      throw new Error(`Handler must export default function: ${uri}`);
+    }
+
+    this._handlers[route] = { def, handler: module.default };
+  }
+
+  private async _safeUpdateRegistration(): Promise<void> {
+    invariant(this._functionServiceRegistration);
+    try {
+      await this._client.services.services.FunctionRegistryService!.updateRegistration({
+        registrationId: this._functionServiceRegistration,
+        functions: this.functions.map(({ def: { id, route } }) => ({ id, route })),
+      });
+    } catch (err) {
+      log.catch(err);
+    }
+  }
+
+  /**
+   * Invoke function.
+   */
+  public async invoke(path: string, data: any): Promise<number> {
+    const seq = ++this._seq;
+    const now = Date.now();
+
+    log.info('req', { seq, path });
+    const statusCode = await this._invoke(path, { data });
+
+    log.info('res', { seq, path, statusCode, duration: Date.now() - now });
+    this.update.emit(statusCode);
+    return statusCode;
+  }
+
+  private async _invoke(path: string, event: FunctionEvent) {
+    const { handler } = this._handlers[path] ?? {};
+    invariant(handler, `invalid path: ${path}`);
+    const context: FunctionContext = {
+      client: this._client,
+      dataDir: this._options.dataDir,
+    } as any;
+
+    let statusCode = 200;
+    const response: FunctionResponse = {
+      status: (code: number) => {
+        statusCode = code;
+        return response;
+      },
+    };
+
+    await handler({ context, event, response });
+    return statusCode;
+  }
+}
+
+const createContext = () => new Context({ name: 'DevServer' });

package/src/runtime/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+export * from './dev-server';
+export * from './scheduler';

package/src/runtime/scheduler.test.ts

@@ -0,0 +1,152 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+
+import { Trigger } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { TestBuilder } from '@dxos/client/testing';
+import { create } from '@dxos/live-object';
+
+import { Scheduler, type SchedulerOptions } from './scheduler';
+import { FunctionRegistry } from '../function';
+import { createInitializedClients, TestType, triggerWebhook } from '../testing';
+import { TriggerRegistry } from '../trigger';
+import { TriggerKind, type FunctionManifest } from '../types';
+
+// TODO(burdon): Test we can add and remove triggers.
+// Flaky: https://cloud.nx.app/runs/uqhKOBA6JQ/task/functions%3Atest
+describe.skip('scheduler', () => {
+  let testBuilder: TestBuilder;
+  let client: Client;
+
+  beforeAll(async () => {
+    testBuilder = new TestBuilder();
+    client = (await createInitializedClients(testBuilder, 1))[0];
+  });
+
+  afterAll(async () => {
+    await testBuilder.destroy();
+  });
+
+  const createScheduler = (callback: SchedulerOptions['callback']) => {
+    const scheduler = new Scheduler(new FunctionRegistry(client), new TriggerRegistry(client), { callback });
+
+    afterAll(async () => {
+      await scheduler.stop();
+    });
+
+    return scheduler;
+  };
+
+  test('timer', async () => {
+    const manifest: FunctionManifest = {
+      functions: [
+        {
+          uri: 'example.com/function/test',
+          route: '/test',
+          handler: 'test',
+        },
+      ],
+      triggers: [
+        {
+          function: 'example.com/function/test',
+          enabled: true,
+          spec: {
+            type: TriggerKind.Timer,
+            cron: '0/1 * * * * *', // Every 1s.
+          },
+        },
+      ],
+    };
+
+    let count = 0;
+    const done = new Trigger();
+    const scheduler = createScheduler(async () => {
+      if (++count === 3) {
+        done.wake();
+      }
+    });
+    await scheduler.register(client.spaces.default, manifest);
+    await scheduler.start();
+
+    await done.wait({ timeout: 5_000 });
+    expect(count).to.equal(3);
+  });
+
+  // Flaky.
+  test.skip('webhook', async () => {
+    const manifest: FunctionManifest = {
+      functions: [
+        {
+          uri: 'example.com/function/test',
+          route: '/test',
+          handler: 'test',
+        },
+      ],
+      triggers: [
+        {
+          function: 'example.com/function/test',
+          enabled: true,
+          spec: {
+            type: TriggerKind.Webhook,
+            method: 'GET',
+          },
+        },
+      ],
+    };
+
+    const done = new Trigger();
+    const scheduler = createScheduler(async () => {
+      done.wake();
+    });
+    const space = await client.spaces.create();
+    await scheduler.register(space, manifest);
+    await scheduler.start();
+
+    setTimeout(async () => triggerWebhook(space, manifest.functions![0].uri));
+
+    await done.wait();
+  });
+
+  test('subscription', async () => {
+    const manifest: FunctionManifest = {
+      functions: [
+        {
+          uri: 'example.com/function/test',
+          route: '/test',
+          handler: 'test',
+        },
+      ],
+      triggers: [
+        {
+          function: 'example.com/function/test',
+          enabled: true,
+          spec: {
+            type: TriggerKind.Subscription,
+            filter: { type: TestType.typename },
+          },
+        },
+      ],
+    };
+
+    let count = 0;
+    const done = new Trigger();
+    const scheduler = createScheduler(async () => {
+      if (++count === 1) {
+        done.wake();
+      }
+    });
+    await scheduler.register(client.spaces.default, manifest);
+    await scheduler.start();
+
+    setTimeout(() => {
+      const space = client.spaces.default;
+      const object = create(TestType, { title: 'Hello world!' });
+      space.db.add(object);
+    }, 100);
+
+    await done.wait();
+  });
+});