@dxos/functions 0.5.3-main.59db342 → 0.5.3-main.61bbff4

package/src/handler.ts

@@ -8,31 +8,61 @@ import { type EchoReactiveObject } from '@dxos/echo-schema';
 import { log } from '@dxos/log';
 import { nonNullable } from '@dxos/util';
 
-// TODO(burdon): No response?
-export interface Response {
-  status(code: number): Response;
-}
+// TODO(burdon): Model after http request. Ref Lambda/OpenFaaS.
+// https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
+// https://www.serverless.com/framework/docs/providers/aws/guide/serverless.yml/#functions
+// https://www.npmjs.com/package/aws-lambda
+
+/**
+ * Function handler.
+ */
+export type FunctionHandler<TData = {}, TMeta = {}> = (params: {
+  context: FunctionContext;
+  event: FunctionEvent<TData, TMeta>;
+  response: FunctionResponse;
+}) => Promise<FunctionResponse | void>;
 
-// TODO(burdon): Limit access to individual space?
+/**
+ * Function context.
+ */
 export interface FunctionContext {
+  // TODO(burdon): Limit access to individual space.
   client: Client;
+  // TODO(burdon): Replace with storage service abstraction.
   dataDir?: string;
 }
 
-// TODO(burdon): Model after http request. Ref Lambda/OpenFaaS.
-// https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
-export type FunctionHandler<T extends {}> = (params: {
-  event: T;
-  context: FunctionContext;
-  response: Response;
-}) => Promise<Response | void>;
+/**
+ * Event payload.
+ */
+export type FunctionEvent<TData = {}, TMeta = {}> = {
+  data: FunctionEventMeta<TMeta> & TData;
+};
+
+/**
+ * Metadata from trigger.
+ */
+export type FunctionEventMeta<TMeta = {}> = {
+  meta: TMeta;
+};
+
+/**
+ * Function response.
+ */
+export interface FunctionResponse {
+  status(code: number): FunctionResponse;
+}
+
+//
+// Subscription utils.
+//
 
-export type FunctionSubscriptionEvent = {
-  space?: string; // TODO(burdon): Convert to PublicKey.
+export type RawSubscriptionData = {
+  spaceKey?: string;
   objects?: string[];
 };
 
-export type FunctionSubscriptionEvent2 = {
+export type SubscriptionData = {
   space?: Space;
   objects?: EchoReactiveObject<any>[];
 };
@@ -47,22 +77,22 @@ export type FunctionSubscriptionEvent2 = {
  *
  * NOTE: Get space key from devtools or `dx space list --json`
  */
-export const subscriptionHandler = (
-  handler: FunctionHandler<FunctionSubscriptionEvent2>,
-): FunctionHandler<FunctionSubscriptionEvent> => {
-  return ({ event, context, ...rest }) => {
+export const subscriptionHandler = <TMeta>(
+  handler: FunctionHandler<SubscriptionData, TMeta>,
+): FunctionHandler<RawSubscriptionData, TMeta> => {
+  return ({ event: { data }, context, ...rest }) => {
     const { client } = context;
-    const space = event.space ? client.spaces.get(PublicKey.from(event.space)) : undefined;
-    const objects =
-      space &&
-      event.objects?.map<EchoReactiveObject<any> | undefined>((id) => space!.db.getObjectById(id)).filter(nonNullable);
+    const space = data.spaceKey ? client.spaces.get(PublicKey.from(data.spaceKey)) : undefined;
+    const objects = space
+      ? data.objects?.map<EchoReactiveObject<any> | undefined>((id) => space!.db.getObjectById(id)).filter(nonNullable)
+      : [];
 
-    if (!!event.space && !space) {
-      log.warn('invalid space', { event });
+    if (!!data.spaceKey && !space) {
+      log.warn('invalid space', { data });
     } else {
       log.info('handler', { space: space?.key.truncate(), objects: objects?.length });
     }
 
-    return handler({ event: { space, objects }, context, ...rest });
+    return handler({ event: { data: { ...data, space, objects } }, context, ...rest });
   };
 };

package/src/index.ts

@@ -3,5 +3,7 @@
 //
 
 export * from './handler';
-export * from './manifest';
+export * from './registry';
 export * from './runtime';
+export * from './trigger';
+export * from './types';

package/src/registry/function-registry.test.ts

@@ -0,0 +1,105 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { Trigger } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { TestBuilder } from '@dxos/client/testing';
+import { Context } from '@dxos/context';
+import { Filter } from '@dxos/echo-db';
+import { create } from '@dxos/echo-schema';
+import { describe, test } from '@dxos/test';
+import { range } from '@dxos/util';
+
+import { FunctionRegistry } from './function-registry';
+import { createInitializedClients } from '../testing';
+import { FunctionDef, type FunctionManifest } from '../types';
+
+const testManifest: FunctionManifest = {
+  functions: [
+    {
+      uri: 'dxos.functions.test/hello',
+      route: '/hello',
+      handler: 'test',
+    },
+  ],
+};
+
+describe('function registry', () => {
+  let ctx: Context;
+  let testBuilder: TestBuilder;
+  beforeEach(async () => {
+    ctx = new Context();
+    testBuilder = new TestBuilder();
+  });
+  afterEach(async () => {
+    await ctx.dispose();
+    await testBuilder.destroy();
+  });
+
+  describe('register', () => {
+    test('creates new functions', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+      await registry.register(space, testManifest);
+      const { objects: definitions } = await space.db.query(Filter.schema(FunctionDef)).run();
+      expect(definitions.length).to.eq(1);
+      expect(definitions[0].uri).to.eq(testManifest.functions?.[0]?.uri);
+    });
+
+    test('de-duplicates by function URI', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+      const existing = space.db.add(create(FunctionDef, { ...testManifest.functions![0] }));
+      await registry.register(space, testManifest);
+      const { objects: definitions } = await space.db.query(Filter.schema(FunctionDef)).run();
+      expect(definitions.length).to.eq(1);
+      expect(definitions[0].uri).to.eq(existing.uri);
+    });
+  });
+
+  describe('onFunctionsRegistered', () => {
+    test('called with all registered when opened', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+      const definitions = range(3, () => create(FunctionDef, { ...testManifest.functions![0] }));
+      definitions.forEach((def) => space.db.add(def));
+
+      const functionRegistered = new Trigger<FunctionDef[]>();
+      registry.onFunctionsRegistered.on((fn) => {
+        functionRegistered.wake(fn.newFunctions);
+      });
+      void registry.open(ctx);
+      const functions = await functionRegistered.wait();
+      const expected = definitions.map((def) => def.uri).sort();
+      expect(functions.map((fn) => fn.uri).sort()).to.deep.eq(expected);
+    });
+
+    test('called when a new functions is added', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+
+      const functionRegistered = new Trigger<FunctionDef>();
+      registry.onFunctionsRegistered.on((fn) => {
+        expect(fn.newFunctions.length).to.eq(1);
+        functionRegistered.wake(fn.newFunctions[0]);
+      });
+      await registry.open(ctx);
+      await registry.register(space, testManifest);
+      const registered = await functionRegistered.wait();
+      expect(registered.uri).to.eq(testManifest.functions![0].uri);
+    });
+  });
+
+  const createRegistry = (client: Client) => {
+    const registry = new FunctionRegistry(client);
+    ctx.onDispose(() => registry.close());
+    return registry;
+  };
+});

package/src/registry/function-registry.ts

@@ -0,0 +1,84 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { create, Filter, type Space } from '@dxos/client/echo';
+import { type Context, Resource } from '@dxos/context';
+import { PublicKey } from '@dxos/keys';
+import { ComplexMap } from '@dxos/util';
+
+import { FunctionDef, type FunctionManifest } from '../types';
+
+export type FunctionsRegisteredEvent = {
+  space: Space;
+  newFunctions: FunctionDef[];
+};
+
+export class FunctionRegistry extends Resource {
+  private readonly _functionBySpaceKey = new ComplexMap<PublicKey, FunctionDef[]>(PublicKey.hash);
+
+  public readonly onFunctionsRegistered = new Event<FunctionsRegisteredEvent>();
+
+  constructor(private readonly _client: Client) {
+    super();
+  }
+
+  public getFunctions(space: Space): FunctionDef[] {
+    return this._functionBySpaceKey.get(space.key) ?? [];
+  }
+
+  /**
+   * The method loads function definitions from the manifest into the space.
+   * We first load all the definitions from the space to deduplicate by functionId.
+   */
+  // TODO(burdon): This should not be space specific (they are static for the agent).
+  public async register(space: Space, manifest: FunctionManifest): Promise<void> {
+    if (!manifest.functions?.length) {
+      return;
+    }
+    if (!space.db.graph.runtimeSchemaRegistry.hasSchema(FunctionDef)) {
+      space.db.graph.runtimeSchemaRegistry.registerSchema(FunctionDef);
+    }
+
+    const { objects: existingDefinitions } = await space.db.query(Filter.schema(FunctionDef)).run();
+    const newDefinitions = getNewDefinitions(manifest.functions, existingDefinitions);
+    const reactiveObjects = newDefinitions.map((template) => create(FunctionDef, { ...template }));
+    reactiveObjects.forEach((obj) => space.db.add(obj));
+  }
+
+  protected override async _open(): Promise<void> {
+    const spaceListSubscription = this._client.spaces.subscribe(async (spaces) => {
+      for (const space of spaces) {
+        if (this._functionBySpaceKey.has(space.key)) {
+          continue;
+        }
+        const registered: FunctionDef[] = [];
+        this._functionBySpaceKey.set(space.key, registered);
+        await space.waitUntilReady();
+        if (this._ctx.disposed) {
+          break;
+        }
+
+        const functionsSubscription = space.db.query(Filter.schema(FunctionDef)).subscribe((definitions) => {
+          const newFunctions = getNewDefinitions(definitions.objects, registered);
+          if (newFunctions.length > 0) {
+            registered.push(...newFunctions);
+            this.onFunctionsRegistered.emit({ space, newFunctions });
+          }
+        });
+        this._ctx.onDispose(functionsSubscription);
+      }
+    });
+    this._ctx.onDispose(() => spaceListSubscription.unsubscribe());
+  }
+
+  protected override async _close(_: Context): Promise<void> {
+    this._functionBySpaceKey.clear();
+  }
+}
+
+const getNewDefinitions = <T extends { uri: string }>(candidateList: T[], existing: FunctionDef[]): T[] => {
+  return candidateList.filter((candidate) => existing.find((def) => def.uri === candidate.uri) == null);
+};

package/src/registry/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './function-registry';

package/src/runtime/dev-server.test.ts

@@ -0,0 +1,60 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { expect } from 'chai';
+import path from 'path';
+
+import { waitForCondition } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { TestBuilder } from '@dxos/client/testing';
+import { describe, test } from '@dxos/test';
+
+import { DevServer } from './dev-server';
+import { FunctionRegistry } from '../registry';
+import { createFunctionRuntime } from '../testing';
+import { type FunctionManifest } from '../types';
+
+describe('dev server', () => {
+  let client: Client;
+  let testBuilder: TestBuilder;
+  before(async () => {
+    testBuilder = new TestBuilder();
+    client = await createFunctionRuntime(testBuilder);
+    expect(client.services.services.FunctionRegistryService).to.exist;
+  });
+
+  after(async () => {
+    await testBuilder.destroy();
+  });
+
+  test('start/stop', async () => {
+    const manifest: FunctionManifest = {
+      functions: [
+        {
+          uri: 'example.com/function/test',
+          route: 'test',
+          handler: 'test',
+        },
+      ],
+    };
+
+    const registry = new FunctionRegistry(client);
+    const server = new DevServer(client, registry, {
+      baseDir: path.join(__dirname, '../testing'),
+    });
+    const space = await client.spaces.create();
+    await registry.register(space, manifest);
+    await server.start();
+
+    // TODO(burdon): Doesn't shut down cleanly.
+    //  Error: invariant violation [this._client.services.services.FunctionRegistryService]
+    testBuilder.ctx.onDispose(() => server.stop());
+    expect(server).to.exist;
+
+    await waitForCondition({ condition: () => server.functions.length > 0 });
+
+    await server.invoke('test', {});
+    expect(server.stats.seq).to.eq(1);
+  });
+});

package/src/runtime/dev-server.ts

@@ -7,18 +7,19 @@ import { getPort } from 'get-port-please';
 import type http from 'http';
 import { join } from 'node:path';
 
-import { Trigger } from '@dxos/async';
+import { Event, Trigger } from '@dxos/async';
 import { type Client } from '@dxos/client';
+import { Context } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
-import { type FunctionContext, type FunctionHandler, type Response } from '../handler';
-import { type FunctionDef, type FunctionManifest } from '../manifest';
+import { type FunctionContext, type FunctionEvent, type FunctionHandler, type FunctionResponse } from '../handler';
+import { type FunctionRegistry } from '../registry';
+import { type FunctionDef } from '../types';
 
 export type DevServerOptions = {
+  baseDir: string;
   port?: number;
-  directory: string;
-  manifest: FunctionManifest;
   reload?: boolean;
   dataDir?: string;
 };
@@ -27,20 +28,37 @@ export type DevServerOptions = {
  * Functions dev server provides a local HTTP server for testing functions.
  */
 export class DevServer {
+  private _ctx = createContext();
+
   // Function handlers indexed by name (URL path).
   private readonly _handlers: Record<string, { def: FunctionDef; handler: FunctionHandler<any> }> = {};
 
   private _server?: http.Server;
   private _port?: number;
-  private _registrationId?: string;
+  private _functionServiceRegistration?: string;
   private _proxy?: string;
   private _seq = 0;
 
+  public readonly update = new Event<number>();
+
   // prettier-ignore
   constructor(
     private readonly _client: Client,
+    private readonly _functionsRegistry: FunctionRegistry,
     private readonly _options: DevServerOptions,
-  ) {}
+  ) {
+    this._functionsRegistry.onFunctionsRegistered.on(async ({ newFunctions }) => {
+      newFunctions.forEach((def) => this._load(def));
+      await this._safeUpdateRegistration();
+      log('new functions loaded', { newFunctions });
+    });
+  }
+
+  get stats() {
+    return {
+      seq: this._seq,
+    };
+  }
 
   get endpoint() {
     invariant(this._port);
@@ -55,30 +73,26 @@ export class DevServer {
     return Object.values(this._handlers);
   }
 
-  async initialize() {
-    for (const def of this._options.manifest.functions) {
-      try {
-        await this._load(def);
-      } catch (err) {
-        log.error('parsing function (check manifest)', err);
-      }
-    }
-  }
-
   async start() {
+    invariant(!this._server);
+    log.info('starting...');
+    this._ctx = createContext();
+
+    // TODO(burdon): Move to hono.
     const app = express();
     app.use(express.json());
 
-    app.post('/:name', async (req, res) => {
-      const { name } = req.params;
+    app.post('/:path', async (req, res) => {
+      const { path } = req.params;
       try {
-        log.info('calling', { name });
+        log.info('calling', { path });
         if (this._options.reload) {
-          const { def } = this._handlers[name];
+          const { def } = this._handlers['/' + path];
           await this._load(def, true);
         }
 
-        res.statusCode = await this._invoke(name, req.body);
+        // TODO(burdon): Get function context.
+        res.statusCode = await this.invoke('/' + path, req.body);
         res.end();
       } catch (err: any) {
         log.catch(err);
@@ -94,72 +108,110 @@ export class DevServer {
       // Register functions.
       const { registrationId, endpoint } = await this._client.services.services.FunctionRegistryService!.register({
         endpoint: this.endpoint,
-        functions: this.functions.map(({ def: { name } }) => ({ name })),
       });
 
-      log.info('registered', { registrationId, endpoint });
-      this._registrationId = registrationId;
+      log.info('registered', { endpoint });
       this._proxy = endpoint;
+      this._functionServiceRegistration = registrationId;
+
+      // Open after registration, so that it can be updated with the list of function definitions.
+      await this._functionsRegistry.open(this._ctx);
     } catch (err: any) {
       await this.stop();
       throw new Error('FunctionRegistryService not available (check plugin is configured).');
     }
+
+    log.info('started', { port: this._port });
   }
 
   async stop() {
+    invariant(this._server);
+    log.info('stopping...');
+
     const trigger = new Trigger();
-    this._server?.close(async () => {
-      if (this._registrationId) {
-        await this._client.services.services.FunctionRegistryService!.unregister({
-          registrationId: this._registrationId,
-        });
+    this._server.close(async () => {
+      log.info('server stopped');
+      try {
+        if (this._functionServiceRegistration) {
+          invariant(this._client.services.services.FunctionRegistryService);
+          await this._client.services.services.FunctionRegistryService.unregister({
+            registrationId: this._functionServiceRegistration,
+          });
+
+          log.info('unregistered', { registrationId: this._functionServiceRegistration });
+          this._functionServiceRegistration = undefined;
+          this._proxy = undefined;
+        }
 
-        log.info('unregistered', { registrationId: this._registrationId });
-        this._registrationId = undefined;
-        this._proxy = undefined;
+        trigger.wake();
+      } catch (err) {
+        trigger.throw(err as Error);
       }
-
-      trigger.wake();
     });
 
     await trigger.wait();
     this._port = undefined;
     this._server = undefined;
+    log.info('stopped');
   }
 
   /**
    * Load function.
    */
-  private async _load(def: FunctionDef, flush = false) {
-    const { id, name, handler } = def;
-    const path = join(this._options.directory, handler);
-    log.info('loading', { id });
+  private async _load(def: FunctionDef, force = false) {
+    const { uri, route, handler } = def;
+    const filePath = join(this._options.baseDir, handler);
+    log.info('loading', { uri, force });
 
     // Remove from cache.
-    if (flush) {
+    if (force) {
       Object.keys(require.cache)
-        .filter((key) => key.startsWith(path))
-        .forEach((key) => delete require.cache[key]);
+        .filter((key) => key.startsWith(filePath))
+        .forEach((key) => {
+          delete require.cache[key];
+        });
     }
 
+    // TODO(burdon): Import types.
     // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const module = require(path);
+    const module = require(filePath);
     if (typeof module.default !== 'function') {
-      throw new Error(`Handler must export default function: ${id}`);
+      throw new Error(`Handler must export default function: ${uri}`);
     }
 
-    this._handlers[name] = { def, handler: module.default };
+    this._handlers[route] = { def, handler: module.default };
+  }
+
+  private async _safeUpdateRegistration(): Promise<void> {
+    invariant(this._functionServiceRegistration);
+    try {
+      await this._client.services.services.FunctionRegistryService!.updateRegistration({
+        registrationId: this._functionServiceRegistration,
+        functions: this.functions.map(({ def: { id, route } }) => ({ id, route })),
+      });
+    } catch (e) {
+      log.catch(e);
+    }
   }
 
   /**
-   * Invoke function handler.
+   * Invoke function.
    */
-  private async _invoke(name: string, event: any) {
+  public async invoke(path: string, data: any): Promise<number> {
     const seq = ++this._seq;
     const now = Date.now();
 
-    log.info('req', { seq, name });
-    const { handler } = this._handlers[name];
+    log.info('req', { seq, path });
+    const statusCode = await this._invoke(path, { data });
+
+    log.info('res', { seq, path, statusCode, duration: Date.now() - now });
+    this.update.emit(statusCode);
+    return statusCode;
+  }
+
+  private async _invoke(path: string, event: FunctionEvent) {
+    const { handler } = this._handlers[path] ?? {};
+    invariant(handler, `invalid path: ${path}`);
 
     const context: FunctionContext = {
       client: this._client,
@@ -167,7 +219,7 @@ export class DevServer {
     };
 
     let statusCode = 200;
-    const response: Response = {
+    const response: FunctionResponse = {
       status: (code: number) => {
         statusCode = code;
         return response;
@@ -175,8 +227,8 @@ export class DevServer {
     };
 
     await handler({ context, event, response });
-    log.info('res', { seq, name, statusCode, duration: Date.now() - now });
-
     return statusCode;
   }
 }
+
+const createContext = () => new Context({ name: 'DevServer' });