@dxos/functions 0.5.3-main.f752aaa → 0.5.3-next.57eca40

package/src/testing/functions-integration.test.ts

@@ -0,0 +1,99 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+import path from 'path';
+
+import { Trigger, waitForCondition } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { create, type Space } from '@dxos/client/echo';
+import { performInvitation, TestBuilder } from '@dxos/client/testing';
+import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
+import { describe, test } from '@dxos/test';
+
+import { setTestCallHandler } from './test/handler';
+import { FunctionRegistry } from '../function';
+import { DevServer, Scheduler } from '../runtime';
+import { createFunctionRuntime, createInitializedClients, TestType } from '../testing';
+import { TriggerRegistry } from '../trigger';
+import { FunctionDef, FunctionTrigger } from '../types';
+
+describe('functions e2e', () => {
+  let testBuilder: TestBuilder;
+  before(async () => {
+    testBuilder = new TestBuilder();
+  });
+  after(async () => {
+    await testBuilder.destroy();
+  });
+
+  test('a function gets triggered in response to another peer object creations', async () => {
+    // TODO(burdon): Create builder pattern.
+    const functionRuntime = await createFunctionRuntime(testBuilder);
+    const devServer = await startDevServer(functionRuntime);
+    const scheduler = await startScheduler(functionRuntime, devServer);
+
+    const app = (await createInitializedClients(testBuilder, 1))[0];
+    const space = await app.spaces.create();
+    await inviteMember(space, functionRuntime);
+
+    const uri = 'example.com/function/test';
+    space.db.add(create(FunctionDef, { uri, route: '/test', handler: 'test' }));
+    const triggerMeta: FunctionTrigger['meta'] = { name: 'DXOS' };
+    space.db.add(
+      create(FunctionTrigger, {
+        function: uri,
+        meta: triggerMeta,
+        spec: {
+          type: 'subscription',
+          filter: [{ type: TestType.typename }],
+        },
+      }),
+    );
+
+    const called = new Trigger<any>();
+    setTestCallHandler(async (args) => {
+      called.wake(args.event.data);
+      return args.response.status(200);
+    });
+
+    await waitTriggersReplicated(space, scheduler);
+    const addedObject = space.db.add(create(TestType, { title: '42' }));
+
+    const callArgs = await called.wait();
+    expect(callArgs.meta).to.deep.eq(triggerMeta);
+    expect(callArgs.objects).to.deep.eq([addedObject.id]);
+    expect(callArgs.spaceKey).to.eq(space.key.toHex());
+  });
+
+  const waitTriggersReplicated = async (space: Space, scheduler: Scheduler) => {
+    await waitForCondition({ condition: () => scheduler.triggers.getActiveTriggers(space).length > 0 });
+  };
+
+  // TODO(burdon): Factor out utils to builder pattern.
+
+  const startScheduler = async (client: Client, devServer: DevServer) => {
+    const functionRegistry = new FunctionRegistry(client);
+    const triggerRegistry = new TriggerRegistry(client);
+    const scheduler = new Scheduler(functionRegistry, triggerRegistry, { endpoint: devServer.endpoint });
+    await scheduler.start();
+    testBuilder.ctx.onDispose(() => scheduler.stop());
+    return scheduler;
+  };
+
+  const startDevServer = async (client: Client) => {
+    const functionRegistry = new FunctionRegistry(client);
+    const server = new DevServer(client, functionRegistry, {
+      baseDir: path.join(__dirname, '../testing'),
+    });
+    await server.start();
+    testBuilder.ctx.onDispose(() => server.stop());
+    return server;
+  };
+
+  const inviteMember = async (host: Space, guest: Client) => {
+    const [{ invitation: hostInvitation }] = await Promise.all(performInvitation({ host, guest: guest.spaces }));
+    expect(hostInvitation?.state).to.eq(Invitation.State.SUCCESS);
+  };
+});

package/src/testing/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './setup';
+export * from './types';
+export * from './util';

package/src/testing/setup.ts

@@ -0,0 +1,45 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { FunctionsPlugin } from '@dxos/agent';
+import { Client, Config } from '@dxos/client';
+import { type TestBuilder } from '@dxos/client/testing';
+import { range } from '@dxos/util';
+
+import { TestType } from './types';
+import { FunctionDef, FunctionTrigger } from '../types';
+
+// TODO(burdon): Create new or extend existing TestBuilder.
+
+export const createInitializedClients = async (testBuilder: TestBuilder, count: number = 1, config?: Config) => {
+  const clients = range(count).map(() => new Client({ config, services: testBuilder.createLocalClientServices() }));
+  testBuilder.ctx.onDispose(() => Promise.all(clients.map((c) => c.destroy())));
+  return Promise.all(
+    clients.map(async (client, index) => {
+      await client.initialize();
+      await client.halo.createIdentity({ displayName: `Peer ${index}` });
+      client.addSchema(TestType, FunctionDef, FunctionTrigger);
+      return client;
+    }),
+  );
+};
+
+export const createFunctionRuntime = async (testBuilder: TestBuilder): Promise<Client> => {
+  const config = new Config({
+    runtime: {
+      agent: {
+        plugins: [{ id: 'dxos.org/agent/plugin/functions', config: { port: 8080 } }],
+      },
+    },
+  });
+
+  const client = (await createInitializedClients(testBuilder, 1, config))[0];
+
+  // TODO(burdon): Better way to configure plugin? (Rationalize chess.test).
+  const functionsPlugin = new FunctionsPlugin();
+  await functionsPlugin.initialize({ client, clientServices: client.services });
+  await functionsPlugin.open();
+  testBuilder.ctx.onDispose(() => functionsPlugin.close());
+  return client;
+};

package/src/testing/test/handler.ts

@@ -0,0 +1,15 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type FunctionHandler } from '../../handler';
+
+let callHandler: FunctionHandler<any> = async ({ response }) => response.status(200);
+
+export const setTestCallHandler = (handler: FunctionHandler<any>) => {
+  callHandler = handler;
+};
+
+export const handler: FunctionHandler<any> = async (args) => {
+  return callHandler(args);
+};

package/src/testing/test/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { handler } from './handler';
+
+export default handler;

package/src/testing/types.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { S, TypedObject } from '@dxos/echo-schema';
+
+export class TestType extends TypedObject({ typename: 'example.com/type/Test', version: '0.1.0' })({
+  title: S.string,
+}) {}

package/src/testing/util.ts

@@ -0,0 +1,16 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Filter, type Space } from '@dxos/client/echo';
+import { invariant } from '@dxos/invariant';
+
+import { FunctionTrigger } from '../types';
+
+export const triggerWebhook = async (space: Space, uri: string) => {
+  const trigger = (
+    await space.db.query(Filter.schema(FunctionTrigger, (t: FunctionTrigger) => t.function === uri)).run()
+  ).objects[0];
+  invariant(trigger.spec.type === 'webhook');
+  void fetch(`http://localhost:${trigger.spec.port}`);
+};

package/src/trigger/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './trigger-registry';

package/src/trigger/trigger-registry.test.ts

@@ -0,0 +1,255 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+
+import { sleep, Trigger, waitForCondition } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { type Space } from '@dxos/client/echo';
+import { TestBuilder } from '@dxos/client/testing';
+import { Context } from '@dxos/context';
+import { Filter } from '@dxos/echo-db';
+import { create, splitMeta } from '@dxos/echo-schema';
+import { describe, test } from '@dxos/test';
+import { range } from '@dxos/util';
+
+import { TriggerRegistry } from './trigger-registry';
+import { createInitializedClients, TestType, triggerWebhook } from '../testing';
+import { type FunctionManifest, FunctionTrigger } from '../types';
+
+const manifest: FunctionManifest = {
+  triggers: [
+    {
+      '@meta': {
+        keys: [
+          {
+            source: 'example.com',
+            id: 'trigger-1',
+          },
+        ],
+      },
+      function: 'example.com/function/webhook-test',
+      spec: {
+        type: 'webhook',
+        method: 'GET',
+      },
+    },
+    {
+      '@meta': {
+        keys: [
+          {
+            source: 'example.com',
+            id: 'trigger-2',
+          },
+        ],
+      },
+      function: 'example.com/function/subscription-test',
+      spec: {
+        type: 'subscription',
+        filter: [
+          {
+            type: TestType.typename,
+          },
+        ],
+      },
+    },
+  ],
+};
+
+chai.use(chaiAsPromised);
+
+describe('trigger registry', () => {
+  let ctx: Context;
+  let testBuilder: TestBuilder;
+  beforeEach(async () => {
+    ctx = new Context();
+    testBuilder = new TestBuilder();
+  });
+  afterEach(async () => {
+    await ctx.dispose();
+    await testBuilder.destroy();
+  });
+
+  describe('register', () => {
+    test('creates new triggers', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+      await registry.register(space, manifest);
+      const { objects } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      expect(objects.length).to.eq(manifest.triggers?.length);
+
+      const expected = manifest.triggers?.map((trigger) => trigger.function).sort();
+      expect(objects.map((object: FunctionTrigger) => object.function).sort()).to.deep.eq(expected);
+    });
+  });
+
+  describe('activate', () => {
+    test('invokes the provided callback', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const callbackInvoked = new Trigger();
+      const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      const webhookTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'webhook')!;
+      await registry.activate({ space }, webhookTrigger, async () => {
+        callbackInvoked.wake();
+        return 200;
+      });
+
+      setTimeout(() => triggerWebhook(space, webhookTrigger.function));
+      await callbackInvoked.wait();
+    });
+
+    test('removes from inactive list', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const inactiveTrigger = registry.getInactiveTriggers(space)[0];
+      await registry.activate({ space }, inactiveTrigger, async () => 200);
+
+      const updatedInactiveList = registry.getInactiveTriggers(space);
+      expect(updatedInactiveList.find((trigger: FunctionTrigger) => trigger.function === inactiveTrigger.function)).to
+        .be.undefined;
+    });
+  });
+
+  describe('deactivate', () => {
+    test('trigger object deletion deactivates a trigger', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      const echoTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'subscription')!;
+      let count = 0;
+      await registry.activate({ space }, echoTrigger, async () => {
+        count++;
+        return 200;
+      });
+
+      space.db.add(create(TestType, { title: '1' }));
+      await sleep(20);
+      expect(count).to.eq(1);
+
+      space.db.remove(echoTrigger);
+      space.db.add(create(TestType, { title: '2' }));
+      await sleep(20);
+      expect(count).to.eq(1);
+    });
+
+    test('registry closing deactivates a trigger', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      const echoTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'subscription')!;
+      let count = 0;
+      await registry.activate({ space }, echoTrigger, async () => {
+        count++;
+        return 200;
+      });
+
+      await registry.close();
+
+      space.db.add(create(TestType, { title: '1' }));
+      await sleep(20);
+      expect(count).to.eq(0);
+    });
+  });
+
+  describe('trigger events', () => {
+    test.only('event fired when all registered when opened', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const triggers = createTriggers(client.spaces.default, 3);
+
+      const triggersRegistered = new Trigger<FunctionTrigger[]>();
+      registry.registered.on((fn) => {
+        expect(fn.space.key.toHex()).to.eq(client.spaces.default.key.toHex());
+        triggersRegistered.wake(fn.triggers);
+      });
+
+      void registry.open(ctx);
+      const functions = await triggersRegistered.wait();
+      const expected = triggers.map((object) => object.id).sort();
+      expect(functions.map((fn) => fn.id).sort()).to.deep.eq(expected);
+    });
+
+    test('event fired when a new trigger is added', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+
+      const triggerRegistered = new Trigger<FunctionTrigger>();
+      registry.registered.on((fn) => {
+        expect(fn.triggers.length).to.eq(1);
+        triggerRegistered.wake(fn.triggers[0]);
+      });
+      await registry.open(ctx);
+      await registry.register(space, { triggers: manifest?.triggers?.slice(0, 1) });
+      const registered = await triggerRegistered.wait();
+      expect(registered.function).to.eq(manifest.triggers![0].function);
+    });
+
+    test('event fired when a new trigger is removed', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+      const triggers = createTriggers(space, 3);
+
+      const triggerLoaded = new Trigger();
+      registry.registered.on((fn) => triggerLoaded.wake());
+
+      const triggerRemoved = new Trigger<FunctionTrigger>();
+      registry.removed.on((fn) => {
+        expect(fn.triggers.length).to.eq(1);
+        triggerRemoved.wake(fn.triggers[0]);
+      });
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await triggerLoaded.wait();
+
+      space.db.remove(triggers[0]);
+      const removedTrigger = await triggerRemoved.wait();
+      expect(removedTrigger.id).to.eq(triggers[0].id);
+    });
+  });
+
+  const createRegistry = (client: Client) => {
+    const registry = new TriggerRegistry(client);
+    ctx.onDispose(() => registry.close());
+    return registry;
+  };
+
+  const createTriggers = (space: Space, count: number) => {
+    const triggers = range(count, () => {
+      const { meta, object } = splitMeta(manifest.triggers![0]);
+      return create(FunctionTrigger, object, meta);
+    });
+
+    triggers.forEach((trigger) => space.db.add(trigger));
+    return triggers;
+  };
+
+  const waitForInactiveTriggers = async (registry: TriggerRegistry, space: Space) => {
+    await waitForCondition({ condition: () => registry.getInactiveTriggers(space).length > 0 });
+  };
+});

package/src/trigger/trigger-registry.ts

@@ -0,0 +1,189 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { create, Filter, getMeta, type Space } from '@dxos/client/echo';
+import { Context, Resource } from '@dxos/context';
+import { ECHO_ATTR_META, foreignKey, foreignKeyEquals, splitMeta } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { ComplexMap } from '@dxos/util';
+
+import { createSubscriptionTrigger, createTimerTrigger, createWebhookTrigger, createWebsocketTrigger } from './type';
+import { type FunctionManifest, FunctionTrigger, type FunctionTriggerType, type TriggerSpec } from '../types';
+import { diff, intersection } from '../util';
+
+type ResponseCode = number;
+
+export type TriggerCallback = (args: object) => Promise<ResponseCode>;
+
+export type TriggerContext = { space: Space };
+
+// TODO(burdon): Make object?
+export type TriggerFactory<Spec extends TriggerSpec, Options = any> = (
+  ctx: Context,
+  context: TriggerContext,
+  spec: Spec,
+  callback: TriggerCallback,
+  options?: Options,
+) => Promise<void>;
+
+export type TriggerHandlerMap = { [type in FunctionTriggerType]: TriggerFactory<any> };
+
+const triggerHandlers: TriggerHandlerMap = {
+  subscription: createSubscriptionTrigger,
+  timer: createTimerTrigger,
+  webhook: createWebhookTrigger,
+  websocket: createWebsocketTrigger,
+};
+
+export type TriggerEvent = {
+  space: Space;
+  triggers: FunctionTrigger[];
+};
+
+type RegisteredTrigger = {
+  activationCtx?: Context;
+  trigger: FunctionTrigger;
+};
+
+export class TriggerRegistry extends Resource {
+  private readonly _triggersBySpaceKey = new ComplexMap<PublicKey, RegisteredTrigger[]>(PublicKey.hash);
+
+  public readonly registered = new Event<TriggerEvent>();
+  public readonly removed = new Event<TriggerEvent>();
+
+  constructor(
+    private readonly _client: Client,
+    private readonly _options?: TriggerHandlerMap,
+  ) {
+    super();
+  }
+
+  public getActiveTriggers(space: Space): FunctionTrigger[] {
+    return this._getTriggers(space, (t) => t.activationCtx != null);
+  }
+
+  public getInactiveTriggers(space: Space): FunctionTrigger[] {
+    return this._getTriggers(space, (t) => t.activationCtx == null);
+  }
+
+  async activate(triggerCtx: TriggerContext, trigger: FunctionTrigger, callback: TriggerCallback): Promise<void> {
+    log('activate', { space: triggerCtx.space.key, trigger });
+    const activationCtx = new Context({ name: `trigger_${trigger.function}` });
+    this._ctx.onDispose(() => activationCtx.dispose());
+    const registeredTrigger = this._triggersBySpaceKey
+      .get(triggerCtx.space.key)
+      ?.find((reg) => reg.trigger.id === trigger.id);
+    invariant(registeredTrigger, `Trigger is not registered: ${trigger.function}`);
+    registeredTrigger.activationCtx = activationCtx;
+
+    try {
+      const options = this._options?.[trigger.spec.type];
+      await triggerHandlers[trigger.spec.type](activationCtx, triggerCtx, trigger.spec, callback, options);
+    } catch (err) {
+      delete registeredTrigger.activationCtx;
+      throw err;
+    }
+  }
+
+  /**
+   * Loads triggers from the manifest into the space.
+   */
+  public async register(space: Space, manifest: FunctionManifest): Promise<void> {
+    log('register', { space: space.key });
+    if (!manifest.triggers?.length) {
+      return;
+    }
+    if (!space.db.graph.runtimeSchemaRegistry.hasSchema(FunctionTrigger)) {
+      space.db.graph.runtimeSchemaRegistry.registerSchema(FunctionTrigger);
+    }
+
+    // Sync triggers.
+    const { objects: existing } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+    const { added, removed } = diff(existing, manifest.triggers, (a, b) => {
+      // Create FK to enable syncing if none are set.
+      // TODO(burdon): Warn if not unique.
+      const keys = b[ECHO_ATTR_META]?.keys ?? [foreignKey('manifest', [b.function, b.spec.type].join('-'))];
+      return intersection(getMeta(a)?.keys ?? [], keys, foreignKeyEquals).length > 0;
+    });
+
+    added.forEach((trigger) => {
+      const { meta, object } = splitMeta(trigger);
+      space.db.add(create(FunctionTrigger, object, meta));
+    });
+    // TODO(burdon): Update existing triggers.
+    removed.forEach((trigger) => space.db.remove(trigger));
+  }
+
+  protected override async _open(): Promise<void> {
+    const spaceListSubscription = this._client.spaces.subscribe(async (spaces) => {
+      for (const space of spaces) {
+        if (this._triggersBySpaceKey.has(space.key)) {
+          continue;
+        }
+
+        const registered: RegisteredTrigger[] = [];
+        this._triggersBySpaceKey.set(space.key, registered);
+        await space.waitUntilReady();
+        if (this._ctx.disposed) {
+          break;
+        }
+        const functionsSubscription = space.db.query(Filter.schema(FunctionTrigger)).subscribe(async (triggers) => {
+          await this._handleRemovedTriggers(space, triggers.objects, registered);
+          this._handleNewTriggers(space, triggers.objects, registered);
+        });
+
+        this._ctx.onDispose(functionsSubscription);
+      }
+    });
+
+    this._ctx.onDispose(() => spaceListSubscription.unsubscribe());
+  }
+
+  protected override async _close(_: Context): Promise<void> {
+    this._triggersBySpaceKey.clear();
+  }
+
+  private _handleNewTriggers(space: Space, allTriggers: FunctionTrigger[], registered: RegisteredTrigger[]) {
+    const newTriggers = allTriggers.filter((candidate) => {
+      return registered.find((reg) => reg.trigger.id === candidate.id) == null;
+    });
+
+    if (newTriggers.length > 0) {
+      const newRegisteredTriggers: RegisteredTrigger[] = newTriggers.map((trigger) => ({ trigger }));
+      registered.push(...newRegisteredTriggers);
+      log('registered new triggers', () => ({ spaceKey: space.key, functions: newTriggers.map((t) => t.function) }));
+      this.registered.emit({ space, triggers: newTriggers });
+    }
+  }
+
+  private async _handleRemovedTriggers(
+    space: Space,
+    allTriggers: FunctionTrigger[],
+    registered: RegisteredTrigger[],
+  ): Promise<void> {
+    const removed: FunctionTrigger[] = [];
+    for (let i = registered.length - 1; i >= 0; i--) {
+      const wasRemoved =
+        allTriggers.find((trigger: FunctionTrigger) => trigger.id === registered[i].trigger.id) == null;
+      if (wasRemoved) {
+        const unregistered = registered.splice(i, 1)[0];
+        await unregistered.activationCtx?.dispose();
+        removed.push(unregistered.trigger);
+      }
+    }
+
+    if (removed.length > 0) {
+      this.removed.emit({ space, triggers: removed });
+    }
+  }
+
+  private _getTriggers(space: Space, predicate: (trigger: RegisteredTrigger) => boolean): FunctionTrigger[] {
+    const allSpaceTriggers = this._triggersBySpaceKey.get(space.key) ?? [];
+    return allSpaceTriggers.filter(predicate).map((trigger) => trigger.trigger);
+  }
+}

package/src/trigger/type/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './subscription-trigger';
+export * from './timer-trigger';
+export * from './webhook-trigger';
+export * from './websocket-trigger';