@dxos/functions 0.5.3-main.6f2dfea → 0.5.3-main.79e0565

package/src/trigger/trigger-registry.test.ts

@@ -0,0 +1,255 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+
+import { sleep, Trigger, waitForCondition } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { type Space } from '@dxos/client/echo';
+import { TestBuilder } from '@dxos/client/testing';
+import { Context } from '@dxos/context';
+import { Filter } from '@dxos/echo-db';
+import { create, splitMeta } from '@dxos/echo-schema';
+import { describe, test } from '@dxos/test';
+import { range } from '@dxos/util';
+
+import { TriggerRegistry } from './trigger-registry';
+import { createInitializedClients, TestType, triggerWebhook } from '../testing';
+import { type FunctionManifest, FunctionTrigger } from '../types';
+
+const manifest: FunctionManifest = {
+  triggers: [
+    {
+      '@meta': {
+        keys: [
+          {
+            source: 'example.com',
+            id: 'trigger-1',
+          },
+        ],
+      },
+      function: 'example.com/function/webhook-test',
+      spec: {
+        type: 'webhook',
+        method: 'GET',
+      },
+    },
+    {
+      '@meta': {
+        keys: [
+          {
+            source: 'example.com',
+            id: 'trigger-2',
+          },
+        ],
+      },
+      function: 'example.com/function/subscription-test',
+      spec: {
+        type: 'subscription',
+        filter: [
+          {
+            type: TestType.typename,
+          },
+        ],
+      },
+    },
+  ],
+};
+
+chai.use(chaiAsPromised);
+
+describe('trigger registry', () => {
+  let ctx: Context;
+  let testBuilder: TestBuilder;
+  beforeEach(async () => {
+    ctx = new Context();
+    testBuilder = new TestBuilder();
+  });
+  afterEach(async () => {
+    await ctx.dispose();
+    await testBuilder.destroy();
+  });
+
+  describe('register', () => {
+    test('creates new triggers', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+      await registry.register(space, manifest);
+      const { objects } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      expect(objects.length).to.eq(manifest.triggers?.length);
+
+      const expected = manifest.triggers?.map((trigger) => trigger.function).sort();
+      expect(objects.map((object: FunctionTrigger) => object.function).sort()).to.deep.eq(expected);
+    });
+  });
+
+  describe('activate', () => {
+    test('invokes the provided callback', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const callbackInvoked = new Trigger();
+      const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      const webhookTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'webhook')!;
+      await registry.activate({ space }, webhookTrigger, async () => {
+        callbackInvoked.wake();
+        return 200;
+      });
+
+      setTimeout(() => triggerWebhook(space, webhookTrigger.function));
+      await callbackInvoked.wait();
+    });
+
+    test('removes from inactive list', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const inactiveTrigger = registry.getInactiveTriggers(space)[0];
+      await registry.activate({ space }, inactiveTrigger, async () => 200);
+
+      const updatedInactiveList = registry.getInactiveTriggers(space);
+      expect(updatedInactiveList.find((trigger: FunctionTrigger) => trigger.function === inactiveTrigger.function)).to
+        .be.undefined;
+    });
+  });
+
+  describe('deactivate', () => {
+    test('trigger object deletion deactivates a trigger', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      const echoTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'subscription')!;
+      let count = 0;
+      await registry.activate({ space }, echoTrigger, async () => {
+        count++;
+        return 200;
+      });
+
+      space.db.add(create(TestType, { title: '1' }));
+      await sleep(20);
+      expect(count).to.eq(1);
+
+      space.db.remove(echoTrigger);
+      space.db.add(create(TestType, { title: '2' }));
+      await sleep(20);
+      expect(count).to.eq(1);
+    });
+
+    test('registry closing deactivates a trigger', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const space = await client.spaces.create();
+      const registry = createRegistry(client);
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await waitForInactiveTriggers(registry, space);
+
+      const { objects: allTriggers } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+      const echoTrigger = allTriggers.find((trigger: FunctionTrigger) => trigger.spec.type === 'subscription')!;
+      let count = 0;
+      await registry.activate({ space }, echoTrigger, async () => {
+        count++;
+        return 200;
+      });
+
+      await registry.close();
+
+      space.db.add(create(TestType, { title: '1' }));
+      await sleep(20);
+      expect(count).to.eq(0);
+    });
+  });
+
+  describe('trigger events', () => {
+    test.only('event fired when all registered when opened', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const triggers = createTriggers(client.spaces.default, 3);
+
+      const triggersRegistered = new Trigger<FunctionTrigger[]>();
+      registry.registered.on((fn) => {
+        expect(fn.space.key.toHex()).to.eq(client.spaces.default.key.toHex());
+        triggersRegistered.wake(fn.triggers);
+      });
+
+      void registry.open(ctx);
+      const functions = await triggersRegistered.wait();
+      const expected = triggers.map((object) => object.id).sort();
+      expect(functions.map((fn) => fn.id).sort()).to.deep.eq(expected);
+    });
+
+    test('event fired when a new trigger is added', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+
+      const triggerRegistered = new Trigger<FunctionTrigger>();
+      registry.registered.on((fn) => {
+        expect(fn.triggers.length).to.eq(1);
+        triggerRegistered.wake(fn.triggers[0]);
+      });
+      await registry.open(ctx);
+      await registry.register(space, { triggers: manifest?.triggers?.slice(0, 1) });
+      const registered = await triggerRegistered.wait();
+      expect(registered.function).to.eq(manifest.triggers![0].function);
+    });
+
+    test('event fired when a new trigger is removed', async () => {
+      const client = (await createInitializedClients(testBuilder))[0];
+      const registry = createRegistry(client);
+      const space = await client.spaces.create();
+      const triggers = createTriggers(space, 3);
+
+      const triggerLoaded = new Trigger();
+      registry.registered.on((fn) => triggerLoaded.wake());
+
+      const triggerRemoved = new Trigger<FunctionTrigger>();
+      registry.removed.on((fn) => {
+        expect(fn.triggers.length).to.eq(1);
+        triggerRemoved.wake(fn.triggers[0]);
+      });
+      await registry.register(space, manifest);
+      await registry.open(ctx);
+      await triggerLoaded.wait();
+
+      space.db.remove(triggers[0]);
+      const removedTrigger = await triggerRemoved.wait();
+      expect(removedTrigger.id).to.eq(triggers[0].id);
+    });
+  });
+
+  const createRegistry = (client: Client) => {
+    const registry = new TriggerRegistry(client);
+    ctx.onDispose(() => registry.close());
+    return registry;
+  };
+
+  const createTriggers = (space: Space, count: number) => {
+    const triggers = range(count, () => {
+      const { meta, object } = splitMeta(manifest.triggers![0]);
+      return create(FunctionTrigger, object, meta);
+    });
+
+    triggers.forEach((trigger) => space.db.add(trigger));
+    return triggers;
+  };
+
+  const waitForInactiveTriggers = async (registry: TriggerRegistry, space: Space) => {
+    await waitForCondition({ condition: () => registry.getInactiveTriggers(space).length > 0 });
+  };
+});

package/src/trigger/trigger-registry.ts

@@ -0,0 +1,189 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { create, Filter, getMeta, type Space } from '@dxos/client/echo';
+import { Context, Resource } from '@dxos/context';
+import { ECHO_ATTR_META, foreignKey, foreignKeyEquals, splitMeta } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { ComplexMap } from '@dxos/util';
+
+import { createSubscriptionTrigger, createTimerTrigger, createWebhookTrigger, createWebsocketTrigger } from './type';
+import { type FunctionManifest, FunctionTrigger, type FunctionTriggerType, type TriggerSpec } from '../types';
+import { diff, intersection } from '../util';
+
+type ResponseCode = number;
+
+export type TriggerCallback = (args: object) => Promise<ResponseCode>;
+
+export type TriggerContext = { space: Space };
+
+// TODO(burdon): Make object?
+export type TriggerFactory<Spec extends TriggerSpec, Options = any> = (
+  ctx: Context,
+  context: TriggerContext,
+  spec: Spec,
+  callback: TriggerCallback,
+  options?: Options,
+) => Promise<void>;
+
+export type TriggerHandlerMap = { [type in FunctionTriggerType]: TriggerFactory<any> };
+
+const triggerHandlers: TriggerHandlerMap = {
+  subscription: createSubscriptionTrigger,
+  timer: createTimerTrigger,
+  webhook: createWebhookTrigger,
+  websocket: createWebsocketTrigger,
+};
+
+export type TriggerEvent = {
+  space: Space;
+  triggers: FunctionTrigger[];
+};
+
+type RegisteredTrigger = {
+  activationCtx?: Context;
+  trigger: FunctionTrigger;
+};
+
+export class TriggerRegistry extends Resource {
+  private readonly _triggersBySpaceKey = new ComplexMap<PublicKey, RegisteredTrigger[]>(PublicKey.hash);
+
+  public readonly registered = new Event<TriggerEvent>();
+  public readonly removed = new Event<TriggerEvent>();
+
+  constructor(
+    private readonly _client: Client,
+    private readonly _options?: TriggerHandlerMap,
+  ) {
+    super();
+  }
+
+  public getActiveTriggers(space: Space): FunctionTrigger[] {
+    return this._getTriggers(space, (t) => t.activationCtx != null);
+  }
+
+  public getInactiveTriggers(space: Space): FunctionTrigger[] {
+    return this._getTriggers(space, (t) => t.activationCtx == null);
+  }
+
+  async activate(triggerCtx: TriggerContext, trigger: FunctionTrigger, callback: TriggerCallback): Promise<void> {
+    log('activate', { space: triggerCtx.space.key, trigger });
+    const activationCtx = new Context({ name: `trigger_${trigger.function}` });
+    this._ctx.onDispose(() => activationCtx.dispose());
+    const registeredTrigger = this._triggersBySpaceKey
+      .get(triggerCtx.space.key)
+      ?.find((reg) => reg.trigger.id === trigger.id);
+    invariant(registeredTrigger, `Trigger is not registered: ${trigger.function}`);
+    registeredTrigger.activationCtx = activationCtx;
+
+    try {
+      const options = this._options?.[trigger.spec.type];
+      await triggerHandlers[trigger.spec.type](activationCtx, triggerCtx, trigger.spec, callback, options);
+    } catch (err) {
+      delete registeredTrigger.activationCtx;
+      throw err;
+    }
+  }
+
+  /**
+   * Loads triggers from the manifest into the space.
+   */
+  public async register(space: Space, manifest: FunctionManifest): Promise<void> {
+    log('register', { space: space.key });
+    if (!manifest.triggers?.length) {
+      return;
+    }
+    if (!space.db.graph.runtimeSchemaRegistry.hasSchema(FunctionTrigger)) {
+      space.db.graph.runtimeSchemaRegistry.registerSchema(FunctionTrigger);
+    }
+
+    // Sync triggers.
+    const { objects: existing } = await space.db.query(Filter.schema(FunctionTrigger)).run();
+    const { added, removed } = diff(existing, manifest.triggers, (a, b) => {
+      // Create FK to enable syncing if none are set.
+      // TODO(burdon): Warn if not unique.
+      const keys = b[ECHO_ATTR_META]?.keys ?? [foreignKey('manifest', [b.function, b.spec.type].join('-'))];
+      return intersection(getMeta(a)?.keys ?? [], keys, foreignKeyEquals).length > 0;
+    });
+
+    added.forEach((trigger) => {
+      const { meta, object } = splitMeta(trigger);
+      space.db.add(create(FunctionTrigger, object, meta));
+    });
+    // TODO(burdon): Update existing triggers.
+    removed.forEach((trigger) => space.db.remove(trigger));
+  }
+
+  protected override async _open(): Promise<void> {
+    const spaceListSubscription = this._client.spaces.subscribe(async (spaces) => {
+      for (const space of spaces) {
+        if (this._triggersBySpaceKey.has(space.key)) {
+          continue;
+        }
+
+        const registered: RegisteredTrigger[] = [];
+        this._triggersBySpaceKey.set(space.key, registered);
+        await space.waitUntilReady();
+        if (this._ctx.disposed) {
+          break;
+        }
+        const functionsSubscription = space.db.query(Filter.schema(FunctionTrigger)).subscribe(async (triggers) => {
+          await this._handleRemovedTriggers(space, triggers.objects, registered);
+          this._handleNewTriggers(space, triggers.objects, registered);
+        });
+
+        this._ctx.onDispose(functionsSubscription);
+      }
+    });
+
+    this._ctx.onDispose(() => spaceListSubscription.unsubscribe());
+  }
+
+  protected override async _close(_: Context): Promise<void> {
+    this._triggersBySpaceKey.clear();
+  }
+
+  private _handleNewTriggers(space: Space, allTriggers: FunctionTrigger[], registered: RegisteredTrigger[]) {
+    const newTriggers = allTriggers.filter((candidate) => {
+      return registered.find((reg) => reg.trigger.id === candidate.id) == null;
+    });
+
+    if (newTriggers.length > 0) {
+      const newRegisteredTriggers: RegisteredTrigger[] = newTriggers.map((trigger) => ({ trigger }));
+      registered.push(...newRegisteredTriggers);
+      log('registered new triggers', () => ({ spaceKey: space.key, functions: newTriggers.map((t) => t.function) }));
+      this.registered.emit({ space, triggers: newTriggers });
+    }
+  }
+
+  private async _handleRemovedTriggers(
+    space: Space,
+    allTriggers: FunctionTrigger[],
+    registered: RegisteredTrigger[],
+  ): Promise<void> {
+    const removed: FunctionTrigger[] = [];
+    for (let i = registered.length - 1; i >= 0; i--) {
+      const wasRemoved =
+        allTriggers.find((trigger: FunctionTrigger) => trigger.id === registered[i].trigger.id) == null;
+      if (wasRemoved) {
+        const unregistered = registered.splice(i, 1)[0];
+        await unregistered.activationCtx?.dispose();
+        removed.push(unregistered.trigger);
+      }
+    }
+
+    if (removed.length > 0) {
+      this.removed.emit({ space, triggers: removed });
+    }
+  }
+
+  private _getTriggers(space: Space, predicate: (trigger: RegisteredTrigger) => boolean): FunctionTrigger[] {
+    const allSpaceTriggers = this._triggersBySpaceKey.get(space.key) ?? [];
+    return allSpaceTriggers.filter(predicate).map((trigger) => trigger.trigger);
+  }
+}

package/src/trigger/type/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './subscription-trigger';
+export * from './timer-trigger';
+export * from './webhook-trigger';
+export * from './websocket-trigger';

package/src/trigger/type/subscription-trigger.ts

@@ -0,0 +1,80 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { TextV0Type } from '@braneframe/types';
+import { debounce, UpdateScheduler } from '@dxos/async';
+import { type Context } from '@dxos/context';
+import { createSubscription, Filter, getAutomergeObjectCore, type Query } from '@dxos/echo-db';
+import { log } from '@dxos/log';
+
+import type { SubscriptionTrigger } from '../../types';
+import { type TriggerCallback, type TriggerContext, type TriggerFactory } from '../trigger-registry';
+
+export const createSubscriptionTrigger: TriggerFactory<SubscriptionTrigger> = async (
+  ctx: Context,
+  triggerCtx: TriggerContext,
+  spec: SubscriptionTrigger,
+  callback: TriggerCallback,
+) => {
+  const objectIds = new Set<string>();
+  const task = new UpdateScheduler(
+    ctx,
+    async () => {
+      if (objectIds.size > 0) {
+        const objects = Array.from(objectIds);
+        objectIds.clear();
+        await callback({ objects });
+      }
+    },
+    { maxFrequency: 4 },
+  );
+
+  // TODO(burdon): Don't fire initially?
+  // TODO(burdon): Create queue. Only allow one invocation per trigger at a time?
+  const subscriptions: (() => void)[] = [];
+  const subscription = createSubscription(({ added, updated }) => {
+    const sizeBefore = objectIds.size;
+    for (const object of added) {
+      objectIds.add(object.id);
+    }
+    for (const object of updated) {
+      objectIds.add(object.id);
+    }
+    if (objectIds.size > sizeBefore) {
+      log.info('updated', { added: added.length, updated: updated.length });
+      task.trigger();
+    }
+  });
+
+  subscriptions.push(() => subscription.unsubscribe());
+
+  // TODO(burdon): Disable trigger if keeps failing.
+  const { filter, options: { deep, delay } = {} } = spec;
+  const update = ({ objects }: Query) => {
+    subscription.update(objects);
+
+    // TODO(burdon): Hack to monitor changes to Document's text object.
+    if (deep) {
+      log.info('update', { objects: objects.length });
+      for (const object of objects) {
+        const content = object.content;
+        if (content instanceof TextV0Type) {
+          subscriptions.push(
+            getAutomergeObjectCore(content).updates.on(debounce(() => subscription.update([object]), 1_000)),
+          );
+        }
+      }
+    }
+  };
+
+  // TODO(burdon): Is Filter.or implemented?
+  // TODO(burdon): [Bug]: all callbacks are fired on the first mutation.
+  // TODO(burdon): [Bug]: not updated when document is deleted (either top or hierarchically).
+  const query = triggerCtx.space.db.query(Filter.or(filter.map(({ type, props }) => Filter.typename(type, props))));
+  subscriptions.push(query.subscribe(delay ? debounce(update, delay) : update));
+
+  ctx.onDispose(() => {
+    subscriptions.forEach((unsubscribe) => unsubscribe());
+  });
+};

package/src/trigger/type/timer-trigger.ts

@@ -0,0 +1,44 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { CronJob } from 'cron';
+
+import { DeferredTask } from '@dxos/async';
+import { type Context } from '@dxos/context';
+import { log } from '@dxos/log';
+
+import type { TimerTrigger } from '../../types';
+import { type TriggerCallback, type TriggerContext, type TriggerFactory } from '../trigger-registry';
+
+export const createTimerTrigger: TriggerFactory<TimerTrigger> = async (
+  ctx: Context,
+  triggerContext: TriggerContext,
+  spec: TimerTrigger,
+  callback: TriggerCallback,
+) => {
+  const task = new DeferredTask(ctx, async () => {
+    await callback({});
+  });
+
+  let last = 0;
+  let run = 0;
+  // https://www.npmjs.com/package/cron#constructor
+  const job = CronJob.from({
+    cronTime: spec.cron,
+    runOnInit: false,
+    onTick: () => {
+      // TODO(burdon): Check greater than 30s (use cron-parser).
+      const now = Date.now();
+      const delta = last ? now - last : 0;
+      last = now;
+
+      run++;
+      log.info('tick', { space: triggerContext.space.key.truncate(), count: run, delta });
+      task.schedule();
+    },
+  });
+
+  job.start();
+  ctx.onDispose(() => job.stop());
+};

package/src/trigger/type/webhook-trigger.ts

@@ -0,0 +1,47 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { getPort } from 'get-port-please';
+import http from 'node:http';
+
+import { type Context } from '@dxos/context';
+import { log } from '@dxos/log';
+
+import type { WebhookTrigger } from '../../types';
+import { type TriggerCallback, type TriggerContext, type TriggerFactory } from '../trigger-registry';
+
+export const createWebhookTrigger: TriggerFactory<WebhookTrigger> = async (
+  ctx: Context,
+  _: TriggerContext,
+  spec: WebhookTrigger,
+  callback: TriggerCallback,
+) => {
+  // TODO(burdon): Enable POST hook with payload.
+  const server = http.createServer(async (req, res) => {
+    if (req.method !== spec.method) {
+      res.statusCode = 405;
+      return res.end();
+    }
+    res.statusCode = await callback({});
+    res.end();
+  });
+
+  // TODO(burdon): Not used.
+  // const DEF_PORT_RANGE = { min: 7500, max: 7599 };
+  // const portRange = Object.assign({}, trigger.port, DEF_PORT_RANGE) as WebhookTrigger['port'];
+  const port = await getPort({
+    random: true,
+    // portRange: [portRange!.min, portRange!.max],
+  });
+
+  // TODO(burdon): Update trigger object with actual port.
+  server.listen(port, () => {
+    log.info('started webhook', { port });
+    spec.port = port;
+  });
+
+  ctx.onDispose(() => {
+    server.close();
+  });
+};