@dxos/functions 0.5.2 → 0.5.3-main.056e7da

package/src/runtime/scheduler.ts

@@ -2,206 +2,143 @@
 // Copyright 2023 DXOS.org
 //
 
-import { CronJob } from 'cron';
+import path from 'node:path';
 
-import { TextV0Type } from '@braneframe/types';
-import { debounce, DeferredTask } from '@dxos/async';
-import { type Client, type PublicKey } from '@dxos/client';
-import { type Space, Filter, createSubscription, type Query, getAutomergeObjectCore } from '@dxos/client/echo';
+import { Mutex } from '@dxos/async';
+import { type Space } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
-import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
-import { ComplexMap } from '@dxos/util';
 
-import { type FunctionSubscriptionEvent } from '../handler';
-import { type FunctionDef, type FunctionManifest, type FunctionTrigger, type TriggerSubscription } from '../manifest';
+import { type FunctionRegistry } from '../function';
+import { type FunctionEventMeta } from '../handler';
+import { type TriggerRegistry } from '../trigger';
+import { type FunctionDef, type FunctionManifest, type FunctionTrigger } from '../types';
 
-type Callback = (data: FunctionSubscriptionEvent) => Promise<number>;
+export type Callback = (data: any) => Promise<void | number>;
 
-type SchedulerOptions = {
+export type SchedulerOptions = {
   endpoint?: string;
   callback?: Callback;
 };
 
 /**
- * Functions scheduler.
+ * The scheduler triggers function execution based on various triggers.
  */
-// TODO(burdon): Create tests.
 export class Scheduler {
-  // Map of mounted functions.
-  private readonly _mounts = new ComplexMap<
-    { id: string; spaceKey: PublicKey },
-    { ctx: Context; trigger: FunctionTrigger }
-  >(({ id, spaceKey }) => `${spaceKey.toHex()}:${id}`);
+  private _ctx = createContext();
+
+  private readonly _functionUriToCallMutex = new Map<string, Mutex>();
 
   constructor(
-    private readonly _client: Client,
-    private readonly _manifest: FunctionManifest,
+    public readonly functions: FunctionRegistry,
+    public readonly triggers: TriggerRegistry,
     private readonly _options: SchedulerOptions = {},
-  ) {}
-
-  async start() {
-    this._client.spaces.subscribe(async (spaces) => {
-      for (const space of spaces) {
-        await space.waitUntilReady();
-        for (const trigger of this._manifest.triggers ?? []) {
-          await this.mount(new Context(), space, trigger);
-        }
-      }
+  ) {
+    this.functions.registered.on(async ({ space, added }) => {
+      await this._safeActivateTriggers(space, this.triggers.getInactiveTriggers(space), added);
+    });
+    this.triggers.registered.on(async ({ space, triggers }) => {
+      await this._safeActivateTriggers(space, triggers, this.functions.getFunctions(space));
     });
   }
 
-  async stop() {
-    for (const { id, spaceKey } of this._mounts.keys()) {
-      await this.unmount(id, spaceKey);
-    }
+  async start() {
+    await this._ctx.dispose();
+    this._ctx = createContext();
+    await this.functions.open(this._ctx);
+    await this.triggers.open(this._ctx);
   }
 
-  private async mount(ctx: Context, space: Space, trigger: FunctionTrigger) {
-    const key = { id: trigger.function, spaceKey: space.key };
-    const def = this._manifest.functions.find((config) => config.id === trigger.function);
-    invariant(def, `Function not found: ${trigger.function}`);
-
-    // Currently supports only one trigger declaration per function.
-    const exists = this._mounts.get(key);
-    if (!exists) {
-      this._mounts.set(key, { ctx, trigger });
-      log('mount', { space: space.key, trigger });
-      if (ctx.disposed) {
-        return;
-      }
-
-      // Timer.
-      if (trigger.schedule) {
-        this._createTimer(ctx, space, def, trigger);
-      }
-
-      // Subscription.
-      for (const triggerSubscription of trigger.subscriptions ?? []) {
-        this._createSubscription(ctx, space, def, triggerSubscription);
-      }
-    }
+  async stop() {
+    await this._ctx.dispose();
+    await this.functions.close();
+    await this.triggers.close();
   }
 
-  private async unmount(id: string, spaceKey: PublicKey) {
-    const key = { id, spaceKey };
-    const { ctx } = this._mounts.get(key) ?? {};
-    if (ctx) {
-      this._mounts.delete(key);
-      await ctx.dispose();
-    }
+  // TODO(burdon): Remove and update registries directly.
+  public async register(space: Space, manifest: FunctionManifest) {
+    await this.functions.register(space, manifest.functions);
+    await this.triggers.register(space, manifest);
   }
 
-  private _createTimer(ctx: Context, space: Space, def: FunctionDef, trigger: FunctionTrigger) {
-    const task = new DeferredTask(ctx, async () => {
-      await this._execFunction(def, {
-        space: space.key,
-      });
+  private async _safeActivateTriggers(
+    space: Space,
+    triggers: FunctionTrigger[],
+    functions: FunctionDef[],
+  ): Promise<void> {
+    const mountTasks = triggers.map((trigger) => {
+      return this.activate(space, functions, trigger);
     });
-
-    invariant(trigger.schedule);
-    let last = 0;
-    let run = 0;
-    // https://www.npmjs.com/package/cron#constructor
-    const job = CronJob.from({
-      cronTime: trigger.schedule,
-      runOnInit: false,
-      onTick: () => {
-        // TODO(burdon): Check greater than 30s (use cron-parser).
-        const now = Date.now();
-        const delta = last ? now - last : 0;
-        last = now;
-
-        run++;
-        log.info('tick', { space: space.key.truncate(), count: run, delta });
-        task.schedule();
-      },
-    });
-
-    job.start();
-    ctx.onDispose(() => job.stop());
+    await Promise.all(mountTasks).catch(log.catch);
   }
 
-  private _createSubscription(ctx: Context, space: Space, def: FunctionDef, triggerSubscription: TriggerSubscription) {
-    log.info('subscription', { space: space.key, triggerSubscription });
-    const objectIds = new Set<string>();
-    const task = new DeferredTask(ctx, async () => {
-      await this._execFunction(def, {
-        space: space.key,
-        objects: Array.from(objectIds),
-      });
-    });
+  private async activate(space: Space, functions: FunctionDef[], trigger: FunctionTrigger) {
+    const definition = functions.find((def) => def.uri === trigger.function);
+    if (!definition) {
+      log.info('function is not found for trigger', { trigger });
+      return;
+    }
 
-    // TODO(burdon): Don't fire initially.
-    // TODO(burdon): Standardize subscription handles.
-    const subscriptions: (() => void)[] = [];
-    const subscription = createSubscription(({ added, updated }) => {
-      log.info('updated', { added: added.length, updated: updated.length });
-      for (const object of added) {
-        objectIds.add(object.id);
-      }
-      for (const object of updated) {
-        objectIds.add(object.id);
-      }
+    await this.triggers.activate(space, trigger, async (args) => {
+      const mutex = this._functionUriToCallMutex.get(definition.uri) ?? new Mutex();
+      this._functionUriToCallMutex.set(definition.uri, mutex);
 
-      task.schedule();
+      log.info('function triggered, waiting for mutex', { uri: definition.uri });
+      return mutex.executeSynchronized(() => {
+        log.info('mutex acquired', { uri: definition.uri });
+        return this._execFunction(definition, trigger, {
+          meta: trigger.meta ?? {},
+          data: { ...args, spaceKey: space.key },
+        });
+      });
     });
-    subscriptions.push(() => subscription.unsubscribe());
-
-    // TODO(burdon): Create queue. Only allow one invocation per trigger at a time?
-    // TODO(burdon): Disable trigger if keeps failing.
-    const { type, props, deep, delay } = triggerSubscription;
-    const update = ({ objects }: Query) => {
-      subscription.update(objects);
-
-      // TODO(burdon): Hack to monitor changes to Document's text object.
-      if (deep) {
-        log.info('update', { type, deep, objects: objects.length });
-        for (const object of objects) {
-          const content = object.content;
-          if (content instanceof TextV0Type) {
-            subscriptions.push(
-              getAutomergeObjectCore(content).updates.on(debounce(() => subscription.update([object]), 1_000)),
-            );
-          }
-        }
-      }
-    };
-
-    // TODO(burdon): [Bug]: all callbacks are fired on the first mutation.
-    // TODO(burdon): [Bug]: not updated when document is deleted (either top or hierarchically).
-    const query = space.db.query(Filter.typename(type, props));
-    subscriptions.push(query.subscribe(delay ? debounce(update, delay * 1_000) : update));
 
-    ctx.onDispose(() => {
-      subscriptions.forEach((unsubscribe) => unsubscribe());
-    });
+    log('activated trigger', { space: space.key, trigger });
   }
 
-  private async _execFunction(def: FunctionDef, data: any) {
+  private async _execFunction<TData, TMeta>(
+    def: FunctionDef,
+    trigger: FunctionTrigger,
+    { data, meta }: { data: TData; meta?: TMeta },
+  ): Promise<number> {
+    let status = 0;
     try {
-      log('request', { function: def.id });
+      // TODO(burdon): Pass in Space key (common context)?
+      const payload = Object.assign({}, meta && ({ meta } satisfies FunctionEventMeta<TMeta>), data);
+
       const { endpoint, callback } = this._options;
-      let status = 0;
       if (endpoint) {
         // TODO(burdon): Move out of scheduler (generalize as callback).
-        const response = await fetch(`${this._options.endpoint}/${def.name}`, {
+        const url = path.join(endpoint, def.route);
+        log.info('exec', { function: def.uri, url, triggerType: trigger.spec.type });
+        const response = await fetch(url, {
           method: 'POST',
           headers: {
             'Content-Type': 'application/json',
           },
-          body: JSON.stringify(data),
+          body: JSON.stringify(payload),
         });
 
         status = response.status;
       } else if (callback) {
-        status = await callback(data);
+        log.info('exec', { function: def.uri });
+        status = (await callback(payload)) ?? 200;
+      }
+
+      // Check errors.
+      if (status && status >= 400) {
+        throw new Error(`Response: ${status}`);
       }
 
       // const result = await response.json();
-      log('result', { function: def.id, result: status });
+      log.info('done', { function: def.uri, status });
     } catch (err: any) {
-      log.error('error', { function: def.id, error: err.message });
+      log.error('error', { function: def.uri, error: err.message });
+      status = 500;
     }
+
+    return status;
   }
 }
+
+const createContext = () => new Context({ name: 'FunctionScheduler' });

package/src/testing/functions-integration.test.ts

@@ -0,0 +1,100 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+import path from 'path';
+
+import { Trigger, waitForCondition } from '@dxos/async';
+import { type Client } from '@dxos/client';
+import { create, type Space } from '@dxos/client/echo';
+import { performInvitation, TestBuilder } from '@dxos/client/testing';
+import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
+import { describe, test } from '@dxos/test';
+
+import { setTestCallHandler } from './test/handler';
+import { FunctionRegistry } from '../function';
+import { DevServer, Scheduler } from '../runtime';
+import { createFunctionRuntime, createInitializedClients, TestType } from '../testing';
+import { TriggerRegistry } from '../trigger';
+import { FunctionDef, FunctionTrigger } from '../types';
+
+describe('functions e2e', () => {
+  let testBuilder: TestBuilder;
+  before(async () => {
+    testBuilder = new TestBuilder();
+  });
+  after(async () => {
+    await testBuilder.destroy();
+  });
+
+  test('a function gets triggered in response to another peer object creations', async () => {
+    // TODO(burdon): Create builder pattern.
+    const functionRuntime = await createFunctionRuntime(testBuilder);
+    const devServer = await startDevServer(functionRuntime);
+    const scheduler = await startScheduler(functionRuntime, devServer);
+
+    const app = (await createInitializedClients(testBuilder, 1))[0];
+    const space = await app.spaces.create();
+    await inviteMember(space, functionRuntime);
+
+    const uri = 'example.com/function/test';
+    space.db.add(create(FunctionDef, { uri, route: '/test', handler: 'test' }));
+    const triggerMeta: FunctionTrigger['meta'] = { name: 'DXOS' };
+    space.db.add(
+      create(FunctionTrigger, {
+        function: uri,
+        enabled: true,
+        meta: triggerMeta,
+        spec: {
+          type: 'subscription',
+          filter: [{ type: TestType.typename }],
+        },
+      }),
+    );
+
+    const called = new Trigger<any>();
+    setTestCallHandler(async (args) => {
+      called.wake(args.event.data);
+      return args.response.status(200);
+    });
+
+    await waitTriggersReplicated(space, scheduler);
+    const addedObject = space.db.add(create(TestType, { title: '42' }));
+
+    const callArgs = await called.wait();
+    expect(callArgs.meta).to.deep.eq(triggerMeta);
+    expect(callArgs.objects).to.deep.eq([addedObject.id]);
+    expect(callArgs.spaceKey).to.eq(space.key.toHex());
+  });
+
+  const waitTriggersReplicated = async (space: Space, scheduler: Scheduler) => {
+    await waitForCondition({ condition: () => scheduler.triggers.getActiveTriggers(space).length > 0 });
+  };
+
+  // TODO(burdon): Factor out utils to builder pattern.
+
+  const startScheduler = async (client: Client, devServer: DevServer) => {
+    const functionRegistry = new FunctionRegistry(client);
+    const triggerRegistry = new TriggerRegistry(client);
+    const scheduler = new Scheduler(functionRegistry, triggerRegistry, { endpoint: devServer.endpoint });
+    await scheduler.start();
+    testBuilder.ctx.onDispose(() => scheduler.stop());
+    return scheduler;
+  };
+
+  const startDevServer = async (client: Client) => {
+    const functionRegistry = new FunctionRegistry(client);
+    const server = new DevServer(client, functionRegistry, {
+      baseDir: path.join(__dirname, '../testing'),
+    });
+    await server.start();
+    testBuilder.ctx.onDispose(() => server.stop());
+    return server;
+  };
+
+  const inviteMember = async (host: Space, guest: Client) => {
+    const [{ invitation: hostInvitation }] = await Promise.all(performInvitation({ host, guest: guest.spaces }));
+    expect(hostInvitation?.state).to.eq(Invitation.State.SUCCESS);
+  };
+});

package/src/testing/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './setup';
+export * from './types';
+export * from './util';

package/src/testing/setup.ts

@@ -0,0 +1,43 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { FunctionsPlugin } from '@dxos/agent';
+import { Client, Config } from '@dxos/client';
+import { type TestBuilder } from '@dxos/client/testing';
+import { range } from '@dxos/util';
+
+import { TestType } from './types';
+import { FunctionDef, FunctionTrigger } from '../types';
+
+// TODO(burdon): Extend/wrap TestBuilder.
+
+export const createInitializedClients = async (testBuilder: TestBuilder, count: number = 1, config?: Config) => {
+  const clients = range(count).map(() => new Client({ config, services: testBuilder.createLocalClientServices() }));
+  testBuilder.ctx.onDispose(() => Promise.all(clients.map((client) => client.destroy())));
+  return Promise.all(
+    clients.map(async (client, index) => {
+      await client.initialize();
+      await client.halo.createIdentity({ displayName: `Peer ${index}` });
+      client.addSchema(FunctionDef, FunctionTrigger, TestType);
+      return client;
+    }),
+  );
+};
+
+export const createFunctionRuntime = async (testBuilder: TestBuilder): Promise<Client> => {
+  const config = new Config({
+    runtime: {
+      agent: {
+        plugins: [{ id: 'dxos.org/agent/plugin/functions', config: { port: 8080 } }],
+      },
+    },
+  });
+
+  const [client] = await createInitializedClients(testBuilder, 1, config);
+  const plugin = new FunctionsPlugin();
+  await plugin.initialize({ client, clientServices: client.services });
+  await plugin.open();
+  testBuilder.ctx.onDispose(() => plugin.close());
+  return client;
+};

package/src/testing/test/handler.ts

@@ -0,0 +1,15 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type FunctionHandler } from '../../handler';
+
+let callHandler: FunctionHandler<any> = async ({ response }) => response.status(200);
+
+export const setTestCallHandler = (handler: FunctionHandler<any>) => {
+  callHandler = handler;
+};
+
+export const handler: FunctionHandler<any> = async (args) => {
+  return callHandler(args);
+};

package/src/testing/test/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { handler } from './handler';
+
+export default handler;

package/src/testing/types.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { S, TypedObject } from '@dxos/echo-schema';
+
+export class TestType extends TypedObject({ typename: 'example.com/type/Test', version: '0.1.0' })({
+  title: S.string,
+}) {}

package/src/testing/util.ts

@@ -0,0 +1,16 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Filter, type Space } from '@dxos/client/echo';
+import { invariant } from '@dxos/invariant';
+
+import { FunctionTrigger } from '../types';
+
+export const triggerWebhook = async (space: Space, uri: string) => {
+  const trigger = (
+    await space.db.query(Filter.schema(FunctionTrigger, (t: FunctionTrigger) => t.function === uri)).run()
+  ).objects[0];
+  invariant(trigger.spec.type === 'webhook');
+  void fetch(`http://localhost:${trigger.spec.port}`);
+};

package/src/trigger/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './trigger-registry';