@dxos/functions 0.5.2 → 0.5.3-main.088a2c8

package/src/runtime/scheduler.ts

@@ -3,36 +3,39 @@
 //
 
 import { CronJob } from 'cron';
+import { getPort } from 'get-port-please';
+import http from 'node:http';
+import path from 'node:path';
+import WebSocket from 'ws';
 
 import { TextV0Type } from '@braneframe/types';
-import { debounce, DeferredTask } from '@dxos/async';
+import { debounce, DeferredTask, sleep, Trigger } from '@dxos/async';
 import { type Client, type PublicKey } from '@dxos/client';
-import { type Space, Filter, createSubscription, type Query, getAutomergeObjectCore } from '@dxos/client/echo';
+import { createSubscription, Filter, getAutomergeObjectCore, type Query, type Space } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { ComplexMap } from '@dxos/util';
 
-import { type FunctionSubscriptionEvent } from '../handler';
-import { type FunctionDef, type FunctionManifest, type FunctionTrigger, type TriggerSubscription } from '../manifest';
+import { type FunctionEventMeta } from '../handler';
+import { type FunctionDef, type FunctionManifest, type FunctionTrigger } from '../types';
 
-type Callback = (data: FunctionSubscriptionEvent) => Promise<number>;
+export type Callback = (data: any) => Promise<void | number>;
 
-type SchedulerOptions = {
+export type SchedulerOptions = {
   endpoint?: string;
   callback?: Callback;
 };
 
 /**
- * Functions scheduler.
+ * The scheduler triggers function execution based on various triggers.
  */
-// TODO(burdon): Create tests.
 export class Scheduler {
   // Map of mounted functions.
   private readonly _mounts = new ComplexMap<
-    { id: string; spaceKey: PublicKey },
+    { spaceKey: PublicKey; id: string },
     { ctx: Context; trigger: FunctionTrigger }
-  >(({ id, spaceKey }) => `${spaceKey.toHex()}:${id}`);
+  >(({ spaceKey, id }) => `${spaceKey.toHex()}:${id}`);
 
   constructor(
     private readonly _client: Client,
@@ -40,6 +43,13 @@ export class Scheduler {
     private readonly _options: SchedulerOptions = {},
   ) {}
 
+  get mounts() {
+    return Array.from(this._mounts.values()).reduce<FunctionTrigger[]>((acc, { trigger }) => {
+      acc.push(trigger);
+      return acc;
+    }, []);
+  }
+
   async start() {
     this._client.spaces.subscribe(async (spaces) => {
       for (const space of spaces) {
@@ -57,12 +67,15 @@ export class Scheduler {
     }
   }
 
+  /**
+   * Mount trigger.
+   */
   private async mount(ctx: Context, space: Space, trigger: FunctionTrigger) {
-    const key = { id: trigger.function, spaceKey: space.key };
+    const key = { spaceKey: space.key, id: trigger.function };
     const def = this._manifest.functions.find((config) => config.id === trigger.function);
     invariant(def, `Function not found: ${trigger.function}`);
 
-    // Currently supports only one trigger declaration per function.
+    // TODO(burdon): Currently supports only one trigger declaration per function.
     const exists = this._mounts.get(key);
     if (!exists) {
       this._mounts.set(key, { ctx, trigger });
@@ -71,14 +84,24 @@ export class Scheduler {
         return;
       }
 
-      // Timer.
-      if (trigger.schedule) {
-        this._createTimer(ctx, space, def, trigger);
+      //
+      // Triggers types.
+      //
+
+      if (trigger.timer) {
+        await this._createTimer(ctx, space, def, trigger);
+      }
+
+      if (trigger.webhook) {
+        await this._createWebhook(ctx, space, def, trigger);
+      }
+
+      if (trigger.websocket) {
+        await this._createWebsocket(ctx, space, def, trigger);
       }
 
-      // Subscription.
-      for (const triggerSubscription of trigger.subscriptions ?? []) {
-        this._createSubscription(ctx, space, def, triggerSubscription);
+      if (trigger.subscription) {
+        await this._createSubscription(ctx, space, def, trigger);
       }
     }
   }
@@ -92,19 +115,66 @@ export class Scheduler {
     }
   }
 
-  private _createTimer(ctx: Context, space: Space, def: FunctionDef, trigger: FunctionTrigger) {
+  private async _execFunction<TData, TMeta>(def: FunctionDef, trigger: FunctionTrigger, data: TData): Promise<number> {
+    let status = 0;
+    try {
+      // TODO(burdon): Pass in Space key (common context)?
+      const payload = Object.assign({}, { meta: trigger.meta as TMeta } satisfies FunctionEventMeta<TMeta>, data);
+
+      const { endpoint, callback } = this._options;
+      if (endpoint) {
+        // TODO(burdon): Move out of scheduler (generalize as callback).
+        const url = path.join(endpoint, def.path);
+        log.info('exec', { function: def.id, url });
+        const response = await fetch(url, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(payload),
+        });
+
+        status = response.status;
+      } else if (callback) {
+        log.info('exec', { function: def.id });
+        status = (await callback(payload)) ?? 200;
+      }
+
+      // Check errors.
+      if (status && status >= 400) {
+        throw new Error(`Response: ${status}`);
+      }
+
+      // const result = await response.json();
+      log.info('done', { function: def.id, status });
+    } catch (err: any) {
+      log.error('error', { function: def.id, error: err.message });
+      status = 500;
+    }
+
+    return status;
+  }
+
+  //
+  // Triggers
+  //
+
+  /**
+   * Cron timer.
+   */
+  private async _createTimer(ctx: Context, space: Space, def: FunctionDef, trigger: FunctionTrigger) {
+    log.info('timer', { space: space.key, trigger });
+    const spec = trigger.timer!;
+
     const task = new DeferredTask(ctx, async () => {
-      await this._execFunction(def, {
-        space: space.key,
-      });
+      await this._execFunction(def, trigger, { spaceKey: space.key });
     });
 
-    invariant(trigger.schedule);
     let last = 0;
     let run = 0;
     // https://www.npmjs.com/package/cron#constructor
     const job = CronJob.from({
-      cronTime: trigger.schedule,
+      cronTime: spec.cron,
       runOnInit: false,
       onTick: () => {
         // TODO(burdon): Check greater than 30s (use cron-parser).
@@ -122,18 +192,138 @@ export class Scheduler {
     ctx.onDispose(() => job.stop());
   }
 
-  private _createSubscription(ctx: Context, space: Space, def: FunctionDef, triggerSubscription: TriggerSubscription) {
-    log.info('subscription', { space: space.key, triggerSubscription });
+  /**
+   * Webhook.
+   */
+  private async _createWebhook(ctx: Context, space: Space, def: FunctionDef, trigger: FunctionTrigger) {
+    log.info('webhook', { space: space.key, trigger });
+    const spec = trigger.webhook!;
+
+    // TODO(burdon): Enable POST hook with payload.
+    const server = http.createServer(async (req, res) => {
+      if (req.method !== spec.method) {
+        res.statusCode = 405;
+        return res.end();
+      }
+
+      res.statusCode = await this._execFunction(def, trigger, { spaceKey: space.key });
+      res.end();
+    });
+
+    // TODO(burdon): Not used.
+    // const DEF_PORT_RANGE = { min: 7500, max: 7599 };
+    // const portRange = Object.assign({}, trigger.port, DEF_PORT_RANGE) as WebhookTrigger['port'];
+    const port = await getPort({
+      random: true,
+      // portRange: [portRange!.min, portRange!.max],
+    });
+
+    // TODO(burdon): Update trigger object with actual port.
+    server.listen(port, () => {
+      log.info('started webhook', { port });
+      spec.port = port;
+    });
+
+    ctx.onDispose(() => {
+      server.close();
+    });
+  }
+
+  /**
+   * Websocket.
+   * NOTE: The port must be unique, so the same hook cannot be used for multiple spaces.
+   */
+  private async _createWebsocket(
+    ctx: Context,
+    space: Space,
+    def: FunctionDef,
+    trigger: FunctionTrigger,
+    options: {
+      retryDelay: number;
+      maxAttempts: number;
+    } = {
+      retryDelay: 2,
+      maxAttempts: 5,
+    },
+  ) {
+    log.info('websocket', { space: space.key, trigger });
+    const spec = trigger.websocket!;
+    const { url, init } = spec;
+
+    let ws: WebSocket;
+    for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
+      const open = new Trigger<boolean>();
+
+      ws = new WebSocket(url);
+      Object.assign(ws, {
+        onopen: () => {
+          log.info('opened', { url });
+          if (spec.init) {
+            ws.send(new TextEncoder().encode(JSON.stringify(init)));
+          }
+
+          open.wake(true);
+        },
+
+        onclose: (event) => {
+          log.info('closed', { url, code: event.code });
+          // Reconnect if server closes (e.g., CF restart).
+          // https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent/code
+          if (event.code === 1006) {
+            setTimeout(async () => {
+              log.info(`reconnecting in ${options.retryDelay}s...`, { url });
+              await this._createWebsocket(ctx, space, def, trigger, options);
+            }, options.retryDelay * 1_000);
+          }
+
+          open.wake(false);
+        },
+
+        onerror: (event) => {
+          log.catch(event.error, { url });
+        },
+
+        onmessage: async (event) => {
+          try {
+            const data = JSON.parse(new TextDecoder().decode(event.data as Uint8Array));
+            await this._execFunction(def, trigger, { spaceKey: space.key, data });
+          } catch (err) {
+            log.catch(err, { url });
+          }
+        },
+      } satisfies Partial<WebSocket>);
+
+      const isOpen = await open.wait();
+      if (isOpen) {
+        break;
+      } else {
+        const wait = Math.pow(attempt, 2) * options.retryDelay;
+        if (attempt < options.maxAttempts) {
+          log.warn(`failed to connect; trying again in ${wait}s`, { attempt });
+          await sleep(wait * 1_000);
+        }
+      }
+    }
+
+    ctx.onDispose(() => {
+      ws?.close();
+    });
+  }
+
+  /**
+   * ECHO subscription.
+   */
+  private async _createSubscription(ctx: Context, space: Space, def: FunctionDef, trigger: FunctionTrigger) {
+    log.info('subscription', { space: space.key, trigger });
+    const spec = trigger.subscription!;
+
     const objectIds = new Set<string>();
     const task = new DeferredTask(ctx, async () => {
-      await this._execFunction(def, {
-        space: space.key,
-        objects: Array.from(objectIds),
-      });
+      await this._execFunction(def, trigger, { spaceKey: space.key, objects: Array.from(objectIds) });
     });
 
-    // TODO(burdon): Don't fire initially.
-    // TODO(burdon): Standardize subscription handles.
+    // TODO(burdon): Don't fire initially?
+    // TODO(burdon): Create queue. Only allow one invocation per trigger at a time?
     const subscriptions: (() => void)[] = [];
     const subscription = createSubscription(({ added, updated }) => {
       log.info('updated', { added: added.length, updated: updated.length });
@@ -146,17 +336,17 @@ export class Scheduler {
 
       task.schedule();
     });
+
     subscriptions.push(() => subscription.unsubscribe());
 
-    // TODO(burdon): Create queue. Only allow one invocation per trigger at a time?
     // TODO(burdon): Disable trigger if keeps failing.
-    const { type, props, deep, delay } = triggerSubscription;
+    const { filter, options: { deep, delay } = {} } = spec;
     const update = ({ objects }: Query) => {
       subscription.update(objects);
 
       // TODO(burdon): Hack to monitor changes to Document's text object.
       if (deep) {
-        log.info('update', { type, deep, objects: objects.length });
+        log.info('update', { objects: objects.length });
         for (const object of objects) {
           const content = object.content;
           if (content instanceof TextV0Type) {
@@ -168,40 +358,14 @@ export class Scheduler {
       }
     };
 
+    // TODO(burdon): Is Filter.or implemented?
     // TODO(burdon): [Bug]: all callbacks are fired on the first mutation.
     // TODO(burdon): [Bug]: not updated when document is deleted (either top or hierarchically).
-    const query = space.db.query(Filter.typename(type, props));
-    subscriptions.push(query.subscribe(delay ? debounce(update, delay * 1_000) : update));
+    const query = space.db.query(Filter.or(filter.map(({ type, props }) => Filter.typename(type, props))));
+    subscriptions.push(query.subscribe(delay ? debounce(update, delay) : update));
 
     ctx.onDispose(() => {
       subscriptions.forEach((unsubscribe) => unsubscribe());
     });
   }
-
-  private async _execFunction(def: FunctionDef, data: any) {
-    try {
-      log('request', { function: def.id });
-      const { endpoint, callback } = this._options;
-      let status = 0;
-      if (endpoint) {
-        // TODO(burdon): Move out of scheduler (generalize as callback).
-        const response = await fetch(`${this._options.endpoint}/${def.name}`, {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-          },
-          body: JSON.stringify(data),
-        });
-
-        status = response.status;
-      } else if (callback) {
-        status = await callback(data);
-      }
-
-      // const result = await response.json();
-      log('result', { function: def.id, result: status });
-    } catch (err: any) {
-      log.error('error', { function: def.id, error: err.message });
-    }
-  }
 }

package/src/testing/test/handler.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type FunctionHandler } from '../../handler';
+
+export const handler: FunctionHandler<any> = async ({ response }) => {
+  return response.status(200);
+};

package/src/testing/test/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { handler } from './handler';
+
+export default handler;

package/src/types.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import * as S from '@effect/schema/Schema';
+
+const TimerTriggerSchema = S.struct({
+  cron: S.string,
+});
+
+const WebhookTriggerSchema = S.mutable(
+  S.struct({
+    method: S.string,
+    // Assigned port.
+    port: S.optional(S.number),
+  }),
+);
+
+const WebsocketTriggerSchema = S.struct({
+  url: S.string,
+  init: S.optional(S.record(S.string, S.any)),
+});
+
+const SubscriptionTriggerSchema = S.struct({
+  spaceKey: S.optional(S.string),
+  // TODO(burdon): Define query DSL.
+  filter: S.array(
+    S.struct({
+      type: S.string,
+      props: S.optional(S.record(S.string, S.any)),
+    }),
+  ),
+  options: S.optional(
+    S.struct({
+      // Watch changes to object (not just creation).
+      deep: S.optional(S.boolean),
+      // Debounce changes (delay in ms).
+      delay: S.optional(S.number),
+    }),
+  ),
+});
+
+const FunctionTriggerSchema = S.struct({
+  function: S.string.pipe(S.description('Function ID/URI.')),
+
+  // Context passed to function.
+  meta: S.optional(S.record(S.string, S.any)),
+
+  // Triggers.
+  timer: S.optional(TimerTriggerSchema),
+  webhook: S.optional(WebhookTriggerSchema),
+  websocket: S.optional(WebsocketTriggerSchema),
+  subscription: S.optional(SubscriptionTriggerSchema),
+});
+
+export type FunctionTrigger = S.Schema.Type<typeof FunctionTriggerSchema>;
+
+export type TimerTrigger = S.Schema.Type<typeof TimerTriggerSchema>;
+export type WebhookTrigger = S.Schema.Type<typeof WebhookTriggerSchema>;
+export type WebsocketTrigger = S.Schema.Type<typeof WebsocketTriggerSchema>;
+export type SubscriptionTrigger = S.Schema.Type<typeof SubscriptionTriggerSchema>;
+
+/**
+ * Function definition.
+ */
+// TODO(burdon): Name vs. path?
+const FunctionDefSchema = S.struct({
+  id: S.string,
+  // name: S.string,
+  description: S.optional(S.string),
+  // TODO(burdon): Rename route?
+  path: S.string,
+  // TODO(burdon): NPM/GitHub/Docker/CF URL?
+  handler: S.string,
+});
+
+export type FunctionDef = S.Schema.Type<typeof FunctionDefSchema>;
+
+/**
+ * Function manifest file.
+ */
+export const FunctionManifestSchema = S.struct({
+  functions: S.mutable(S.array(FunctionDefSchema)),
+  triggers: S.optional(S.mutable(S.array(FunctionTriggerSchema))),
+});
+
+export type FunctionManifest = S.Schema.Type<typeof FunctionManifestSchema>;

package/dist/types/src/manifest.d.ts

@@ -1,26 +0,0 @@
-export type FunctionDef = {
-    id: string;
-    name: string;
-    handler: string;
-    description?: string;
-};
-export type TriggerSubscription = {
-    type: string;
-    spaceKey: string;
-    props?: Record<string, any>;
-    deep?: boolean;
-    delay?: number;
-};
-export type FunctionTrigger = {
-    function: string;
-    schedule?: string;
-    subscriptions?: TriggerSubscription[];
-};
-/**
- * Function manifest file.
- */
-export type FunctionManifest = {
-    functions: FunctionDef[];
-    triggers: FunctionTrigger[];
-};
-//# sourceMappingURL=manifest.d.ts.map

package/dist/types/src/manifest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../../src/manifest.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,WAAW,GAAG;IAExB,EAAE,EAAE,MAAM,CAAC;IAEX,IAAI,EAAE,MAAM,CAAC;IAEb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAGF,MAAM,MAAM,mBAAmB,GAAG;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAKF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B,CAAC"}

package/src/manifest.ts

@@ -1,42 +0,0 @@
-//
-// Copyright 2023 DXOS.org
-//
-
-// Lambda-like function definitions.
-// See: https://www.serverless.com/framework/docs/providers/aws/guide/serverless.yml/#functions
-
-export type FunctionDef = {
-  // FQ function name.
-  id: string;
-  // URL path.
-  name: string;
-  // File path of handler.
-  handler: string;
-  description?: string;
-};
-
-// TODO(burdon): Query DSL.
-export type TriggerSubscription = {
-  type: string;
-  spaceKey: string;
-  props?: Record<string, any>;
-  deep?: boolean; // Watch changes to object (not just creation).
-  delay?: number;
-};
-
-// TODO(burdon): Generalize binding.
-// https://www.npmjs.com/package/aws-lambda
-// https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
-export type FunctionTrigger = {
-  function: string;
-  schedule?: string;
-  subscriptions?: TriggerSubscription[];
-};
-
-/**
- * Function manifest file.
- */
-export type FunctionManifest = {
-  functions: FunctionDef[];
-  triggers: FunctionTrigger[];
-};