@dxos/functions 0.5.3-main.f1ddd61 → 0.5.3-main.f752aaa

package/src/runtime/scheduler.test.ts

@@ -3,22 +3,31 @@
 //
 
 import { expect } from 'chai';
+import WebSocket from 'ws';
 
 import { Trigger } from '@dxos/async';
 import { Client } from '@dxos/client';
 import { TestBuilder } from '@dxos/client/testing';
+import { create, S, TypedObject } from '@dxos/echo-schema';
 import { describe, test } from '@dxos/test';
 
 import { Scheduler } from './scheduler';
-import { type FunctionManifest } from '../manifest';
+import { type FunctionManifest } from '../types';
 
+// TODO(burdon): Test we can add and remove triggers.
 describe('scheduler', () => {
-  test.only('callback', async () => {
+  let client: Client;
+  before(async () => {
     const testBuilder = new TestBuilder();
-    const client = new Client({ services: testBuilder.createLocal() });
+    client = new Client({ services: testBuilder.createLocal() });
     await client.initialize();
     await client.halo.createIdentity();
+  });
+  after(async () => {
+    await client.destroy();
+  });
 
+  test('timer', async () => {
     const manifest: FunctionManifest = {
       functions: [
         {
@@ -30,7 +39,9 @@ describe('scheduler', () => {
       triggers: [
         {
           function: 'example.com/function/test',
-          schedule: '0/1 * * * * *', // Every 1s.
+          timer: {
+            cron: '0/1 * * * * *', // Every 1s.
+          },
         },
       ],
     };
@@ -42,17 +53,156 @@ describe('scheduler', () => {
         if (++count === 3) {
           done.wake();
         }
+      },
+    });
+
+    await scheduler.start();
+    after(async () => {
+      await scheduler.stop();
+    });
+
+    await done.wait({ timeout: 5_000 });
+    expect(count).to.equal(3);
+  });
+
+  test('webhook', async () => {
+    const manifest: FunctionManifest = {
+      functions: [
+        {
+          id: 'example.com/function/test',
+          name: 'test',
+          handler: 'test',
+        },
+      ],
+      triggers: [
+        {
+          function: 'example.com/function/test',
+          webhook: {
+            port: 8080,
+          },
+        },
+      ],
+    };
 
-        return 200;
+    const done = new Trigger();
+    const scheduler = new Scheduler(client, manifest, {
+      callback: async () => {
+        done.wake();
       },
     });
 
     await scheduler.start();
+    after(async () => {
+      await scheduler.stop();
+    });
 
+    setTimeout(() => {
+      void fetch('http://localhost:8080');
+    });
     await done.wait();
-    expect(count).to.equal(3);
+  });
 
-    await scheduler.stop();
-    await client.destroy();
+  test.only('websocket', async () => {
+    const manifest: FunctionManifest = {
+      functions: [
+        {
+          id: 'example.com/function/test',
+          name: 'test',
+          handler: 'test',
+        },
+      ],
+      triggers: [
+        {
+          function: 'example.com/function/test',
+          websocket: {
+            // url: 'https://hub.dxos.network/api/mailbox/test',
+            url: 'http://localhost:8081',
+            init: {
+              type: 'sync',
+            },
+          },
+        },
+      ],
+    };
+
+    const done = new Trigger();
+    const scheduler = new Scheduler(client, manifest, {
+      callback: async (data) => {
+        done.wake();
+      },
+    });
+
+    await scheduler.start();
+    after(async () => {
+      await scheduler.stop();
+    });
+
+    // Test server.
+    setTimeout(() => {
+      const wss = new WebSocket.Server({ port: 8081 });
+      wss.on('connection', (ws: WebSocket) => {
+        ws.on('message', (data) => {
+          const info = JSON.parse(new TextDecoder().decode(data as ArrayBuffer));
+          expect(info.type).to.equal('sync');
+          done.wake();
+        });
+      });
+    }, 500);
+
+    await done.wait();
+  });
+
+  test('subscription', async () => {
+    class TestType extends TypedObject({ typename: 'example.com/type/Test', version: '0.1.0' })({
+      title: S.string,
+    }) {}
+    client.addSchema(TestType);
+
+    const manifest: FunctionManifest = {
+      functions: [
+        {
+          id: 'example.com/function/test',
+          name: 'test',
+          handler: 'test',
+        },
+      ],
+      triggers: [
+        {
+          function: 'example.com/function/test',
+          subscription: {
+            spaceKey: client.spaces.default.key.toHex(),
+            filter: [
+              {
+                type: TestType.typename,
+              },
+            ],
+          },
+        },
+      ],
+    };
+
+    let count = 0;
+    const done = new Trigger();
+    const scheduler = new Scheduler(client, manifest, {
+      callback: async () => {
+        if (++count === 2) {
+          done.wake();
+        }
+      },
+    });
+
+    await scheduler.start();
+    after(async () => {
+      await scheduler.stop();
+    });
+
+    // TODO(burdon): Query for Expando?
+    setTimeout(() => {
+      const space = client.spaces.default;
+      const object = create(TestType, { title: 'Hello world!' });
+      space.db.add(object);
+    }, 100);
+
+    await done.wait();
   });
 });

package/src/runtime/scheduler.ts

@@ -3,30 +3,39 @@
 //
 
 import { CronJob } from 'cron';
+import http from 'node:http';
+import WebSocket from 'ws';
 
 import { TextV0Type } from '@braneframe/types';
-import { debounce, DeferredTask } from '@dxos/async';
+import { debounce, DeferredTask, sleep, Trigger } from '@dxos/async';
 import { type Client, type PublicKey } from '@dxos/client';
-import { type Space, Filter, createSubscription, type Query, getAutomergeObjectCore } from '@dxos/client/echo';
+import { createSubscription, Filter, getAutomergeObjectCore, type Query, type Space } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { ComplexMap } from '@dxos/util';
 
 import { type FunctionSubscriptionEvent } from '../handler';
-import { type FunctionDef, type FunctionManifest, type FunctionTrigger, type TriggerSubscription } from '../manifest';
+import {
+  type FunctionDef,
+  type FunctionManifest,
+  type FunctionTrigger,
+  type SubscriptionTrigger,
+  type TimerTrigger,
+  type WebhookTrigger,
+  type WebsocketTrigger,
+} from '../types';
 
-type Callback = (data: FunctionSubscriptionEvent) => Promise<number>;
+type Callback = (data: FunctionSubscriptionEvent) => Promise<void>;
 
-type SchedulerOptions = {
+export type SchedulerOptions = {
   endpoint?: string;
   callback?: Callback;
 };
 
 /**
- * Functions scheduler.
+ * The scheduler triggers function exectuion based on various triggers.
  */
-// TODO(burdon): Create tests.
 export class Scheduler {
   // Map of mounted functions.
   private readonly _mounts = new ComplexMap<
@@ -62,7 +71,7 @@ export class Scheduler {
     const def = this._manifest.functions.find((config) => config.id === trigger.function);
     invariant(def, `Function not found: ${trigger.function}`);
 
-    // Currently supports only one trigger declaration per function.
+    // TODO(burdon): Currently supports only one trigger declaration per function.
     const exists = this._mounts.get(key);
     if (!exists) {
       this._mounts.set(key, { ctx, trigger });
@@ -71,14 +80,24 @@ export class Scheduler {
         return;
       }
 
-      // Timer.
-      if (trigger.schedule) {
-        this._createTimer(ctx, space, def, trigger);
+      //
+      // Triggers types.
+      //
+
+      if (trigger.timer) {
+        await this._createTimer(ctx, space, def, trigger.timer);
+      }
+
+      if (trigger.webhook) {
+        await this._createWebhook(ctx, space, def, trigger.webhook);
+      }
+
+      if (trigger.websocket) {
+        await this._createWebsocket(ctx, space, def, trigger.websocket);
       }
 
-      // Subscription.
-      for (const triggerSubscription of trigger.subscriptions ?? []) {
-        this._createSubscription(ctx, space, def, triggerSubscription);
+      if (trigger.subscription) {
+        await this._createSubscription(ctx, space, def, trigger.subscription);
       }
     }
   }
@@ -92,19 +111,51 @@ export class Scheduler {
     }
   }
 
-  private _createTimer(ctx: Context, space: Space, def: FunctionDef, trigger: FunctionTrigger) {
+  // TODO(burdon): Pass in Space key (common context).
+  private async _execFunction(def: FunctionDef, data: any) {
+    try {
+      log.info('exec', { function: def.id });
+      const { endpoint, callback } = this._options;
+      if (endpoint) {
+        // TODO(burdon): Move out of scheduler (generalize as callback).
+        await fetch(`${this._options.endpoint}/${def.name}`, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(data),
+        });
+      } else if (callback) {
+        await callback(data);
+      }
+
+      // const result = await response.json();
+      log.info('done', { function: def.id });
+    } catch (err: any) {
+      log.error('error', { function: def.id, error: err.message });
+    }
+  }
+
+  //
+  // Triggers
+  //
+
+  /**
+   * Cron timer.
+   */
+  private async _createTimer(ctx: Context, space: Space, def: FunctionDef, trigger: TimerTrigger) {
+    log.info('timer', { space: space.key, trigger });
+    const { cron } = trigger;
+
     const task = new DeferredTask(ctx, async () => {
-      await this._execFunction(def, {
-        space: space.key,
-      });
+      await this._execFunction(def, { space: space.key });
     });
 
-    invariant(trigger.schedule);
     let last = 0;
     let run = 0;
     // https://www.npmjs.com/package/cron#constructor
     const job = CronJob.from({
-      cronTime: trigger.schedule,
+      cronTime: cron,
       runOnInit: false,
       onTick: () => {
         // TODO(burdon): Check greater than 30s (use cron-parser).
@@ -122,18 +173,109 @@ export class Scheduler {
     ctx.onDispose(() => job.stop());
   }
 
-  private _createSubscription(ctx: Context, space: Space, def: FunctionDef, triggerSubscription: TriggerSubscription) {
-    log.info('subscription', { space: space.key, triggerSubscription });
+  /**
+   * Webhook.
+   */
+  private async _createWebhook(ctx: Context, space: Space, def: FunctionDef, trigger: WebhookTrigger) {
+    log.info('webhook', { space: space.key, trigger });
+    const { port } = trigger;
+
+    // TODO(burdon): POST JSON.
+    const server = http.createServer(async (req, res) => {
+      await this._execFunction(def, { space: space.key });
+    });
+
+    server.listen(port, () => {
+      log.info('started webhook', { port });
+    });
+
+    ctx.onDispose(() => {
+      server.close();
+    });
+  }
+
+  /**
+   * Websocket.
+   */
+  private async _createWebsocket(
+    ctx: Context,
+    space: Space,
+    def: FunctionDef,
+    trigger: WebsocketTrigger,
+    options: {
+      retryDelay: number;
+      maxAttempts: number;
+    } = {
+      retryDelay: 2,
+      maxAttempts: 5,
+    },
+  ) {
+    log.info('websocket', { space: space.key, trigger });
+    const { url } = trigger;
+
+    let ws: WebSocket;
+    for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
+      const open = new Trigger<boolean>();
+
+      ws = new WebSocket(url);
+      Object.assign(ws, {
+        onopen: () => {
+          log.info('opened', { url });
+          if (trigger.init) {
+            ws.send(new TextEncoder().encode(JSON.stringify(trigger.init)));
+          }
+
+          open.wake(true);
+        },
+
+        onclose: () => {
+          log.info('closed', { url });
+          open.wake(false);
+        },
+
+        onerror: (event) => {
+          log.catch(event.error, { url });
+        },
+
+        onmessage: async (event) => {
+          try {
+            const data = JSON.parse(new TextDecoder().decode(event.data as Uint8Array));
+            await this._execFunction(def, { space: space.key, data });
+          } catch (err) {
+            log.catch(err, { url });
+          }
+        },
+      } satisfies Partial<WebSocket>);
+
+      const isOpen = await open.wait();
+      if (isOpen) {
+        break;
+      } else {
+        const wait = Math.pow(attempt, 2) * options.retryDelay;
+        if (attempt < options.maxAttempts) {
+          log.warn(`failed to connect; trying again in ${wait}s`, { attempt });
+          await sleep(wait * 1_000);
+        }
+      }
+    }
+
+    ctx.onDispose(() => {
+      ws?.close();
+    });
+  }
+
+  /**
+   * ECHO subscription.
+   */
+  private async _createSubscription(ctx: Context, space: Space, def: FunctionDef, trigger: SubscriptionTrigger) {
+    log.info('subscription', { space: space.key, trigger });
     const objectIds = new Set<string>();
     const task = new DeferredTask(ctx, async () => {
-      await this._execFunction(def, {
-        space: space.key,
-        objects: Array.from(objectIds),
-      });
+      await this._execFunction(def, { space: space.key, objects: Array.from(objectIds) });
     });
 
     // TODO(burdon): Don't fire initially.
-    // TODO(burdon): Standardize subscription handles.
+    // TODO(burdon): Subscription is called THREE times.
     const subscriptions: (() => void)[] = [];
     const subscription = createSubscription(({ added, updated }) => {
       log.info('updated', { added: added.length, updated: updated.length });
@@ -150,13 +292,13 @@ export class Scheduler {
 
     // TODO(burdon): Create queue. Only allow one invocation per trigger at a time?
     // TODO(burdon): Disable trigger if keeps failing.
-    const { type, props, deep, delay } = triggerSubscription;
+    const { filter, options: { deep, delay } = {} } = trigger;
     const update = ({ objects }: Query) => {
       subscription.update(objects);
 
       // TODO(burdon): Hack to monitor changes to Document's text object.
       if (deep) {
-        log.info('update', { type, deep, objects: objects.length });
+        log.info('update', { objects: objects.length });
         for (const object of objects) {
           const content = object.content;
           if (content instanceof TextV0Type) {
@@ -168,40 +310,14 @@ export class Scheduler {
       }
     };
 
+    // TODO(burdon): Is Filter.or implemented?
     // TODO(burdon): [Bug]: all callbacks are fired on the first mutation.
     // TODO(burdon): [Bug]: not updated when document is deleted (either top or hierarchically).
-    const query = space.db.query(Filter.typename(type, props));
-    subscriptions.push(query.subscribe(delay ? debounce(update, delay * 1_000) : update));
+    const query = space.db.query(Filter.or(filter.map(({ type, props }) => Filter.typename(type, props))));
+    subscriptions.push(query.subscribe(delay ? debounce(update, delay) : update));
 
     ctx.onDispose(() => {
       subscriptions.forEach((unsubscribe) => unsubscribe());
     });
   }
-
-  private async _execFunction(def: FunctionDef, data: any) {
-    try {
-      log('request', { function: def.id });
-      const { endpoint, callback } = this._options;
-      let status = 0;
-      if (endpoint) {
-        // TODO(burdon): Move out of scheduler (generalize as callback).
-        const response = await fetch(`${this._options.endpoint}/${def.name}`, {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-          },
-          body: JSON.stringify(data),
-        });
-
-        status = response.status;
-      } else if (callback) {
-        status = await callback(data);
-      }
-
-      // const result = await response.json();
-      log('result', { function: def.id, result: status });
-    } catch (err: any) {
-      log.error('error', { function: def.id, error: err.message });
-    }
-  }
 }

package/src/types.ts

@@ -0,0 +1,76 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import * as S from '@effect/schema/Schema';
+
+const TimerTriggerSchema = S.struct({
+  cron: S.string,
+});
+
+const WebhookTriggerSchema = S.struct({
+  port: S.number,
+});
+
+const WebsocketTriggerSchema = S.struct({
+  url: S.string,
+  init: S.optional(S.record(S.string, S.any)),
+});
+
+const SubscriptionTriggerSchema = S.struct({
+  spaceKey: S.optional(S.string),
+  // TODO(burdon): Define query DSL.
+  filter: S.array(
+    S.struct({
+      type: S.string,
+      props: S.optional(S.record(S.string, S.any)),
+    }),
+  ),
+  options: S.optional(
+    S.struct({
+      // Watch changes to object (not just creation).
+      deep: S.optional(S.boolean),
+      // Debounce changes (delay in ms).
+      delay: S.optional(S.number),
+    }),
+  ),
+});
+
+const FunctionTriggerSchema = S.struct({
+  function: S.string.pipe(S.description('Function ID/URI.')),
+
+  timer: S.optional(TimerTriggerSchema),
+  webhook: S.optional(WebhookTriggerSchema),
+  websocket: S.optional(WebsocketTriggerSchema),
+  subscription: S.optional(SubscriptionTriggerSchema),
+});
+
+export type TimerTrigger = S.Schema.Type<typeof TimerTriggerSchema>;
+export type WebhookTrigger = S.Schema.Type<typeof WebhookTriggerSchema>;
+export type WebsocketTrigger = S.Schema.Type<typeof WebsocketTriggerSchema>;
+export type SubscriptionTrigger = S.Schema.Type<typeof SubscriptionTriggerSchema>;
+export type FunctionTrigger = S.Schema.Type<typeof FunctionTriggerSchema>;
+
+/**
+ * Function definition.
+ */
+// TODO(burdon): Name vs. path?
+const FunctionDefSchema = S.struct({
+  id: S.string,
+  description: S.optional(S.string),
+  name: S.string,
+  // TODO(burdon): NPM/GitHub URL?
+  handler: S.string,
+});
+
+export type FunctionDef = S.Schema.Type<typeof FunctionDefSchema>;
+
+/**
+ * Function manifest file.
+ */
+export const FunctionManifestSchema = S.struct({
+  functions: S.mutable(S.array(FunctionDefSchema)),
+  triggers: S.mutable(S.array(FunctionTriggerSchema)),
+});
+
+export type FunctionManifest = S.Schema.Type<typeof FunctionManifestSchema>;

package/dist/types/src/manifest.d.ts

@@ -1,26 +0,0 @@
-export type FunctionDef = {
-    id: string;
-    name: string;
-    handler: string;
-    description?: string;
-};
-export type TriggerSubscription = {
-    type: string;
-    spaceKey: string;
-    props?: Record<string, any>;
-    deep?: boolean;
-    delay?: number;
-};
-export type FunctionTrigger = {
-    function: string;
-    schedule?: string;
-    subscriptions?: TriggerSubscription[];
-};
-/**
- * Function manifest file.
- */
-export type FunctionManifest = {
-    functions: FunctionDef[];
-    triggers: FunctionTrigger[];
-};
-//# sourceMappingURL=manifest.d.ts.map

package/dist/types/src/manifest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../../src/manifest.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,WAAW,GAAG;IAExB,EAAE,EAAE,MAAM,CAAC;IAEX,IAAI,EAAE,MAAM,CAAC;IAEb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAGF,MAAM,MAAM,mBAAmB,GAAG;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAKF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B,CAAC"}

package/src/manifest.ts

@@ -1,42 +0,0 @@
-//
-// Copyright 2023 DXOS.org
-//
-
-// Lambda-like function definitions.
-// See: https://www.serverless.com/framework/docs/providers/aws/guide/serverless.yml/#functions
-
-export type FunctionDef = {
-  // FQ function name.
-  id: string;
-  // URL path.
-  name: string;
-  // File path of handler.
-  handler: string;
-  description?: string;
-};
-
-// TODO(burdon): Query DSL.
-export type TriggerSubscription = {
-  type: string;
-  spaceKey: string;
-  props?: Record<string, any>;
-  deep?: boolean; // Watch changes to object (not just creation).
-  delay?: number;
-};
-
-// TODO(burdon): Generalize binding.
-// https://www.npmjs.com/package/aws-lambda
-// https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
-export type FunctionTrigger = {
-  function: string;
-  schedule?: string;
-  subscriptions?: TriggerSubscription[];
-};
-
-/**
- * Function manifest file.
- */
-export type FunctionManifest = {
-  functions: FunctionDef[];
-  triggers: FunctionTrigger[];
-};