@dxos/functions 0.5.3-main.f752aaa → 0.5.3-next.57eca40

package/src/trigger/type/subscription-trigger.ts

@@ -0,0 +1,80 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { TextV0Type } from '@braneframe/types';
+import { debounce, UpdateScheduler } from '@dxos/async';
+import { type Context } from '@dxos/context';
+import { createSubscription, Filter, getAutomergeObjectCore, type Query } from '@dxos/echo-db';
+import { log } from '@dxos/log';
+
+import type { SubscriptionTrigger } from '../../types';
+import { type TriggerCallback, type TriggerContext, type TriggerFactory } from '../trigger-registry';
+
+export const createSubscriptionTrigger: TriggerFactory<SubscriptionTrigger> = async (
+  ctx: Context,
+  triggerCtx: TriggerContext,
+  spec: SubscriptionTrigger,
+  callback: TriggerCallback,
+) => {
+  const objectIds = new Set<string>();
+  const task = new UpdateScheduler(
+    ctx,
+    async () => {
+      if (objectIds.size > 0) {
+        const objects = Array.from(objectIds);
+        objectIds.clear();
+        await callback({ objects });
+      }
+    },
+    { maxFrequency: 4 },
+  );
+
+  // TODO(burdon): Don't fire initially?
+  // TODO(burdon): Create queue. Only allow one invocation per trigger at a time?
+  const subscriptions: (() => void)[] = [];
+  const subscription = createSubscription(({ added, updated }) => {
+    const sizeBefore = objectIds.size;
+    for (const object of added) {
+      objectIds.add(object.id);
+    }
+    for (const object of updated) {
+      objectIds.add(object.id);
+    }
+    if (objectIds.size > sizeBefore) {
+      log.info('updated', { added: added.length, updated: updated.length });
+      task.trigger();
+    }
+  });
+
+  subscriptions.push(() => subscription.unsubscribe());
+
+  // TODO(burdon): Disable trigger if keeps failing.
+  const { filter, options: { deep, delay } = {} } = spec;
+  const update = ({ objects }: Query) => {
+    subscription.update(objects);
+
+    // TODO(burdon): Hack to monitor changes to Document's text object.
+    if (deep) {
+      log.info('update', { objects: objects.length });
+      for (const object of objects) {
+        const content = object.content;
+        if (content instanceof TextV0Type) {
+          subscriptions.push(
+            getAutomergeObjectCore(content).updates.on(debounce(() => subscription.update([object]), 1_000)),
+          );
+        }
+      }
+    }
+  };
+
+  // TODO(burdon): Is Filter.or implemented?
+  // TODO(burdon): [Bug]: all callbacks are fired on the first mutation.
+  // TODO(burdon): [Bug]: not updated when document is deleted (either top or hierarchically).
+  const query = triggerCtx.space.db.query(Filter.or(filter.map(({ type, props }) => Filter.typename(type, props))));
+  subscriptions.push(query.subscribe(delay ? debounce(update, delay) : update));
+
+  ctx.onDispose(() => {
+    subscriptions.forEach((unsubscribe) => unsubscribe());
+  });
+};

package/src/trigger/type/timer-trigger.ts

@@ -0,0 +1,44 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { CronJob } from 'cron';
+
+import { DeferredTask } from '@dxos/async';
+import { type Context } from '@dxos/context';
+import { log } from '@dxos/log';
+
+import type { TimerTrigger } from '../../types';
+import { type TriggerCallback, type TriggerContext, type TriggerFactory } from '../trigger-registry';
+
+export const createTimerTrigger: TriggerFactory<TimerTrigger> = async (
+  ctx: Context,
+  triggerContext: TriggerContext,
+  spec: TimerTrigger,
+  callback: TriggerCallback,
+) => {
+  const task = new DeferredTask(ctx, async () => {
+    await callback({});
+  });
+
+  let last = 0;
+  let run = 0;
+  // https://www.npmjs.com/package/cron#constructor
+  const job = CronJob.from({
+    cronTime: spec.cron,
+    runOnInit: false,
+    onTick: () => {
+      // TODO(burdon): Check greater than 30s (use cron-parser).
+      const now = Date.now();
+      const delta = last ? now - last : 0;
+      last = now;
+
+      run++;
+      log.info('tick', { space: triggerContext.space.key.truncate(), count: run, delta });
+      task.schedule();
+    },
+  });
+
+  job.start();
+  ctx.onDispose(() => job.stop());
+};

package/src/trigger/type/webhook-trigger.ts

@@ -0,0 +1,47 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { getPort } from 'get-port-please';
+import http from 'node:http';
+
+import { type Context } from '@dxos/context';
+import { log } from '@dxos/log';
+
+import type { WebhookTrigger } from '../../types';
+import { type TriggerCallback, type TriggerContext, type TriggerFactory } from '../trigger-registry';
+
+export const createWebhookTrigger: TriggerFactory<WebhookTrigger> = async (
+  ctx: Context,
+  _: TriggerContext,
+  spec: WebhookTrigger,
+  callback: TriggerCallback,
+) => {
+  // TODO(burdon): Enable POST hook with payload.
+  const server = http.createServer(async (req, res) => {
+    if (req.method !== spec.method) {
+      res.statusCode = 405;
+      return res.end();
+    }
+    res.statusCode = await callback({});
+    res.end();
+  });
+
+  // TODO(burdon): Not used.
+  // const DEF_PORT_RANGE = { min: 7500, max: 7599 };
+  // const portRange = Object.assign({}, trigger.port, DEF_PORT_RANGE) as WebhookTrigger['port'];
+  const port = await getPort({
+    random: true,
+    // portRange: [portRange!.min, portRange!.max],
+  });
+
+  // TODO(burdon): Update trigger object with actual port.
+  server.listen(port, () => {
+    log.info('started webhook', { port });
+    spec.port = port;
+  });
+
+  ctx.onDispose(() => {
+    server.close();
+  });
+};

package/src/trigger/type/websocket-trigger.ts

@@ -0,0 +1,91 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import WebSocket from 'ws';
+
+import { sleep, Trigger } from '@dxos/async';
+import { type Context } from '@dxos/context';
+import { log } from '@dxos/log';
+
+import { type WebsocketTrigger } from '../../types';
+import { type TriggerCallback, type TriggerContext, type TriggerFactory } from '../trigger-registry';
+
+interface WebsocketTriggerOptions {
+  retryDelay: number;
+  maxAttempts: number;
+}
+
+/**
+ * Websocket.
+ * NOTE: The port must be unique, so the same hook cannot be used for multiple spaces.
+ */
+export const createWebsocketTrigger: TriggerFactory<WebsocketTrigger, WebsocketTriggerOptions> = async (
+  ctx: Context,
+  triggerCtx: TriggerContext,
+  spec: WebsocketTrigger,
+  callback: TriggerCallback,
+  options: WebsocketTriggerOptions = { retryDelay: 2, maxAttempts: 5 },
+) => {
+  const { url, init } = spec;
+
+  let ws: WebSocket;
+  for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
+    const open = new Trigger<boolean>();
+
+    ws = new WebSocket(url);
+    Object.assign(ws, {
+      onopen: () => {
+        log.info('opened', { url });
+        if (spec.init) {
+          ws.send(new TextEncoder().encode(JSON.stringify(init)));
+        }
+
+        open.wake(true);
+      },
+
+      onclose: (event) => {
+        log.info('closed', { url, code: event.code });
+        // Reconnect if server closes (e.g., CF restart).
+        // https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent/code
+        if (event.code === 1006) {
+          setTimeout(async () => {
+            log.info(`reconnecting in ${options.retryDelay}s...`, { url });
+            await createWebsocketTrigger(ctx, triggerCtx, spec, callback, options);
+          }, options.retryDelay * 1_000);
+        }
+
+        open.wake(false);
+      },
+
+      onerror: (event) => {
+        log.catch(event.error, { url });
+      },
+
+      onmessage: async (event) => {
+        try {
+          log.info('message');
+          const data = JSON.parse(new TextDecoder().decode(event.data as Uint8Array));
+          await callback({ data });
+        } catch (err) {
+          log.catch(err, { url });
+        }
+      },
+    } satisfies Partial<WebSocket>);
+
+    const isOpen = await open.wait();
+    if (isOpen) {
+      break;
+    } else {
+      const wait = Math.pow(attempt, 2) * options.retryDelay;
+      if (attempt < options.maxAttempts) {
+        log.warn(`failed to connect; trying again in ${wait}s`, { attempt });
+        await sleep(wait * 1_000);
+      }
+    }
+  }
+
+  ctx.onDispose(() => {
+    ws?.close();
+  });
+};

package/src/types.ts

@@ -2,24 +2,19 @@
 // Copyright 2023 DXOS.org
 //
 
-import * as S from '@effect/schema/Schema';
+import { RawObject, S, TypedObject } from '@dxos/echo-schema';
 
-const TimerTriggerSchema = S.struct({
-  cron: S.string,
-});
-
-const WebhookTriggerSchema = S.struct({
-  port: S.number,
-});
-
-const WebsocketTriggerSchema = S.struct({
-  url: S.string,
-  init: S.optional(S.record(S.string, S.any)),
-});
+/**
+ * Type discriminator for TriggerSpec.
+ * Every spec has a type field of type FunctionTriggerType that we can use to understand which
+ * type we're working with.
+ * https://www.typescriptlang.org/docs/handbook/2/narrowing.html#discriminated-unions
+ */
+export type FunctionTriggerType = 'subscription' | 'timer' | 'webhook' | 'websocket';
 
 const SubscriptionTriggerSchema = S.struct({
-  spaceKey: S.optional(S.string),
-  // TODO(burdon): Define query DSL.
+  type: S.literal('subscription'),
+  // TODO(burdon): Define query DSL (from ECHO).
   filter: S.array(
     S.struct({
       type: S.string,
@@ -36,41 +31,73 @@ const SubscriptionTriggerSchema = S.struct({
   ),
 });
 
-const FunctionTriggerSchema = S.struct({
-  function: S.string.pipe(S.description('Function ID/URI.')),
+export type SubscriptionTrigger = S.Schema.Type<typeof SubscriptionTriggerSchema>;
 
-  timer: S.optional(TimerTriggerSchema),
-  webhook: S.optional(WebhookTriggerSchema),
-  websocket: S.optional(WebsocketTriggerSchema),
-  subscription: S.optional(SubscriptionTriggerSchema),
+const TimerTriggerSchema = S.struct({
+  type: S.literal('timer'),
+  cron: S.string,
 });
 
 export type TimerTrigger = S.Schema.Type<typeof TimerTriggerSchema>;
+
+const WebhookTriggerSchema = S.mutable(
+  S.struct({
+    type: S.literal('webhook'),
+    method: S.string,
+    // Assigned port.
+    port: S.optional(S.number),
+  }),
+);
+
 export type WebhookTrigger = S.Schema.Type<typeof WebhookTriggerSchema>;
+
+const WebsocketTriggerSchema = S.struct({
+  type: S.literal('websocket'),
+  url: S.string,
+  init: S.optional(S.record(S.string, S.any)),
+});
+
 export type WebsocketTrigger = S.Schema.Type<typeof WebsocketTriggerSchema>;
-export type SubscriptionTrigger = S.Schema.Type<typeof SubscriptionTriggerSchema>;
-export type FunctionTrigger = S.Schema.Type<typeof FunctionTriggerSchema>;
+
+const TriggerSpecSchema = S.union(
+  TimerTriggerSchema,
+  WebhookTriggerSchema,
+  WebsocketTriggerSchema,
+  SubscriptionTriggerSchema,
+);
+
+export type TriggerSpec = TimerTrigger | WebhookTrigger | WebsocketTrigger | SubscriptionTrigger;
 
 /**
  * Function definition.
  */
-// TODO(burdon): Name vs. path?
-const FunctionDefSchema = S.struct({
-  id: S.string,
+export class FunctionDef extends TypedObject({
+  typename: 'dxos.org/type/FunctionDef',
+  version: '0.1.0',
+})({
+  uri: S.string,
   description: S.optional(S.string),
-  name: S.string,
-  // TODO(burdon): NPM/GitHub URL?
+  route: S.string,
+  // TODO(burdon): NPM/GitHub/Docker/CF URL?
   handler: S.string,
-});
+}) {}
 
-export type FunctionDef = S.Schema.Type<typeof FunctionDefSchema>;
+export class FunctionTrigger extends TypedObject({
+  typename: 'dxos.org/type/FunctionTrigger',
+  version: '0.1.0',
+})({
+  function: S.string.pipe(S.description('Function URI.')),
+  // Context is merged into the event data passed to the function.
+  meta: S.optional(S.object),
+  spec: TriggerSpecSchema,
+}) {}
 
 /**
  * Function manifest file.
  */
 export const FunctionManifestSchema = S.struct({
-  functions: S.mutable(S.array(FunctionDefSchema)),
-  triggers: S.mutable(S.array(FunctionTriggerSchema)),
+  functions: S.optional(S.mutable(S.array(RawObject(FunctionDef)))),
+  triggers: S.optional(S.mutable(S.array(RawObject(FunctionTrigger)))),
 });
 
 export type FunctionManifest = S.Schema.Type<typeof FunctionManifestSchema>;

package/src/util.test.ts

@@ -0,0 +1,43 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { describe, test } from '@dxos/test';
+
+import { diff, intersection } from './util';
+
+describe('diff', () => {
+  test('returns the difference between two sets', () => {
+    {
+      const { added, updated, removed } = diff<number>([], [], (a, b) => a === b);
+      expect(added).to.deep.eq([]);
+      expect(updated).to.deep.eq([]);
+      expect(removed).to.deep.eq([]);
+    }
+    {
+      const previous = [1, 2, 3];
+      const next = [2, 3, 4];
+      const { added, updated, removed } = diff(previous, next, (a, b) => a === b);
+      expect(added).to.deep.eq([4]);
+      expect(updated).to.deep.eq([2, 3]);
+      expect(removed).to.deep.eq([1]);
+    }
+    {
+      const previous = [{ x: 1 }, { x: 2 }, { x: 3 }];
+      const next = [{ x: 2 }, { x: 3 }, { x: 4 }];
+      const { added, updated, removed } = diff(previous, next, (a, b) => a.x === b.x);
+      expect(added).to.deep.eq([{ x: 4 }]);
+      expect(updated).to.deep.eq([{ x: 2 }, { x: 3 }]);
+      expect(removed).to.deep.eq([{ x: 1 }]);
+    }
+  });
+
+  test('intersection', () => {
+    expect(intersection([1, 2, 3], [2, 3, 4], (a, b) => a === b)).to.deep.eq([2, 3]);
+    expect(
+      intersection([{ x: 1 }, { x: 2 }, { x: 3 }], [{ x: 2 }, { x: 3 }, { x: 4 }], (a, b) => a.x === b.x),
+    ).to.deep.eq([{ x: 2 }, { x: 3 }]);
+  });
+});

package/src/util.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export type Comparator<A, B = A> = (a: A, b: B) => boolean;
+
+export type DiffResult<A, B = A> = {
+  added: B[];
+  updated: A[];
+  removed: A[];
+};
+
+/**
+ *
+ * @param previous
+ * @param next
+ * @param comparator
+ */
+// TODO(burdon): Factor out.
+export const diff = <A, B = A>(
+  previous: readonly A[],
+  next: readonly B[],
+  comparator: Comparator<A, B>,
+): DiffResult<A, B> => {
+  const remaining = [...previous];
+  const result: DiffResult<A, B> = {
+    added: [],
+    updated: [],
+    removed: remaining,
+  };
+
+  // TODO(burdon): Mark and sweep.
+  for (const object of next) {
+    const index = remaining.findIndex((item) => comparator(item, object));
+    if (index === -1) {
+      result.added.push(object);
+    } else {
+      result.updated.push(remaining[index]);
+      remaining.splice(index, 1);
+    }
+  }
+
+  return result;
+};
+
+// TODO(burdon): Factor out.
+export const intersection = <A, B = A>(a: A[], b: B[], comparator: Comparator<A, B>): A[] =>
+  a.filter((a) => b.find((b) => comparator(a, b)) !== undefined);