@dxos/functions 0.3.8-next.f4e0086 → 0.3.9-main.03b62b6

package/src/runtime/dev-server.ts

@@ -3,33 +3,38 @@
 //
 
 import express from 'express';
+import { getPort } from 'get-port-please';
 import type http from 'http';
 import { join } from 'node:path';
-import { getPortPromise } from 'portfinder';
 
 import { Trigger } from '@dxos/async';
 import { type Client } from '@dxos/client';
+import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
 import { type FunctionContext, type FunctionHandler, type Response } from '../handler';
 import { type FunctionDef, type FunctionManifest } from '../manifest';
 
-const DEFAULT_PORT = 7001;
-
 export type DevServerOptions = {
+  port?: number;
   directory: string;
   manifest: FunctionManifest;
+  reload?: boolean;
+  dataDir?: string;
 };
 
 /**
  * Functions dev server provides a local HTTP server for testing functions.
  */
 export class DevServer {
+  // Function handlers indexed by name (URL path).
   private readonly _handlers: Record<string, { def: FunctionDef; handler: FunctionHandler<any> }> = {};
 
   private _server?: http.Server;
   private _port?: number;
   private _registrationId?: string;
+  private _proxy?: string;
+  private _seq = 0;
 
   // prettier-ignore
   constructor(
@@ -37,12 +42,13 @@ export class DevServer {
     private readonly _options: DevServerOptions,
   ) {}
 
-  get port() {
-    return this._port;
+  get endpoint() {
+    invariant(this._port);
+    return `http://localhost:${this._port}`;
   }
 
-  get endpoint() {
-    return this._port ? `http://localhost:${this._port}` : undefined;
+  get proxy() {
+    return this._proxy;
   }
 
   get functions() {
@@ -51,20 +57,8 @@ export class DevServer {
 
   async initialize() {
     for (const def of this._options.manifest.functions) {
-      const { id, endpoint, handler: path } = def;
       try {
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
-        const module = require(join(this._options.directory, path));
-        const handler = module.default;
-        if (typeof handler !== 'function') {
-          throw new Error(`Handler must export default function: ${id}`);
-        }
-
-        if (this._handlers[endpoint]) {
-          log.warn(`Function already registered: ${id}`);
-        }
-
-        this._handlers[endpoint] = { def, handler };
+        await this._load(def);
       } catch (err) {
         log.error('parsing function (check manifest)', err);
       }
@@ -75,54 +69,39 @@ export class DevServer {
     const app = express();
     app.use(express.json());
 
-    app.post('/:endpoint', async (req, res) => {
-      const { endpoint } = req.params;
-
-      const response: Response = {
-        status: (code: number) => {
-          res.statusCode = code;
-          return response;
-        },
-
-        succeed: (result = {}) => {
-          res.end(JSON.stringify(result));
-          return response;
-        },
-      };
-
-      const context: FunctionContext = {
-        client: this._client,
-        status: response.status.bind(response),
-      };
-
-      void (async () => {
-        try {
-          log('invoking', { endpoint });
-          const { handler } = this._handlers[endpoint];
-          const response = await handler({ context, event: req.body });
-          log('done', { response });
-        } catch (err: any) {
-          res.statusCode = 500;
-          res.end(err.message);
+    app.post('/:name', async (req, res) => {
+      const { name } = req.params;
+      try {
+        if (this._options.reload) {
+          const { def } = this._handlers[name];
+          await this._load(def, true);
         }
-      })();
+
+        res.statusCode = await this._invoke(name, req.body);
+        res.end();
+      } catch (err: any) {
+        log.catch(err);
+        res.statusCode = 500;
+        res.end();
+      }
     });
 
-    this._port = await getPortPromise({ startPort: DEFAULT_PORT });
+    this._port = await getPort({ host: 'localhost', port: 7200, portRange: [7200, 7299] });
     this._server = app.listen(this._port);
 
-    // TODO(burdon): Check plugin is registered.
-    //  TypeError: Cannot read properties of undefined (reading 'register')
     try {
-      const { registrationId } = await this._client.services.services.FunctionRegistryService!.register({
-        endpoint: this.endpoint!,
-        functions: this.functions.map(({ def: { endpoint } }) => ({ name: endpoint })), // TODO(burdon): Change proto name => id.
+      // Register functions.
+      const { registrationId, endpoint } = await this._client.services.services.FunctionRegistryService!.register({
+        endpoint: this.endpoint,
+        functions: this.functions.map(({ def: { name } }) => ({ name })),
       });
 
+      log.info('registered', { registrationId, endpoint });
       this._registrationId = registrationId;
+      this._proxy = endpoint;
     } catch (err: any) {
       await this.stop();
-      throw new Error('FunctionRegistryService not available; check config (agent.plugins.functions).');
+      throw new Error('FunctionRegistryService not available (check plugin is configured).');
     }
   }
 
@@ -133,14 +112,70 @@ export class DevServer {
         await this._client.services.services.FunctionRegistryService!.unregister({
           registrationId: this._registrationId,
         });
+
+        log.info('unregistered', { registrationId: this._registrationId });
         this._registrationId = undefined;
+        this._proxy = undefined;
       }
 
       trigger.wake();
     });
 
     await trigger.wait();
-    this._server = undefined;
     this._port = undefined;
+    this._server = undefined;
+  }
+
+  /**
+   * Load function.
+   */
+  private async _load(def: FunctionDef, flush = false) {
+    const { id, name, handler } = def;
+    const path = join(this._options.directory, handler);
+    log.info('loading', { id });
+
+    // Remove from cache.
+    if (flush) {
+      Object.keys(require.cache)
+        .filter((key) => key.startsWith(path))
+        .forEach((key) => delete require.cache[key]);
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const module = require(path);
+    if (typeof module.default !== 'function') {
+      throw new Error(`Handler must export default function: ${id}`);
+    }
+
+    this._handlers[name] = { def, handler: module.default };
+  }
+
+  /**
+   * Invoke function handler.
+   */
+  private async _invoke(name: string, event: any) {
+    const seq = ++this._seq;
+    const now = Date.now();
+
+    log.info('req', { seq, name });
+    const { handler } = this._handlers[name];
+
+    const context: FunctionContext = {
+      client: this._client,
+      dataDir: this._options.dataDir,
+    };
+
+    let statusCode = 200;
+    const response: Response = {
+      status: (code: number) => {
+        statusCode = code;
+        return response;
+      },
+    };
+
+    await handler({ context, event, response });
+    log.info('res', { seq, name, statusCode, duration: Date.now() - now });
+
+    return statusCode;
   }
 }

package/src/runtime/index.ts

@@ -3,4 +3,4 @@
 //
 
 export * from './dev-server';
-export * from './trigger-manager';
+export * from './scheduler';

package/src/runtime/scheduler.ts

@@ -0,0 +1,162 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { CronJob } from 'cron';
+
+import { debounce, DeferredTask } from '@dxos/async';
+import { type Client, type PublicKey } from '@dxos/client';
+import { type Space, TextObject } from '@dxos/client/echo';
+import { Context } from '@dxos/context';
+import { Filter, createSubscription, type Query, subscribe } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { ComplexMap } from '@dxos/util';
+
+import { type FunctionDef, type FunctionManifest, type FunctionTrigger } from '../manifest';
+
+type SchedulerOptions = {
+  endpoint: string;
+};
+
+/**
+ * Functions scheduler.
+ */
+// TODO(burdon): Create tests.
+export class Scheduler {
+  // Map of mounted functions.
+  private readonly _mounts = new ComplexMap<
+    { id: string; spaceKey: PublicKey },
+    { ctx: Context; trigger: FunctionTrigger }
+  >(({ id, spaceKey }) => `${spaceKey.toHex()}:${id}`);
+
+  constructor(
+    private readonly _client: Client,
+    private readonly _manifest: FunctionManifest,
+    private readonly _options: SchedulerOptions,
+  ) {}
+
+  async start() {
+    this._client.spaces.subscribe(async (spaces) => {
+      for (const space of spaces) {
+        await space.waitUntilReady();
+        for (const trigger of this._manifest.triggers ?? []) {
+          await this.mount(new Context(), space, trigger);
+        }
+      }
+    });
+  }
+
+  async stop() {
+    for (const { id, spaceKey } of this._mounts.keys()) {
+      await this.unmount(id, spaceKey);
+    }
+  }
+
+  private async mount(ctx: Context, space: Space, trigger: FunctionTrigger) {
+    const key = { id: trigger.function, spaceKey: space.key };
+    const def = this._manifest.functions.find((config) => config.id === trigger.function);
+    invariant(def, `Function not found: ${trigger.function}`);
+
+    const exists = this._mounts.get(key);
+    if (!exists) {
+      this._mounts.set(key, { ctx, trigger });
+      log('mount', { space: space.key, trigger });
+      if (ctx.disposed) {
+        return;
+      }
+
+      // Cron schedule.
+      if (trigger.schedule) {
+        const task = new DeferredTask(ctx, async () => {
+          await this.execFunction(def, {
+            space: space.key,
+          });
+        });
+
+        // TODO(burdon): Check greater than 30s min (use cron-parser).
+        const job = new CronJob(trigger.schedule, () => task.schedule());
+
+        job.start();
+        ctx.onDispose(() => job.stop());
+      }
+
+      // ECHO subscription.
+      if (trigger.subscription) {
+        const objectIds = new Set<string>();
+        const task = new DeferredTask(ctx, async () => {
+          await this.execFunction(def, {
+            space: space.key,
+            objects: Array.from(objectIds),
+          });
+        });
+
+        // TODO(burdon): Standardize subscription handles.
+        const subscriptions: (() => void)[] = [];
+        const subscription = createSubscription(({ added, updated }) => {
+          for (const object of added) {
+            objectIds.add(object.id);
+          }
+          for (const object of updated) {
+            objectIds.add(object.id);
+          }
+
+          task.schedule();
+        });
+        subscriptions.push(() => subscription.unsubscribe());
+
+        const { type, props, deep, delay } = trigger.subscription;
+        const update = ({ objects }: Query) => {
+          subscription.update(objects);
+
+          // TODO(burdon): Hack to monitor changes to Document's text object.
+          if (deep) {
+            log.info('update', { type, deep, objects: objects.length });
+            for (const object of objects) {
+              const content = object.content;
+              if (content instanceof TextObject) {
+                subscriptions.push(content[subscribe](debounce(() => subscription.update([object]), 1_000)));
+              }
+            }
+          }
+        };
+
+        // TODO(burdon): [Bug]: all callbacks are fired on the first mutation.
+        // TODO(burdon): [Bug]: not updated when document is deleted (either top or hierarchically).
+        const query = space.db.query(Filter.typename(type, props));
+        subscriptions.push(query.subscribe(delay ? debounce(update, delay * 1_000) : update));
+
+        ctx.onDispose(() => {
+          subscriptions.forEach((unsubscribe) => unsubscribe());
+        });
+      }
+    }
+  }
+
+  private async unmount(id: string, spaceKey: PublicKey) {
+    const key = { id, spaceKey };
+    const { ctx } = this._mounts.get(key) ?? {};
+    if (ctx) {
+      this._mounts.delete(key);
+      await ctx.dispose();
+    }
+  }
+
+  private async execFunction(def: FunctionDef, data: any) {
+    try {
+      log('request', { function: def.id });
+      const response = await fetch(`${this._options.endpoint}/${def.name}`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(data),
+      });
+
+      // const result = await response.json();
+      log('result', { function: def.id, result: response.status });
+    } catch (err: any) {
+      log.error('error', { function: def.id, error: err.message });
+    }
+  }
+}

package/dist/types/src/runtime/trigger-manager.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"trigger-manager.d.ts","sourceRoot":"","sources":["../../../../src/runtime/trigger-manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,MAAM,EAAkB,MAAM,cAAc,CAAC;AAQ3D,OAAO,EAAE,KAAK,gBAAgB,EAAwB,MAAM,aAAa,CAAC;AAG1E,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,qBAAa,cAAc;IASvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc;IAVjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAGiC;IAEzD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;gBAG1B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,gBAAgB,EAC3B,cAAc,EAAE,aAAa;IAG1C,KAAK;IAaL,IAAI;YAMI,KAAK;YAuDL,OAAO;YASP,YAAY;CAqB3B"}

package/src/runtime/trigger-manager.ts

@@ -1,140 +0,0 @@
-//
-// Copyright 2023 DXOS.org
-//
-
-import { DeferredTask } from '@dxos/async';
-import { type Client, type PublicKey } from '@dxos/client';
-import type { Query, Space } from '@dxos/client/echo';
-import { Context } from '@dxos/context';
-import { Filter, createSubscription } from '@dxos/echo-schema';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { ComplexMap } from '@dxos/util';
-
-import { type FunctionManifest, type FunctionTrigger } from '../manifest';
-
-// TODO(burdon): Rename.
-export type InvokeOptions = {
-  endpoint: string;
-  runtime: string;
-};
-
-export class TriggerManager {
-  private readonly _mounts = new ComplexMap<
-    { name: string; spaceKey: PublicKey },
-    { ctx: Context; trigger: FunctionTrigger }
-  >(({ name, spaceKey }) => `${spaceKey.toHex()}:${name}`);
-
-  private readonly _queries = new Set<Query>();
-
-  constructor(
-    private readonly _client: Client,
-    private readonly _manifest: FunctionManifest,
-    private readonly _invokeOptions: InvokeOptions,
-  ) {}
-
-  async start() {
-    // TODO(burdon): Make runtime configurable (via CLI)?
-    this._client.spaces.subscribe(async (spaces) => {
-      for (const space of spaces) {
-        await space.waitUntilReady();
-        for (const trigger of this._manifest.triggers ?? []) {
-          // TODO(burdon): New context? Shared?
-          await this.mount(new Context(), space, trigger);
-        }
-      }
-    });
-  }
-
-  async stop() {
-    for (const { name, spaceKey } of this._mounts.keys()) {
-      await this.unmount(name, spaceKey);
-    }
-  }
-
-  private async mount(ctx: Context, space: Space, trigger: FunctionTrigger) {
-    const key = { name: trigger.function, spaceKey: space.key };
-    const config = this._manifest.functions.find((config) => config.id === trigger.function);
-    invariant(config, `Function not found: ${trigger.function}`);
-    const exists = this._mounts.get(key);
-    if (!exists) {
-      this._mounts.set(key, { ctx, trigger });
-      log('mount', { space: space.key, trigger });
-      if (ctx.disposed) {
-        return;
-      }
-
-      // TODO(burdon): Why DeferredTask? How to pass objectIds to function?
-      const objectIds = new Set<string>();
-      const task = new DeferredTask(ctx, async () => {
-        await this.execFunction(this._invokeOptions, config.endpoint, {
-          space: space.key,
-          objects: Array.from(objectIds),
-        });
-      });
-
-      let count = 0;
-      const subscription = createSubscription(({ added, updated }) => {
-        for (const object of added) {
-          objectIds.add(object.id);
-        }
-        for (const object of updated) {
-          objectIds.add(object.id);
-        }
-
-        log('updated', {
-          trigger,
-          space: space.key,
-          objects: objectIds.size,
-          count,
-        });
-
-        task.schedule();
-        count++;
-      });
-      // TODO(burdon): DSL for query (replace props).
-      const query = space.db.query(Filter.typename(trigger.subscription.type, trigger.subscription.props));
-      this._queries.add(query);
-      const unsubscribe = query.subscribe(({ objects }) => {
-        subscription.update(objects);
-      }, true);
-
-      ctx.onDispose(() => {
-        subscription.unsubscribe();
-        unsubscribe();
-        this._queries.delete(query);
-      });
-    }
-  }
-
-  private async unmount(name: string, spaceKey: PublicKey) {
-    const key = { name, spaceKey };
-    const { ctx } = this._mounts.get(key) ?? {};
-    if (ctx) {
-      this._mounts.delete(key);
-      await ctx.dispose();
-    }
-  }
-
-  private async execFunction(options: InvokeOptions, functionName: string, data: any) {
-    const { endpoint, runtime } = options;
-    invariant(endpoint, 'Missing endpoint');
-    invariant(runtime, 'Missing runtime');
-
-    try {
-      log('invoke', { function: functionName });
-      const url = `${endpoint}/${runtime}/${functionName}`;
-      const res = await fetch(url, {
-        method: 'POST',
-        body: JSON.stringify(data),
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      });
-
-      log('result', { function: functionName, result: await res.json() });
-    } catch (err: any) {
-      log.error('error', { function: functionName, error: err.message });
-    }
-  }
-}