@openfn/ws-worker 0.1.5 → 0.1.8

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # ws-worker
 
+## 0.1.8
+
+### Patch Changes
+
+- b8fd13d: Fix undefined log output
+
+## 0.1.7
+
+### Patch Changes
+
+- 8f7f57b: Send timestamps as strings in microsecond precision
+- Updated dependencies [ca701e8]
+  - @openfn/logger@0.0.19
+  - @openfn/engine-multi@0.1.5
+  - @openfn/runtime@0.0.33
+
+## 0.1.6
+
+### Patch Changes
+
+- 5037c68: Support maxWorkers flag
+  Enforce minBackoff before claiming the next job
+- Updated dependencies [ac7b0ca]
+  - @openfn/engine-multi@0.1.4
+
 ## 0.1.5
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,6 +1,50 @@
 import Koa from 'koa';
-import { Logger } from '@openfn/logger';
+import { SanitizePolicies, Logger } from '@openfn/logger';
 import { RuntimeEngine } from '@openfn/engine-multi';
+import { Channel as Channel$1 } from 'phoenix';
+import { ExecutionPlan } from '@openfn/runtime';
+
+type AttemptOptions = {
+  timeout?: number;
+  sanitize?: SanitizePolicies;
+};
+
+type ReceiveHook = {
+  receive: (
+    status: 'ok' | 'timeout' | 'error',
+    callback: (payload?: any) => void
+  ) => ReceiveHook;
+};
+
+// export declare class Socket extends PhxSocket {
+//   constructor(endpoint: string, options: { params: any });
+//   onOpen(callback: () => void): void;
+//   connect(): void;
+//   channel(channelName: string, params: any): Channel;
+// }
+
+interface Channel extends Channel$1 {
+  // on: (event: string, fn: (evt: any) => void) => void;
+
+  // TODO it would be super nice to infer the event from the payload
+  push: <P = any>(event: string, payload?: P) => ReceiveHook;
+  // join: () => ReceiveHook;
+}
+
+declare type AttemptState = {
+    activeRun?: string;
+    activeJob?: string;
+    plan: ExecutionPlan;
+    options: AttemptOptions;
+    dataclips: Record<string, any>;
+    lastDataclipId?: string;
+};
+declare type Context = {
+    channel: Channel;
+    state: AttemptState;
+    logger: Logger;
+    onComplete: (result: any) => void;
+};
 
 declare type CLAIM_ATTEMPT = {
     id: string;
@@ -22,7 +66,8 @@ declare type ServerOptions = {
 interface ServerApp extends Koa {
     id: string;
     socket: any;
-    channel: any;
+    channel: Channel;
+    workflows: Record<string, true | Context>;
     execute: ({ id, token }: CLAIM_ATTEMPT) => Promise<void>;
     destroy: () => void;
     killWorkloop: () => void;

package/dist/index.js

@@ -63,17 +63,21 @@ var RUN_COMPLETE = "run:complete";
 
 // src/api/claim.ts
 var mockLogger = createMockLogger();
-var claim = (channel, execute2, logger = mockLogger) => {
+var claim = (app, logger = mockLogger, maxWorkers = 5) => {
   return new Promise((resolve, reject) => {
+    const activeWorkers = Object.keys(app.workflows).length;
+    if (activeWorkers >= maxWorkers) {
+      return reject(new Error("Server at capacity"));
+    }
     logger.debug("requesting attempt...");
-    channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
+    app.channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
       logger.debug(`pulled ${attempts.length} attempts`);
       if (!attempts?.length) {
-        return reject(new Error("claim failed"));
+        return reject(new Error("No attempts returned"));
       }
       attempts.forEach((attempt) => {
         logger.debug("starting attempt", attempt.id);
-        execute2(attempt);
+        app.execute(attempt);
         resolve();
       });
     });
@@ -82,18 +86,18 @@ var claim = (channel, execute2, logger = mockLogger) => {
 var claim_default = claim;
 
 // src/api/workloop.ts
-var startWorkloop = (channel, execute2, logger, minBackoff, maxBackoff) => {
+var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
   let promise;
   let cancelled = false;
   const workLoop = () => {
     if (!cancelled) {
-      promise = try_with_backoff_default(() => claim_default(channel, execute2, logger), {
+      promise = try_with_backoff_default(() => claim_default(app, logger, maxWorkers), {
         min: minBackoff,
         max: maxBackoff
       });
       promise.then(() => {
         if (!cancelled) {
-          workLoop();
+          setTimeout(workLoop, minBackoff);
         }
       });
     }
@@ -212,62 +216,65 @@ var eventMap = {
   "workflow-start": ATTEMPT_START,
   "job-start": RUN_START,
   "job-complete": RUN_COMPLETE,
-  log: ATTEMPT_LOG,
+  "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-function execute(channel, engine, logger, plan, options = {}) {
-  return new Promise(async (resolve, reject) => {
-    logger.info("execute...");
-    const state = {
-      plan,
-      lastDataclipId: plan.initialState,
-      dataclips: {},
-      options
+function execute(channel, engine, logger, plan, options = {}, onComplete = (_result) => {
+}) {
+  logger.info("execute...");
+  const state = {
+    plan,
+    lastDataclipId: plan.initialState,
+    dataclips: {},
+    options
+  };
+  const context = { channel, state, logger, onComplete };
+  const addEvent = (eventName, handler) => {
+    const wrappedFn = async (event) => {
+      const lightningEvent = eventMap[eventName] ?? eventName;
+      try {
+        await handler(context, event);
+        logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
+      } catch (e) {
+        logger.error(
+          `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
+        );
+        logger.error(e);
+      }
     };
-    const context = { channel, state, logger, onComplete: resolve };
-    const addEvent = (eventName, handler) => {
-      const wrappedFn = async (event) => {
-        const lightningEvent = eventMap[eventName];
-        try {
-          await handler(context, event);
-          logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
-        } catch (e) {
-          logger.error(
-            `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
-          );
-          logger.error(e);
-        }
-      };
-      return {
-        [eventName]: wrappedFn
-      };
-    };
-    const listeners = Object.assign(
-      {},
-      addEvent("workflow-start", onWorkflowStart),
-      addEvent("job-start", onJobStart),
-      addEvent("job-complete", onJobComplete),
-      addEvent("workflow-log", onJobLog),
-      addEvent("workflow-complete", onWorkflowComplete),
-      addEvent("workflow-error", onWorkflowError)
-    );
-    engine.listen(plan.id, listeners);
-    const resolvers = {
-      credential: (id) => loadCredential(channel, id)
+    return {
+      [eventName]: wrappedFn
     };
+  };
+  const listeners = Object.assign(
+    {},
+    addEvent("workflow-start", onWorkflowStart),
+    addEvent("job-start", onJobStart),
+    addEvent("job-complete", onJobComplete),
+    addEvent("workflow-log", onJobLog),
+    addEvent("workflow-complete", onWorkflowComplete),
+    addEvent("workflow-error", onWorkflowError)
+  );
+  engine.listen(plan.id, listeners);
+  const resolvers = {
+    credential: (id) => loadCredential(channel, id)
+  };
+  Promise.resolve().then(async () => {
     if (typeof plan.initialState === "string") {
       logger.debug("loading dataclip", plan.initialState);
       plan.initialState = await loadDataclip(channel, plan.initialState);
       logger.success("dataclip loaded");
       logger.debug(plan.initialState);
     }
+    return plan;
+  }).then(() => {
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
       onWorkflowError(context, { workflowId: plan.id, message: e.message });
-      reject(e);
     }
   });
+  return context;
 }
 var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
   channel.push(event, payload).receive("error", reject).receive("timeout", () => reject(new Error("timeout"))).receive("ok", resolve);
@@ -321,12 +328,13 @@ async function onWorkflowError({ state, channel, onComplete }, event) {
   onComplete({});
 }
 function onJobLog({ channel, state }, event) {
+  const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
   const log = {
     attempt_id: state.plan.id,
     message: event.message,
     source: event.name,
     level: event.level,
-    timestamp: event.time || Date.now()
+    timestamp: timeInMicroseconds.toString()
   };
   if (state.activeRun) {
     log.run_id = state.activeRun;
@@ -443,11 +451,11 @@ function connect(app, engine, logger, options = {}) {
       }
       logger.info("Starting workloop");
       app.killWorkloop = workloop_default(
-        channel,
-        app.execute,
+        app,
         logger,
         options.backoff?.min || MIN_BACKOFF,
-        options.backoff?.max || MAX_BACKOFF
+        options.backoff?.max || MAX_BACKOFF,
+        options.maxWorkflows
       );
     } else {
       logger.break();
@@ -484,23 +492,36 @@ function createServer(engine, options = {}) {
       logger.debug(str);
     })
   );
+  app.workflows = {};
   const server = app.listen(port);
   logger.success(`ws-worker ${app.id} listening on ${port}`);
   app.execute = async ({ id, token }) => {
     if (app.socket) {
+      app.workflows[id] = true;
       const {
         channel: attemptChannel,
         plan,
         options: options2
       } = await attempt_default(app.socket, token, id, logger);
-      execute(attemptChannel, engine, logger, plan, options2);
+      const onComplete = () => {
+        delete app.workflows[id];
+      };
+      const context = execute(
+        attemptChannel,
+        engine,
+        logger,
+        plan,
+        options2,
+        onComplete
+      );
+      app.workflows[id] = context;
     } else {
       logger.error("No lightning socket established");
     }
   };
   router.post("/claim", async (ctx) => {
     logger.info("triggering claim from POST request");
-    return claim_default(app.channel, app.execute, logger).then(() => {
+    return claim_default(app, logger, options.maxWorkflows).then(() => {
       logger.info("claim complete: 1 attempt claimed");
       ctx.body = "complete";
       ctx.status = 200;

package/dist/start.js

@@ -4917,19 +4917,27 @@ async function createMock() {
         workflowId,
         message,
         level: "info",
-        timestamp: Date.now(),
+        time: (BigInt(Date.now()) * BigInt(1e3)).toString(),
         name: "mck"
       });
     };
     dispatch("job-start", { workflowId, jobId, runId });
     info("Running job " + jobId);
     let nextState = initialState;
-    try {
-      nextState = JSON.parse(expression);
-      info("Parsing expression as JSON state");
-      info(nextState);
-    } catch (e) {
+    if (expression?.startsWith?.("wait@")) {
+      const [_, delay] = expression.split("@");
       nextState = initialState;
+      await new Promise((resolve5) => {
+        setTimeout(() => resolve5(), parseInt(delay));
+      });
+    } else {
+      try {
+        nextState = JSON.parse(expression);
+        info("Parsing expression as JSON state");
+        info(nextState);
+      } catch (e) {
+        nextState = initialState;
+      }
     }
     dispatch("job-complete", { workflowId, jobId, state: nextState, runId });
     return nextState;
@@ -5037,17 +5045,21 @@ var RUN_COMPLETE = "run:complete";
 
 // src/api/claim.ts
 var mockLogger = createMockLogger();
-var claim = (channel, execute2, logger2 = mockLogger) => {
+var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
   return new Promise((resolve5, reject) => {
+    const activeWorkers = Object.keys(app.workflows).length;
+    if (activeWorkers >= maxWorkers) {
+      return reject(new Error("Server at capacity"));
+    }
     logger2.debug("requesting attempt...");
-    channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
+    app.channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
       logger2.debug(`pulled ${attempts.length} attempts`);
       if (!attempts?.length) {
-        return reject(new Error("claim failed"));
+        return reject(new Error("No attempts returned"));
       }
       attempts.forEach((attempt) => {
         logger2.debug("starting attempt", attempt.id);
-        execute2(attempt);
+        app.execute(attempt);
         resolve5();
       });
     });
@@ -5056,18 +5068,18 @@ var claim = (channel, execute2, logger2 = mockLogger) => {
 var claim_default = claim;
 
 // src/api/workloop.ts
-var startWorkloop = (channel, execute2, logger2, minBackoff2, maxBackoff2) => {
+var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
   let promise;
   let cancelled = false;
   const workLoop = () => {
     if (!cancelled) {
-      promise = try_with_backoff_default(() => claim_default(channel, execute2, logger2), {
+      promise = try_with_backoff_default(() => claim_default(app, logger2, maxWorkers), {
         min: minBackoff2,
         max: maxBackoff2
       });
       promise.then(() => {
         if (!cancelled) {
-          workLoop();
+          setTimeout(workLoop, minBackoff2);
         }
       });
     }
@@ -5186,62 +5198,65 @@ var eventMap = {
   "workflow-start": ATTEMPT_START,
   "job-start": RUN_START,
   "job-complete": RUN_COMPLETE,
-  log: ATTEMPT_LOG,
+  "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-function execute(channel, engine, logger2, plan, options = {}) {
-  return new Promise(async (resolve5, reject) => {
-    logger2.info("execute...");
-    const state = {
-      plan,
-      lastDataclipId: plan.initialState,
-      dataclips: {},
-      options
-    };
-    const context = { channel, state, logger: logger2, onComplete: resolve5 };
-    const addEvent = (eventName, handler) => {
-      const wrappedFn = async (event) => {
-        const lightningEvent = eventMap[eventName];
-        try {
-          await handler(context, event);
-          logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
-        } catch (e) {
-          logger2.error(
-            `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
-          );
-          logger2.error(e);
-        }
-      };
-      return {
-        [eventName]: wrappedFn
-      };
+function execute(channel, engine, logger2, plan, options = {}, onComplete = (_result) => {
+}) {
+  logger2.info("execute...");
+  const state = {
+    plan,
+    lastDataclipId: plan.initialState,
+    dataclips: {},
+    options
+  };
+  const context = { channel, state, logger: logger2, onComplete };
+  const addEvent = (eventName, handler) => {
+    const wrappedFn = async (event) => {
+      const lightningEvent = eventMap[eventName] ?? eventName;
+      try {
+        await handler(context, event);
+        logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
+      } catch (e) {
+        logger2.error(
+          `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
+        );
+        logger2.error(e);
+      }
     };
-    const listeners = Object.assign(
-      {},
-      addEvent("workflow-start", onWorkflowStart),
-      addEvent("job-start", onJobStart),
-      addEvent("job-complete", onJobComplete),
-      addEvent("workflow-log", onJobLog),
-      addEvent("workflow-complete", onWorkflowComplete),
-      addEvent("workflow-error", onWorkflowError)
-    );
-    engine.listen(plan.id, listeners);
-    const resolvers = {
-      credential: (id) => loadCredential(channel, id)
+    return {
+      [eventName]: wrappedFn
     };
+  };
+  const listeners = Object.assign(
+    {},
+    addEvent("workflow-start", onWorkflowStart),
+    addEvent("job-start", onJobStart),
+    addEvent("job-complete", onJobComplete),
+    addEvent("workflow-log", onJobLog),
+    addEvent("workflow-complete", onWorkflowComplete),
+    addEvent("workflow-error", onWorkflowError)
+  );
+  engine.listen(plan.id, listeners);
+  const resolvers = {
+    credential: (id) => loadCredential(channel, id)
+  };
+  Promise.resolve().then(async () => {
     if (typeof plan.initialState === "string") {
       logger2.debug("loading dataclip", plan.initialState);
       plan.initialState = await loadDataclip(channel, plan.initialState);
       logger2.success("dataclip loaded");
       logger2.debug(plan.initialState);
     }
+    return plan;
+  }).then(() => {
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
       onWorkflowError(context, { workflowId: plan.id, message: e.message });
-      reject(e);
     }
   });
+  return context;
 }
 var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
   channel.push(event, payload).receive("error", reject).receive("timeout", () => reject(new Error("timeout"))).receive("ok", resolve5);
@@ -5295,12 +5310,13 @@ async function onWorkflowError({ state, channel, onComplete }, event) {
   onComplete({});
 }
 function onJobLog({ channel, state }, event) {
+  const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
   const log = {
     attempt_id: state.plan.id,
     message: event.message,
     source: event.name,
     level: event.level,
-    timestamp: event.time || Date.now()
+    timestamp: timeInMicroseconds.toString()
   };
   if (state.activeRun) {
     log.run_id = state.activeRun;
@@ -5417,11 +5433,11 @@ function connect(app, engine, logger2, options = {}) {
       }
       logger2.info("Starting workloop");
       app.killWorkloop = workloop_default(
-        channel,
-        app.execute,
+        app,
         logger2,
         options.backoff?.min || MIN_BACKOFF,
-        options.backoff?.max || MAX_BACKOFF
+        options.backoff?.max || MAX_BACKOFF,
+        options.maxWorkflows
       );
     } else {
       logger2.break();
@@ -5458,23 +5474,36 @@ function createServer(engine, options = {}) {
       logger2.debug(str);
     })
   );
+  app.workflows = {};
   const server = app.listen(port);
   logger2.success(`ws-worker ${app.id} listening on ${port}`);
   app.execute = async ({ id, token }) => {
     if (app.socket) {
+      app.workflows[id] = true;
       const {
         channel: attemptChannel,
         plan,
         options: options2
       } = await attempt_default(app.socket, token, id, logger2);
-      execute(attemptChannel, engine, logger2, plan, options2);
+      const onComplete = () => {
+        delete app.workflows[id];
+      };
+      const context = execute(
+        attemptChannel,
+        engine,
+        logger2,
+        plan,
+        options2,
+        onComplete
+      );
+      app.workflows[id] = context;
     } else {
       logger2.error("No lightning socket established");
     }
   };
   router.post("/claim", async (ctx) => {
     logger2.info("triggering claim from POST request");
-    return claim_default(app.channel, app.execute, logger2).then(() => {
+    return claim_default(app, logger2, options.maxWorkflows).then(() => {
       logger2.info("claim complete: 1 attempt claimed");
       ctx.body = "complete";
       ctx.status = 200;
@@ -5535,6 +5564,10 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
 }).option("backoff", {
   description: "Claim backoff rules: min/max (s)",
   default: "1/10"
+}).option("capacity", {
+  description: "max concurrent workers",
+  default: 5,
+  type: "number"
 }).parse();
 var logger = createLogger("SRV", { level: args.log });
 if (args.lightning === "mock") {
@@ -5551,7 +5584,6 @@ if (args.lightning === "mock") {
   args.secret = WORKER_SECRET;
 }
 var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseInt(n, 10) * 1e3);
-console.log(minBackoff, maxBackoff);
 function engineReady(engine) {
   server_default(engine, {
     port: args.port,
@@ -5562,7 +5594,8 @@ function engineReady(engine) {
     backoff: {
       min: minBackoff,
       max: maxBackoff
-    }
+    },
+    maxWorkflows: args.capacity
   });
 }
 if (args.mock) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.1.5",
+  "version": "0.1.8",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,9 +21,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.1.3",
-    "@openfn/logger": "0.0.18",
-    "@openfn/runtime": "0.0.32"
+    "@openfn/engine-multi": "0.1.5",
+    "@openfn/logger": "0.0.19",
+    "@openfn/runtime": "0.0.33"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.0.4"
+    "@openfn/lightning-mock": "1.0.6"
   },
   "files": [
     "dist",