@openfn/ws-worker 0.2.6 → 0.2.9

package/CHANGELOG.md

@@ -1,5 +1,32 @@
 # ws-worker
 
+## 0.2.9
+
+### Patch Changes
+
+- 54d0017: Start ws-worker using node (not pnpm) by default
+- 6f78b7a: Add env var for WORKER_REPO_DIR
+- Updated dependencies [4a17048]
+  - @openfn/engine-multi@0.2.0
+  - @openfn/runtime@0.2.0
+
+## 0.2.8
+
+### Patch Changes
+
+- Tweak typings
+
+## 0.2.7
+
+### Patch Changes
+
+- d542aa9: worker: leave attempt channel when finished working
+- Updated dependencies [793d523]
+- Updated dependencies [857c42b]
+- Updated dependencies [f17fb4a]
+  - @openfn/engine-multi@0.1.11
+  - @openfn/runtime@0.1.4
+
 ## 0.2.6
 
 ### Patch Changes

package/README.md

@@ -48,7 +48,7 @@ You can start a dev server (which rebuilds on save) by running:
 pnpm start:watch
 ```
 
-This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `ENGINE_REPO_DIR` or `/tmp/openfn/repo`.
+This will wrap a real runtime engine into the server. It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `WORKER_REPO_DIR` (or a default path in /tmp)
 
 ### Disabling auto-fetch
 

package/dist/index.d.ts

@@ -17,6 +17,40 @@ type ExitReason = {
   error_type: string | null;
 };
 
+type Node = {
+  id: string;
+  body?: string;
+  adaptor?: string;
+  credential_id?: any; // TODO tighten this up, string or object
+  type?: 'webhook' | 'cron'; // trigger only
+  state?: any; // Initial state / defaults
+};
+
+interface Edge {
+  id: string;
+  source_job_id?: string;
+  source_trigger_id?: string;
+  target_job_id: string;
+  name?: string;
+  condition?: string;
+  error_path?: boolean;
+  errors?: any;
+  enabled?: boolean;
+}
+
+// An attempt object returned by Lightning
+type Attempt = {
+  id: string;
+  dataclip_id: string;
+  starting_node_id: string;
+
+  triggers: Node[];
+  jobs: Node[];
+  edges: Edge[];
+
+  options?: AttemptOptions;
+};
+
 type AttemptOptions = {
   timeout?: number;
   sanitize?: SanitizePolicies;
@@ -45,13 +79,6 @@ type ReceiveHook = {
   ) => ReceiveHook;
 };
 
-// export declare class Socket extends PhxSocket {
-//   constructor(endpoint: string, options: { params: any });
-//   onOpen(callback: () => void): void;
-//   connect(): void;
-//   channel(channelName: string, params: any): Channel;
-// }
-
 interface Channel extends Channel$1 {
   // on: (event: string, fn: (evt: any) => void) => void;
 
@@ -64,13 +91,69 @@ declare type Context = {
     channel: Channel;
     state: AttemptState;
     logger: Logger;
-    onComplete: (result: any) => void;
+    onFinish: (result: any) => void;
 };
 
-declare type CLAIM_ATTEMPT = {
+declare const CLAIM = "claim";
+declare type ClaimPayload = {
+    demand?: number;
+};
+declare type ClaimReply = {
+    attempts: Array<ClaimAttempt>;
+};
+declare type ClaimAttempt = {
     id: string;
     token: string;
 };
+declare const GET_ATTEMPT = "fetch:attempt";
+declare type GetAttemptPayload = void;
+declare type GetAttemptReply = Attempt;
+declare const GET_CREDENTIAL = "fetch:credential";
+declare type GetCredentialPayload = {
+    id: string;
+};
+declare type GetCredentialReply = {};
+declare const GET_DATACLIP = "fetch:dataclip";
+declare type GetDataclipPayload = {
+    id: string;
+};
+declare type GetDataClipReply = Uint8Array;
+declare const ATTEMPT_START = "attempt:start";
+declare type AttemptStartPayload = void;
+declare type AttemptStartReply = {};
+declare const ATTEMPT_COMPLETE = "attempt:complete";
+declare type AttemptCompletePayload = ExitReason & {
+    final_dataclip_id?: string;
+};
+declare type AttemptCompleteReply = undefined;
+declare const ATTEMPT_LOG = "attempt:log";
+declare type AttemptLogPayload = {
+    message: Array<string | object>;
+    timestamp: string;
+    attempt_id: string;
+    level?: string;
+    source?: string;
+    job_id?: string;
+    run_id?: string;
+};
+declare type AttemptLogReply = void;
+declare const RUN_START = "run:start";
+declare type RunStartPayload = {
+    job_id: string;
+    run_id: string;
+    attempt_id?: string;
+    input_dataclip_id?: string;
+};
+declare type RunStartReply = void;
+declare const RUN_COMPLETE = "run:complete";
+declare type RunCompletePayload = ExitReason & {
+    attempt_id?: string;
+    job_id: string;
+    run_id: string;
+    output_dataclip?: string;
+    output_dataclip_id?: string;
+};
+declare type RunCompleteReply = void;
 
 declare type ServerOptions = {
     maxWorkflows?: number;
@@ -89,10 +172,10 @@ interface ServerApp extends Koa {
     socket: any;
     channel: Channel;
     workflows: Record<string, true | Context>;
-    execute: ({ id, token }: CLAIM_ATTEMPT) => Promise<void>;
+    execute: ({ id, token }: ClaimAttempt) => Promise<void>;
     destroy: () => void;
     killWorkloop?: () => void;
 }
 declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
 
-export { createServer as default };
+export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, RUN_COMPLETE, RUN_START, RunCompletePayload, RunCompleteReply, RunStartPayload, RunStartReply, createServer as default };

package/dist/index.js

@@ -286,11 +286,11 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-function execute(channel, engine, logger, plan, options = {}, onComplete = (_result) => {
+function execute(channel, engine, logger, plan, options = {}, onFinish = (_result) => {
 }) {
   logger.info("executing ", plan.id);
   const state = create_attempt_state_default(plan, options);
-  const context = { channel, state, logger, onComplete };
+  const context = { channel, state, logger, onFinish };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
       const lightningEvent = eventMap[eventName] ?? eventName;
@@ -337,7 +337,8 @@ function execute(channel, engine, logger, plan, options = {}, onComplete = (_res
       onWorkflowError(context, {
         workflowId: plan.id,
         message: e.message,
-        type: e.type
+        type: e.type,
+        severity: e.severity
       });
     }
   });
@@ -398,22 +399,27 @@ function onJobComplete({ channel, state }, event, error) {
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onComplete }, _event) {
+async function onWorkflowComplete({ state, channel, onFinish }, _event) {
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateAttemptExitReason(state);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
     final_dataclip_id: state.lastDataclipId,
     ...reason
   });
-  onComplete({ reason, state: result });
+  onFinish({ reason, state: result });
 }
-async function onWorkflowError({ state, channel, onComplete }, event) {
-  const reason = calculateJobExitReason("", { data: {} }, event);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onComplete({ reason });
+async function onWorkflowError({ state, channel, logger, onFinish }, event) {
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger.error("ERROR in workflow-error handler:", e.message);
+    logger.error(e);
+  }
 }
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
@@ -603,8 +609,9 @@ function createServer(engine, options = {}) {
         plan,
         options: options2
       } = await attempt_default(app.socket, token, id, logger);
-      const onComplete = () => {
+      const onFinish = () => {
         delete app.workflows[id];
+        attemptChannel.leave();
       };
       const context = execute(
         attemptChannel,
@@ -612,7 +619,7 @@ function createServer(engine, options = {}) {
         logger,
         plan,
         options2,
-        onComplete
+        onFinish
       );
       app.workflows[id] = context;
     } else {
@@ -654,5 +661,14 @@ var server_default = createServer;
 // src/index.ts
 var src_default = server_default;
 export {
+  ATTEMPT_COMPLETE,
+  ATTEMPT_LOG,
+  ATTEMPT_START,
+  CLAIM,
+  GET_ATTEMPT,
+  GET_CREDENTIAL,
+  GET_DATACLIP,
+  RUN_COMPLETE,
+  RUN_START,
   src_default as default
 };

package/dist/start.js

@@ -4864,8 +4864,8 @@ import createLogger from "@openfn/logger";
 import createRTE from "@openfn/engine-multi";
 
 // src/mock/runtime-engine.ts
-import crypto from "node:crypto";
 import { EventEmitter } from "node:events";
+import run from "@openfn/runtime";
 
 // src/mock/resolvers.ts
 var mockResolveCredential = (_credId) => new Promise(
@@ -4887,6 +4887,10 @@ var resolvers_default = {
 };
 
 // src/mock/runtime-engine.ts
+var helpers = {
+  fn: (f) => (s) => f(s),
+  wait: (duration) => (s) => new Promise((resolve5) => setTimeout(() => resolve5(s), duration))
+};
 async function createMock() {
   const activeWorkflows = {};
   const bus = new EventEmitter();
@@ -4902,73 +4906,59 @@ async function createMock() {
   const listen = (planId, events) => {
     listeners[planId] = events;
   };
-  const executeJob = async (workflowId, job, initialState = {}, resolvers = resolvers_default) => {
-    const { id, expression, configuration, adaptor } = job;
-    if (!expression && !adaptor) {
-      return initialState;
-    }
-    const runId = crypto.randomUUID();
-    const jobId = id;
-    if (typeof configuration === "string") {
-      await resolvers.credential?.(configuration);
-    }
-    const info = (...message) => {
-      dispatch("workflow-log", {
-        workflowId,
-        message,
-        level: "info",
-        time: (BigInt(Date.now()) * BigInt(1e3)).toString(),
-        name: "mck"
-      });
-    };
-    dispatch("job-start", { workflowId, jobId, runId });
-    info("Running job " + jobId);
-    let nextState = initialState;
-    if (expression?.startsWith?.("wait@")) {
-      const [_, delay] = expression.split("@");
-      nextState = initialState;
-      await new Promise((resolve5) => {
-        setTimeout(() => resolve5(), parseInt(delay));
-      });
-    } else {
-      try {
-        nextState = JSON.parse(expression);
-        info("Parsing expression as JSON state");
-        info(nextState);
-      } catch (e) {
-        nextState = initialState;
-      }
-    }
-    dispatch("job-complete", {
-      workflowId,
-      jobId,
-      state: nextState,
-      runId,
-      next: []
-    });
-    return nextState;
-  };
-  const execute2 = (xplan, options = {
+  const execute2 = async (xplan, options = {
     resolvers: resolvers_default
   }) => {
-    if (options.throw) {
-      throw new Error("test error");
-    }
-    const { id, jobs, initialState } = xplan;
-    const workflowId = id;
+    const { id, jobs } = xplan;
     activeWorkflows[id] = true;
-    setTimeout(() => {
-      dispatch("workflow-start", { workflowId });
-      setTimeout(async () => {
-        let state = initialState || {};
-        for (const job of jobs) {
-          state = await executeJob(id, job, state, options.resolvers);
+    for (const job of jobs) {
+      if (typeof job.configuration === "string") {
+        job.configuration = await options.resolvers?.credential?.(
+          job.configuration
+        );
+      }
+      if (typeof job.expression === "string" && !job.expression.match(/export default \[/)) {
+        job.expression = `export default [${job.expression}];`;
+      }
+    }
+    const jobLogger = {
+      log: (...args2) => {
+        dispatch("workflow-log", {
+          workflowId: id,
+          level: "info",
+          json: true,
+          message: args2,
+          time: Date.now()
+        });
+      }
+    };
+    const opts = {
+      strict: false,
+      jobLogger,
+      ...options,
+      globals: helpers,
+      callbacks: {
+        notify: (name, payload) => {
+          dispatch(name, {
+            workflowId: id,
+            ...payload
+          });
         }
-        setTimeout(() => {
-          delete activeWorkflows[id];
-          dispatch("workflow-complete", { workflowId });
-        }, 1);
-      }, 1);
+      }
+    };
+    setTimeout(async () => {
+      dispatch("workflow-start", { workflowId: id });
+      try {
+        await run(xplan, void 0, opts);
+      } catch (e) {
+        dispatch("workflow-error", {
+          workflowId: id,
+          type: e.name,
+          message: e.message
+        });
+      }
+      delete activeWorkflows[id];
+      dispatch("workflow-complete", { workflowId: id });
     }, 1);
   };
   const getStatus = () => {
@@ -5105,10 +5095,10 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
 var workloop_default = startWorkloop;
 
 // src/api/execute.ts
-import crypto3 from "node:crypto";
+import crypto2 from "node:crypto";
 
 // src/util/convert-attempt.ts
-import crypto2 from "node:crypto";
+import crypto from "node:crypto";
 var conditions = {
   on_job_success: "!state.errors",
   on_job_failure: "state.errors",
@@ -5153,7 +5143,7 @@ var convert_attempt_default = (attempt) => {
   }
   if (attempt.jobs?.length) {
     attempt.jobs.forEach((job) => {
-      const id = job.id || crypto2.randomUUID();
+      const id = job.id || crypto.randomUUID();
       nodes[id] = {
         id,
         configuration: job.credential_id,
@@ -5274,11 +5264,11 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-function execute(channel, engine, logger2, plan, options = {}, onComplete = (_result) => {
+function execute(channel, engine, logger2, plan, options = {}, onFinish = (_result) => {
 }) {
   logger2.info("executing ", plan.id);
   const state = create_attempt_state_default(plan, options);
-  const context = { channel, state, logger: logger2, onComplete };
+  const context = { channel, state, logger: logger2, onFinish };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
       const lightningEvent = eventMap[eventName] ?? eventName;
@@ -5325,7 +5315,8 @@ function execute(channel, engine, logger2, plan, options = {}, onComplete = (_re
       onWorkflowError(context, {
         workflowId: plan.id,
         message: e.message,
-        type: e.type
+        type: e.type,
+        severity: e.severity
       });
     }
   });
@@ -5335,7 +5326,7 @@ var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
   channel.push(event, payload).receive("error", reject).receive("timeout", () => reject(new Error("timeout"))).receive("ok", resolve5);
 });
 function onJobStart({ channel, state }, event) {
-  state.activeRun = crypto3.randomUUID();
+  state.activeRun = crypto2.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
   return sendEvent(channel, RUN_START, {
@@ -5353,7 +5344,7 @@ function onJobError(context, event) {
   }
 }
 function onJobComplete({ channel, state }, event, error) {
-  const dataclipId = crypto3.randomUUID();
+  const dataclipId = crypto2.randomUUID();
   const run_id = state.activeRun;
   const job_id = state.activeJob;
   if (!state.dataclips) {
@@ -5386,22 +5377,27 @@ function onJobComplete({ channel, state }, event, error) {
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onComplete }, _event) {
+async function onWorkflowComplete({ state, channel, onFinish }, _event) {
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateAttemptExitReason(state);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
     final_dataclip_id: state.lastDataclipId,
     ...reason
   });
-  onComplete({ reason, state: result });
+  onFinish({ reason, state: result });
 }
-async function onWorkflowError({ state, channel, onComplete }, event) {
-  const reason = calculateJobExitReason("", { data: {} }, event);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onComplete({ reason });
+async function onWorkflowError({ state, channel, logger: logger2, onFinish }, event) {
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger2.error("ERROR in workflow-error handler:", e.message);
+    logger2.error(e);
+  }
 }
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
@@ -5591,8 +5587,9 @@ function createServer(engine, options = {}) {
         plan,
         options: options2
       } = await attempt_default(app.socket, token, id, logger2);
-      const onComplete = () => {
+      const onFinish = () => {
         delete app.workflows[id];
+        attemptChannel.leave();
       };
       const context = execute(
         attemptChannel,
@@ -5600,7 +5597,7 @@ function createServer(engine, options = {}) {
         logger2,
         plan,
         options2,
-        onComplete
+        onFinish
       );
       app.workflows[id] = context;
     } else {
@@ -5640,6 +5637,7 @@ function createServer(engine, options = {}) {
 var server_default = createServer;
 
 // src/start.ts
+var { WORKER_REPO_DIR, WORKER_SECRET } = process.env;
 var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-worker server").option("port", {
   alias: "p",
   description: "Port to run the server on",
@@ -5651,7 +5649,8 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
   default: "ws://localhost:4000/worker"
 }).option("repo-dir", {
   alias: "d",
-  description: "Path to the runtime repo (where modules will be installed)"
+  description: "Path to the runtime repo (where modules will be installed)",
+  default: WORKER_REPO_DIR
 }).option("secret", {
   alias: "s",
   description: "Worker secret (comes from WORKER_SECRET by default)"
@@ -5682,7 +5681,6 @@ if (args.lightning === "mock") {
     args.secret = "abdefg";
   }
 } else if (!args.secret) {
-  const { WORKER_SECRET } = process.env;
   if (!WORKER_SECRET) {
     logger.error("WORKER_SECRET is not set");
     process.exit(1);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.2.6",
+  "version": "0.2.9",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,9 +21,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.1.10",
+    "@openfn/engine-multi": "0.2.0",
     "@openfn/logger": "0.0.19",
-    "@openfn/runtime": "0.1.3"
+    "@openfn/runtime": "0.2.0"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.0.11"
+    "@openfn/lightning-mock": "1.1.2"
   },
   "files": [
     "dist",