@openfn/ws-worker 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,12 @@
+# ws-worker
+
+## 0.1.0
+
+First release of the websocket worker, which handles comm between Lightning and the multi-threaded engine.
+
+Features:
+
+- Websocket integration with JWT auth
+- Eventing between Lightning and the Worker
+- Eventing between the Worker and the Engine
+- Placeholder exit reasons

package/README.md

@@ -0,0 +1,73 @@
+# Websocket Worker
+
+The Websocket Worker `ws-worker` provides a Websocket interface between Lightning and a Runtime Engine.
+
+It is a fairly thin layer between the two systems, designed to transport messages and convert Lightning data structures into runtime-friendly ones.
+
+This package contains:
+
+- A server which connects Lightning to an Engine (exposing dev APIs to http and node.js)
+- A mock runtime engine implementation
+
+The mock services allow lightweight and controlled testing of the interfaces between them.
+
+## Getting started
+
+To use this server:
+
+- Start a lightning instance (you can use the mock if you like, see `../lightning-mock`)
+- Start the worker server with `pnpm start`
+
+The worker will use the WORKER_SECRET env var (which you should have set for Lightning already). Check WORKERS.md in Lightning and run this in Lightning if you haven't already:
+
+```
+mix lightning.gen_worker_keys
+```
+
+### WS Server
+
+To start a `ws-worker` server, run:
+
+```
+pnpm start
+```
+
+You may want to add `--log debug` or disable the work loop, see below.
+
+The default settings will try and connect to lightning at `localhost:4000`.
+
+Pass a custom lightining url with `-l ws://localhost:1234`. You need to include the websocket endpoint, which at the time of writing is `/worker`.
+
+Use `-l mock` to connect to a lightning mock server (on the default port).
+
+## Watched Server
+
+You can start a dev server (which rebuilds on save) by running:
+
+```
+pnpm start:watch
+```
+
+This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `ENGINE_REPO_DIR` or `/tmp/openfn/repo`.
+
+### Disabling auto-fetch
+
+When working in dev it is convinient to disable the workloop. To switch it off, run:
+
+```
+pnpm start --no-loop
+```
+
+To manually trigger a claim, post to `/claim`:
+
+```
+curl -X POST http://localhost:2222/claim
+```
+
+## Architecture
+
+Lightning is expected to maintain a queue of attempts. The Worker pulls those attempts from the queue, via websocket, and sends them off to the Engine for execution.
+
+While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements.
+
+The ws-worker server is designed for zero persistence. It does not have any database, does not use the file system. Should the server crash, tracking of any active jobs will be lost (Lightning is expected to time these runs out).

package/dist/index.d.ts

@@ -0,0 +1,30 @@
+import Koa from 'koa';
+import { Logger } from '@openfn/logger';
+import { RuntimeEngine } from '@openfn/engine-multi';
+
+declare type CLAIM_ATTEMPT = {
+    id: string;
+    token: string;
+};
+
+declare type ServerOptions = {
+    backoff?: number;
+    maxBackoff?: number;
+    maxWorkflows?: number;
+    port?: number;
+    lightning?: string;
+    logger?: Logger;
+    noLoop?: boolean;
+    secret?: string;
+};
+interface ServerApp extends Koa {
+    id: string;
+    socket: any;
+    channel: any;
+    execute: ({ id, token }: CLAIM_ATTEMPT) => Promise<void>;
+    destroy: () => void;
+    killWorkloop: () => void;
+}
+declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
+
+export { createServer as default };

package/dist/index.js

@@ -0,0 +1,524 @@
+// src/server.ts
+import Koa from "koa";
+import bodyParser from "koa-bodyparser";
+import koaLogger from "koa-logger";
+import Router from "@koa/router";
+import { humanId } from "human-id";
+import { createMockLogger as createMockLogger2 } from "@openfn/logger";
+
+// src/util/try-with-backoff.ts
+var MAX_BACKOFF = 1e3 * 30;
+var tryWithBackoff = (fn, opts = {}) => {
+  if (!opts.timeout) {
+    opts.timeout = 100;
+  }
+  if (!opts.attempts) {
+    opts.attempts = 1;
+  }
+  if (!opts.maxBackoff) {
+    opts.maxBackoff = MAX_BACKOFF;
+  }
+  let { timeout, attempts, maxAttempts } = opts;
+  timeout = timeout;
+  attempts = attempts;
+  let cancelled = false;
+  if (!opts.isCancelled) {
+    opts.isCancelled = () => cancelled;
+  }
+  const promise = new Promise(async (resolve, reject) => {
+    try {
+      await fn();
+      resolve();
+    } catch (e) {
+      if (opts.isCancelled()) {
+        return resolve();
+      }
+      if (!isNaN(maxAttempts) && attempts >= maxAttempts) {
+        return reject(new Error("max attempts exceeded"));
+      }
+      setTimeout(() => {
+        if (opts.isCancelled()) {
+          return resolve();
+        }
+        const nextOpts = {
+          maxAttempts,
+          attempts: attempts + 1,
+          timeout: Math.min(opts.maxBackoff, timeout * 1.2),
+          isCancelled: opts.isCancelled
+        };
+        tryWithBackoff(fn, nextOpts).then(resolve).catch(reject);
+      }, timeout);
+    }
+  });
+  promise.cancel = () => {
+    cancelled = true;
+  };
+  return promise;
+};
+var try_with_backoff_default = tryWithBackoff;
+
+// src/api/claim.ts
+import { createMockLogger } from "@openfn/logger";
+
+// src/events.ts
+var CLAIM = "claim";
+var GET_ATTEMPT = "fetch:attempt";
+var GET_CREDENTIAL = "fetch:credential";
+var GET_DATACLIP = "fetch:dataclip";
+var ATTEMPT_START = "attempt:start";
+var ATTEMPT_COMPLETE = "attempt:complete";
+var ATTEMPT_LOG = "attempt:log";
+var RUN_START = "run:start";
+var RUN_COMPLETE = "run:complete";
+
+// src/api/claim.ts
+var mockLogger = createMockLogger();
+var claim = (channel, execute2, logger = mockLogger) => {
+  return new Promise((resolve, reject) => {
+    logger.debug("requesting attempt...");
+    channel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
+      logger.debug(`pulled ${attempts.length} attempts`);
+      if (!attempts?.length) {
+        return reject(new Error("claim failed"));
+      }
+      attempts.forEach((attempt) => {
+        logger.debug("starting attempt", attempt.id);
+        execute2(attempt);
+        resolve();
+      });
+    });
+  });
+};
+var claim_default = claim;
+
+// src/api/workloop.ts
+var startWorkloop = (channel, execute2, logger, options = {}) => {
+  let promise;
+  let cancelled = false;
+  const workLoop = () => {
+    if (!cancelled) {
+      promise = try_with_backoff_default(() => claim_default(channel, execute2, logger), {
+        timeout: options.timeout,
+        maxBackoff: options.maxBackoff
+      });
+      promise.then(() => {
+        if (!cancelled) {
+          workLoop();
+        }
+      });
+    }
+  };
+  workLoop();
+  return () => {
+    logger.debug("cancelling workloop");
+    cancelled = true;
+    promise.cancel();
+  };
+};
+var workloop_default = startWorkloop;
+
+// src/api/execute.ts
+import crypto2 from "node:crypto";
+
+// src/util/convert-attempt.ts
+import crypto from "node:crypto";
+var convert_attempt_default = (attempt) => {
+  const options = attempt.options || {};
+  const plan = {
+    id: attempt.id
+  };
+  if (attempt.dataclip_id) {
+    plan.initialState = attempt.dataclip_id;
+  }
+  if (attempt.starting_node_id) {
+    plan.start = attempt.starting_node_id;
+  }
+  const nodes = {};
+  const edges = attempt.edges ?? [];
+  if (attempt.triggers?.length) {
+    attempt.triggers.forEach((trigger) => {
+      const id = trigger.id || "trigger";
+      nodes[id] = {
+        id
+      };
+      const connectedEdges = edges.filter((e) => e.source_trigger_id === id);
+      if (connectedEdges.length) {
+        nodes[id].next = connectedEdges.reduce((obj, edge) => {
+          if (edge.enabled !== false) {
+            obj[edge.target_job_id] = true;
+          }
+          return obj;
+        }, {});
+      } else {
+      }
+    });
+  }
+  if (attempt.jobs?.length) {
+    attempt.jobs.forEach((job) => {
+      const id = job.id || crypto.randomUUID();
+      nodes[id] = {
+        id,
+        configuration: job.credential,
+        expression: job.body,
+        adaptor: job.adaptor
+      };
+      if (job.state) {
+        nodes[id].state = job.state;
+      }
+      const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
+        const newEdge = {};
+        if (edge.condition) {
+          newEdge.condition = edge.condition;
+        }
+        if (edge.enabled === false) {
+          newEdge.disabled = true;
+        }
+        obj[edge.target_job_id] = Object.keys(newEdge).length ? newEdge : true;
+        return obj;
+      }, {});
+      if (Object.keys(next).length) {
+        nodes[id].next = next;
+      }
+    });
+  }
+  plan.jobs = Object.values(nodes);
+  return {
+    plan,
+    options
+  };
+};
+
+// src/util/get-with-reply.ts
+var get_with_reply_default = (channel, event, payload) => new Promise((resolve) => {
+  channel.push(event, payload).receive("ok", (evt) => {
+    resolve(evt);
+  });
+});
+
+// src/util/stringify.ts
+import stringify from "fast-safe-stringify";
+var stringify_default = (obj) => stringify(obj, (_key, value) => {
+  if (value instanceof Uint8Array) {
+    return Array.from(value);
+  }
+  return value;
+});
+
+// src/api/execute.ts
+var enc = new TextDecoder("utf-8");
+var eventMap = {
+  "workflow-start": ATTEMPT_START,
+  "job-start": RUN_START,
+  "job-complete": RUN_COMPLETE,
+  log: ATTEMPT_LOG,
+  "workflow-complete": ATTEMPT_COMPLETE
+};
+function execute(channel, engine, logger, plan, options = {}) {
+  return new Promise(async (resolve, reject) => {
+    logger.info("execute...");
+    const state = {
+      plan,
+      lastDataclipId: plan.initialState,
+      dataclips: {},
+      options
+    };
+    const context = { channel, state, logger, onComplete: resolve };
+    const addEvent = (eventName, handler) => {
+      const wrappedFn = async (event) => {
+        const lightningEvent = eventMap[eventName];
+        try {
+          await handler(context, event);
+          logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
+        } catch (e) {
+          logger.error(
+            `${plan.id} :: ${lightningEvent} :: ERR: ${e.message || e.toString()}`
+          );
+          logger.error(e);
+        }
+      };
+      return {
+        [eventName]: wrappedFn
+      };
+    };
+    const listeners = Object.assign(
+      {},
+      addEvent("workflow-start", onWorkflowStart),
+      addEvent("job-start", onJobStart),
+      addEvent("job-complete", onJobComplete),
+      addEvent("log", onJobLog),
+      addEvent("workflow-complete", onWorkflowComplete),
+      addEvent("workflow-error", onWorkflowError)
+    );
+    engine.listen(plan.id, listeners);
+    const resolvers = {
+      credential: (id) => loadCredential(channel, id)
+    };
+    if (typeof plan.initialState === "string") {
+      logger.debug("loading dataclip", plan.initialState);
+      plan.initialState = await loadDataclip(channel, plan.initialState);
+      logger.success("dataclip loaded");
+      logger.debug(plan.initialState);
+    }
+    try {
+      engine.execute(plan, { resolvers, ...options });
+    } catch (e) {
+      onWorkflowError(context, { workflowId: plan.id, message: e.message });
+      reject(e);
+    }
+  });
+}
+var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
+  channel.push(event, payload).receive("error", reject).receive("timeout", () => reject(new Error("timeout"))).receive("ok", resolve);
+});
+function onJobStart({ channel, state }, event) {
+  state.activeRun = crypto2.randomUUID();
+  state.activeJob = event.jobId;
+  return sendEvent(channel, RUN_START, {
+    run_id: state.activeRun,
+    job_id: state.activeJob,
+    input_dataclip_id: state.lastDataclipId
+  });
+}
+function onJobComplete({ channel, state }, event) {
+  const dataclipId = crypto2.randomUUID();
+  const run_id = state.activeRun;
+  const job_id = state.activeJob;
+  if (!state.dataclips) {
+    state.dataclips = {};
+  }
+  state.dataclips[dataclipId] = event.state;
+  state.lastDataclipId = dataclipId;
+  delete state.activeRun;
+  delete state.activeJob;
+  return sendEvent(channel, RUN_COMPLETE, {
+    run_id,
+    job_id,
+    output_dataclip_id: dataclipId,
+    output_dataclip: stringify_default(event.state),
+    reason: "success"
+  });
+}
+function onWorkflowStart({ channel }, _event) {
+  return sendEvent(channel, ATTEMPT_START);
+}
+async function onWorkflowComplete({ state, channel, onComplete }, _event) {
+  const result = state.dataclips[state.lastDataclipId];
+  await sendEvent(channel, ATTEMPT_COMPLETE, {
+    final_dataclip_id: state.lastDataclipId,
+    status: "success",
+    reason: "ok"
+  });
+  onComplete(result);
+}
+async function onWorkflowError({ state, channel, onComplete }, event) {
+  await sendEvent(channel, ATTEMPT_COMPLETE, {
+    reason: "fail",
+    final_dataclip_id: state.lastDataclipId,
+    message: event.message
+  });
+  onComplete({});
+}
+function onJobLog({ channel, state }, event) {
+  const log = {
+    attempt_id: state.plan.id,
+    message: event.message,
+    source: event.name,
+    level: event.level,
+    timestamp: event.time || Date.now()
+  };
+  if (state.activeRun) {
+    log.run_id = state.activeRun;
+  }
+  return sendEvent(channel, ATTEMPT_LOG, log);
+}
+async function loadDataclip(channel, stateId) {
+  const result = await get_with_reply_default(channel, GET_DATACLIP, {
+    id: stateId
+  });
+  const str = enc.decode(new Uint8Array(result));
+  return JSON.parse(str);
+}
+async function loadCredential(channel, credentialId) {
+  return get_with_reply_default(channel, GET_CREDENTIAL, { id: credentialId });
+}
+
+// src/channels/attempt.ts
+var joinAttemptChannel = (socket, token, attemptId, logger) => {
+  return new Promise((resolve, reject) => {
+    let didReceiveOk = false;
+    const channelName = `attempt:${attemptId}`;
+    logger.debug("connecting to ", channelName);
+    const channel = socket.channel(channelName, { token });
+    channel.join().receive("ok", async (e) => {
+      if (!didReceiveOk) {
+        didReceiveOk = true;
+        logger.success(`connected to ${channelName}`, e);
+        const { plan, options } = await loadAttempt(channel);
+        logger.debug("converted attempt as execution plan:", plan);
+        resolve({ channel, plan, options });
+      }
+    }).receive("error", (err) => {
+      logger.error(`error connecting to ${channelName}`, err);
+      reject(err);
+    });
+  });
+};
+var attempt_default = joinAttemptChannel;
+async function loadAttempt(channel) {
+  const attemptBody = await get_with_reply_default(channel, GET_ATTEMPT);
+  return convert_attempt_default(attemptBody);
+}
+
+// src/channels/worker-queue.ts
+import { Socket as PhxSocket } from "phoenix";
+import { WebSocket } from "ws";
+
+// src/util/worker-token.ts
+import * as jose from "jose";
+var alg = "HS256";
+var generateWorkerToken = async (secret, workerId) => {
+  if (!secret) {
+    console.warn();
+    console.warn("WARNING: Worker Secret not provided!");
+    console.warn(
+      "This worker will attempt to connect to Lightning with default secret"
+    );
+    console.warn();
+  }
+  const encodedSecret = new TextEncoder().encode(secret || "<secret>");
+  const claims = {
+    worker_id: workerId
+  };
+  const jwt = await new jose.SignJWT(claims).setProtectedHeader({ alg }).setIssuedAt().setIssuer("urn:example:issuer").setAudience("urn:example:audience").sign(encodedSecret);
+  return jwt;
+};
+var worker_token_default = generateWorkerToken;
+
+// src/channels/worker-queue.ts
+var connectToWorkerQueue = (endpoint, serverId, secret, SocketConstructor = PhxSocket) => {
+  return new Promise(async (done, reject) => {
+    const token = await worker_token_default(secret, serverId);
+    const socket = new SocketConstructor(endpoint, {
+      params: { token },
+      transport: WebSocket
+    });
+    let didOpen = false;
+    socket.onOpen(() => {
+      didOpen = true;
+      const channel = socket.channel("worker:queue");
+      channel.join().receive("ok", () => {
+        done({ socket, channel });
+      }).receive("error", (e) => {
+        console.log("ERROR", e);
+      }).receive("timeout", (e) => {
+        console.log("TIMEOUT", e);
+      });
+    });
+    socket.onError((e) => {
+      if (!didOpen) {
+        reject(e);
+      }
+    });
+    socket.connect();
+  });
+};
+var worker_queue_default = connectToWorkerQueue;
+
+// src/server.ts
+var DEFAULT_PORT = 1234;
+function connect(app, engine, logger, options = {}) {
+  logger.debug("Connecting to Lightning at", options.lightning);
+  worker_queue_default(options.lightning, app.id, options.secret).then(({ socket, channel }) => {
+    logger.success("Connected to Lightning at", options.lightning);
+    app.socket = socket;
+    app.channel = channel;
+    if (!options.noLoop) {
+      logger.info("Starting workloop");
+      app.killWorkloop = workloop_default(channel, app.execute, logger, {
+        maxBackoff: options.maxBackoff
+      });
+    } else {
+      logger.break();
+      logger.warn("Workloop not starting");
+      logger.info("This server will not auto-pull work from lightning.");
+      logger.info("You can manually claim by posting to /claim, eg:");
+      logger.info(
+        `  curl -X POST http://locahost:${options.port || DEFAULT_PORT}/claim`
+      );
+      logger.break();
+    }
+  }).catch((e) => {
+    logger.error(
+      "CRITICAL ERROR: could not connect to lightning at",
+      options.lightning
+    );
+    logger.debug(e);
+    app.killWorkloop?.();
+    setTimeout(() => {
+      connect(app, engine, logger, options);
+    }, 1e4);
+  });
+}
+function createServer(engine, options = {}) {
+  const logger = options.logger || createMockLogger2();
+  const port = options.port || DEFAULT_PORT;
+  logger.debug("Starting server");
+  const app = new Koa();
+  app.id = humanId({ separator: "-", capitalize: false });
+  const router = new Router();
+  app.use(bodyParser());
+  app.use(
+    koaLogger((str, _args) => {
+      logger.debug(str);
+    })
+  );
+  const server = app.listen(port);
+  logger.success(`ws-worker ${app.id} listening on ${port}`);
+  app.execute = async ({ id, token }) => {
+    if (app.socket) {
+      const {
+        channel: attemptChannel,
+        plan,
+        options: options2
+      } = await attempt_default(app.socket, token, id, logger);
+      execute(attemptChannel, engine, logger, plan, options2);
+    } else {
+      logger.error("No lightning socket established");
+    }
+  };
+  router.post("/claim", async (ctx) => {
+    logger.info("triggering claim from POST request");
+    return claim_default(app.channel, app.execute, logger).then(() => {
+      logger.info("claim complete: 1 attempt claimed");
+      ctx.body = "complete";
+      ctx.status = 200;
+    }).catch(() => {
+      logger.info("claim complete: no attempts");
+      ctx.body = "no attempts";
+      ctx.status = 204;
+    });
+  });
+  app.destroy = () => {
+    logger.info("Closing server...");
+    server.close();
+    app.killWorkloop?.();
+    logger.success("Server closed");
+  };
+  app.use(router.routes());
+  if (options.lightning) {
+    connect(app, engine, logger, options);
+  } else {
+    logger.warn("No lightning URL provided");
+  }
+  app.on = (...args) => {
+    return engine.on(...args);
+  };
+  return app;
+}
+var server_default = createServer;
+
+// src/index.ts
+var src_default = server_default;
+export {
+  src_default as default
+};

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@openfn/ws-worker",
+  "version": "0.1.0",
+  "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
+  "main": "dist/index.js",
+  "type": "module",
+  "author": "Open Function Group <admin@openfn.org>",
+  "license": "ISC",
+  "dependencies": {
+    "@koa/router": "^12.0.0",
+    "@types/koa-logger": "^3.1.2",
+    "@types/ws": "^8.5.6",
+    "fast-safe-stringify": "^2.1.1",
+    "human-id": "^4.1.0",
+    "jose": "^4.14.6",
+    "koa": "^2.13.4",
+    "koa-bodyparser": "^4.4.0",
+    "koa-logger": "^3.2.1",
+    "phoenix": "^1.7.7",
+    "ws": "^8.14.1",
+    "@openfn/engine-multi": "0.1.0",
+    "@openfn/logger": "0.0.18",
+    "@openfn/runtime": "0.0.32"
+  },
+  "devDependencies": {
+    "@types/koa": "^2.13.5",
+    "@types/koa-bodyparser": "^4.3.10",
+    "@types/koa__router": "^12.0.1",
+    "@types/node": "^18.15.3",
+    "@types/nodemon": "1.19.3",
+    "@types/phoenix": "^1.6.2",
+    "@types/yargs": "^17.0.12",
+    "ava": "5.1.0",
+    "nodemon": "3.0.1",
+    "ts-node": "^10.9.1",
+    "tslib": "^2.4.0",
+    "tsup": "^6.2.3",
+    "typescript": "^4.6.4",
+    "yargs": "^17.6.2",
+    "@openfn/lightning-mock": "1.0.1"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "test": "pnpm ava --serial",
+    "test:types": "pnpm tsc --noEmit --project tsconfig.json",
+    "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting",
+    "build:watch": "pnpm build --watch",
+    "start": "NODE_OPTIONS=\"--experimental-vm-modules --no-warnings\" ts-node-esm --transpile-only src/start.ts",
+    "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'",
+    "pack": "pnpm pack --pack-destination ../../dist"
+  }
+}