@openfn/ws-worker 1.6.7 → 1.8.0

package/CHANGELOG.md

@@ -1,5 +1,44 @@
 # ws-worker
 
+## 1.8.0
+
+### Minor Changes
+
+- fd0e499: Support collections
+- bcd82e9: Accept collection version at startup (as arg or auto-looked-up from npm)
+- Support @local adaptor versions (which map to the monorepo)
+
+### Patch Changes
+
+- 1c79dc1: Append the collections adaptor to steps that need it
+- b15f151: Update worker to use adaptors as an array on xplans. Internal only change.
+- Updated dependencies [3463ff9]
+- Updated dependencies [7245bf7]
+  - @openfn/runtime@1.5.0
+  - @openfn/engine-multi@1.4.0
+
+## 1.7.1
+
+### Patch Changes
+
+- 1c79dc1: Append the collections adaptor to steps that need it
+- b15f151: Update worker to use adaptors as an array on xplans. Internal only change.
+- Updated dependencies [3463ff9]
+- Updated dependencies [7245bf7]
+  - @openfn/runtime@2.0.0
+  - @openfn/engine-multi@1.4.0
+
+## 1.7.0
+
+### Minor Changes
+
+- ae55a6a: Include timestamp on step complete even if the step failed
+
+### Patch Changes
+
+- Updated dependencies [ae55a6a]
+  - @openfn/engine-multi@1.3.0
+
 ## 1.6.7
 
 ### Patch Changes

package/dist/index.d.ts

@@ -74,6 +74,9 @@ declare type ServerOptions = {
     };
     socketTimeoutSeconds?: number;
     payloadLimitMb?: number;
+    collectionsVersion?: string;
+    collectionsUrl?: string;
+    monorepoDir?: string;
 };
 interface ServerApp extends Koa {
     id: string;
@@ -86,6 +89,7 @@ interface ServerApp extends Koa {
     engine: RuntimeEngine;
     options: ServerOptions;
     workloop?: Workloop;
+    collectionsVersion?: string;
     execute: ({ id, token }: ClaimRun) => Promise<void>;
     destroy: () => void;
     resumeWorkloop: () => void;

package/dist/index.js

@@ -29,7 +29,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.7";
+    version = "1.8.0";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -107,6 +107,8 @@ var init_package = __esm({
 
 // src/server.ts
 import { EventEmitter as EventEmitter2 } from "node:events";
+import { promisify } from "node:util";
+import { exec as _exec } from "node:child_process";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -316,6 +318,8 @@ var workloop_default = startWorkloop;
 
 // src/util/convert-lightning-plan.ts
 import crypto2 from "node:crypto";
+import path from "node:path";
+import { getNameAndVersion } from "@openfn/runtime";
 var conditions = {
   on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
   on_job_failure: (upstreamId) => `Boolean(state?.errors && state.errors["${upstreamId}"])`,
@@ -335,7 +339,37 @@ var mapTriggerEdgeCondition = (edge) => {
     return true;
   return condition;
 };
-var convert_lightning_plan_default = (run) => {
+var convert_lightning_plan_default = (run, options = {}) => {
+  const { collectionsVersion, monorepoPath } = options;
+  const appendLocalVersions = (job) => {
+    if (monorepoPath && job.adaptors) {
+      for (const adaptor of job.adaptors) {
+        const { name: name2, version: version2 } = getNameAndVersion(adaptor);
+        if (monorepoPath && version2 === "local") {
+          const shortName = name2.replace("@openfn/language-", "");
+          const localPath = path.resolve(monorepoPath, "packages", shortName);
+          job.linker ??= {};
+          job.linker[name2] = {
+            path: localPath,
+            version: "local"
+          };
+        }
+      }
+    }
+    return job;
+  };
+  const appendCollectionsAdaptor = (plan2, collectionsVersion2 = "latest") => {
+    let hasCollections;
+    plan2.workflow.steps.forEach((step) => {
+      const job = step;
+      if (job.expression?.match(/(collections\.)/)) {
+        hasCollections = true;
+        job.adaptors ??= [];
+        job.adaptors.push(`@openfn/language-collections@${collectionsVersion2}`);
+      }
+    });
+    return hasCollections;
+  };
   const runtimeOpts = {};
   const engineOpts = {};
   if (run.options) {
@@ -396,7 +430,7 @@ var convert_lightning_plan_default = (run) => {
         id,
         configuration: step.credential || step.credential_id,
         expression: step.body,
-        adaptor: step.adaptor
+        adaptors: step.adaptor ? [step.adaptor] : []
       };
       if (step.name) {
         job.name = step.name;
@@ -428,6 +462,21 @@ var convert_lightning_plan_default = (run) => {
   if (run.name) {
     plan.workflow.name = run.name;
   }
+  if (collectionsVersion) {
+    const hasCollections = appendCollectionsAdaptor(
+      plan,
+      collectionsVersion
+    );
+    if (hasCollections) {
+      plan.workflow.credentials = {
+        collections_token: true,
+        collections_endpoint: true
+      };
+    }
+  }
+  for (const step of plan.workflow.steps) {
+    appendLocalVersions(step);
+  }
   return {
     plan,
     options: engineOpts,
@@ -469,8 +518,8 @@ var create_run_state_default = (plan, input) => {
   if (typeof input === "string") {
     const jobs = plan.workflow.steps;
     let startNode = jobs[0];
-    if (plan.options.start) {
-      startNode = jobs.find(({ id }) => id === plan.options.start);
+    if (plan.options?.start) {
+      startNode = jobs.find(({ id }) => id === plan.options?.start);
     }
     const initialRuns = [];
     if (startNode) {
@@ -910,9 +959,8 @@ var joinRunChannel = (socket, token, runId, logger) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger.success(`connected to ${channelName}`, e);
-        const { plan, options, input } = await loadRun(channel);
-        logger.debug("converted run as execution plan:", plan);
-        resolve({ channel, plan, options, input });
+        const run = await get_with_reply_default(channel, GET_PLAN);
+        resolve({ channel, run });
       }
     }).receive("error", (err) => {
       logger.error(`error connecting to ${channelName}`, err);
@@ -931,10 +979,6 @@ var joinRunChannel = (socket, token, runId, logger) => {
   });
 };
 var run_default = joinRunChannel;
-async function loadRun(channel) {
-  const runBody = await get_with_reply_default(channel, GET_PLAN);
-  return convert_lightning_plan_default(runBody);
-}
 
 // src/channels/worker-queue.ts
 import EventEmitter from "node:events";
@@ -1008,6 +1052,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
 var worker_queue_default = connectToWorkerQueue;
 
 // src/server.ts
+var exec = promisify(_exec);
 var DEFAULT_PORT = 2222;
 var MIN_BACKOFF = 1e3;
 var MAX_BACKOFF = 1e3 * 30;
@@ -1060,6 +1105,31 @@ function connect(app, logger, options = {}) {
     logger
   ).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
 }
+async function setupCollections(options, logger) {
+  if (options.collectionsUrl) {
+    logger.log("Using collections endpoint at ", options.collectionsUrl);
+  } else {
+    logger.warn(
+      "WARNING: no collections URL provided. Collections service will not be enabled."
+    );
+    logger.warn(
+      "Pass --collections-version or set WORKER_COLLECTIONS_URL to set the url"
+    );
+    return;
+  }
+  if (options.collectionsVersion && options.collectionsVersion !== "latest") {
+    logger.log(
+      "Using collections version from CLI/env: ",
+      options.collectionsVersion
+    );
+    return options.collectionsVersion;
+  }
+  const { stdout: version2 } = await exec(
+    "npm view @openfn/language-collections@latest version"
+  );
+  logger.log("Using collections version from @latest: ", version2);
+  return version2;
+}
 function createServer(engine, options = {}) {
   const logger = options.logger || createMockLogger2();
   const port = options.port || DEFAULT_PORT;
@@ -1102,12 +1172,23 @@ function createServer(engine, options = {}) {
       try {
         const start = Date.now();
         app.workflows[id] = true;
-        const {
-          channel: runChannel,
-          plan,
-          options: options2 = {},
-          input
-        } = await run_default(app.socket, token, id, logger);
+        const { channel: runChannel, run } = await run_default(
+          app.socket,
+          token,
+          id,
+          logger
+        );
+        const { plan, options: options2, input } = convert_lightning_plan_default(run, {
+          collectionsVersion: app.options.collectionsVersion,
+          monorepoPath: app.options.monorepoDir
+        });
+        logger.debug("converted run body into execution plan:", plan);
+        if (plan.workflow.credentials?.collections_token) {
+          plan.workflow.credentials.collections_token = token;
+        }
+        if (plan.workflow.credentials?.collections_endpoint) {
+          plan.workflow.credentials.collections_endpoint = app.options.collectionsUrl;
+        }
         if (!("payloadLimitMb" in options2)) {
           options2.payloadLimitMb = app.options.payloadLimitMb;
         }
@@ -1156,7 +1237,10 @@ function createServer(engine, options = {}) {
   app.destroy = () => destroy_default(app, logger);
   app.use(router.routes());
   if (options.lightning) {
-    connect(app, logger, options);
+    setupCollections(options, logger).then((version2) => {
+      app.collectionsVersion = version2;
+      connect(app, logger, options);
+    });
   } else {
     logger.warn("No lightning URL provided");
   }

package/dist/start.js

@@ -37,7 +37,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.7";
+    version = "1.8.0";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -246,6 +246,8 @@ var runtime_engine_default = createMock;
 
 // src/server.ts
 import { EventEmitter as EventEmitter3 } from "node:events";
+import { promisify } from "node:util";
+import { exec as _exec } from "node:child_process";
 import Koa from "koa";
 import bodyParser from "koa-bodyparser";
 import koaLogger from "koa-logger";
@@ -455,6 +457,8 @@ var workloop_default = startWorkloop;
 
 // src/util/convert-lightning-plan.ts
 import crypto3 from "node:crypto";
+import path from "node:path";
+import { getNameAndVersion } from "@openfn/runtime";
 var conditions = {
   on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
   on_job_failure: (upstreamId) => `Boolean(state?.errors && state.errors["${upstreamId}"])`,
@@ -474,7 +478,37 @@ var mapTriggerEdgeCondition = (edge) => {
     return true;
   return condition;
 };
-var convert_lightning_plan_default = (run2) => {
+var convert_lightning_plan_default = (run2, options = {}) => {
+  const { collectionsVersion, monorepoPath } = options;
+  const appendLocalVersions = (job) => {
+    if (monorepoPath && job.adaptors) {
+      for (const adaptor of job.adaptors) {
+        const { name: name2, version: version2 } = getNameAndVersion(adaptor);
+        if (monorepoPath && version2 === "local") {
+          const shortName = name2.replace("@openfn/language-", "");
+          const localPath = path.resolve(monorepoPath, "packages", shortName);
+          job.linker ??= {};
+          job.linker[name2] = {
+            path: localPath,
+            version: "local"
+          };
+        }
+      }
+    }
+    return job;
+  };
+  const appendCollectionsAdaptor = (plan2, collectionsVersion2 = "latest") => {
+    let hasCollections;
+    plan2.workflow.steps.forEach((step) => {
+      const job = step;
+      if (job.expression?.match(/(collections\.)/)) {
+        hasCollections = true;
+        job.adaptors ??= [];
+        job.adaptors.push(`@openfn/language-collections@${collectionsVersion2}`);
+      }
+    });
+    return hasCollections;
+  };
   const runtimeOpts = {};
   const engineOpts = {};
   if (run2.options) {
@@ -535,7 +569,7 @@ var convert_lightning_plan_default = (run2) => {
         id,
         configuration: step.credential || step.credential_id,
         expression: step.body,
-        adaptor: step.adaptor
+        adaptors: step.adaptor ? [step.adaptor] : []
       };
       if (step.name) {
         job.name = step.name;
@@ -567,6 +601,21 @@ var convert_lightning_plan_default = (run2) => {
   if (run2.name) {
     plan.workflow.name = run2.name;
   }
+  if (collectionsVersion) {
+    const hasCollections = appendCollectionsAdaptor(
+      plan,
+      collectionsVersion
+    );
+    if (hasCollections) {
+      plan.workflow.credentials = {
+        collections_token: true,
+        collections_endpoint: true
+      };
+    }
+  }
+  for (const step of plan.workflow.steps) {
+    appendLocalVersions(step);
+  }
   return {
     plan,
     options: engineOpts,
@@ -608,8 +657,8 @@ var create_run_state_default = (plan, input) => {
   if (typeof input === "string") {
     const jobs = plan.workflow.steps;
     let startNode = jobs[0];
-    if (plan.options.start) {
-      startNode = jobs.find(({ id }) => id === plan.options.start);
+    if (plan.options?.start) {
+      startNode = jobs.find(({ id }) => id === plan.options?.start);
     }
     const initialRuns = [];
     if (startNode) {
@@ -1049,9 +1098,8 @@ var joinRunChannel = (socket, token, runId, logger2) => {
       if (!didReceiveOk) {
         didReceiveOk = true;
         logger2.success(`connected to ${channelName}`, e);
-        const { plan, options, input } = await loadRun(channel);
-        logger2.debug("converted run as execution plan:", plan);
-        resolve5({ channel, plan, options, input });
+        const run2 = await get_with_reply_default(channel, GET_PLAN);
+        resolve5({ channel, run: run2 });
       }
     }).receive("error", (err) => {
       logger2.error(`error connecting to ${channelName}`, err);
@@ -1070,10 +1118,6 @@ var joinRunChannel = (socket, token, runId, logger2) => {
   });
 };
 var run_default = joinRunChannel;
-async function loadRun(channel) {
-  const runBody = await get_with_reply_default(channel, GET_PLAN);
-  return convert_lightning_plan_default(runBody);
-}
 
 // src/channels/worker-queue.ts
 import EventEmitter2 from "node:events";
@@ -1147,6 +1191,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
 var worker_queue_default = connectToWorkerQueue;
 
 // src/server.ts
+var exec = promisify(_exec);
 var DEFAULT_PORT = 2222;
 var MIN_BACKOFF = 1e3;
 var MAX_BACKOFF = 1e3 * 30;
@@ -1199,6 +1244,31 @@ function connect(app, logger2, options = {}) {
     logger2
   ).on("connect", onConnect).on("disconnect", onDisconnect).on("error", onError);
 }
+async function setupCollections(options, logger2) {
+  if (options.collectionsUrl) {
+    logger2.log("Using collections endpoint at ", options.collectionsUrl);
+  } else {
+    logger2.warn(
+      "WARNING: no collections URL provided. Collections service will not be enabled."
+    );
+    logger2.warn(
+      "Pass --collections-version or set WORKER_COLLECTIONS_URL to set the url"
+    );
+    return;
+  }
+  if (options.collectionsVersion && options.collectionsVersion !== "latest") {
+    logger2.log(
+      "Using collections version from CLI/env: ",
+      options.collectionsVersion
+    );
+    return options.collectionsVersion;
+  }
+  const { stdout: version2 } = await exec(
+    "npm view @openfn/language-collections@latest version"
+  );
+  logger2.log("Using collections version from @latest: ", version2);
+  return version2;
+}
 function createServer(engine, options = {}) {
   const logger2 = options.logger || createMockLogger2();
   const port = options.port || DEFAULT_PORT;
@@ -1241,12 +1311,23 @@ function createServer(engine, options = {}) {
       try {
         const start = Date.now();
         app.workflows[id] = true;
-        const {
-          channel: runChannel,
-          plan,
-          options: options2 = {},
-          input
-        } = await run_default(app.socket, token, id, logger2);
+        const { channel: runChannel, run: run2 } = await run_default(
+          app.socket,
+          token,
+          id,
+          logger2
+        );
+        const { plan, options: options2, input } = convert_lightning_plan_default(run2, {
+          collectionsVersion: app.options.collectionsVersion,
+          monorepoPath: app.options.monorepoDir
+        });
+        logger2.debug("converted run body into execution plan:", plan);
+        if (plan.workflow.credentials?.collections_token) {
+          plan.workflow.credentials.collections_token = token;
+        }
+        if (plan.workflow.credentials?.collections_endpoint) {
+          plan.workflow.credentials.collections_endpoint = app.options.collectionsUrl;
+        }
         if (!("payloadLimitMb" in options2)) {
           options2.payloadLimitMb = app.options.payloadLimitMb;
         }
@@ -1295,7 +1376,10 @@ function createServer(engine, options = {}) {
   app.destroy = () => destroy_default(app, logger2);
   app.use(router.routes());
   if (options.lightning) {
-    connect(app, logger2, options);
+    setupCollections(options, logger2).then((version2) => {
+      app.collectionsVersion = version2;
+      connect(app, logger2, options);
+    });
   } else {
     logger2.warn("No lightning URL provided");
   }
@@ -2591,11 +2675,11 @@ var parser = new YargsParser({
   format,
   normalize,
   resolve: resolve2,
-  require: (path) => {
+  require: (path2) => {
     if (typeof __require !== "undefined") {
-      return __require(path);
-    } else if (path.match(/\.json$/)) {
-      return JSON.parse(readFileSync(path, "utf8"));
+      return __require(path2);
+    } else if (path2.match(/\.json$/)) {
+      return JSON.parse(readFileSync(path2, "utf8"));
     } else {
       throw Error("only .json config files are supported in ESM");
     }
@@ -6186,6 +6270,8 @@ function parseArgs(argv) {
   const {
     WORKER_BACKOFF,
     WORKER_CAPACITY,
+    WORKER_COLLECTIONS_VERSION,
+    WORKER_COLLECTIONS_URL,
     WORKER_LIGHTNING_PUBLIC_KEY,
     WORKER_LIGHTNING_SERVICE_URL,
     WORKER_LOG_LEVEL,
@@ -6196,7 +6282,8 @@ function parseArgs(argv) {
     WORKER_REPO_DIR,
     WORKER_SECRET,
     WORKER_STATE_PROPS_TO_REMOVE,
-    WORKER_SOCKET_TIMEOUT_SECONDS
+    WORKER_SOCKET_TIMEOUT_SECONDS,
+    OPENFN_ADAPTORS_REPO
   } = process.env;
   const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("port", {
     alias: "p",
@@ -6208,6 +6295,9 @@ function parseArgs(argv) {
   }).option("repo-dir", {
     alias: "d",
     description: "Path to the runtime repo (where modules will be installed). Env: WORKER_REPO_DIR"
+  }).option("monorepo-dir", {
+    alias: "m",
+    description: "Path to the adaptors mono repo, from where @local adaptors will be loaded. Env: OPENFN_ADAPTORS_REPO"
   }).option("secret", {
     alias: "s",
     description: "Worker secret. (comes from WORKER_SECRET by default). Env: WORKER_SECRET"
@@ -6243,6 +6333,12 @@ function parseArgs(argv) {
     alias: "t",
     description: "Default run timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
     type: "number"
+  }).option("collections-url", {
+    alias: ["c"],
+    description: "URL to the Collections service endpoint. Required for Collections, eg, https://app.openfn.org/collections. Env: WORKER_COLLECTIONS_URL"
+  }).option("collections-version", {
+    description: "The version of the collections adaptor to use for all runs on this worker instance.Env: WORKER_COLLECTIONS_VERSION",
+    type: "string"
   });
   const args2 = parser2.parse();
   return {
@@ -6254,6 +6350,7 @@ function parseArgs(argv) {
       "ws://localhost:4000/worker"
     ),
     repoDir: setArg(args2.repoDir, WORKER_REPO_DIR),
+    monorepoDir: setArg(args2.monorepoDir, OPENFN_ADAPTORS_REPO),
     secret: setArg(args2.secret, WORKER_SECRET),
     lightningPublicKey: setArg(
       args2.lightningPublicKey,
@@ -6278,7 +6375,12 @@ function parseArgs(argv) {
       args2.socketTimeoutSeconds,
       WORKER_SOCKET_TIMEOUT_SECONDS,
       DEFAULT_SOCKET_TIMEOUT_SECONDS
-    )
+    ),
+    collectionsVersion: setArg(
+      args2.collectionsVersion,
+      WORKER_COLLECTIONS_VERSION
+    ),
+    collectionsUrl: setArg(args2.collectionsUrl, WORKER_COLLECTIONS_URL)
   };
 }
 
@@ -6309,7 +6411,10 @@ function engineReady(engine) {
       max: maxBackoff
     },
     maxWorkflows: args.capacity,
-    payloadLimitMb: args.payloadMemory
+    payloadLimitMb: args.payloadMemory,
+    collectionsVersion: args.collectionsVersion,
+    collectionsUrl: args.collectionsUrl,
+    monorepoDir: args.monorepoDir
   };
   if (args.lightningPublicKey) {
     logger.info(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.6.7",
+  "version": "1.8.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -22,10 +22,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.2.5",
+    "@openfn/engine-multi": "1.4.0",
     "@openfn/lexicon": "^1.1.0",
     "@openfn/logger": "1.0.2",
-    "@openfn/runtime": "1.4.2"
+    "@openfn/runtime": "1.5.0"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -42,7 +42,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.0.19"
+    "@openfn/lightning-mock": "2.0.21"
   },
   "files": [
     "dist",