@openfn/ws-worker 1.6.3 → 1.6.6

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # ws-worker
 
+## 1.6.6
+
+### Patch Changes
+
+- Log claim event duration
+
+## 1.6.5
+
+### Patch Changes
+
+- 5db5862: Dont log compiled job code
+- f581c6b: log duration of runs and server capacity
+- 3e6eba2: Trap errors coming out of the websocket
+  - @openfn/engine-multi@1.2.5
+
+## 1.6.4
+
+### Patch Changes
+
+- 0cf7198: Do not send the input_dataclip_id in step:start if the dataclip was witheld
+
 ## 1.6.3
 
 ### Patch Changes

package/dist/index.d.ts

@@ -18,6 +18,8 @@ type RunState = {
   // For each run, map the input ids
   // TODO better name maybe?
   inputDataclips: Record<string, string>;
+  // If for any reason a dataclip was not sent to lightning, track it
+  withheldDataclips: Record<string, true>;
   reasons: Record<string, ExitReason>;
 
   // final dataclip id

package/dist/index.js

@@ -29,7 +29,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.3";
+    version = "1.6.6";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -232,18 +232,21 @@ var claim = (app, logger = mockLogger, options = {}) => {
     const { maxWorkers = 5 } = options;
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
-      logger.debug("skipping claim attempt: server at capacity");
+      logger.debug(
+        `skipping claim attempt: server at capacity (${activeWorkers}/${maxWorkers})`
+      );
       return reject(new Error("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger.debug("skipping claim attempt: websocket unavailable");
       return reject(new Error("No websocket available"));
     }
-    logger.debug("requesting run...");
+    logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
+    const start = Date.now();
     app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+      const duration = Date.now() - start;
       logger.debug(
-        `claimed ${runs.length} runs: `,
-        runs.map((r) => r.id).join(",")
+        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -456,6 +459,7 @@ var create_run_state_default = (plan, input) => {
     lastDataclipId: "",
     dataclips: {},
     inputDataclips: {},
+    withheldDataclips: {},
     reasons: {},
     plan,
     input
@@ -660,6 +664,7 @@ async function onStepComplete(context, event, error) {
     }
     evt.output_dataclip_id = dataclipId;
   } catch (e) {
+    state.withheldDataclips[dataclipId] = true;
     evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
     const time = (timestamp2() - BigInt(1e7)).toString();
     await onJobLog(context, {
@@ -684,12 +689,15 @@ async function onStepStart(context, event) {
   state.activeStep = crypto4.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
-  await sendEvent(channel, STEP_START, {
+  const evt = {
     step_id: state.activeStep,
     job_id: state.activeJob,
-    input_dataclip_id,
     timestamp: timeInMicroseconds(event.time)
-  });
+  };
+  if (!state.withheldDataclips[input_dataclip_id]) {
+    evt.input_dataclip_id = input_dataclip_id;
+  }
+  await sendEvent(channel, STEP_START, evt);
 }
 
 // src/util/log-final-reason.ts
@@ -1083,32 +1091,41 @@ function createServer(engine, options = {}) {
   app.options = options;
   app.execute = async ({ id, token }) => {
     if (app.socket) {
-      app.workflows[id] = true;
-      const {
-        channel: runChannel,
-        plan,
-        options: options2 = {},
-        input
-      } = await run_default(app.socket, token, id, logger);
-      if (!("payloadLimitMb" in options2)) {
-        options2.payloadLimitMb = app.options.payloadLimitMb;
+      try {
+        const start = Date.now();
+        app.workflows[id] = true;
+        const {
+          channel: runChannel,
+          plan,
+          options: options2 = {},
+          input
+        } = await run_default(app.socket, token, id, logger);
+        if (!("payloadLimitMb" in options2)) {
+          options2.payloadLimitMb = app.options.payloadLimitMb;
+        }
+        const onFinish = () => {
+          const duration = (Date.now() - start) / 1e3;
+          logger.debug(
+            `workflow ${id} complete in ${duration}s: releasing worker`
+          );
+          delete app.workflows[id];
+          runChannel.leave();
+          app.events.emit(INTERNAL_RUN_COMPLETE);
+        };
+        const context = execute(
+          runChannel,
+          engine,
+          logger,
+          plan,
+          input,
+          options2,
+          onFinish
+        );
+        app.workflows[id] = context;
+      } catch (e) {
+        logger.error(`Unexpected error executing ${id}`);
+        logger.error(e);
       }
-      const onFinish = () => {
-        logger.debug(`workflow ${id} complete: releasing worker`);
-        delete app.workflows[id];
-        runChannel.leave();
-        app.events.emit(INTERNAL_RUN_COMPLETE);
-      };
-      const context = execute(
-        runChannel,
-        engine,
-        logger,
-        plan,
-        input,
-        options2,
-        onFinish
-      );
-      app.workflows[id] = context;
     } else {
       logger.error("No lightning socket established");
     }

package/dist/start.js

@@ -37,7 +37,7 @@ var name, version, description, main, type, scripts, bin, author, license, depen
 var init_package = __esm({
   "package.json"() {
     name = "@openfn/ws-worker";
-    version = "1.6.3";
+    version = "1.6.6";
     description = "A Websocket Worker to connect Lightning to a Runtime Engine";
     main = "dist/index.js";
     type = "module";
@@ -371,18 +371,21 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     const { maxWorkers = 5 } = options;
     const activeWorkers = Object.keys(app.workflows).length;
     if (activeWorkers >= maxWorkers) {
-      logger2.debug("skipping claim attempt: server at capacity");
+      logger2.debug(
+        `skipping claim attempt: server at capacity (${activeWorkers}/${maxWorkers})`
+      );
       return reject(new Error("Server at capacity"));
     }
     if (!app.queueChannel) {
       logger2.debug("skipping claim attempt: websocket unavailable");
       return reject(new Error("No websocket available"));
     }
-    logger2.debug("requesting run...");
+    logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
+    const start = Date.now();
     app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
+      const duration = Date.now() - start;
       logger2.debug(
-        `claimed ${runs.length} runs: `,
-        runs.map((r) => r.id).join(",")
+        `claimed ${runs.length} runs in ${duration}ms (${runs.length ? runs.map((r) => r.id).join(",") : "-"})`
       );
       if (!runs?.length) {
         return reject(new Error("No runs returned"));
@@ -595,6 +598,7 @@ var create_run_state_default = (plan, input) => {
     lastDataclipId: "",
     dataclips: {},
     inputDataclips: {},
+    withheldDataclips: {},
     reasons: {},
     plan,
     input
@@ -799,6 +803,7 @@ async function onStepComplete(context, event, error) {
     }
     evt.output_dataclip_id = dataclipId;
   } catch (e) {
+    state.withheldDataclips[dataclipId] = true;
     evt.output_dataclip_error = "DATACLIP_TOO_LARGE";
     const time = (timestamp2() - BigInt(1e7)).toString();
     await onJobLog(context, {
@@ -823,12 +828,15 @@ async function onStepStart(context, event) {
   state.activeStep = crypto5.randomUUID();
   state.activeJob = event.jobId;
   const input_dataclip_id = state.inputDataclips[event.jobId];
-  await sendEvent(channel, STEP_START, {
+  const evt = {
     step_id: state.activeStep,
     job_id: state.activeJob,
-    input_dataclip_id,
     timestamp: timeInMicroseconds(event.time)
-  });
+  };
+  if (!state.withheldDataclips[input_dataclip_id]) {
+    evt.input_dataclip_id = input_dataclip_id;
+  }
+  await sendEvent(channel, STEP_START, evt);
 }
 
 // src/util/log-final-reason.ts
@@ -1222,32 +1230,41 @@ function createServer(engine, options = {}) {
   app.options = options;
   app.execute = async ({ id, token }) => {
     if (app.socket) {
-      app.workflows[id] = true;
-      const {
-        channel: runChannel,
-        plan,
-        options: options2 = {},
-        input
-      } = await run_default(app.socket, token, id, logger2);
-      if (!("payloadLimitMb" in options2)) {
-        options2.payloadLimitMb = app.options.payloadLimitMb;
-      }
-      const onFinish = () => {
-        logger2.debug(`workflow ${id} complete: releasing worker`);
-        delete app.workflows[id];
-        runChannel.leave();
-        app.events.emit(INTERNAL_RUN_COMPLETE);
-      };
-      const context = execute(
-        runChannel,
-        engine,
-        logger2,
-        plan,
-        input,
-        options2,
-        onFinish
-      );
-      app.workflows[id] = context;
+      try {
+        const start = Date.now();
+        app.workflows[id] = true;
+        const {
+          channel: runChannel,
+          plan,
+          options: options2 = {},
+          input
+        } = await run_default(app.socket, token, id, logger2);
+        if (!("payloadLimitMb" in options2)) {
+          options2.payloadLimitMb = app.options.payloadLimitMb;
+        }
+        const onFinish = () => {
+          const duration = (Date.now() - start) / 1e3;
+          logger2.debug(
+            `workflow ${id} complete in ${duration}s: releasing worker`
+          );
+          delete app.workflows[id];
+          runChannel.leave();
+          app.events.emit(INTERNAL_RUN_COMPLETE);
+        };
+        const context = execute(
+          runChannel,
+          engine,
+          logger2,
+          plan,
+          input,
+          options2,
+          onFinish
+        );
+        app.workflows[id] = context;
+      } catch (e) {
+        logger2.error(`Unexpected error executing ${id}`);
+        logger2.error(e);
+      }
     } else {
       logger2.error("No lightning socket established");
     }
@@ -6259,6 +6276,7 @@ function parseArgs(argv) {
 // src/start.ts
 var args = parseArgs(process.argv);
 var logger = createLogger("SRV", { level: args.log });
+logger.info("Starting worker server...");
 if (args.lightning === "mock") {
   args.lightning = "ws://localhost:8888/worker";
   if (!args.secret) {
@@ -6270,7 +6288,7 @@ if (args.lightning === "mock") {
 }
 var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseInt(n, 10) * 1e3);
 function engineReady(engine) {
-  logger.debug("Creating worker server...");
+  logger.debug("Creating worker instance");
   const workerOptions = {
     port: args.port,
     lightning: args.lightning,
@@ -6301,6 +6319,7 @@ function engineReady(engine) {
   } = workerOptions;
   logger.debug("Worker options:", humanOptions);
   server_default(engine, workerOptions);
+  logger.success("Worker started OK");
 }
 if (args.mock) {
   runtime_engine_default().then((engine) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.6.3",
+  "version": "1.6.6",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -22,10 +22,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.0",
-    "@openfn/engine-multi": "1.2.4",
+    "@openfn/engine-multi": "1.2.5",
     "@openfn/lexicon": "^1.1.0",
-    "@openfn/logger": "1.0.2",
-    "@openfn/runtime": "1.4.2"
+    "@openfn/runtime": "1.4.2",
+    "@openfn/logger": "1.0.2"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -42,7 +42,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "2.0.18"
+    "@openfn/lightning-mock": "2.0.19"
   },
   "files": [
     "dist",