@openfn/ws-worker 1.18.0 → 1.19.0

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # ws-worker
 
+## 1.19.0
+
+### Minor Changes
+
+- 475f8cf: Add options to profile memory usage in workflows. This should give a more accurate reading of the memory used by a run, but may have a small impact on performance
+
+### Patch Changes
+
+- 34edb8b: Improved logging of available memory on claim
+- Updated dependencies [3cd3cd5]
+- Updated dependencies [475f8cf]
+  - @openfn/runtime@1.7.4
+  - @openfn/engine-multi@1.8.0
+
+## 1.18.1
+
+### Patch Changes
+
+- b61bf9b: Fix an issue where memory may not be released after runs
+- Updated dependencies [b61bf9b]
+  - @openfn/engine-multi@1.7.1
+
 ## 1.18.0
 
 ### Minor Changes

package/dist/index.js

@@ -120,6 +120,7 @@ var tryWithBackoff = (fn, opts = {}) => {
 var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
+import v8 from "node:v8";
 import * as Sentry from "@sentry/node";
 import crypto from "node:crypto";
 import * as jose from "jose";
@@ -177,7 +178,13 @@ var claim = (app, logger = mockLogger, options = {}) => {
     }
     const claimId = ++claimIdGen;
     app.openClaims[claimId] = demand;
-    logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
+    const { used_heap_size, heap_size_limit } = v8.getHeapStatistics();
+    const usedHeapMb = Math.round(used_heap_size / 1024 / 1024);
+    const totalHeapMb = Math.round(heap_size_limit / 1024 / 1024);
+    const memPercent = Math.round(usedHeapMb / totalHeapMb * 100);
+    logger.debug(
+      `Claiming runs :: demand ${demand} | capacity ${activeWorkers}/${maxWorkers} | memory ${memPercent}% (${usedHeapMb}/${totalHeapMb}mb)`
+    );
     app.events.emit(INTERNAL_CLAIM_START);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
@@ -889,8 +896,11 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
         }
         const lightningEvent = eventMap[eventName] ?? eventName;
         try {
+          let start = Date.now();
           await handler(context, event);
-          logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
+          logger.info(
+            `${plan.id} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+          );
         } catch (e) {
           if (!e.reportedToSentry) {
             Sentry3.captureException(e);

package/dist/start.js

@@ -269,6 +269,7 @@ var tryWithBackoff = (fn, opts = {}) => {
 var try_with_backoff_default = tryWithBackoff;
 
 // src/api/claim.ts
+import v8 from "node:v8";
 import * as Sentry from "@sentry/node";
 import crypto2 from "node:crypto";
 import * as jose from "jose";
@@ -326,7 +327,13 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
     }
     const claimId = ++claimIdGen;
     app.openClaims[claimId] = demand;
-    logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
+    const { used_heap_size, heap_size_limit } = v8.getHeapStatistics();
+    const usedHeapMb = Math.round(used_heap_size / 1024 / 1024);
+    const totalHeapMb = Math.round(heap_size_limit / 1024 / 1024);
+    const memPercent = Math.round(usedHeapMb / totalHeapMb * 100);
+    logger2.debug(
+      `Claiming runs :: demand ${demand} | capacity ${activeWorkers}/${maxWorkers} | memory ${memPercent}% (${usedHeapMb}/${totalHeapMb}mb)`
+    );
     app.events.emit(INTERNAL_CLAIM_START);
     const start = Date.now();
     app.queueChannel.push(CLAIM, {
@@ -1038,8 +1045,11 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
         }
         const lightningEvent = eventMap[eventName] ?? eventName;
         try {
+          let start = Date.now();
           await handler(context, event);
-          logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
+          logger2.info(
+            `${plan.id} :: sent ${lightningEvent} :: OK :: ${Date.now() - start}ms`
+          );
         } catch (e) {
           if (!e.reportedToSentry) {
             Sentry3.captureException(e);
@@ -6418,6 +6428,9 @@ function setArg(argValue, envValue, defaultValue) {
   if (typeof defaultValue === "number" && envValue && !argValue) {
     return parseInt(envValue);
   }
+  if (typeof defaultValue === "boolean" && envValue && argValue === void 0) {
+    return envValue === "true" || envValue === "1";
+  }
   return argValue ?? envValue ?? defaultValue;
 }
 function parseArgs(argv) {
@@ -6436,6 +6449,8 @@ function parseArgs(argv) {
     WORKER_MAX_RUN_MEMORY_MB,
     WORKER_MESSAGE_TIMEOUT_SECONDS,
     WORKER_PORT,
+    WORKER_PROFILE,
+    WORKER_PROFILE_POLL_INTERVAL_MS,
     WORKER_REPO_DIR,
     WORKER_SECRET,
     WORKER_SENTRY_DSN,
@@ -6518,6 +6533,12 @@ function parseArgs(argv) {
   }).option("engine-validation-retries", {
     description: "The number of times to retry engine validation. Useful in hosted environments. Default 3. ENV: WORKER_VALIDATION_RETRIES",
     type: "number"
+  }).option("profile", {
+    description: "Enable profiling for runs. Default false. Env: WORKER_PROFILE",
+    type: "boolean"
+  }).option("profile-poll-interval-ms", {
+    description: "Interval for polling profile data, in milliseconds. Default 10. Env: WORKER_PROFILE_POLL_INTERVAL_MS",
+    type: "number"
   });
   const args2 = parser2.parse();
   return {
@@ -6580,6 +6601,12 @@ function parseArgs(argv) {
       args2.engineValidationTimeoutMs,
       WORKER_VALIDATION_TIMEOUT_MS,
       5e3
+    ),
+    profile: setArg(args2.profile, WORKER_PROFILE, false),
+    profilePollIntervalMs: setArg(
+      args2.profilePollIntervalMs,
+      WORKER_PROFILE_POLL_INTERVAL_MS,
+      10
     )
   };
 }
@@ -6659,7 +6686,9 @@ if (args.mock) {
     statePropsToRemove: args.statePropsToRemove,
     runTimeoutMs: args.maxRunDurationSeconds * 1e3,
     workerValidationTimeout: args.engineValidationTimeoutMs,
-    workerValidationRetries: args.engineValidationRetries
+    workerValidationRetries: args.engineValidationRetries,
+    profile: args.profile,
+    profilePollInterval: args.profilePollIntervalMs
   };
   logger.debug("Creating runtime engine...");
   logger.debug("Engine options:", engineOptions);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.18.0",
+  "version": "1.19.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.3",
-    "@openfn/engine-multi": "1.7.0",
-    "@openfn/runtime": "1.7.3",
     "@openfn/lexicon": "^1.2.5",
-    "@openfn/logger": "1.0.6"
+    "@openfn/runtime": "1.7.4",
+    "@openfn/logger": "1.0.6",
+    "@openfn/engine-multi": "1.8.0"
   },
   "devDependencies": {
     "@types/koa": "^2.15.0",
@@ -43,7 +43,7 @@
     "tsup": "^6.7.0",
     "typescript": "^4.9.5",
     "yargs": "^17.7.2",
-    "@openfn/lightning-mock": "2.3.1"
+    "@openfn/lightning-mock": "2.3.3"
   },
   "files": [
     "dist",