@openfn/ws-worker 1.16.1 → 1.18.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # ws-worker
 
+## 1.18.0
+
+### Minor Changes
+
+- cf8d3c2: Add retry with backoff to worker validation
+
+### Patch Changes
+
+- Updated dependencies [cf8d3c2]
+  - @openfn/engine-multi@1.7.0
+
+## 1.17.0
+
+### Minor Changes
+
+- 09dd4b2: - Add `final_state` object to `workflow:complete` event
+  - Remove unused `final_dataclip_id` from `workflow:complete` payload
+
+### Patch Changes
+
+- Updated dependencies [09dd4b2]
+  - @openfn/lexicon@1.2.4
+
 ## 1.16.1
 
 ### Patch Changes

package/dist/index.js

@@ -348,10 +348,6 @@ var convert_lightning_plan_default = (run, options = {}) => {
       engineOpts.jobLogLevel = run.options.job_log_level;
     }
   }
-  const plan = {
-    id: run.id,
-    options: runtimeOpts
-  };
   let initialState;
   if (run.dataclip_id) {
     initialState = run.dataclip_id;
@@ -415,8 +411,14 @@ var convert_lightning_plan_default = (run, options = {}) => {
       nodes[id] = job;
     });
   }
-  plan.workflow = {
-    steps: Object.values(nodes)
+  const plan = {
+    id: run.id,
+    options: runtimeOpts,
+    workflow: {
+      // id: run.workflowId, // TODO: lightning does not send this
+      // name: run.name, // TODO: lightning does not send this
+      steps: Object.values(nodes)
+    }
   };
   if (run.globals && typeof run.globals === "string")
     plan.workflow.globals = run.globals;
@@ -806,12 +808,12 @@ var log_final_reason_default = async (context, reason) => {
 // src/events/run-complete.ts
 async function onWorkflowComplete(context, event) {
   const { state, onFinish, logger } = context;
-  const result = state.dataclips[state.lastDataclipId];
+  const result = event.state;
   const reason = calculateRunExitReason(state);
   await log_final_reason_default(context, reason);
   try {
     await sendEvent(context, RUN_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
+      final_state: result,
       timestamp: timeInMicroseconds(event.time),
       ...reason
     });
@@ -834,7 +836,6 @@ async function onRunError(context, event) {
     }
     await log_final_reason_default(context, reason);
     await sendEvent(context, RUN_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
       ...reason
     });
     onFinish({ reason });

package/dist/start.js

@@ -105,10 +105,19 @@ async function createMock() {
       }
     };
     setTimeout(async () => {
+      const startTime = Date.now();
       dispatch("workflow-start", { workflowId: id, threadId });
       try {
-        await run(xplan, input, opts);
-        dispatch("workflow-complete", { workflowId: id, threadId });
+        const result = await run(xplan, input, opts);
+        const duration = Date.now() - startTime;
+        dispatch("workflow-complete", {
+          workflowId: id,
+          threadId,
+          state: result,
+          duration,
+          time: BigInt(Date.now()) * BigInt(1e6)
+          // Convert to nanoseconds
+        });
       } catch (e) {
         dispatch("workflow-error", {
           threadId,
@@ -488,10 +497,6 @@ var convert_lightning_plan_default = (run2, options = {}) => {
       engineOpts.jobLogLevel = run2.options.job_log_level;
     }
   }
-  const plan = {
-    id: run2.id,
-    options: runtimeOpts
-  };
   let initialState;
   if (run2.dataclip_id) {
     initialState = run2.dataclip_id;
@@ -555,8 +560,14 @@ var convert_lightning_plan_default = (run2, options = {}) => {
       nodes[id] = job;
     });
   }
-  plan.workflow = {
-    steps: Object.values(nodes)
+  const plan = {
+    id: run2.id,
+    options: runtimeOpts,
+    workflow: {
+      // id: run.workflowId, // TODO: lightning does not send this
+      // name: run.name, // TODO: lightning does not send this
+      steps: Object.values(nodes)
+    }
   };
   if (run2.globals && typeof run2.globals === "string")
     plan.workflow.globals = run2.globals;
@@ -946,12 +957,12 @@ var log_final_reason_default = async (context, reason) => {
 // src/events/run-complete.ts
 async function onWorkflowComplete(context, event) {
   const { state, onFinish, logger: logger2 } = context;
-  const result = state.dataclips[state.lastDataclipId];
+  const result = event.state;
   const reason = calculateRunExitReason(state);
   await log_final_reason_default(context, reason);
   try {
     await sendEvent(context, RUN_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
+      final_state: result,
       timestamp: timeInMicroseconds(event.time),
       ...reason
     });
@@ -974,7 +985,6 @@ async function onRunError(context, event) {
     }
     await log_final_reason_default(context, reason);
     await sendEvent(context, RUN_COMPLETE, {
-      final_dataclip_id: state.lastDataclipId,
       ...reason
     });
     onFinish({ reason });
@@ -6431,7 +6441,9 @@ function parseArgs(argv) {
     WORKER_SENTRY_DSN,
     WORKER_SENTRY_ENV,
     WORKER_SOCKET_TIMEOUT_SECONDS,
-    WORKER_STATE_PROPS_TO_REMOVE
+    WORKER_STATE_PROPS_TO_REMOVE,
+    WORKER_VALIDATION_RETRIES,
+    WORKER_VALIDATION_TIMEOUT_MS
   } = process.env;
   const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("debug", {
     hidden: true,
@@ -6500,6 +6512,12 @@ function parseArgs(argv) {
   }).option("collections-version", {
     description: "The version of the collections adaptor to use for all runs on this worker instance.Env: WORKER_COLLECTIONS_VERSION",
     type: "string"
+  }).option("engine-validation-timeout-ms", {
+    description: "The timeout used to run the validation task within the engine, in milliseconds. Env: WORKER_VALIDATION_TIMEOUT_MS",
+    type: "number"
+  }).option("engine-validation-retries", {
+    description: "The number of times to retry engine validation. Useful in hosted environments. Default 3. ENV: WORKER_VALIDATION_RETRIES",
+    type: "number"
   });
   const args2 = parser2.parse();
   return {
@@ -6552,7 +6570,17 @@ function parseArgs(argv) {
       args2.collectionsVersion,
       WORKER_COLLECTIONS_VERSION
     ),
-    collectionsUrl: setArg(args2.collectionsUrl, WORKER_COLLECTIONS_URL)
+    collectionsUrl: setArg(args2.collectionsUrl, WORKER_COLLECTIONS_URL),
+    engineValidationRetries: setArg(
+      args2.engineValidationRetries,
+      WORKER_VALIDATION_RETRIES,
+      3
+    ),
+    engineValidationTimeoutMs: setArg(
+      args2.engineValidationTimeoutMs,
+      WORKER_VALIDATION_TIMEOUT_MS,
+      5e3
+    )
   };
 }
 
@@ -6629,7 +6657,9 @@ if (args.mock) {
     memoryLimitMb: args.runMemory,
     maxWorkers: args.capacity,
     statePropsToRemove: args.statePropsToRemove,
-    runTimeoutMs: args.maxRunDurationSeconds * 1e3
+    runTimeoutMs: args.maxRunDurationSeconds * 1e3,
+    workerValidationTimeout: args.engineValidationTimeoutMs,
+    workerValidationRetries: args.engineValidationRetries
   };
   logger.debug("Creating runtime engine...");
   logger.debug("Engine options:", engineOptions);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.16.1",
+  "version": "1.18.0",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -23,10 +23,10 @@
     "koa-logger": "^3.2.1",
     "phoenix": "1.7.10",
     "ws": "^8.18.3",
-    "@openfn/engine-multi": "1.6.14",
-    "@openfn/lexicon": "^1.2.3",
-    "@openfn/logger": "1.0.6",
-    "@openfn/runtime": "1.7.3"
+    "@openfn/engine-multi": "1.7.0",
+    "@openfn/runtime": "1.7.3",
+    "@openfn/lexicon": "^1.2.5",
+    "@openfn/logger": "1.0.6"
   },
   "devDependencies": {
     "@types/koa": "^2.15.0",
@@ -43,7 +43,7 @@
     "tsup": "^6.7.0",
     "typescript": "^4.9.5",
     "yargs": "^17.7.2",
-    "@openfn/lightning-mock": "2.2.9"
+    "@openfn/lightning-mock": "2.3.1"
   },
   "files": [
     "dist",