@openfn/ws-worker 0.2.4 → 0.2.6

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # ws-worker
 
+## 0.2.6
+
+### Patch Changes
+
+- 0fb2d58: correctly handle input_dataclip_id for runs
+- Updated dependencies [c8e9d51]
+- Updated dependencies [7f352d2]
+  - @openfn/engine-multi@0.1.10
+  - @openfn/runtime@0.1.3
+
+## 0.2.5
+
+### Patch Changes
+
+- 36337d7: When calculating exit reasons, exclude non-executed downstream nodes in leaf calculation. (i.e., look at the final executed node in each branch when determining the attempt exit reason.)
+
 ## 0.2.4
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,7 +1,6 @@
 import Koa from 'koa';
 import { SanitizePolicies, Logger } from '@openfn/logger';
 import { Channel as Channel$1 } from 'phoenix';
-import { ExecutionPlan } from '@openfn/runtime';
 import { RuntimeEngine } from '@openfn/engine-multi';
 
 type ExitReasonStrings =
@@ -23,6 +22,22 @@ type AttemptOptions = {
   sanitize?: SanitizePolicies;
 };
 
+// Internal server state for each attempt
+type AttemptState = {
+  activeRun?: string;
+  activeJob?: string;
+  plan: ExecutionPlan;
+  options: AttemptOptions;
+  dataclips: Record<string, any>;
+  // For each run, map the input ids
+  // TODO better name maybe?
+  inputDataclips: Record<string, string>;
+  reasons: Record<string, ExitReason>;
+
+  // final dataclip id
+  lastDataclipId?: string;
+};
+
 type ReceiveHook = {
   receive: (
     status: 'ok' | 'timeout' | 'error',
@@ -45,15 +60,6 @@ interface Channel extends Channel$1 {
   // join: () => ReceiveHook;
 }
 
-declare type AttemptState = {
-    activeRun?: string;
-    activeJob?: string;
-    plan: ExecutionPlan;
-    options: AttemptOptions;
-    dataclips: Record<string, any>;
-    reasons: Record<string, ExitReason>;
-    lastDataclipId?: string;
-};
 declare type Context = {
     channel: Channel;
     state: AttemptState;

package/dist/index.js

@@ -215,6 +215,35 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
+// src/util/create-attempt-state.ts
+var create_attempt_state_default = (plan, options = {}) => {
+  const state = {
+    plan,
+    lastDataclipId: "",
+    dataclips: {},
+    inputDataclips: {},
+    reasons: {},
+    options
+  };
+  if (typeof plan.initialState === "string") {
+    let startNode = plan.jobs[0];
+    if (plan.start) {
+      startNode = plan.jobs.find(({ id }) => id === plan.start);
+    }
+    const initialRuns = [];
+    if (!startNode.expression) {
+      initialRuns.push(...Object.keys(startNode.next));
+    } else {
+      initialRuns.push(startNode.id);
+    }
+    initialRuns.forEach((id) => {
+      state.inputDataclips[id] = plan.initialState;
+    });
+  } else {
+  }
+  return state;
+};
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -230,9 +259,16 @@ var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   }
   return { reason, error_type, error_message };
 };
+var isLeafNode = (state, job) => {
+  if (!job.next || Object.keys(job.next).length == 0) {
+    return true;
+  }
+  const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
+  return !hasDownstream;
+};
 var calculateAttemptExitReason = (state) => {
   if (state.plan && state.reasons) {
-    const leafJobReasons = state.plan.jobs.filter(({ next }) => !next || Object.keys(next).length == 0).map(({ id }) => state.reasons[id]);
+    const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
     const fail = leafJobReasons.find((r) => r && r.reason === "fail");
     if (fail) {
       return fail;
@@ -250,17 +286,10 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-var createAttemptState = (plan, options = {}) => ({
-  plan,
-  lastDataclipId: plan.initialState,
-  dataclips: {},
-  reasons: {},
-  options
-});
 function execute(channel, engine, logger, plan, options = {}, onComplete = (_result) => {
 }) {
-  logger.info("execute...");
-  const state = createAttemptState(plan, options);
+  logger.info("executing ", plan.id);
+  const state = create_attempt_state_default(plan, options);
   const context = { channel, state, logger, onComplete };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
@@ -305,7 +334,11 @@ function execute(channel, engine, logger, plan, options = {}, onComplete = (_res
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, { workflowId: plan.id, message: e.message });
+      onWorkflowError(context, {
+        workflowId: plan.id,
+        message: e.message,
+        type: e.type
+      });
     }
   });
   return context;
@@ -316,10 +349,11 @@ var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
 function onJobStart({ channel, state }, event) {
   state.activeRun = crypto2.randomUUID();
   state.activeJob = event.jobId;
+  const input_dataclip_id = state.inputDataclips[event.jobId];
   return sendEvent(channel, RUN_START, {
     run_id: state.activeRun,
     job_id: state.activeJob,
-    input_dataclip_id: state.lastDataclipId
+    input_dataclip_id
   });
 }
 function onJobError(context, event) {
@@ -339,27 +373,27 @@ function onJobComplete({ channel, state }, event, error) {
   }
   state.dataclips[dataclipId] = event.state;
   state.lastDataclipId = dataclipId;
+  event.next?.forEach((nextJobId) => {
+    state.inputDataclips[nextJobId] = dataclipId;
+  });
   delete state.activeRun;
   delete state.activeJob;
-  try {
-    const { reason, error_message, error_type } = calculateJobExitReason(
-      job_id,
-      event.state,
-      error
-    );
-    state.reasons[job_id] = { reason, error_message, error_type };
-    return sendEvent(channel, RUN_COMPLETE, {
-      run_id,
-      job_id,
-      output_dataclip_id: dataclipId,
-      output_dataclip: stringify_default(event.state),
-      reason,
-      error_message,
-      error_type
-    });
-  } catch (e) {
-    console.log(e);
-  }
+  const { reason, error_message, error_type } = calculateJobExitReason(
+    job_id,
+    event.state,
+    error
+  );
+  state.reasons[job_id] = { reason, error_message, error_type };
+  const evt = {
+    run_id,
+    job_id,
+    output_dataclip_id: dataclipId,
+    output_dataclip: stringify_default(event.state),
+    reason,
+    error_message,
+    error_type
+  };
+  return sendEvent(channel, RUN_COMPLETE, evt);
 }
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);

package/dist/start.js

@@ -4939,7 +4939,13 @@ async function createMock() {
         nextState = initialState;
       }
     }
-    dispatch("job-complete", { workflowId, jobId, state: nextState, runId });
+    dispatch("job-complete", {
+      workflowId,
+      jobId,
+      state: nextState,
+      runId,
+      next: []
+    });
     return nextState;
   };
   const execute2 = (xplan, options = {
@@ -5197,6 +5203,35 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
+// src/util/create-attempt-state.ts
+var create_attempt_state_default = (plan, options = {}) => {
+  const state = {
+    plan,
+    lastDataclipId: "",
+    dataclips: {},
+    inputDataclips: {},
+    reasons: {},
+    options
+  };
+  if (typeof plan.initialState === "string") {
+    let startNode = plan.jobs[0];
+    if (plan.start) {
+      startNode = plan.jobs.find(({ id }) => id === plan.start);
+    }
+    const initialRuns = [];
+    if (!startNode.expression) {
+      initialRuns.push(...Object.keys(startNode.next));
+    } else {
+      initialRuns.push(startNode.id);
+    }
+    initialRuns.forEach((id) => {
+      state.inputDataclips[id] = plan.initialState;
+    });
+  } else {
+  }
+  return state;
+};
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -5212,9 +5247,16 @@ var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   }
   return { reason, error_type, error_message };
 };
+var isLeafNode = (state, job) => {
+  if (!job.next || Object.keys(job.next).length == 0) {
+    return true;
+  }
+  const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
+  return !hasDownstream;
+};
 var calculateAttemptExitReason = (state) => {
   if (state.plan && state.reasons) {
-    const leafJobReasons = state.plan.jobs.filter(({ next }) => !next || Object.keys(next).length == 0).map(({ id }) => state.reasons[id]);
+    const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
     const fail = leafJobReasons.find((r) => r && r.reason === "fail");
     if (fail) {
       return fail;
@@ -5232,17 +5274,10 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-var createAttemptState = (plan, options = {}) => ({
-  plan,
-  lastDataclipId: plan.initialState,
-  dataclips: {},
-  reasons: {},
-  options
-});
 function execute(channel, engine, logger2, plan, options = {}, onComplete = (_result) => {
 }) {
-  logger2.info("execute...");
-  const state = createAttemptState(plan, options);
+  logger2.info("executing ", plan.id);
+  const state = create_attempt_state_default(plan, options);
   const context = { channel, state, logger: logger2, onComplete };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
@@ -5287,7 +5322,11 @@ function execute(channel, engine, logger2, plan, options = {}, onComplete = (_re
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, { workflowId: plan.id, message: e.message });
+      onWorkflowError(context, {
+        workflowId: plan.id,
+        message: e.message,
+        type: e.type
+      });
     }
   });
   return context;
@@ -5298,10 +5337,11 @@ var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
 function onJobStart({ channel, state }, event) {
   state.activeRun = crypto3.randomUUID();
   state.activeJob = event.jobId;
+  const input_dataclip_id = state.inputDataclips[event.jobId];
   return sendEvent(channel, RUN_START, {
     run_id: state.activeRun,
     job_id: state.activeJob,
-    input_dataclip_id: state.lastDataclipId
+    input_dataclip_id
   });
 }
 function onJobError(context, event) {
@@ -5321,27 +5361,27 @@ function onJobComplete({ channel, state }, event, error) {
   }
   state.dataclips[dataclipId] = event.state;
   state.lastDataclipId = dataclipId;
+  event.next?.forEach((nextJobId) => {
+    state.inputDataclips[nextJobId] = dataclipId;
+  });
   delete state.activeRun;
   delete state.activeJob;
-  try {
-    const { reason, error_message, error_type } = calculateJobExitReason(
-      job_id,
-      event.state,
-      error
-    );
-    state.reasons[job_id] = { reason, error_message, error_type };
-    return sendEvent(channel, RUN_COMPLETE, {
-      run_id,
-      job_id,
-      output_dataclip_id: dataclipId,
-      output_dataclip: stringify_default(event.state),
-      reason,
-      error_message,
-      error_type
-    });
-  } catch (e) {
-    console.log(e);
-  }
+  const { reason, error_message, error_type } = calculateJobExitReason(
+    job_id,
+    event.state,
+    error
+  );
+  state.reasons[job_id] = { reason, error_message, error_type };
+  const evt = {
+    run_id,
+    job_id,
+    output_dataclip_id: dataclipId,
+    output_dataclip: stringify_default(event.state),
+    reason,
+    error_message,
+    error_type
+  };
+  return sendEvent(channel, RUN_COMPLETE, evt);
 }
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,9 +21,9 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.1.9",
+    "@openfn/engine-multi": "0.1.10",
     "@openfn/logger": "0.0.19",
-    "@openfn/runtime": "0.1.2"
+    "@openfn/runtime": "0.1.3"
   },
   "devDependencies": {
     "@types/koa": "^2.13.5",
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.0.10"
+    "@openfn/lightning-mock": "1.0.11"
   },
   "files": [
     "dist",