@openfn/ws-worker 0.2.5 → 0.2.7

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # ws-worker
 
+## 0.2.7
+
+### Patch Changes
+
+- d542aa9: worker: leave attempt channel when finished working
+- Updated dependencies [793d523]
+- Updated dependencies [857c42b]
+- Updated dependencies [f17fb4a]
+  - @openfn/engine-multi@0.1.11
+  - @openfn/runtime@0.1.4
+
+## 0.2.6
+
+### Patch Changes
+
+- 0fb2d58: correctly handle input_dataclip_id for runs
+- Updated dependencies [c8e9d51]
+- Updated dependencies [7f352d2]
+  - @openfn/engine-multi@0.1.10
+  - @openfn/runtime@0.1.3
+
 ## 0.2.5
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,7 +1,6 @@
 import Koa from 'koa';
 import { SanitizePolicies, Logger } from '@openfn/logger';
 import { Channel as Channel$1 } from 'phoenix';
-import { ExecutionPlan } from '@openfn/runtime';
 import { RuntimeEngine } from '@openfn/engine-multi';
 
 type ExitReasonStrings =
@@ -23,6 +22,22 @@ type AttemptOptions = {
   sanitize?: SanitizePolicies;
 };
 
+// Internal server state for each attempt
+type AttemptState = {
+  activeRun?: string;
+  activeJob?: string;
+  plan: ExecutionPlan;
+  options: AttemptOptions;
+  dataclips: Record<string, any>;
+  // For each run, map the input ids
+  // TODO better name maybe?
+  inputDataclips: Record<string, string>;
+  reasons: Record<string, ExitReason>;
+
+  // final dataclip id
+  lastDataclipId?: string;
+};
+
 type ReceiveHook = {
   receive: (
     status: 'ok' | 'timeout' | 'error',
@@ -45,20 +60,11 @@ interface Channel extends Channel$1 {
   // join: () => ReceiveHook;
 }
 
-declare type AttemptState = {
-    activeRun?: string;
-    activeJob?: string;
-    plan: ExecutionPlan;
-    options: AttemptOptions;
-    dataclips: Record<string, any>;
-    reasons: Record<string, ExitReason>;
-    lastDataclipId?: string;
-};
 declare type Context = {
     channel: Channel;
     state: AttemptState;
     logger: Logger;
-    onComplete: (result: any) => void;
+    onFinish: (result: any) => void;
 };
 
 declare type CLAIM_ATTEMPT = {

package/dist/index.js

@@ -215,6 +215,35 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
+// src/util/create-attempt-state.ts
+var create_attempt_state_default = (plan, options = {}) => {
+  const state = {
+    plan,
+    lastDataclipId: "",
+    dataclips: {},
+    inputDataclips: {},
+    reasons: {},
+    options
+  };
+  if (typeof plan.initialState === "string") {
+    let startNode = plan.jobs[0];
+    if (plan.start) {
+      startNode = plan.jobs.find(({ id }) => id === plan.start);
+    }
+    const initialRuns = [];
+    if (!startNode.expression) {
+      initialRuns.push(...Object.keys(startNode.next));
+    } else {
+      initialRuns.push(startNode.id);
+    }
+    initialRuns.forEach((id) => {
+      state.inputDataclips[id] = plan.initialState;
+    });
+  } else {
+  }
+  return state;
+};
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -257,18 +286,11 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-var createAttemptState = (plan, options = {}) => ({
-  plan,
-  lastDataclipId: plan.initialState,
-  dataclips: {},
-  reasons: {},
-  options
-});
-function execute(channel, engine, logger, plan, options = {}, onComplete = (_result) => {
+function execute(channel, engine, logger, plan, options = {}, onFinish = (_result) => {
 }) {
-  logger.info("execute...");
-  const state = createAttemptState(plan, options);
-  const context = { channel, state, logger, onComplete };
+  logger.info("executing ", plan.id);
+  const state = create_attempt_state_default(plan, options);
+  const context = { channel, state, logger, onFinish };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
       const lightningEvent = eventMap[eventName] ?? eventName;
@@ -312,7 +334,12 @@ function execute(channel, engine, logger, plan, options = {}, onComplete = (_res
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, { workflowId: plan.id, message: e.message });
+      onWorkflowError(context, {
+        workflowId: plan.id,
+        message: e.message,
+        type: e.type,
+        severity: e.severity
+      });
     }
   });
   return context;
@@ -323,10 +350,11 @@ var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
 function onJobStart({ channel, state }, event) {
   state.activeRun = crypto2.randomUUID();
   state.activeJob = event.jobId;
+  const input_dataclip_id = state.inputDataclips[event.jobId];
   return sendEvent(channel, RUN_START, {
     run_id: state.activeRun,
     job_id: state.activeJob,
-    input_dataclip_id: state.lastDataclipId
+    input_dataclip_id
   });
 }
 function onJobError(context, event) {
@@ -346,47 +374,52 @@ function onJobComplete({ channel, state }, event, error) {
   }
   state.dataclips[dataclipId] = event.state;
   state.lastDataclipId = dataclipId;
+  event.next?.forEach((nextJobId) => {
+    state.inputDataclips[nextJobId] = dataclipId;
+  });
   delete state.activeRun;
   delete state.activeJob;
-  try {
-    const { reason, error_message, error_type } = calculateJobExitReason(
-      job_id,
-      event.state,
-      error
-    );
-    state.reasons[job_id] = { reason, error_message, error_type };
-    return sendEvent(channel, RUN_COMPLETE, {
-      run_id,
-      job_id,
-      output_dataclip_id: dataclipId,
-      output_dataclip: stringify_default(event.state),
-      reason,
-      error_message,
-      error_type
-    });
-  } catch (e) {
-    console.log(e);
-  }
+  const { reason, error_message, error_type } = calculateJobExitReason(
+    job_id,
+    event.state,
+    error
+  );
+  state.reasons[job_id] = { reason, error_message, error_type };
+  const evt = {
+    run_id,
+    job_id,
+    output_dataclip_id: dataclipId,
+    output_dataclip: stringify_default(event.state),
+    reason,
+    error_message,
+    error_type
+  };
+  return sendEvent(channel, RUN_COMPLETE, evt);
 }
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onComplete }, _event) {
+async function onWorkflowComplete({ state, channel, onFinish }, _event) {
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateAttemptExitReason(state);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
     final_dataclip_id: state.lastDataclipId,
     ...reason
   });
-  onComplete({ reason, state: result });
+  onFinish({ reason, state: result });
 }
-async function onWorkflowError({ state, channel, onComplete }, event) {
-  const reason = calculateJobExitReason("", { data: {} }, event);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onComplete({ reason });
+async function onWorkflowError({ state, channel, logger, onFinish }, event) {
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger.error("ERROR in workflow-error handler:", e.message);
+    logger.error(e);
+  }
 }
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
@@ -576,8 +609,9 @@ function createServer(engine, options = {}) {
         plan,
         options: options2
       } = await attempt_default(app.socket, token, id, logger);
-      const onComplete = () => {
+      const onFinish = () => {
         delete app.workflows[id];
+        attemptChannel.leave();
       };
       const context = execute(
         attemptChannel,
@@ -585,7 +619,7 @@ function createServer(engine, options = {}) {
         logger,
         plan,
         options2,
-        onComplete
+        onFinish
       );
       app.workflows[id] = context;
     } else {

package/dist/start.js

@@ -4939,7 +4939,13 @@ async function createMock() {
         nextState = initialState;
       }
     }
-    dispatch("job-complete", { workflowId, jobId, state: nextState, runId });
+    dispatch("job-complete", {
+      workflowId,
+      jobId,
+      state: nextState,
+      runId,
+      next: []
+    });
     return nextState;
   };
   const execute2 = (xplan, options = {
@@ -5197,6 +5203,35 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
   return value;
 });
 
+// src/util/create-attempt-state.ts
+var create_attempt_state_default = (plan, options = {}) => {
+  const state = {
+    plan,
+    lastDataclipId: "",
+    dataclips: {},
+    inputDataclips: {},
+    reasons: {},
+    options
+  };
+  if (typeof plan.initialState === "string") {
+    let startNode = plan.jobs[0];
+    if (plan.start) {
+      startNode = plan.jobs.find(({ id }) => id === plan.start);
+    }
+    const initialRuns = [];
+    if (!startNode.expression) {
+      initialRuns.push(...Object.keys(startNode.next));
+    } else {
+      initialRuns.push(startNode.id);
+    }
+    initialRuns.forEach((id) => {
+      state.inputDataclips[id] = plan.initialState;
+    });
+  } else {
+  }
+  return state;
+};
+
 // src/api/reasons.ts
 var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
   let reason = "success";
@@ -5239,18 +5274,11 @@ var eventMap = {
   "workflow-log": ATTEMPT_LOG,
   "workflow-complete": ATTEMPT_COMPLETE
 };
-var createAttemptState = (plan, options = {}) => ({
-  plan,
-  lastDataclipId: plan.initialState,
-  dataclips: {},
-  reasons: {},
-  options
-});
-function execute(channel, engine, logger2, plan, options = {}, onComplete = (_result) => {
+function execute(channel, engine, logger2, plan, options = {}, onFinish = (_result) => {
 }) {
-  logger2.info("execute...");
-  const state = createAttemptState(plan, options);
-  const context = { channel, state, logger: logger2, onComplete };
+  logger2.info("executing ", plan.id);
+  const state = create_attempt_state_default(plan, options);
+  const context = { channel, state, logger: logger2, onFinish };
   const addEvent = (eventName, handler) => {
     const wrappedFn = async (event) => {
       const lightningEvent = eventMap[eventName] ?? eventName;
@@ -5294,7 +5322,12 @@ function execute(channel, engine, logger2, plan, options = {}, onComplete = (_re
     try {
       engine.execute(plan, { resolvers, ...options });
     } catch (e) {
-      onWorkflowError(context, { workflowId: plan.id, message: e.message });
+      onWorkflowError(context, {
+        workflowId: plan.id,
+        message: e.message,
+        type: e.type,
+        severity: e.severity
+      });
     }
   });
   return context;
@@ -5305,10 +5338,11 @@ var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
 function onJobStart({ channel, state }, event) {
   state.activeRun = crypto3.randomUUID();
   state.activeJob = event.jobId;
+  const input_dataclip_id = state.inputDataclips[event.jobId];
   return sendEvent(channel, RUN_START, {
     run_id: state.activeRun,
     job_id: state.activeJob,
-    input_dataclip_id: state.lastDataclipId
+    input_dataclip_id
   });
 }
 function onJobError(context, event) {
@@ -5328,47 +5362,52 @@ function onJobComplete({ channel, state }, event, error) {
   }
   state.dataclips[dataclipId] = event.state;
   state.lastDataclipId = dataclipId;
+  event.next?.forEach((nextJobId) => {
+    state.inputDataclips[nextJobId] = dataclipId;
+  });
   delete state.activeRun;
   delete state.activeJob;
-  try {
-    const { reason, error_message, error_type } = calculateJobExitReason(
-      job_id,
-      event.state,
-      error
-    );
-    state.reasons[job_id] = { reason, error_message, error_type };
-    return sendEvent(channel, RUN_COMPLETE, {
-      run_id,
-      job_id,
-      output_dataclip_id: dataclipId,
-      output_dataclip: stringify_default(event.state),
-      reason,
-      error_message,
-      error_type
-    });
-  } catch (e) {
-    console.log(e);
-  }
+  const { reason, error_message, error_type } = calculateJobExitReason(
+    job_id,
+    event.state,
+    error
+  );
+  state.reasons[job_id] = { reason, error_message, error_type };
+  const evt = {
+    run_id,
+    job_id,
+    output_dataclip_id: dataclipId,
+    output_dataclip: stringify_default(event.state),
+    reason,
+    error_message,
+    error_type
+  };
+  return sendEvent(channel, RUN_COMPLETE, evt);
 }
 function onWorkflowStart({ channel }, _event) {
   return sendEvent(channel, ATTEMPT_START);
 }
-async function onWorkflowComplete({ state, channel, onComplete }, _event) {
+async function onWorkflowComplete({ state, channel, onFinish }, _event) {
   const result = state.dataclips[state.lastDataclipId];
   const reason = calculateAttemptExitReason(state);
   await sendEvent(channel, ATTEMPT_COMPLETE, {
     final_dataclip_id: state.lastDataclipId,
     ...reason
   });
-  onComplete({ reason, state: result });
+  onFinish({ reason, state: result });
 }
-async function onWorkflowError({ state, channel, onComplete }, event) {
-  const reason = calculateJobExitReason("", { data: {} }, event);
-  await sendEvent(channel, ATTEMPT_COMPLETE, {
-    final_dataclip_id: state.lastDataclipId,
-    ...reason
-  });
-  onComplete({ reason });
+async function onWorkflowError({ state, channel, logger: logger2, onFinish }, event) {
+  try {
+    const reason = calculateJobExitReason("", { data: {} }, event);
+    await sendEvent(channel, ATTEMPT_COMPLETE, {
+      final_dataclip_id: state.lastDataclipId,
+      ...reason
+    });
+    onFinish({ reason });
+  } catch (e) {
+    logger2.error("ERROR in workflow-error handler:", e.message);
+    logger2.error(e);
+  }
 }
 function onJobLog({ channel, state }, event) {
   const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
@@ -5558,8 +5597,9 @@ function createServer(engine, options = {}) {
         plan,
         options: options2
       } = await attempt_default(app.socket, token, id, logger2);
-      const onComplete = () => {
+      const onFinish = () => {
         delete app.workflows[id];
+        attemptChannel.leave();
       };
       const context = execute(
         attemptChannel,
@@ -5567,7 +5607,7 @@ function createServer(engine, options = {}) {
         logger2,
         plan,
         options2,
-        onComplete
+        onFinish
       );
       app.workflows[id] = context;
     } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.2.5",
+  "version": "0.2.7",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",
@@ -21,8 +21,8 @@
     "koa-logger": "^3.2.1",
     "phoenix": "^1.7.7",
     "ws": "^8.14.1",
-    "@openfn/engine-multi": "0.1.9",
-    "@openfn/runtime": "0.1.2",
+    "@openfn/engine-multi": "0.1.11",
+    "@openfn/runtime": "0.1.4",
     "@openfn/logger": "0.0.19"
   },
   "devDependencies": {
@@ -40,7 +40,7 @@
     "tsup": "^6.2.3",
     "typescript": "^4.6.4",
     "yargs": "^17.6.2",
-    "@openfn/lightning-mock": "1.0.10"
+    "@openfn/lightning-mock": "1.0.12"
   },
   "files": [
     "dist",