@openfn/ws-worker 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,16 @@
1
1
  # ws-worker
2
2
 
3
+ ## 0.8.0
4
+
5
+ ### Minor Changes
6
+
7
+ - 7e4c159: Rename attempts to runs
8
+
9
+ ### Patch Changes
10
+
11
+ - Updated dependencies [7e4c159]
12
+ - @openfn/engine-multi@0.4.0
13
+
3
14
  ## 0.7.0
4
15
 
5
16
  ### Minor Changes
package/README.md CHANGED
@@ -68,7 +68,7 @@ curl -X POST http://localhost:2222/claim
68
68
 
69
69
  ## Architecture
70
70
 
71
- Lightning is expected to maintain a queue of attempts. The Worker pulls those attempts from the queue, via websocket, and sends them off to the Engine for execution.
71
+ Lightning is expected to maintain a queue of runs. The Worker pulls those runs from the queue, via websocket, and sends them off to the Engine for execution.
72
72
 
73
73
  While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements.
74
74
 
package/dist/index.d.ts CHANGED
@@ -42,8 +42,8 @@ interface Edge {
42
42
  enabled?: boolean;
43
43
  }
44
44
 
45
- // An attempt object returned by Lightning
46
- type Attempt = {
45
+ // An run object returned by Lightning
46
+ type Run = {
47
47
  id: string;
48
48
  dataclip_id: string;
49
49
  starting_node_id: string;
@@ -52,31 +52,21 @@ type Attempt = {
52
52
  jobs: Node[];
53
53
  edges: Edge[];
54
54
 
55
- options?: AttemptOptions;
55
+ options?: RunOptions;
56
56
  };
57
57
 
58
- type AttemptOptions = {
59
- // This is what Lightning will ssend us
60
- // Note that this is the NEW terminology, so it's the timeout for the whole "attempt"
61
- runTimeout?: number;
62
-
63
- // this is the internal old terminology, which will be deprecated soon
64
- attemptTimeoutMs?: number;
65
-
66
- attemptTimeout?: number; // deprecated
67
-
68
- // deprecated alias for timeout. Maps to "attemptTimeout" internally
69
- timeout?: number;
58
+ type RunOptions = {
59
+ runTimeoutMs?: number;
70
60
 
71
61
  sanitize?: SanitizePolicies;
72
62
  };
73
63
 
74
- // Internal server state for each attempt
75
- type AttemptState = {
64
+ // Internal server state for each run
65
+ type RunState = {
76
66
  activeStep?: string;
77
67
  activeJob?: string;
78
68
  plan: ExecutionPlan;
79
- options: AttemptOptions;
69
+ options: RunOptions;
80
70
  dataclips: Record<string, any>;
81
71
  // For each run, map the input ids
82
72
  // TODO better name maybe?
@@ -107,15 +97,15 @@ declare type ClaimPayload = {
107
97
  demand?: number;
108
98
  };
109
99
  declare type ClaimReply = {
110
- attempts: Array<ClaimAttempt>;
100
+ runs: Array<ClaimRun>;
111
101
  };
112
- declare type ClaimAttempt = {
102
+ declare type ClaimRun = {
113
103
  id: string;
114
104
  token: string;
115
105
  };
116
- declare const GET_ATTEMPT = "fetch:attempt";
117
- declare type GetAttemptPayload = void;
118
- declare type GetAttemptReply = Attempt;
106
+ declare const GET_PLAN = "fetch:plan";
107
+ declare type GetPlanPayload = void;
108
+ declare type GetPlanReply = Run;
119
109
  declare const GET_CREDENTIAL = "fetch:credential";
120
110
  declare type GetCredentialPayload = {
121
111
  id: string;
@@ -126,48 +116,48 @@ declare type GetDataclipPayload = {
126
116
  id: string;
127
117
  };
128
118
  declare type GetDataClipReply = Uint8Array;
129
- declare const ATTEMPT_START = "attempt:start";
130
- declare type AttemptStartPayload = void;
131
- declare type AttemptStartReply = {};
132
- declare const ATTEMPT_COMPLETE = "attempt:complete";
133
- declare type AttemptCompletePayload = ExitReason & {
119
+ declare const RUN_START = "run:start";
120
+ declare type RunStartPayload = void;
121
+ declare type RunStartReply = {};
122
+ declare const RUN_COMPLETE = "run:complete";
123
+ declare type RunCompletePayload = ExitReason & {
134
124
  final_dataclip_id?: string;
135
125
  };
136
- declare type AttemptCompleteReply = undefined;
137
- declare const ATTEMPT_LOG = "attempt:log";
138
- declare type AttemptLogPayload = {
126
+ declare type RunCompleteReply = undefined;
127
+ declare const RUN_LOG = "run:log";
128
+ declare type RunLogPayload = {
139
129
  message: Array<string | object>;
140
130
  timestamp: string;
141
- attempt_id: string;
131
+ run_id: string;
142
132
  level?: string;
143
133
  source?: string;
144
134
  job_id?: string;
145
135
  step_id?: string;
146
136
  };
147
- declare type AttemptLogReply = void;
137
+ declare type RunLogReply = void;
148
138
  declare const STEP_START = "step:start";
149
139
  declare type StepStartPayload = {
150
140
  job_id: string;
151
141
  step_id: string;
152
- attempt_id?: string;
142
+ run_id?: string;
153
143
  input_dataclip_id?: string;
154
144
  versions: Record<string, string>;
155
145
  };
156
146
  declare type StepStartReply = void;
157
147
  declare const STEP_COMPLETE = "step:complete";
158
148
  declare type StepCompletePayload = ExitReason & {
159
- attempt_id?: string;
149
+ run_id?: string;
160
150
  job_id: string;
161
151
  step_id: string;
162
152
  output_dataclip?: string;
163
153
  output_dataclip_id?: string;
164
154
  };
165
155
  declare type StepCompleteReply = void;
166
- declare const INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
156
+ declare const INTERNAL_RUN_COMPLETE = "server:run-complete";
167
157
 
168
158
  declare type Context = {
169
159
  channel: Channel;
170
- state: AttemptState;
160
+ state: RunState;
171
161
  logger: Logger;
172
162
  engine: RuntimeEngine;
173
163
  onFinish: (result: any) => void;
@@ -194,10 +184,10 @@ interface ServerApp extends Koa {
194
184
  events: EventEmitter;
195
185
  server: Server;
196
186
  engine: RuntimeEngine;
197
- execute: ({ id, token }: ClaimAttempt) => Promise<void>;
187
+ execute: ({ id, token }: ClaimRun) => Promise<void>;
198
188
  destroy: () => void;
199
189
  killWorkloop?: () => void;
200
190
  }
201
191
  declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
202
192
 
203
- export { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, AttemptStartPayload, AttemptStartReply, CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, INTERNAL_ATTEMPT_COMPLETE, STEP_COMPLETE, STEP_START, StepCompletePayload, StepCompleteReply, StepStartPayload, StepStartReply, createServer as default };
193
+ export { CLAIM, ClaimPayload, ClaimReply, ClaimRun, GET_CREDENTIAL, GET_DATACLIP, GET_PLAN, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, GetPlanPayload, GetPlanReply, INTERNAL_RUN_COMPLETE, RUN_COMPLETE, RUN_LOG, RUN_START, RunCompletePayload, RunCompleteReply, RunLogPayload, RunLogReply, RunStartPayload, RunStartReply, STEP_COMPLETE, STEP_START, StepCompletePayload, StepCompleteReply, StepStartPayload, StepStartReply, createServer as default };
package/dist/index.js CHANGED
@@ -9,15 +9,15 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
9
9
 
10
10
  // src/events.ts
11
11
  var CLAIM = "claim";
12
- var GET_ATTEMPT = "fetch:attempt";
12
+ var GET_PLAN = "fetch:plan";
13
13
  var GET_CREDENTIAL = "fetch:credential";
14
14
  var GET_DATACLIP = "fetch:dataclip";
15
- var ATTEMPT_START = "attempt:start";
16
- var ATTEMPT_COMPLETE = "attempt:complete";
17
- var ATTEMPT_LOG = "attempt:log";
15
+ var RUN_START = "run:start";
16
+ var RUN_COMPLETE = "run:complete";
17
+ var RUN_LOG = "run:log";
18
18
  var STEP_START = "step:start";
19
19
  var STEP_COMPLETE = "step:complete";
20
- var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
20
+ var INTERNAL_RUN_COMPLETE = "server:run-complete";
21
21
 
22
22
  // src/api/destroy.ts
23
23
  var destroy = async (app, logger) => {
@@ -32,7 +32,7 @@ var destroy = async (app, logger) => {
32
32
  });
33
33
  }),
34
34
  new Promise(async (resolve) => {
35
- await waitForAttempts(app, logger);
35
+ await waitForRuns(app, logger);
36
36
  await app.engine.destroy();
37
37
  app.socket?.disconnect();
38
38
  resolve();
@@ -40,16 +40,16 @@ var destroy = async (app, logger) => {
40
40
  ]);
41
41
  logger.success("Server closed");
42
42
  };
43
- var waitForAttempts = (app, logger) => new Promise((resolve) => {
43
+ var waitForRuns = (app, logger) => new Promise((resolve) => {
44
44
  const log = () => {
45
45
  logger.debug(
46
- `Waiting for ${Object.keys(app.workflows).length} attempts to complete...`
46
+ `Waiting for ${Object.keys(app.workflows).length} runs to complete...`
47
47
  );
48
48
  };
49
- const onAttemptComplete = () => {
49
+ const onRunComplete = () => {
50
50
  if (Object.keys(app.workflows).length === 0) {
51
- logger.debug("All attempts completed!");
52
- app.events.off(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
51
+ logger.debug("All runs completed!");
52
+ app.events.off(INTERNAL_RUN_COMPLETE, onRunComplete);
53
53
  resolve();
54
54
  } else {
55
55
  log();
@@ -57,7 +57,7 @@ var waitForAttempts = (app, logger) => new Promise((resolve) => {
57
57
  };
58
58
  if (Object.keys(app.workflows).length) {
59
59
  log();
60
- app.events.on(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
60
+ app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
61
61
  } else {
62
62
  resolve();
63
63
  }
@@ -67,7 +67,7 @@ var destroy_default = destroy;
67
67
  // src/util/try-with-backoff.ts
68
68
  var BACKOFF_MULTIPLIER = 1.15;
69
69
  var tryWithBackoff = (fn, opts = {}) => {
70
- const { min = 1e3, max = 1e4, maxAttempts, attempts = 1 } = opts;
70
+ const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
71
71
  let cancelled = false;
72
72
  if (!opts.isCancelled) {
73
73
  opts.isCancelled = () => cancelled;
@@ -80,16 +80,16 @@ var tryWithBackoff = (fn, opts = {}) => {
80
80
  if (opts.isCancelled()) {
81
81
  return resolve();
82
82
  }
83
- if (!isNaN(maxAttempts) && attempts >= maxAttempts) {
84
- return reject(new Error("max attempts exceeded"));
83
+ if (!isNaN(maxRuns) && runs >= maxRuns) {
84
+ return reject(new Error("max runs exceeded"));
85
85
  }
86
86
  setTimeout(() => {
87
87
  if (opts.isCancelled()) {
88
88
  return resolve();
89
89
  }
90
90
  const nextOpts = {
91
- maxAttempts,
92
- attempts: attempts + 1,
91
+ maxRuns,
92
+ runs: runs + 1,
93
93
  min: Math.min(max, min * BACKOFF_MULTIPLIER),
94
94
  max,
95
95
  isCancelled: opts.isCancelled
@@ -117,15 +117,15 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
117
117
  if (!app.queueChannel) {
118
118
  return reject(new Error("No websocket available"));
119
119
  }
120
- logger.debug("requesting attempt...");
121
- app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
122
- logger.debug(`pulled ${attempts.length} attempts`);
123
- if (!attempts?.length) {
124
- return reject(new Error("No attempts returned"));
120
+ logger.debug("requesting run...");
121
+ app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
122
+ logger.debug(`pulled ${runs.length} runs`);
123
+ if (!runs?.length) {
124
+ return reject(new Error("No runs returned"));
125
125
  }
126
- attempts.forEach((attempt) => {
127
- logger.debug("starting attempt", attempt.id);
128
- app.execute(attempt);
126
+ runs.forEach((run) => {
127
+ logger.debug("starting run", run.id);
128
+ app.execute(run);
129
129
  resolve();
130
130
  });
131
131
  }).receive("error", () => {
@@ -165,7 +165,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
165
165
  };
166
166
  var workloop_default = startWorkloop;
167
167
 
168
- // src/util/convert-attempt.ts
168
+ // src/util/convert-run.ts
169
169
  import crypto from "node:crypto";
170
170
  var conditions = {
171
171
  on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
@@ -187,28 +187,23 @@ var mapTriggerEdgeCondition = (edge) => {
187
187
  return condition;
188
188
  };
189
189
  var mapOptions = (options) => {
190
- const { attemptTimeout, timeout, runTimeout, ...opts } = options;
191
- const to = runTimeout || attemptTimeout || timeout;
192
- if (to) {
193
- opts.attemptTimeoutMs = to;
194
- }
195
- return opts;
190
+ return options;
196
191
  };
197
- var convert_attempt_default = (attempt) => {
198
- const options = attempt.options || {};
192
+ var convert_run_default = (run) => {
193
+ const options = run.options || {};
199
194
  const plan = {
200
- id: attempt.id
195
+ id: run.id
201
196
  };
202
- if (attempt.dataclip_id) {
203
- plan.initialState = attempt.dataclip_id;
197
+ if (run.dataclip_id) {
198
+ plan.initialState = run.dataclip_id;
204
199
  }
205
- if (attempt.starting_node_id) {
206
- plan.start = attempt.starting_node_id;
200
+ if (run.starting_node_id) {
201
+ plan.start = run.starting_node_id;
207
202
  }
208
203
  const nodes = {};
209
- const edges = attempt.edges ?? [];
210
- if (attempt.triggers?.length) {
211
- attempt.triggers.forEach((trigger) => {
204
+ const edges = run.edges ?? [];
205
+ if (run.triggers?.length) {
206
+ run.triggers.forEach((trigger) => {
212
207
  const id = trigger.id || "trigger";
213
208
  nodes[id] = {
214
209
  id
@@ -225,8 +220,8 @@ var convert_attempt_default = (attempt) => {
225
220
  }
226
221
  });
227
222
  }
228
- if (attempt.jobs?.length) {
229
- attempt.jobs.forEach((job) => {
223
+ if (run.jobs?.length) {
224
+ run.jobs.forEach((job) => {
230
225
  const id = job.id || crypto.randomUUID();
231
226
  nodes[id] = {
232
227
  id,
@@ -277,8 +272,8 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
277
272
  return value;
278
273
  });
279
274
 
280
- // src/util/create-attempt-state.ts
281
- var create_attempt_state_default = (plan, options = {}) => {
275
+ // src/util/create-run-state.ts
276
+ var create_run_state_default = (plan, options = {}) => {
282
277
  const state = {
283
278
  plan,
284
279
  lastDataclipId: "",
@@ -331,7 +326,7 @@ var isLeafNode = (state, job) => {
331
326
  const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
332
327
  return !hasDownstream;
333
328
  };
334
- var calculateAttemptExitReason = (state) => {
329
+ var calculateRunExitReason = (state) => {
335
330
  if (state.plan && state.reasons) {
336
331
  const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
337
332
  const fail = leafJobReasons.find((r) => r && r.reason === "fail");
@@ -386,7 +381,7 @@ import { timestamp } from "@openfn/logger";
386
381
  // package.json
387
382
  var package_default = {
388
383
  name: "@openfn/ws-worker",
389
- version: "0.7.0",
384
+ version: "0.8.0",
390
385
  description: "A Websocket Worker to connect Lightning to a Runtime Engine",
391
386
  main: "dist/index.js",
392
387
  type: "module",
@@ -526,21 +521,21 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
526
521
  });
527
522
  };
528
523
 
529
- // src/events/attempt-complete.ts
524
+ // src/events/run-complete.ts
530
525
  async function onWorkflowComplete(context, _event) {
531
526
  const { state, channel, onFinish } = context;
532
527
  const result = state.dataclips[state.lastDataclipId];
533
- const reason = calculateAttemptExitReason(state);
528
+ const reason = calculateRunExitReason(state);
534
529
  await log_final_reason_default(context, reason);
535
- await sendEvent(channel, ATTEMPT_COMPLETE, {
530
+ await sendEvent(channel, RUN_COMPLETE, {
536
531
  final_dataclip_id: state.lastDataclipId,
537
532
  ...reason
538
533
  });
539
534
  onFinish({ reason, state: result });
540
535
  }
541
536
 
542
- // src/events/attempt-error.ts
543
- async function onAttemptError(context, event) {
537
+ // src/events/run-error.ts
538
+ async function onRunError(context, event) {
544
539
  const { state, channel, logger, onFinish } = context;
545
540
  try {
546
541
  const reason = calculateJobExitReason("", { data: {} }, event);
@@ -548,7 +543,7 @@ async function onAttemptError(context, event) {
548
543
  await onJobError(context, { error: event });
549
544
  }
550
545
  await log_final_reason_default(context, reason);
551
- await sendEvent(channel, ATTEMPT_COMPLETE, {
546
+ await sendEvent(channel, RUN_COMPLETE, {
552
547
  final_dataclip_id: state.lastDataclipId,
553
548
  ...reason
554
549
  });
@@ -590,16 +585,16 @@ var throttle_default = createThrottler;
590
585
  // src/api/execute.ts
591
586
  var enc = new TextDecoder("utf-8");
592
587
  var eventMap = {
593
- "workflow-start": ATTEMPT_START,
588
+ "workflow-start": RUN_START,
594
589
  "job-start": STEP_START,
595
590
  "job-complete": STEP_COMPLETE,
596
- "workflow-log": ATTEMPT_LOG,
597
- "workflow-complete": ATTEMPT_COMPLETE
591
+ "workflow-log": RUN_LOG,
592
+ "workflow-complete": RUN_COMPLETE
598
593
  };
599
594
  function execute(channel, engine, logger, plan, options = {}, onFinish = (_result) => {
600
595
  }) {
601
596
  logger.info("executing ", plan.id);
602
- const state = create_attempt_state_default(plan, options);
597
+ const state = create_run_state_default(plan, options);
603
598
  const context = { channel, state, logger, engine, onFinish };
604
599
  const throttle = throttle_default();
605
600
  const addEvent = (eventName, handler) => {
@@ -627,7 +622,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
627
622
  addEvent("job-error", throttle(onJobError)),
628
623
  addEvent("workflow-log", throttle(onJobLog)),
629
624
  addEvent("workflow-complete", throttle(onWorkflowComplete)),
630
- addEvent("workflow-error", throttle(onAttemptError))
625
+ addEvent("workflow-error", throttle(onRunError))
631
626
  );
632
627
  engine.listen(plan.id, listeners);
633
628
  const resolvers = {
@@ -645,7 +640,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
645
640
  try {
646
641
  engine.execute(plan, { resolvers, ...options });
647
642
  } catch (e) {
648
- onAttemptError(context, {
643
+ onRunError(context, {
649
644
  workflowId: plan.id,
650
645
  message: e.message,
651
646
  type: e.type,
@@ -669,12 +664,12 @@ function onJobError(context, event) {
669
664
  }
670
665
  }
671
666
  function onWorkflowStart({ channel }, _event) {
672
- return sendEvent(channel, ATTEMPT_START);
667
+ return sendEvent(channel, RUN_START);
673
668
  }
674
669
  function onJobLog({ channel, state }, event) {
675
670
  const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
676
671
  const log = {
677
- attempt_id: state.plan.id,
672
+ run_id: state.plan.id,
678
673
  message: event.message,
679
674
  source: event.name,
680
675
  level: event.level,
@@ -683,7 +678,7 @@ function onJobLog({ channel, state }, event) {
683
678
  if (state.activeStep) {
684
679
  log.step_id = state.activeStep;
685
680
  }
686
- return sendEvent(channel, ATTEMPT_LOG, log);
681
+ return sendEvent(channel, RUN_LOG, log);
687
682
  }
688
683
  async function loadDataclip(channel, stateId) {
689
684
  const result = await get_with_reply_default(channel, GET_DATACLIP, {
@@ -701,19 +696,19 @@ var healthcheck_default = (ctx) => {
701
696
  ctx.status = 200;
702
697
  };
703
698
 
704
- // src/channels/attempt.ts
705
- var joinAttemptChannel = (socket, token, attemptId, logger) => {
699
+ // src/channels/run.ts
700
+ var joinRunChannel = (socket, token, runId, logger) => {
706
701
  return new Promise((resolve, reject) => {
707
702
  let didReceiveOk = false;
708
- const channelName = `attempt:${attemptId}`;
703
+ const channelName = `run:${runId}`;
709
704
  logger.debug("connecting to ", channelName);
710
705
  const channel = socket.channel(channelName, { token });
711
706
  channel.join().receive("ok", async (e) => {
712
707
  if (!didReceiveOk) {
713
708
  didReceiveOk = true;
714
709
  logger.success(`connected to ${channelName}`, e);
715
- const { plan, options } = await loadAttempt(channel);
716
- logger.debug("converted attempt as execution plan:", plan);
710
+ const { plan, options } = await loadRun(channel);
711
+ logger.debug("converted run as execution plan:", plan);
717
712
  resolve({ channel, plan, options });
718
713
  }
719
714
  }).receive("error", (err) => {
@@ -722,10 +717,10 @@ var joinAttemptChannel = (socket, token, attemptId, logger) => {
722
717
  });
723
718
  });
724
719
  };
725
- var attempt_default = joinAttemptChannel;
726
- async function loadAttempt(channel) {
727
- const attemptBody = await get_with_reply_default(channel, GET_ATTEMPT);
728
- return convert_attempt_default(attemptBody);
720
+ var run_default = joinRunChannel;
721
+ async function loadRun(channel) {
722
+ const runBody = await get_with_reply_default(channel, GET_PLAN);
723
+ return convert_run_default(runBody);
729
724
  }
730
725
 
731
726
  // src/channels/worker-queue.ts
@@ -741,7 +736,7 @@ var generateWorkerToken = async (secret, workerId, logger) => {
741
736
  logger.warn();
742
737
  logger.warn("WARNING: Worker Secret not provided!");
743
738
  logger.warn(
744
- "This worker will attempt to connect to Lightning with default secret"
739
+ "This worker will try to connect to Lightning with default secret"
745
740
  );
746
741
  logger.warn();
747
742
  }
@@ -865,17 +860,17 @@ function createServer(engine, options = {}) {
865
860
  if (app.socket) {
866
861
  app.workflows[id] = true;
867
862
  const {
868
- channel: attemptChannel,
863
+ channel: runChannel,
869
864
  plan,
870
865
  options: options2
871
- } = await attempt_default(app.socket, token, id, logger);
866
+ } = await run_default(app.socket, token, id, logger);
872
867
  const onFinish = () => {
873
868
  delete app.workflows[id];
874
- attemptChannel.leave();
875
- app.events.emit(INTERNAL_ATTEMPT_COMPLETE);
869
+ runChannel.leave();
870
+ app.events.emit(INTERNAL_RUN_COMPLETE);
876
871
  };
877
872
  const context = execute(
878
- attemptChannel,
873
+ runChannel,
879
874
  engine,
880
875
  logger,
881
876
  plan,
@@ -890,12 +885,12 @@ function createServer(engine, options = {}) {
890
885
  router.post("/claim", async (ctx) => {
891
886
  logger.info("triggering claim from POST request");
892
887
  return claim_default(app, logger, options.maxWorkflows).then(() => {
893
- logger.info("claim complete: 1 attempt claimed");
888
+ logger.info("claim complete: 1 run claimed");
894
889
  ctx.body = "complete";
895
890
  ctx.status = 200;
896
891
  }).catch(() => {
897
- logger.info("claim complete: no attempts");
898
- ctx.body = "no attempts";
892
+ logger.info("claim complete: no runs");
893
+ ctx.body = "no runs";
899
894
  ctx.status = 204;
900
895
  });
901
896
  });
@@ -927,14 +922,14 @@ var server_default = createServer;
927
922
  // src/index.ts
928
923
  var src_default = server_default;
929
924
  export {
930
- ATTEMPT_COMPLETE,
931
- ATTEMPT_LOG,
932
- ATTEMPT_START,
933
925
  CLAIM,
934
- GET_ATTEMPT,
935
926
  GET_CREDENTIAL,
936
927
  GET_DATACLIP,
937
- INTERNAL_ATTEMPT_COMPLETE,
928
+ GET_PLAN,
929
+ INTERNAL_RUN_COMPLETE,
930
+ RUN_COMPLETE,
931
+ RUN_LOG,
932
+ RUN_START,
938
933
  STEP_COMPLETE,
939
934
  STEP_START,
940
935
  src_default as default
package/dist/start.js CHANGED
@@ -4998,15 +4998,15 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
4998
4998
 
4999
4999
  // src/events.ts
5000
5000
  var CLAIM = "claim";
5001
- var GET_ATTEMPT = "fetch:attempt";
5001
+ var GET_PLAN = "fetch:plan";
5002
5002
  var GET_CREDENTIAL = "fetch:credential";
5003
5003
  var GET_DATACLIP = "fetch:dataclip";
5004
- var ATTEMPT_START = "attempt:start";
5005
- var ATTEMPT_COMPLETE = "attempt:complete";
5006
- var ATTEMPT_LOG = "attempt:log";
5004
+ var RUN_START = "run:start";
5005
+ var RUN_COMPLETE = "run:complete";
5006
+ var RUN_LOG = "run:log";
5007
5007
  var STEP_START = "step:start";
5008
5008
  var STEP_COMPLETE = "step:complete";
5009
- var INTERNAL_ATTEMPT_COMPLETE = "server:attempt-complete";
5009
+ var INTERNAL_RUN_COMPLETE = "server:run-complete";
5010
5010
 
5011
5011
  // src/api/destroy.ts
5012
5012
  var destroy = async (app, logger2) => {
@@ -5021,7 +5021,7 @@ var destroy = async (app, logger2) => {
5021
5021
  });
5022
5022
  }),
5023
5023
  new Promise(async (resolve5) => {
5024
- await waitForAttempts(app, logger2);
5024
+ await waitForRuns(app, logger2);
5025
5025
  await app.engine.destroy();
5026
5026
  app.socket?.disconnect();
5027
5027
  resolve5();
@@ -5029,16 +5029,16 @@ var destroy = async (app, logger2) => {
5029
5029
  ]);
5030
5030
  logger2.success("Server closed");
5031
5031
  };
5032
- var waitForAttempts = (app, logger2) => new Promise((resolve5) => {
5032
+ var waitForRuns = (app, logger2) => new Promise((resolve5) => {
5033
5033
  const log = () => {
5034
5034
  logger2.debug(
5035
- `Waiting for ${Object.keys(app.workflows).length} attempts to complete...`
5035
+ `Waiting for ${Object.keys(app.workflows).length} runs to complete...`
5036
5036
  );
5037
5037
  };
5038
- const onAttemptComplete = () => {
5038
+ const onRunComplete = () => {
5039
5039
  if (Object.keys(app.workflows).length === 0) {
5040
- logger2.debug("All attempts completed!");
5041
- app.events.off(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
5040
+ logger2.debug("All runs completed!");
5041
+ app.events.off(INTERNAL_RUN_COMPLETE, onRunComplete);
5042
5042
  resolve5();
5043
5043
  } else {
5044
5044
  log();
@@ -5046,7 +5046,7 @@ var waitForAttempts = (app, logger2) => new Promise((resolve5) => {
5046
5046
  };
5047
5047
  if (Object.keys(app.workflows).length) {
5048
5048
  log();
5049
- app.events.on(INTERNAL_ATTEMPT_COMPLETE, onAttemptComplete);
5049
+ app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
5050
5050
  } else {
5051
5051
  resolve5();
5052
5052
  }
@@ -5056,7 +5056,7 @@ var destroy_default = destroy;
5056
5056
  // src/util/try-with-backoff.ts
5057
5057
  var BACKOFF_MULTIPLIER = 1.15;
5058
5058
  var tryWithBackoff = (fn, opts = {}) => {
5059
- const { min = 1e3, max = 1e4, maxAttempts, attempts = 1 } = opts;
5059
+ const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
5060
5060
  let cancelled = false;
5061
5061
  if (!opts.isCancelled) {
5062
5062
  opts.isCancelled = () => cancelled;
@@ -5069,16 +5069,16 @@ var tryWithBackoff = (fn, opts = {}) => {
5069
5069
  if (opts.isCancelled()) {
5070
5070
  return resolve5();
5071
5071
  }
5072
- if (!isNaN(maxAttempts) && attempts >= maxAttempts) {
5073
- return reject(new Error("max attempts exceeded"));
5072
+ if (!isNaN(maxRuns) && runs >= maxRuns) {
5073
+ return reject(new Error("max runs exceeded"));
5074
5074
  }
5075
5075
  setTimeout(() => {
5076
5076
  if (opts.isCancelled()) {
5077
5077
  return resolve5();
5078
5078
  }
5079
5079
  const nextOpts = {
5080
- maxAttempts,
5081
- attempts: attempts + 1,
5080
+ maxRuns,
5081
+ runs: runs + 1,
5082
5082
  min: Math.min(max, min * BACKOFF_MULTIPLIER),
5083
5083
  max,
5084
5084
  isCancelled: opts.isCancelled
@@ -5106,15 +5106,15 @@ var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
5106
5106
  if (!app.queueChannel) {
5107
5107
  return reject(new Error("No websocket available"));
5108
5108
  }
5109
- logger2.debug("requesting attempt...");
5110
- app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ attempts }) => {
5111
- logger2.debug(`pulled ${attempts.length} attempts`);
5112
- if (!attempts?.length) {
5113
- return reject(new Error("No attempts returned"));
5109
+ logger2.debug("requesting run...");
5110
+ app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
5111
+ logger2.debug(`pulled ${runs.length} runs`);
5112
+ if (!runs?.length) {
5113
+ return reject(new Error("No runs returned"));
5114
5114
  }
5115
- attempts.forEach((attempt) => {
5116
- logger2.debug("starting attempt", attempt.id);
5117
- app.execute(attempt);
5115
+ runs.forEach((run2) => {
5116
+ logger2.debug("starting run", run2.id);
5117
+ app.execute(run2);
5118
5118
  resolve5();
5119
5119
  });
5120
5120
  }).receive("error", () => {
@@ -5154,7 +5154,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
5154
5154
  };
5155
5155
  var workloop_default = startWorkloop;
5156
5156
 
5157
- // src/util/convert-attempt.ts
5157
+ // src/util/convert-run.ts
5158
5158
  import crypto2 from "node:crypto";
5159
5159
  var conditions = {
5160
5160
  on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
@@ -5176,28 +5176,23 @@ var mapTriggerEdgeCondition = (edge) => {
5176
5176
  return condition;
5177
5177
  };
5178
5178
  var mapOptions = (options) => {
5179
- const { attemptTimeout, timeout, runTimeout, ...opts } = options;
5180
- const to = runTimeout || attemptTimeout || timeout;
5181
- if (to) {
5182
- opts.attemptTimeoutMs = to;
5183
- }
5184
- return opts;
5179
+ return options;
5185
5180
  };
5186
- var convert_attempt_default = (attempt) => {
5187
- const options = attempt.options || {};
5181
+ var convert_run_default = (run2) => {
5182
+ const options = run2.options || {};
5188
5183
  const plan = {
5189
- id: attempt.id
5184
+ id: run2.id
5190
5185
  };
5191
- if (attempt.dataclip_id) {
5192
- plan.initialState = attempt.dataclip_id;
5186
+ if (run2.dataclip_id) {
5187
+ plan.initialState = run2.dataclip_id;
5193
5188
  }
5194
- if (attempt.starting_node_id) {
5195
- plan.start = attempt.starting_node_id;
5189
+ if (run2.starting_node_id) {
5190
+ plan.start = run2.starting_node_id;
5196
5191
  }
5197
5192
  const nodes = {};
5198
- const edges = attempt.edges ?? [];
5199
- if (attempt.triggers?.length) {
5200
- attempt.triggers.forEach((trigger) => {
5193
+ const edges = run2.edges ?? [];
5194
+ if (run2.triggers?.length) {
5195
+ run2.triggers.forEach((trigger) => {
5201
5196
  const id = trigger.id || "trigger";
5202
5197
  nodes[id] = {
5203
5198
  id
@@ -5214,8 +5209,8 @@ var convert_attempt_default = (attempt) => {
5214
5209
  }
5215
5210
  });
5216
5211
  }
5217
- if (attempt.jobs?.length) {
5218
- attempt.jobs.forEach((job) => {
5212
+ if (run2.jobs?.length) {
5213
+ run2.jobs.forEach((job) => {
5219
5214
  const id = job.id || crypto2.randomUUID();
5220
5215
  nodes[id] = {
5221
5216
  id,
@@ -5266,8 +5261,8 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
5266
5261
  return value;
5267
5262
  });
5268
5263
 
5269
- // src/util/create-attempt-state.ts
5270
- var create_attempt_state_default = (plan, options = {}) => {
5264
+ // src/util/create-run-state.ts
5265
+ var create_run_state_default = (plan, options = {}) => {
5271
5266
  const state = {
5272
5267
  plan,
5273
5268
  lastDataclipId: "",
@@ -5320,7 +5315,7 @@ var isLeafNode = (state, job) => {
5320
5315
  const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
5321
5316
  return !hasDownstream;
5322
5317
  };
5323
- var calculateAttemptExitReason = (state) => {
5318
+ var calculateRunExitReason = (state) => {
5324
5319
  if (state.plan && state.reasons) {
5325
5320
  const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
5326
5321
  const fail = leafJobReasons.find((r) => r && r.reason === "fail");
@@ -5375,7 +5370,7 @@ import { timestamp } from "@openfn/logger";
5375
5370
  // package.json
5376
5371
  var package_default = {
5377
5372
  name: "@openfn/ws-worker",
5378
- version: "0.7.0",
5373
+ version: "0.8.0",
5379
5374
  description: "A Websocket Worker to connect Lightning to a Runtime Engine",
5380
5375
  main: "dist/index.js",
5381
5376
  type: "module",
@@ -5515,21 +5510,21 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
5515
5510
  });
5516
5511
  };
5517
5512
 
5518
- // src/events/attempt-complete.ts
5513
+ // src/events/run-complete.ts
5519
5514
  async function onWorkflowComplete(context, _event) {
5520
5515
  const { state, channel, onFinish } = context;
5521
5516
  const result = state.dataclips[state.lastDataclipId];
5522
- const reason = calculateAttemptExitReason(state);
5517
+ const reason = calculateRunExitReason(state);
5523
5518
  await log_final_reason_default(context, reason);
5524
- await sendEvent(channel, ATTEMPT_COMPLETE, {
5519
+ await sendEvent(channel, RUN_COMPLETE, {
5525
5520
  final_dataclip_id: state.lastDataclipId,
5526
5521
  ...reason
5527
5522
  });
5528
5523
  onFinish({ reason, state: result });
5529
5524
  }
5530
5525
 
5531
- // src/events/attempt-error.ts
5532
- async function onAttemptError(context, event) {
5526
+ // src/events/run-error.ts
5527
+ async function onRunError(context, event) {
5533
5528
  const { state, channel, logger: logger2, onFinish } = context;
5534
5529
  try {
5535
5530
  const reason = calculateJobExitReason("", { data: {} }, event);
@@ -5537,7 +5532,7 @@ async function onAttemptError(context, event) {
5537
5532
  await onJobError(context, { error: event });
5538
5533
  }
5539
5534
  await log_final_reason_default(context, reason);
5540
- await sendEvent(channel, ATTEMPT_COMPLETE, {
5535
+ await sendEvent(channel, RUN_COMPLETE, {
5541
5536
  final_dataclip_id: state.lastDataclipId,
5542
5537
  ...reason
5543
5538
  });
@@ -5579,16 +5574,16 @@ var throttle_default = createThrottler;
5579
5574
  // src/api/execute.ts
5580
5575
  var enc = new TextDecoder("utf-8");
5581
5576
  var eventMap = {
5582
- "workflow-start": ATTEMPT_START,
5577
+ "workflow-start": RUN_START,
5583
5578
  "job-start": STEP_START,
5584
5579
  "job-complete": STEP_COMPLETE,
5585
- "workflow-log": ATTEMPT_LOG,
5586
- "workflow-complete": ATTEMPT_COMPLETE
5580
+ "workflow-log": RUN_LOG,
5581
+ "workflow-complete": RUN_COMPLETE
5587
5582
  };
5588
5583
  function execute(channel, engine, logger2, plan, options = {}, onFinish = (_result) => {
5589
5584
  }) {
5590
5585
  logger2.info("executing ", plan.id);
5591
- const state = create_attempt_state_default(plan, options);
5586
+ const state = create_run_state_default(plan, options);
5592
5587
  const context = { channel, state, logger: logger2, engine, onFinish };
5593
5588
  const throttle = throttle_default();
5594
5589
  const addEvent = (eventName, handler) => {
@@ -5616,7 +5611,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
5616
5611
  addEvent("job-error", throttle(onJobError)),
5617
5612
  addEvent("workflow-log", throttle(onJobLog)),
5618
5613
  addEvent("workflow-complete", throttle(onWorkflowComplete)),
5619
- addEvent("workflow-error", throttle(onAttemptError))
5614
+ addEvent("workflow-error", throttle(onRunError))
5620
5615
  );
5621
5616
  engine.listen(plan.id, listeners);
5622
5617
  const resolvers = {
@@ -5634,7 +5629,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
5634
5629
  try {
5635
5630
  engine.execute(plan, { resolvers, ...options });
5636
5631
  } catch (e) {
5637
- onAttemptError(context, {
5632
+ onRunError(context, {
5638
5633
  workflowId: plan.id,
5639
5634
  message: e.message,
5640
5635
  type: e.type,
@@ -5658,12 +5653,12 @@ function onJobError(context, event) {
5658
5653
  }
5659
5654
  }
5660
5655
  function onWorkflowStart({ channel }, _event) {
5661
- return sendEvent(channel, ATTEMPT_START);
5656
+ return sendEvent(channel, RUN_START);
5662
5657
  }
5663
5658
  function onJobLog({ channel, state }, event) {
5664
5659
  const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
5665
5660
  const log = {
5666
- attempt_id: state.plan.id,
5661
+ run_id: state.plan.id,
5667
5662
  message: event.message,
5668
5663
  source: event.name,
5669
5664
  level: event.level,
@@ -5672,7 +5667,7 @@ function onJobLog({ channel, state }, event) {
5672
5667
  if (state.activeStep) {
5673
5668
  log.step_id = state.activeStep;
5674
5669
  }
5675
- return sendEvent(channel, ATTEMPT_LOG, log);
5670
+ return sendEvent(channel, RUN_LOG, log);
5676
5671
  }
5677
5672
  async function loadDataclip(channel, stateId) {
5678
5673
  const result = await get_with_reply_default(channel, GET_DATACLIP, {
@@ -5690,19 +5685,19 @@ var healthcheck_default = (ctx) => {
5690
5685
  ctx.status = 200;
5691
5686
  };
5692
5687
 
5693
- // src/channels/attempt.ts
5694
- var joinAttemptChannel = (socket, token, attemptId, logger2) => {
5688
+ // src/channels/run.ts
5689
+ var joinRunChannel = (socket, token, runId, logger2) => {
5695
5690
  return new Promise((resolve5, reject) => {
5696
5691
  let didReceiveOk = false;
5697
- const channelName = `attempt:${attemptId}`;
5692
+ const channelName = `run:${runId}`;
5698
5693
  logger2.debug("connecting to ", channelName);
5699
5694
  const channel = socket.channel(channelName, { token });
5700
5695
  channel.join().receive("ok", async (e) => {
5701
5696
  if (!didReceiveOk) {
5702
5697
  didReceiveOk = true;
5703
5698
  logger2.success(`connected to ${channelName}`, e);
5704
- const { plan, options } = await loadAttempt(channel);
5705
- logger2.debug("converted attempt as execution plan:", plan);
5699
+ const { plan, options } = await loadRun(channel);
5700
+ logger2.debug("converted run as execution plan:", plan);
5706
5701
  resolve5({ channel, plan, options });
5707
5702
  }
5708
5703
  }).receive("error", (err) => {
@@ -5711,10 +5706,10 @@ var joinAttemptChannel = (socket, token, attemptId, logger2) => {
5711
5706
  });
5712
5707
  });
5713
5708
  };
5714
- var attempt_default = joinAttemptChannel;
5715
- async function loadAttempt(channel) {
5716
- const attemptBody = await get_with_reply_default(channel, GET_ATTEMPT);
5717
- return convert_attempt_default(attemptBody);
5709
+ var run_default = joinRunChannel;
5710
+ async function loadRun(channel) {
5711
+ const runBody = await get_with_reply_default(channel, GET_PLAN);
5712
+ return convert_run_default(runBody);
5718
5713
  }
5719
5714
 
5720
5715
  // src/channels/worker-queue.ts
@@ -5730,7 +5725,7 @@ var generateWorkerToken = async (secret, workerId, logger2) => {
5730
5725
  logger2.warn();
5731
5726
  logger2.warn("WARNING: Worker Secret not provided!");
5732
5727
  logger2.warn(
5733
- "This worker will attempt to connect to Lightning with default secret"
5728
+ "This worker will try to connect to Lightning with default secret"
5734
5729
  );
5735
5730
  logger2.warn();
5736
5731
  }
@@ -5854,17 +5849,17 @@ function createServer(engine, options = {}) {
5854
5849
  if (app.socket) {
5855
5850
  app.workflows[id] = true;
5856
5851
  const {
5857
- channel: attemptChannel,
5852
+ channel: runChannel,
5858
5853
  plan,
5859
5854
  options: options2
5860
- } = await attempt_default(app.socket, token, id, logger2);
5855
+ } = await run_default(app.socket, token, id, logger2);
5861
5856
  const onFinish = () => {
5862
5857
  delete app.workflows[id];
5863
- attemptChannel.leave();
5864
- app.events.emit(INTERNAL_ATTEMPT_COMPLETE);
5858
+ runChannel.leave();
5859
+ app.events.emit(INTERNAL_RUN_COMPLETE);
5865
5860
  };
5866
5861
  const context = execute(
5867
- attemptChannel,
5862
+ runChannel,
5868
5863
  engine,
5869
5864
  logger2,
5870
5865
  plan,
@@ -5879,12 +5874,12 @@ function createServer(engine, options = {}) {
5879
5874
  router.post("/claim", async (ctx) => {
5880
5875
  logger2.info("triggering claim from POST request");
5881
5876
  return claim_default(app, logger2, options.maxWorkflows).then(() => {
5882
- logger2.info("claim complete: 1 attempt claimed");
5877
+ logger2.info("claim complete: 1 run claimed");
5883
5878
  ctx.body = "complete";
5884
5879
  ctx.status = 200;
5885
5880
  }).catch(() => {
5886
- logger2.info("claim complete: no attempts");
5887
- ctx.body = "no attempts";
5881
+ logger2.info("claim complete: no runs");
5882
+ ctx.body = "no runs";
5888
5883
  ctx.status = 204;
5889
5884
  });
5890
5885
  });
@@ -5972,7 +5967,7 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
5972
5967
  default: WORKER_MAX_RUN_MEMORY_MB ? parseInt(WORKER_MAX_RUN_MEMORY_MB) : 500
5973
5968
  }).option("max-run-duration-seconds", {
5974
5969
  alias: "t",
5975
- description: "Default attempt timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
5970
+ description: "Default run timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
5976
5971
  type: "number",
5977
5972
  default: WORKER_MAX_RUN_DURATION_SECONDS || 60 * 5
5978
5973
  }).parse();
@@ -6016,7 +6011,7 @@ if (args.mock) {
6016
6011
  memoryLimitMb: args.runMemory,
6017
6012
  maxWorkers: args.capacity,
6018
6013
  statePropsToRemove: args.statePropsToRemove,
6019
- attemptTimeoutMs: args.maxRunDurationSeconds * 1e3
6014
+ runTimeoutMs: args.maxRunDurationSeconds * 1e3
6020
6015
  };
6021
6016
  logger.debug("Creating runtime engine...");
6022
6017
  logger.debug("Engine options:", engineOptions);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openfn/ws-worker",
3
- "version": "0.7.0",
3
+ "version": "0.8.0",
4
4
  "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
5
5
  "main": "dist/index.js",
6
6
  "type": "module",
@@ -22,7 +22,7 @@
22
22
  "koa-logger": "^3.2.1",
23
23
  "phoenix": "^1.7.7",
24
24
  "ws": "^8.14.1",
25
- "@openfn/engine-multi": "0.3.0",
25
+ "@openfn/engine-multi": "0.4.0",
26
26
  "@openfn/logger": "0.0.19",
27
27
  "@openfn/runtime": "0.2.5"
28
28
  },
@@ -41,7 +41,7 @@
41
41
  "tsup": "^6.2.3",
42
42
  "typescript": "^4.6.4",
43
43
  "yargs": "^17.6.2",
44
- "@openfn/lightning-mock": "1.1.11"
44
+ "@openfn/lightning-mock": "1.2.0"
45
45
  },
46
46
  "files": [
47
47
  "dist",