@openfn/ws-worker 0.8.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,3 +1,110 @@
1
+ var __defProp = Object.defineProperty;
2
+ var __getOwnPropNames = Object.getOwnPropertyNames;
3
+ var __esm = (fn, res) => function __init() {
4
+ return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
5
+ };
6
+ var __export = (target, all) => {
7
+ for (var name2 in all)
8
+ __defProp(target, name2, { get: all[name2], enumerable: true });
9
+ };
10
+
11
+ // package.json
12
+ var package_exports = {};
13
+ __export(package_exports, {
14
+ author: () => author,
15
+ bin: () => bin,
16
+ default: () => package_default,
17
+ dependencies: () => dependencies,
18
+ description: () => description,
19
+ devDependencies: () => devDependencies,
20
+ files: () => files,
21
+ license: () => license,
22
+ main: () => main,
23
+ name: () => name,
24
+ scripts: () => scripts,
25
+ type: () => type,
26
+ version: () => version
27
+ });
28
+ var name, version, description, main, type, scripts, bin, author, license, dependencies, devDependencies, files, package_default;
29
+ var init_package = __esm({
30
+ "package.json"() {
31
+ name = "@openfn/ws-worker";
32
+ version = "1.0.0";
33
+ description = "A Websocket Worker to connect Lightning to a Runtime Engine";
34
+ main = "dist/index.js";
35
+ type = "module";
36
+ scripts = {
37
+ test: "pnpm ava --serial",
38
+ "test:types": "pnpm tsc --noEmit --project tsconfig.json",
39
+ build: "tsup --config tsup.config.js",
40
+ "build:watch": "pnpm build --watch",
41
+ start: "ts-node-esm --transpile-only src/start.ts",
42
+ "start:prod": "node dist/start.js",
43
+ "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'",
44
+ pack: "pnpm pack --pack-destination ../../dist"
45
+ };
46
+ bin = {
47
+ worker: "dist/start.js"
48
+ };
49
+ author = "Open Function Group <admin@openfn.org>";
50
+ license = "ISC";
51
+ dependencies = {
52
+ "@koa/router": "^12.0.0",
53
+ "@openfn/engine-multi": "workspace:*",
54
+ "@openfn/lexicon": "workspace:^",
55
+ "@openfn/logger": "workspace:*",
56
+ "@openfn/runtime": "workspace:*",
57
+ "@types/koa-logger": "^3.1.2",
58
+ "@types/ws": "^8.5.6",
59
+ "fast-safe-stringify": "^2.1.1",
60
+ figures: "^5.0.0",
61
+ "human-id": "^4.1.0",
62
+ jose: "^4.14.6",
63
+ koa: "^2.13.4",
64
+ "koa-bodyparser": "^4.4.0",
65
+ "koa-logger": "^3.2.1",
66
+ phoenix: "1.7.10",
67
+ ws: "^8.14.1"
68
+ };
69
+ devDependencies = {
70
+ "@openfn/lightning-mock": "workspace:*",
71
+ "@types/koa": "^2.13.5",
72
+ "@types/koa-bodyparser": "^4.3.10",
73
+ "@types/koa__router": "^12.0.1",
74
+ "@types/node": "^18.15.3",
75
+ "@types/nodemon": "1.19.3",
76
+ "@types/phoenix": "^1.6.2",
77
+ "@types/yargs": "^17.0.12",
78
+ ava: "5.1.0",
79
+ nodemon: "3.0.1",
80
+ "ts-node": "^10.9.1",
81
+ tslib: "^2.4.0",
82
+ tsup: "^6.2.3",
83
+ typescript: "^4.6.4",
84
+ yargs: "^17.6.2"
85
+ };
86
+ files = [
87
+ "dist",
88
+ "README.md",
89
+ "CHANGELOG.md"
90
+ ];
91
+ package_default = {
92
+ name,
93
+ version,
94
+ description,
95
+ main,
96
+ type,
97
+ scripts,
98
+ bin,
99
+ author,
100
+ license,
101
+ dependencies,
102
+ devDependencies,
103
+ files
104
+ };
105
+ }
106
+ });
107
+
1
108
  // src/server.ts
2
109
  import { EventEmitter as EventEmitter2 } from "node:events";
3
110
  import Koa from "koa";
@@ -10,8 +117,8 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
10
117
  // src/events.ts
11
118
  var CLAIM = "claim";
12
119
  var GET_PLAN = "fetch:plan";
13
- var GET_CREDENTIAL = "fetch:credential";
14
120
  var GET_DATACLIP = "fetch:dataclip";
121
+ var GET_CREDENTIAL = "fetch:credential";
15
122
  var RUN_START = "run:start";
16
123
  var RUN_COMPLETE = "run:complete";
17
124
  var RUN_LOG = "run:log";
@@ -35,6 +142,7 @@ var destroy = async (app, logger) => {
35
142
  await waitForRuns(app, logger);
36
143
  await app.engine.destroy();
37
144
  app.socket?.disconnect();
145
+ logger.info("Server closed....");
38
146
  resolve();
39
147
  })
40
148
  ]);
@@ -59,6 +167,7 @@ var waitForRuns = (app, logger) => new Promise((resolve) => {
59
167
  log();
60
168
  app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
61
169
  } else {
170
+ logger.debug("No active rns detected");
62
171
  resolve();
63
172
  }
64
173
  });
@@ -106,10 +215,22 @@ var tryWithBackoff = (fn, opts = {}) => {
106
215
  var try_with_backoff_default = tryWithBackoff;
107
216
 
108
217
  // src/api/claim.ts
218
+ import crypto from "node:crypto";
219
+ import * as jose from "jose";
109
220
  import { createMockLogger } from "@openfn/logger";
110
221
  var mockLogger = createMockLogger();
111
- var claim = (app, logger = mockLogger, maxWorkers = 5) => {
222
+ var verifyToken = async (token, publicKey) => {
223
+ const key = crypto.createPublicKey(publicKey);
224
+ const { payload } = await jose.jwtVerify(token, key, {
225
+ issuer: "Lightning"
226
+ });
227
+ if (payload) {
228
+ return true;
229
+ }
230
+ };
231
+ var claim = (app, logger = mockLogger, options = {}) => {
112
232
  return new Promise((resolve, reject) => {
233
+ const { maxWorkers = 5 } = options;
113
234
  const activeWorkers = Object.keys(app.workflows).length;
114
235
  if (activeWorkers >= maxWorkers) {
115
236
  return reject(new Error("Server at capacity"));
@@ -123,7 +244,21 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
123
244
  if (!runs?.length) {
124
245
  return reject(new Error("No runs returned"));
125
246
  }
126
- runs.forEach((run) => {
247
+ runs.forEach(async (run) => {
248
+ if (app.options?.runPublicKey) {
249
+ try {
250
+ await verifyToken(run.token, app.options.runPublicKey);
251
+ logger.debug("verified run token for", run.id);
252
+ } catch (e) {
253
+ logger.error("Error validating run token");
254
+ logger.error(e);
255
+ reject();
256
+ app.destroy();
257
+ return;
258
+ }
259
+ } else {
260
+ logger.debug("skipping run token validation for", run.id);
261
+ }
127
262
  logger.debug("starting run", run.id);
128
263
  app.execute(run);
129
264
  resolve();
@@ -144,10 +279,15 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
144
279
  let cancelled = false;
145
280
  const workLoop = () => {
146
281
  if (!cancelled) {
147
- promise = try_with_backoff_default(() => claim_default(app, logger, maxWorkers), {
148
- min: minBackoff,
149
- max: maxBackoff
150
- });
282
+ promise = try_with_backoff_default(
283
+ () => claim_default(app, logger, {
284
+ maxWorkers
285
+ }),
286
+ {
287
+ min: minBackoff,
288
+ max: maxBackoff
289
+ }
290
+ );
151
291
  promise.then(() => {
152
292
  if (!cancelled) {
153
293
  setTimeout(workLoop, minBackoff);
@@ -165,8 +305,8 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
165
305
  };
166
306
  var workloop_default = startWorkloop;
167
307
 
168
- // src/util/convert-run.ts
169
- import crypto from "node:crypto";
308
+ // src/util/convert-lightning-plan.ts
309
+ import crypto2 from "node:crypto";
170
310
  var conditions = {
171
311
  on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
172
312
  on_job_failure: (upstreamId) => `Boolean(state?.errors && state.errors["${upstreamId}"])`,
@@ -186,19 +326,30 @@ var mapTriggerEdgeCondition = (edge) => {
186
326
  return true;
187
327
  return condition;
188
328
  };
189
- var mapOptions = (options) => {
190
- return options;
191
- };
192
- var convert_run_default = (run) => {
193
- const options = run.options || {};
329
+ var convert_lightning_plan_default = (run) => {
330
+ const runtimeOpts = {};
331
+ const engineOpts = {};
332
+ if (run.options) {
333
+ if (run.options.runTimeoutMs) {
334
+ engineOpts.runTimeoutMs = run.options.runTimeoutMs;
335
+ }
336
+ if (run.options.sanitize) {
337
+ engineOpts.sanitize = run.options.sanitize;
338
+ }
339
+ if (run.options.hasOwnProperty("output_dataclips")) {
340
+ engineOpts.outputDataclips = run.options.output_dataclips;
341
+ }
342
+ }
194
343
  const plan = {
195
- id: run.id
344
+ id: run.id,
345
+ options: runtimeOpts
196
346
  };
347
+ let initialState;
197
348
  if (run.dataclip_id) {
198
- plan.initialState = run.dataclip_id;
349
+ initialState = run.dataclip_id;
199
350
  }
200
351
  if (run.starting_node_id) {
201
- plan.start = run.starting_node_id;
352
+ runtimeOpts.start = run.starting_node_id;
202
353
  }
203
354
  const nodes = {};
204
355
  const edges = run.edges ?? [];
@@ -210,27 +361,33 @@ var convert_run_default = (run) => {
210
361
  };
211
362
  const connectedEdges = edges.filter((e) => e.source_trigger_id === id);
212
363
  if (connectedEdges.length) {
213
- nodes[id].next = connectedEdges.reduce((obj, edge) => {
214
- if (edge.enabled !== false) {
215
- obj[edge.target_job_id] = mapTriggerEdgeCondition(edge);
216
- }
217
- return obj;
218
- }, {});
364
+ nodes[id].next = connectedEdges.reduce(
365
+ (obj, edge) => {
366
+ if (edge.enabled !== false) {
367
+ obj[edge.target_job_id] = mapTriggerEdgeCondition(edge);
368
+ }
369
+ return obj;
370
+ },
371
+ {}
372
+ );
219
373
  } else {
220
374
  }
221
375
  });
222
376
  }
223
377
  if (run.jobs?.length) {
224
- run.jobs.forEach((job) => {
225
- const id = job.id || crypto.randomUUID();
226
- nodes[id] = {
378
+ run.jobs.forEach((step) => {
379
+ const id = step.id || crypto2.randomUUID();
380
+ const job = {
227
381
  id,
228
- configuration: job.credential || job.credential_id,
229
- expression: job.body,
230
- adaptor: job.adaptor
382
+ configuration: step.credential || step.credential_id,
383
+ expression: step.body,
384
+ adaptor: step.adaptor
231
385
  };
232
- if (job.state) {
233
- nodes[id].state = job.state;
386
+ if (step.name) {
387
+ job.name = step.name;
388
+ }
389
+ if (step.state) {
390
+ job.state = step.state;
234
391
  }
235
392
  const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
236
393
  const newEdge = {};
@@ -245,21 +402,32 @@ var convert_run_default = (run) => {
245
402
  return obj;
246
403
  }, {});
247
404
  if (Object.keys(next).length) {
248
- nodes[id].next = next;
405
+ job.next = next;
249
406
  }
407
+ nodes[id] = job;
250
408
  });
251
409
  }
252
- plan.jobs = Object.values(nodes);
410
+ plan.workflow = {
411
+ steps: Object.values(nodes)
412
+ };
413
+ if (run.name) {
414
+ plan.workflow.name = run.name;
415
+ }
253
416
  return {
254
417
  plan,
255
- options: mapOptions(options)
418
+ options: engineOpts,
419
+ input: initialState || {}
256
420
  };
257
421
  };
258
422
 
259
423
  // src/util/get-with-reply.ts
260
- var get_with_reply_default = (channel, event, payload) => new Promise((resolve) => {
424
+ var get_with_reply_default = (channel, event, payload) => new Promise((resolve, reject) => {
261
425
  channel.push(event, payload).receive("ok", (evt) => {
262
426
  resolve(evt);
427
+ }).receive("error", (e) => {
428
+ reject(e);
429
+ }).receive("timeout", (e) => {
430
+ reject(e);
263
431
  });
264
432
  });
265
433
 
@@ -273,19 +441,20 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
273
441
  });
274
442
 
275
443
  // src/util/create-run-state.ts
276
- var create_run_state_default = (plan, options = {}) => {
444
+ var create_run_state_default = (plan, input) => {
277
445
  const state = {
278
- plan,
279
446
  lastDataclipId: "",
280
447
  dataclips: {},
281
448
  inputDataclips: {},
282
449
  reasons: {},
283
- options
450
+ plan,
451
+ input
284
452
  };
285
- if (typeof plan.initialState === "string") {
286
- let startNode = plan.jobs[0];
287
- if (plan.start) {
288
- startNode = plan.jobs.find(({ id }) => id === plan.start);
453
+ if (typeof input === "string") {
454
+ const jobs = plan.workflow.steps;
455
+ let startNode = jobs[0];
456
+ if (plan.options.start) {
457
+ startNode = jobs.find(({ id }) => id === plan.options.start);
289
458
  }
290
459
  const initialRuns = [];
291
460
  if (!startNode.expression) {
@@ -294,15 +463,42 @@ var create_run_state_default = (plan, options = {}) => {
294
463
  initialRuns.push(startNode.id);
295
464
  }
296
465
  initialRuns.forEach((id) => {
297
- state.inputDataclips[id] = plan.initialState;
466
+ state.inputDataclips[id] = input;
298
467
  });
299
468
  } else {
300
469
  }
301
470
  return state;
302
471
  };
303
472
 
473
+ // src/util/throttle.ts
474
+ var createThrottler = () => {
475
+ const q = [];
476
+ let activePromise;
477
+ const add = (fn) => {
478
+ return (...args) => new Promise((resolve, reject) => {
479
+ q.push({ fn, args, resolve, reject });
480
+ shift();
481
+ });
482
+ };
483
+ const shift = () => {
484
+ if (activePromise) {
485
+ return;
486
+ }
487
+ const next = q.shift();
488
+ if (next) {
489
+ const { fn, args, resolve, reject } = next;
490
+ activePromise = fn(...args).then(resolve).catch(reject).finally(() => {
491
+ activePromise = void 0;
492
+ shift();
493
+ });
494
+ }
495
+ };
496
+ return add;
497
+ };
498
+ var throttle_default = createThrottler;
499
+
304
500
  // src/events/step-complete.ts
305
- import crypto2 from "node:crypto";
501
+ import crypto3 from "node:crypto";
306
502
 
307
503
  // src/api/reasons.ts
308
504
  var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
@@ -328,7 +524,7 @@ var isLeafNode = (state, job) => {
328
524
  };
329
525
  var calculateRunExitReason = (state) => {
330
526
  if (state.plan && state.reasons) {
331
- const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
527
+ const leafJobReasons = state.plan.workflow.steps.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
332
528
  const fail = leafJobReasons.find((r) => r && r.reason === "fail");
333
529
  if (fail) {
334
530
  return fail;
@@ -338,8 +534,8 @@ var calculateRunExitReason = (state) => {
338
534
  };
339
535
 
340
536
  // src/events/step-complete.ts
341
- function onStepComplete({ channel, state }, event, error) {
342
- const dataclipId = crypto2.randomUUID();
537
+ function onStepComplete({ channel, state, options }, event, error) {
538
+ const dataclipId = crypto3.randomUUID();
343
539
  const step_id = state.activeStep;
344
540
  const job_id = state.activeJob;
345
541
  if (!state.dataclips) {
@@ -363,7 +559,6 @@ function onStepComplete({ channel, state }, event, error) {
363
559
  step_id,
364
560
  job_id,
365
561
  output_dataclip_id: dataclipId,
366
- output_dataclip: stringify_default(outputState),
367
562
  reason,
368
563
  error_message,
369
564
  error_type,
@@ -371,94 +566,34 @@ function onStepComplete({ channel, state }, event, error) {
371
566
  duration: event.duration,
372
567
  thread_id: event.threadId
373
568
  };
569
+ if (!options || options.outputDataclips !== false) {
570
+ evt.output_dataclip = stringify_default(outputState);
571
+ }
374
572
  return sendEvent(channel, STEP_COMPLETE, evt);
375
573
  }
376
574
 
377
575
  // src/events/step-start.ts
378
- import crypto3 from "node:crypto";
576
+ init_package();
577
+ import crypto4 from "node:crypto";
379
578
  import { timestamp } from "@openfn/logger";
380
579
 
381
- // package.json
382
- var package_default = {
383
- name: "@openfn/ws-worker",
384
- version: "0.8.0",
385
- description: "A Websocket Worker to connect Lightning to a Runtime Engine",
386
- main: "dist/index.js",
387
- type: "module",
388
- scripts: {
389
- test: "pnpm ava --serial",
390
- "test:types": "pnpm tsc --noEmit --project tsconfig.json",
391
- build: "tsup --config tsup.config.js",
392
- "build:watch": "pnpm build --watch",
393
- start: "ts-node-esm --transpile-only src/start.ts",
394
- "start:prod": "node dist/start.js",
395
- "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'",
396
- pack: "pnpm pack --pack-destination ../../dist"
397
- },
398
- bin: {
399
- worker: "dist/start.js"
400
- },
401
- author: "Open Function Group <admin@openfn.org>",
402
- license: "ISC",
403
- dependencies: {
404
- "@koa/router": "^12.0.0",
405
- "@openfn/engine-multi": "workspace:*",
406
- "@openfn/logger": "workspace:*",
407
- "@openfn/runtime": "workspace:*",
408
- "@types/koa-logger": "^3.1.2",
409
- "@types/ws": "^8.5.6",
410
- "fast-safe-stringify": "^2.1.1",
411
- figures: "^5.0.0",
412
- "human-id": "^4.1.0",
413
- jose: "^4.14.6",
414
- koa: "^2.13.4",
415
- "koa-bodyparser": "^4.4.0",
416
- "koa-logger": "^3.2.1",
417
- phoenix: "^1.7.7",
418
- ws: "^8.14.1"
419
- },
420
- devDependencies: {
421
- "@openfn/lightning-mock": "workspace:*",
422
- "@types/koa": "^2.13.5",
423
- "@types/koa-bodyparser": "^4.3.10",
424
- "@types/koa__router": "^12.0.1",
425
- "@types/node": "^18.15.3",
426
- "@types/nodemon": "1.19.3",
427
- "@types/phoenix": "^1.6.2",
428
- "@types/yargs": "^17.0.12",
429
- ava: "5.1.0",
430
- nodemon: "3.0.1",
431
- "ts-node": "^10.9.1",
432
- tslib: "^2.4.0",
433
- tsup: "^6.2.3",
434
- typescript: "^4.6.4",
435
- yargs: "^17.6.2"
436
- },
437
- files: [
438
- "dist",
439
- "README.md",
440
- "CHANGELOG.md"
441
- ]
442
- };
443
-
444
580
  // src/util/versions.ts
445
581
  import { mainSymbols } from "figures";
446
582
  var { triangleRightSmall: t } = mainSymbols;
447
583
  var versions_default = (stepId, versions, adaptor) => {
448
- let longest = "compiler".length;
584
+ let longest = "worker".length;
449
585
  for (const v in versions) {
450
586
  longest = Math.max(v.length, longest);
451
587
  }
452
- const { node, compiler, engine, worker, runtime, ...adaptors } = versions;
588
+ const { node, worker, ...adaptors } = versions;
453
589
  const prefix = (str2) => ` ${t} ${str2.padEnd(longest + 4, " ")}`;
454
590
  let str = `Versions for step ${stepId}:
455
591
  ${prefix("node.js")}${versions.node || "unknown"}
456
- ${prefix("worker")}${versions.worker || "unknown"}
457
- ${prefix("engine")}${versions.engine || "unknown"}`;
592
+ ${prefix("worker")}${versions.worker || "unknown"}`;
458
593
  if (Object.keys(adaptors).length) {
459
594
  let allAdaptors = Object.keys(adaptors);
460
595
  if (adaptor) {
461
- allAdaptors = allAdaptors.filter((name) => adaptor.startsWith(name));
596
+ allAdaptors = allAdaptors.filter((name2) => adaptor.startsWith(name2));
462
597
  }
463
598
  str += "\n" + allAdaptors.sort().map((adaptorName) => `${prefix(adaptorName)}${adaptors[adaptorName]}`).join("\n");
464
599
  }
@@ -469,9 +604,11 @@ ${prefix("engine")}${versions.engine || "unknown"}`;
469
604
  async function onStepStart(context, event) {
470
605
  const time = (timestamp() - BigInt(1e7)).toString();
471
606
  const { channel, state } = context;
472
- state.activeStep = crypto3.randomUUID();
607
+ state.activeStep = crypto4.randomUUID();
473
608
  state.activeJob = event.jobId;
474
- const job = state.plan.jobs.find(({ id }) => id === event.jobId);
609
+ const job = state.plan.workflow.steps.find(
610
+ ({ id }) => id === event.jobId
611
+ );
475
612
  const input_dataclip_id = state.inputDataclips[event.jobId];
476
613
  const versions = {
477
614
  worker: package_default.version,
@@ -549,39 +686,12 @@ async function onRunError(context, event) {
549
686
  });
550
687
  onFinish({ reason });
551
688
  } catch (e) {
552
- logger.error("ERROR in workflow-error handler:", e.message);
689
+ logger.error("ERROR in run-error handler:", e.message);
553
690
  logger.error(e);
554
691
  onFinish({});
555
692
  }
556
693
  }
557
694
 
558
- // src/util/throttle.ts
559
- var createThrottler = () => {
560
- const q = [];
561
- let activePromise;
562
- const add = (fn) => {
563
- return (...args) => new Promise((resolve, reject) => {
564
- q.push({ fn, args, resolve, reject });
565
- shift();
566
- });
567
- };
568
- const shift = () => {
569
- if (activePromise) {
570
- return;
571
- }
572
- const next = q.shift();
573
- if (next) {
574
- const { fn, args, resolve, reject } = next;
575
- activePromise = fn(...args).then(resolve).catch(reject).finally(() => {
576
- activePromise = void 0;
577
- shift();
578
- });
579
- }
580
- };
581
- return add;
582
- };
583
- var throttle_default = createThrottler;
584
-
585
695
  // src/api/execute.ts
586
696
  var enc = new TextDecoder("utf-8");
587
697
  var eventMap = {
@@ -591,11 +701,18 @@ var eventMap = {
591
701
  "workflow-log": RUN_LOG,
592
702
  "workflow-complete": RUN_COMPLETE
593
703
  };
594
- function execute(channel, engine, logger, plan, options = {}, onFinish = (_result) => {
704
+ function execute(channel, engine, logger, plan, input, options = {}, onFinish = (_result) => {
595
705
  }) {
596
706
  logger.info("executing ", plan.id);
597
- const state = create_run_state_default(plan, options);
598
- const context = { channel, state, logger, engine, onFinish };
707
+ const state = create_run_state_default(plan, input);
708
+ const context = {
709
+ channel,
710
+ state,
711
+ logger,
712
+ engine,
713
+ options,
714
+ onFinish
715
+ };
599
716
  const throttle = throttle_default();
600
717
  const addEvent = (eventName, handler) => {
601
718
  const wrappedFn = async (event) => {
@@ -628,17 +745,24 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
628
745
  const resolvers = {
629
746
  credential: (id) => loadCredential(channel, id)
630
747
  };
631
- Promise.resolve().then(async () => {
632
- if (typeof plan.initialState === "string") {
633
- logger.debug("loading dataclip", plan.initialState);
634
- plan.initialState = await loadDataclip(channel, plan.initialState);
635
- logger.success("dataclip loaded");
636
- logger.debug(plan.initialState);
748
+ setTimeout(async () => {
749
+ let loadedInput = input;
750
+ if (typeof input === "string") {
751
+ logger.debug("loading dataclip", input);
752
+ try {
753
+ loadedInput = await loadDataclip(channel, input);
754
+ logger.success("dataclip loaded");
755
+ } catch (e) {
756
+ return onRunError(context, {
757
+ workflowId: plan.id,
758
+ message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
759
+ type: "DataClipError",
760
+ severity: "exception"
761
+ });
762
+ }
637
763
  }
638
- return plan;
639
- }).then(() => {
640
764
  try {
641
- engine.execute(plan, { resolvers, ...options });
765
+ engine.execute(plan, loadedInput, { resolvers, ...options });
642
766
  } catch (e) {
643
767
  onRunError(context, {
644
768
  workflowId: plan.id,
@@ -670,7 +794,7 @@ function onJobLog({ channel, state }, event) {
670
794
  const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
671
795
  const log = {
672
796
  run_id: state.plan.id,
673
- message: event.message,
797
+ message: typeof event.message === "string" ? JSON.parse(event.message) : event.message,
674
798
  source: event.name,
675
799
  level: event.level,
676
800
  timestamp: timeInMicroseconds.toString()
@@ -707,9 +831,9 @@ var joinRunChannel = (socket, token, runId, logger) => {
707
831
  if (!didReceiveOk) {
708
832
  didReceiveOk = true;
709
833
  logger.success(`connected to ${channelName}`, e);
710
- const { plan, options } = await loadRun(channel);
834
+ const { plan, options, input } = await loadRun(channel);
711
835
  logger.debug("converted run as execution plan:", plan);
712
- resolve({ channel, plan, options });
836
+ resolve({ channel, plan, options, input });
713
837
  }
714
838
  }).receive("error", (err) => {
715
839
  logger.error(`error connecting to ${channelName}`, err);
@@ -720,16 +844,17 @@ var joinRunChannel = (socket, token, runId, logger) => {
720
844
  var run_default = joinRunChannel;
721
845
  async function loadRun(channel) {
722
846
  const runBody = await get_with_reply_default(channel, GET_PLAN);
723
- return convert_run_default(runBody);
847
+ return convert_lightning_plan_default(runBody);
724
848
  }
725
849
 
726
850
  // src/channels/worker-queue.ts
727
851
  import EventEmitter from "node:events";
728
852
  import { Socket as PhxSocket } from "phoenix";
729
853
  import { WebSocket } from "ws";
854
+ import { API_VERSION } from "@openfn/lexicon/lightning";
730
855
 
731
856
  // src/util/worker-token.ts
732
- import * as jose from "jose";
857
+ import * as jose2 from "jose";
733
858
  var alg = "HS256";
734
859
  var generateWorkerToken = async (secret, workerId, logger) => {
735
860
  if (!secret) {
@@ -744,7 +869,7 @@ var generateWorkerToken = async (secret, workerId, logger) => {
744
869
  const claims = {
745
870
  worker_id: workerId
746
871
  };
747
- const jwt = await new jose.SignJWT(claims).setProtectedHeader({ alg }).setIssuedAt().setIssuer("urn:example:issuer").setAudience("urn:example:audience").sign(encodedSecret);
872
+ const jwt = await new jose2.SignJWT(claims).setProtectedHeader({ alg }).setIssuedAt().setIssuer("urn:openfn:worker").sign(encodedSecret);
748
873
  return jwt;
749
874
  };
750
875
  var worker_token_default = generateWorkerToken;
@@ -752,9 +877,15 @@ var worker_token_default = generateWorkerToken;
752
877
  // src/channels/worker-queue.ts
753
878
  var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructor = PhxSocket) => {
754
879
  const events = new EventEmitter();
755
- worker_token_default(secret, serverId, logger).then((token) => {
880
+ worker_token_default(secret, serverId, logger).then(async (token) => {
881
+ const pkg = await Promise.resolve().then(() => (init_package(), package_exports));
882
+ const params = {
883
+ token,
884
+ api_version: API_VERSION,
885
+ worker_version: pkg.default.version
886
+ };
756
887
  const socket = new SocketConstructor(endpoint, {
757
- params: { token },
888
+ params,
758
889
  transport: WebSocket
759
890
  });
760
891
  let didOpen = false;
@@ -793,6 +924,9 @@ var MAX_BACKOFF = 1e3 * 30;
793
924
  function connect(app, logger, options = {}) {
794
925
  logger.debug("Connecting to Lightning at", options.lightning);
795
926
  const onConnect = ({ socket, channel }) => {
927
+ if (app.destroyed) {
928
+ return;
929
+ }
796
930
  logger.success("Connected to Lightning at", options.lightning);
797
931
  app.socket = socket;
798
932
  app.queueChannel = channel;
@@ -827,6 +961,9 @@ function connect(app, logger, options = {}) {
827
961
  }
828
962
  };
829
963
  const onError = (e) => {
964
+ if (app.destroyed) {
965
+ return;
966
+ }
830
967
  logger.error(
831
968
  "CRITICAL ERROR: could not connect to lightning at",
832
969
  options.lightning
@@ -856,13 +993,15 @@ function createServer(engine, options = {}) {
856
993
  process.send?.("READY");
857
994
  router.get("/livez", healthcheck_default);
858
995
  router.get("/", healthcheck_default);
996
+ app.options = options || {};
859
997
  app.execute = async ({ id, token }) => {
860
998
  if (app.socket) {
861
999
  app.workflows[id] = true;
862
1000
  const {
863
1001
  channel: runChannel,
864
1002
  plan,
865
- options: options2
1003
+ options: options2,
1004
+ input
866
1005
  } = await run_default(app.socket, token, id, logger);
867
1006
  const onFinish = () => {
868
1007
  delete app.workflows[id];
@@ -874,6 +1013,7 @@ function createServer(engine, options = {}) {
874
1013
  engine,
875
1014
  logger,
876
1015
  plan,
1016
+ input,
877
1017
  options2,
878
1018
  onFinish
879
1019
  );
@@ -884,7 +1024,9 @@ function createServer(engine, options = {}) {
884
1024
  };
885
1025
  router.post("/claim", async (ctx) => {
886
1026
  logger.info("triggering claim from POST request");
887
- return claim_default(app, logger, options.maxWorkflows).then(() => {
1027
+ return claim_default(app, logger, {
1028
+ maxWorkers: options.maxWorkflows
1029
+ }).then(() => {
888
1030
  logger.info("claim complete: 1 run claimed");
889
1031
  ctx.body = "complete";
890
1032
  ctx.status = 200;