@openfn/ws-worker 0.8.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/index.d.ts +49 -121
- package/dist/index.js +311 -169
- package/dist/start.js +410 -249
- package/package.json +7 -6
package/dist/index.js
CHANGED
|
@@ -1,3 +1,110 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
3
|
+
var __esm = (fn, res) => function __init() {
|
|
4
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
5
|
+
};
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name2 in all)
|
|
8
|
+
__defProp(target, name2, { get: all[name2], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
// package.json
|
|
12
|
+
var package_exports = {};
|
|
13
|
+
__export(package_exports, {
|
|
14
|
+
author: () => author,
|
|
15
|
+
bin: () => bin,
|
|
16
|
+
default: () => package_default,
|
|
17
|
+
dependencies: () => dependencies,
|
|
18
|
+
description: () => description,
|
|
19
|
+
devDependencies: () => devDependencies,
|
|
20
|
+
files: () => files,
|
|
21
|
+
license: () => license,
|
|
22
|
+
main: () => main,
|
|
23
|
+
name: () => name,
|
|
24
|
+
scripts: () => scripts,
|
|
25
|
+
type: () => type,
|
|
26
|
+
version: () => version
|
|
27
|
+
});
|
|
28
|
+
var name, version, description, main, type, scripts, bin, author, license, dependencies, devDependencies, files, package_default;
|
|
29
|
+
var init_package = __esm({
|
|
30
|
+
"package.json"() {
|
|
31
|
+
name = "@openfn/ws-worker";
|
|
32
|
+
version = "1.0.0";
|
|
33
|
+
description = "A Websocket Worker to connect Lightning to a Runtime Engine";
|
|
34
|
+
main = "dist/index.js";
|
|
35
|
+
type = "module";
|
|
36
|
+
scripts = {
|
|
37
|
+
test: "pnpm ava --serial",
|
|
38
|
+
"test:types": "pnpm tsc --noEmit --project tsconfig.json",
|
|
39
|
+
build: "tsup --config tsup.config.js",
|
|
40
|
+
"build:watch": "pnpm build --watch",
|
|
41
|
+
start: "ts-node-esm --transpile-only src/start.ts",
|
|
42
|
+
"start:prod": "node dist/start.js",
|
|
43
|
+
"start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'",
|
|
44
|
+
pack: "pnpm pack --pack-destination ../../dist"
|
|
45
|
+
};
|
|
46
|
+
bin = {
|
|
47
|
+
worker: "dist/start.js"
|
|
48
|
+
};
|
|
49
|
+
author = "Open Function Group <admin@openfn.org>";
|
|
50
|
+
license = "ISC";
|
|
51
|
+
dependencies = {
|
|
52
|
+
"@koa/router": "^12.0.0",
|
|
53
|
+
"@openfn/engine-multi": "workspace:*",
|
|
54
|
+
"@openfn/lexicon": "workspace:^",
|
|
55
|
+
"@openfn/logger": "workspace:*",
|
|
56
|
+
"@openfn/runtime": "workspace:*",
|
|
57
|
+
"@types/koa-logger": "^3.1.2",
|
|
58
|
+
"@types/ws": "^8.5.6",
|
|
59
|
+
"fast-safe-stringify": "^2.1.1",
|
|
60
|
+
figures: "^5.0.0",
|
|
61
|
+
"human-id": "^4.1.0",
|
|
62
|
+
jose: "^4.14.6",
|
|
63
|
+
koa: "^2.13.4",
|
|
64
|
+
"koa-bodyparser": "^4.4.0",
|
|
65
|
+
"koa-logger": "^3.2.1",
|
|
66
|
+
phoenix: "1.7.10",
|
|
67
|
+
ws: "^8.14.1"
|
|
68
|
+
};
|
|
69
|
+
devDependencies = {
|
|
70
|
+
"@openfn/lightning-mock": "workspace:*",
|
|
71
|
+
"@types/koa": "^2.13.5",
|
|
72
|
+
"@types/koa-bodyparser": "^4.3.10",
|
|
73
|
+
"@types/koa__router": "^12.0.1",
|
|
74
|
+
"@types/node": "^18.15.3",
|
|
75
|
+
"@types/nodemon": "1.19.3",
|
|
76
|
+
"@types/phoenix": "^1.6.2",
|
|
77
|
+
"@types/yargs": "^17.0.12",
|
|
78
|
+
ava: "5.1.0",
|
|
79
|
+
nodemon: "3.0.1",
|
|
80
|
+
"ts-node": "^10.9.1",
|
|
81
|
+
tslib: "^2.4.0",
|
|
82
|
+
tsup: "^6.2.3",
|
|
83
|
+
typescript: "^4.6.4",
|
|
84
|
+
yargs: "^17.6.2"
|
|
85
|
+
};
|
|
86
|
+
files = [
|
|
87
|
+
"dist",
|
|
88
|
+
"README.md",
|
|
89
|
+
"CHANGELOG.md"
|
|
90
|
+
];
|
|
91
|
+
package_default = {
|
|
92
|
+
name,
|
|
93
|
+
version,
|
|
94
|
+
description,
|
|
95
|
+
main,
|
|
96
|
+
type,
|
|
97
|
+
scripts,
|
|
98
|
+
bin,
|
|
99
|
+
author,
|
|
100
|
+
license,
|
|
101
|
+
dependencies,
|
|
102
|
+
devDependencies,
|
|
103
|
+
files
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
|
|
1
108
|
// src/server.ts
|
|
2
109
|
import { EventEmitter as EventEmitter2 } from "node:events";
|
|
3
110
|
import Koa from "koa";
|
|
@@ -10,8 +117,8 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
|
|
|
10
117
|
// src/events.ts
|
|
11
118
|
var CLAIM = "claim";
|
|
12
119
|
var GET_PLAN = "fetch:plan";
|
|
13
|
-
var GET_CREDENTIAL = "fetch:credential";
|
|
14
120
|
var GET_DATACLIP = "fetch:dataclip";
|
|
121
|
+
var GET_CREDENTIAL = "fetch:credential";
|
|
15
122
|
var RUN_START = "run:start";
|
|
16
123
|
var RUN_COMPLETE = "run:complete";
|
|
17
124
|
var RUN_LOG = "run:log";
|
|
@@ -35,6 +142,7 @@ var destroy = async (app, logger) => {
|
|
|
35
142
|
await waitForRuns(app, logger);
|
|
36
143
|
await app.engine.destroy();
|
|
37
144
|
app.socket?.disconnect();
|
|
145
|
+
logger.info("Server closed....");
|
|
38
146
|
resolve();
|
|
39
147
|
})
|
|
40
148
|
]);
|
|
@@ -59,6 +167,7 @@ var waitForRuns = (app, logger) => new Promise((resolve) => {
|
|
|
59
167
|
log();
|
|
60
168
|
app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
61
169
|
} else {
|
|
170
|
+
logger.debug("No active rns detected");
|
|
62
171
|
resolve();
|
|
63
172
|
}
|
|
64
173
|
});
|
|
@@ -106,10 +215,22 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
106
215
|
var try_with_backoff_default = tryWithBackoff;
|
|
107
216
|
|
|
108
217
|
// src/api/claim.ts
|
|
218
|
+
import crypto from "node:crypto";
|
|
219
|
+
import * as jose from "jose";
|
|
109
220
|
import { createMockLogger } from "@openfn/logger";
|
|
110
221
|
var mockLogger = createMockLogger();
|
|
111
|
-
var
|
|
222
|
+
var verifyToken = async (token, publicKey) => {
|
|
223
|
+
const key = crypto.createPublicKey(publicKey);
|
|
224
|
+
const { payload } = await jose.jwtVerify(token, key, {
|
|
225
|
+
issuer: "Lightning"
|
|
226
|
+
});
|
|
227
|
+
if (payload) {
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
var claim = (app, logger = mockLogger, options = {}) => {
|
|
112
232
|
return new Promise((resolve, reject) => {
|
|
233
|
+
const { maxWorkers = 5 } = options;
|
|
113
234
|
const activeWorkers = Object.keys(app.workflows).length;
|
|
114
235
|
if (activeWorkers >= maxWorkers) {
|
|
115
236
|
return reject(new Error("Server at capacity"));
|
|
@@ -123,7 +244,21 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
|
|
|
123
244
|
if (!runs?.length) {
|
|
124
245
|
return reject(new Error("No runs returned"));
|
|
125
246
|
}
|
|
126
|
-
runs.forEach((run) => {
|
|
247
|
+
runs.forEach(async (run) => {
|
|
248
|
+
if (app.options?.runPublicKey) {
|
|
249
|
+
try {
|
|
250
|
+
await verifyToken(run.token, app.options.runPublicKey);
|
|
251
|
+
logger.debug("verified run token for", run.id);
|
|
252
|
+
} catch (e) {
|
|
253
|
+
logger.error("Error validating run token");
|
|
254
|
+
logger.error(e);
|
|
255
|
+
reject();
|
|
256
|
+
app.destroy();
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
} else {
|
|
260
|
+
logger.debug("skipping run token validation for", run.id);
|
|
261
|
+
}
|
|
127
262
|
logger.debug("starting run", run.id);
|
|
128
263
|
app.execute(run);
|
|
129
264
|
resolve();
|
|
@@ -144,10 +279,15 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
|
|
|
144
279
|
let cancelled = false;
|
|
145
280
|
const workLoop = () => {
|
|
146
281
|
if (!cancelled) {
|
|
147
|
-
promise = try_with_backoff_default(
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
282
|
+
promise = try_with_backoff_default(
|
|
283
|
+
() => claim_default(app, logger, {
|
|
284
|
+
maxWorkers
|
|
285
|
+
}),
|
|
286
|
+
{
|
|
287
|
+
min: minBackoff,
|
|
288
|
+
max: maxBackoff
|
|
289
|
+
}
|
|
290
|
+
);
|
|
151
291
|
promise.then(() => {
|
|
152
292
|
if (!cancelled) {
|
|
153
293
|
setTimeout(workLoop, minBackoff);
|
|
@@ -165,8 +305,8 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
|
|
|
165
305
|
};
|
|
166
306
|
var workloop_default = startWorkloop;
|
|
167
307
|
|
|
168
|
-
// src/util/convert-
|
|
169
|
-
import
|
|
308
|
+
// src/util/convert-lightning-plan.ts
|
|
309
|
+
import crypto2 from "node:crypto";
|
|
170
310
|
var conditions = {
|
|
171
311
|
on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
|
|
172
312
|
on_job_failure: (upstreamId) => `Boolean(state?.errors && state.errors["${upstreamId}"])`,
|
|
@@ -186,19 +326,30 @@ var mapTriggerEdgeCondition = (edge) => {
|
|
|
186
326
|
return true;
|
|
187
327
|
return condition;
|
|
188
328
|
};
|
|
189
|
-
var
|
|
190
|
-
|
|
191
|
-
};
|
|
192
|
-
|
|
193
|
-
|
|
329
|
+
var convert_lightning_plan_default = (run) => {
|
|
330
|
+
const runtimeOpts = {};
|
|
331
|
+
const engineOpts = {};
|
|
332
|
+
if (run.options) {
|
|
333
|
+
if (run.options.runTimeoutMs) {
|
|
334
|
+
engineOpts.runTimeoutMs = run.options.runTimeoutMs;
|
|
335
|
+
}
|
|
336
|
+
if (run.options.sanitize) {
|
|
337
|
+
engineOpts.sanitize = run.options.sanitize;
|
|
338
|
+
}
|
|
339
|
+
if (run.options.hasOwnProperty("output_dataclips")) {
|
|
340
|
+
engineOpts.outputDataclips = run.options.output_dataclips;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
194
343
|
const plan = {
|
|
195
|
-
id: run.id
|
|
344
|
+
id: run.id,
|
|
345
|
+
options: runtimeOpts
|
|
196
346
|
};
|
|
347
|
+
let initialState;
|
|
197
348
|
if (run.dataclip_id) {
|
|
198
|
-
|
|
349
|
+
initialState = run.dataclip_id;
|
|
199
350
|
}
|
|
200
351
|
if (run.starting_node_id) {
|
|
201
|
-
|
|
352
|
+
runtimeOpts.start = run.starting_node_id;
|
|
202
353
|
}
|
|
203
354
|
const nodes = {};
|
|
204
355
|
const edges = run.edges ?? [];
|
|
@@ -210,27 +361,33 @@ var convert_run_default = (run) => {
|
|
|
210
361
|
};
|
|
211
362
|
const connectedEdges = edges.filter((e) => e.source_trigger_id === id);
|
|
212
363
|
if (connectedEdges.length) {
|
|
213
|
-
nodes[id].next = connectedEdges.reduce(
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
364
|
+
nodes[id].next = connectedEdges.reduce(
|
|
365
|
+
(obj, edge) => {
|
|
366
|
+
if (edge.enabled !== false) {
|
|
367
|
+
obj[edge.target_job_id] = mapTriggerEdgeCondition(edge);
|
|
368
|
+
}
|
|
369
|
+
return obj;
|
|
370
|
+
},
|
|
371
|
+
{}
|
|
372
|
+
);
|
|
219
373
|
} else {
|
|
220
374
|
}
|
|
221
375
|
});
|
|
222
376
|
}
|
|
223
377
|
if (run.jobs?.length) {
|
|
224
|
-
run.jobs.forEach((
|
|
225
|
-
const id =
|
|
226
|
-
|
|
378
|
+
run.jobs.forEach((step) => {
|
|
379
|
+
const id = step.id || crypto2.randomUUID();
|
|
380
|
+
const job = {
|
|
227
381
|
id,
|
|
228
|
-
configuration:
|
|
229
|
-
expression:
|
|
230
|
-
adaptor:
|
|
382
|
+
configuration: step.credential || step.credential_id,
|
|
383
|
+
expression: step.body,
|
|
384
|
+
adaptor: step.adaptor
|
|
231
385
|
};
|
|
232
|
-
if (
|
|
233
|
-
|
|
386
|
+
if (step.name) {
|
|
387
|
+
job.name = step.name;
|
|
388
|
+
}
|
|
389
|
+
if (step.state) {
|
|
390
|
+
job.state = step.state;
|
|
234
391
|
}
|
|
235
392
|
const next = edges.filter((e) => e.source_job_id === id).reduce((obj, edge) => {
|
|
236
393
|
const newEdge = {};
|
|
@@ -245,21 +402,32 @@ var convert_run_default = (run) => {
|
|
|
245
402
|
return obj;
|
|
246
403
|
}, {});
|
|
247
404
|
if (Object.keys(next).length) {
|
|
248
|
-
|
|
405
|
+
job.next = next;
|
|
249
406
|
}
|
|
407
|
+
nodes[id] = job;
|
|
250
408
|
});
|
|
251
409
|
}
|
|
252
|
-
plan.
|
|
410
|
+
plan.workflow = {
|
|
411
|
+
steps: Object.values(nodes)
|
|
412
|
+
};
|
|
413
|
+
if (run.name) {
|
|
414
|
+
plan.workflow.name = run.name;
|
|
415
|
+
}
|
|
253
416
|
return {
|
|
254
417
|
plan,
|
|
255
|
-
options:
|
|
418
|
+
options: engineOpts,
|
|
419
|
+
input: initialState || {}
|
|
256
420
|
};
|
|
257
421
|
};
|
|
258
422
|
|
|
259
423
|
// src/util/get-with-reply.ts
|
|
260
|
-
var get_with_reply_default = (channel, event, payload) => new Promise((resolve) => {
|
|
424
|
+
var get_with_reply_default = (channel, event, payload) => new Promise((resolve, reject) => {
|
|
261
425
|
channel.push(event, payload).receive("ok", (evt) => {
|
|
262
426
|
resolve(evt);
|
|
427
|
+
}).receive("error", (e) => {
|
|
428
|
+
reject(e);
|
|
429
|
+
}).receive("timeout", (e) => {
|
|
430
|
+
reject(e);
|
|
263
431
|
});
|
|
264
432
|
});
|
|
265
433
|
|
|
@@ -273,19 +441,20 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
|
|
|
273
441
|
});
|
|
274
442
|
|
|
275
443
|
// src/util/create-run-state.ts
|
|
276
|
-
var create_run_state_default = (plan,
|
|
444
|
+
var create_run_state_default = (plan, input) => {
|
|
277
445
|
const state = {
|
|
278
|
-
plan,
|
|
279
446
|
lastDataclipId: "",
|
|
280
447
|
dataclips: {},
|
|
281
448
|
inputDataclips: {},
|
|
282
449
|
reasons: {},
|
|
283
|
-
|
|
450
|
+
plan,
|
|
451
|
+
input
|
|
284
452
|
};
|
|
285
|
-
if (typeof
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
453
|
+
if (typeof input === "string") {
|
|
454
|
+
const jobs = plan.workflow.steps;
|
|
455
|
+
let startNode = jobs[0];
|
|
456
|
+
if (plan.options.start) {
|
|
457
|
+
startNode = jobs.find(({ id }) => id === plan.options.start);
|
|
289
458
|
}
|
|
290
459
|
const initialRuns = [];
|
|
291
460
|
if (!startNode.expression) {
|
|
@@ -294,15 +463,42 @@ var create_run_state_default = (plan, options = {}) => {
|
|
|
294
463
|
initialRuns.push(startNode.id);
|
|
295
464
|
}
|
|
296
465
|
initialRuns.forEach((id) => {
|
|
297
|
-
state.inputDataclips[id] =
|
|
466
|
+
state.inputDataclips[id] = input;
|
|
298
467
|
});
|
|
299
468
|
} else {
|
|
300
469
|
}
|
|
301
470
|
return state;
|
|
302
471
|
};
|
|
303
472
|
|
|
473
|
+
// src/util/throttle.ts
|
|
474
|
+
var createThrottler = () => {
|
|
475
|
+
const q = [];
|
|
476
|
+
let activePromise;
|
|
477
|
+
const add = (fn) => {
|
|
478
|
+
return (...args) => new Promise((resolve, reject) => {
|
|
479
|
+
q.push({ fn, args, resolve, reject });
|
|
480
|
+
shift();
|
|
481
|
+
});
|
|
482
|
+
};
|
|
483
|
+
const shift = () => {
|
|
484
|
+
if (activePromise) {
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
const next = q.shift();
|
|
488
|
+
if (next) {
|
|
489
|
+
const { fn, args, resolve, reject } = next;
|
|
490
|
+
activePromise = fn(...args).then(resolve).catch(reject).finally(() => {
|
|
491
|
+
activePromise = void 0;
|
|
492
|
+
shift();
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
};
|
|
496
|
+
return add;
|
|
497
|
+
};
|
|
498
|
+
var throttle_default = createThrottler;
|
|
499
|
+
|
|
304
500
|
// src/events/step-complete.ts
|
|
305
|
-
import
|
|
501
|
+
import crypto3 from "node:crypto";
|
|
306
502
|
|
|
307
503
|
// src/api/reasons.ts
|
|
308
504
|
var calculateJobExitReason = (jobId, state = { data: {} }, error) => {
|
|
@@ -328,7 +524,7 @@ var isLeafNode = (state, job) => {
|
|
|
328
524
|
};
|
|
329
525
|
var calculateRunExitReason = (state) => {
|
|
330
526
|
if (state.plan && state.reasons) {
|
|
331
|
-
const leafJobReasons = state.plan.
|
|
527
|
+
const leafJobReasons = state.plan.workflow.steps.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
|
|
332
528
|
const fail = leafJobReasons.find((r) => r && r.reason === "fail");
|
|
333
529
|
if (fail) {
|
|
334
530
|
return fail;
|
|
@@ -338,8 +534,8 @@ var calculateRunExitReason = (state) => {
|
|
|
338
534
|
};
|
|
339
535
|
|
|
340
536
|
// src/events/step-complete.ts
|
|
341
|
-
function onStepComplete({ channel, state }, event, error) {
|
|
342
|
-
const dataclipId =
|
|
537
|
+
function onStepComplete({ channel, state, options }, event, error) {
|
|
538
|
+
const dataclipId = crypto3.randomUUID();
|
|
343
539
|
const step_id = state.activeStep;
|
|
344
540
|
const job_id = state.activeJob;
|
|
345
541
|
if (!state.dataclips) {
|
|
@@ -363,7 +559,6 @@ function onStepComplete({ channel, state }, event, error) {
|
|
|
363
559
|
step_id,
|
|
364
560
|
job_id,
|
|
365
561
|
output_dataclip_id: dataclipId,
|
|
366
|
-
output_dataclip: stringify_default(outputState),
|
|
367
562
|
reason,
|
|
368
563
|
error_message,
|
|
369
564
|
error_type,
|
|
@@ -371,94 +566,34 @@ function onStepComplete({ channel, state }, event, error) {
|
|
|
371
566
|
duration: event.duration,
|
|
372
567
|
thread_id: event.threadId
|
|
373
568
|
};
|
|
569
|
+
if (!options || options.outputDataclips !== false) {
|
|
570
|
+
evt.output_dataclip = stringify_default(outputState);
|
|
571
|
+
}
|
|
374
572
|
return sendEvent(channel, STEP_COMPLETE, evt);
|
|
375
573
|
}
|
|
376
574
|
|
|
377
575
|
// src/events/step-start.ts
|
|
378
|
-
|
|
576
|
+
init_package();
|
|
577
|
+
import crypto4 from "node:crypto";
|
|
379
578
|
import { timestamp } from "@openfn/logger";
|
|
380
579
|
|
|
381
|
-
// package.json
|
|
382
|
-
var package_default = {
|
|
383
|
-
name: "@openfn/ws-worker",
|
|
384
|
-
version: "0.8.0",
|
|
385
|
-
description: "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
386
|
-
main: "dist/index.js",
|
|
387
|
-
type: "module",
|
|
388
|
-
scripts: {
|
|
389
|
-
test: "pnpm ava --serial",
|
|
390
|
-
"test:types": "pnpm tsc --noEmit --project tsconfig.json",
|
|
391
|
-
build: "tsup --config tsup.config.js",
|
|
392
|
-
"build:watch": "pnpm build --watch",
|
|
393
|
-
start: "ts-node-esm --transpile-only src/start.ts",
|
|
394
|
-
"start:prod": "node dist/start.js",
|
|
395
|
-
"start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'",
|
|
396
|
-
pack: "pnpm pack --pack-destination ../../dist"
|
|
397
|
-
},
|
|
398
|
-
bin: {
|
|
399
|
-
worker: "dist/start.js"
|
|
400
|
-
},
|
|
401
|
-
author: "Open Function Group <admin@openfn.org>",
|
|
402
|
-
license: "ISC",
|
|
403
|
-
dependencies: {
|
|
404
|
-
"@koa/router": "^12.0.0",
|
|
405
|
-
"@openfn/engine-multi": "workspace:*",
|
|
406
|
-
"@openfn/logger": "workspace:*",
|
|
407
|
-
"@openfn/runtime": "workspace:*",
|
|
408
|
-
"@types/koa-logger": "^3.1.2",
|
|
409
|
-
"@types/ws": "^8.5.6",
|
|
410
|
-
"fast-safe-stringify": "^2.1.1",
|
|
411
|
-
figures: "^5.0.0",
|
|
412
|
-
"human-id": "^4.1.0",
|
|
413
|
-
jose: "^4.14.6",
|
|
414
|
-
koa: "^2.13.4",
|
|
415
|
-
"koa-bodyparser": "^4.4.0",
|
|
416
|
-
"koa-logger": "^3.2.1",
|
|
417
|
-
phoenix: "^1.7.7",
|
|
418
|
-
ws: "^8.14.1"
|
|
419
|
-
},
|
|
420
|
-
devDependencies: {
|
|
421
|
-
"@openfn/lightning-mock": "workspace:*",
|
|
422
|
-
"@types/koa": "^2.13.5",
|
|
423
|
-
"@types/koa-bodyparser": "^4.3.10",
|
|
424
|
-
"@types/koa__router": "^12.0.1",
|
|
425
|
-
"@types/node": "^18.15.3",
|
|
426
|
-
"@types/nodemon": "1.19.3",
|
|
427
|
-
"@types/phoenix": "^1.6.2",
|
|
428
|
-
"@types/yargs": "^17.0.12",
|
|
429
|
-
ava: "5.1.0",
|
|
430
|
-
nodemon: "3.0.1",
|
|
431
|
-
"ts-node": "^10.9.1",
|
|
432
|
-
tslib: "^2.4.0",
|
|
433
|
-
tsup: "^6.2.3",
|
|
434
|
-
typescript: "^4.6.4",
|
|
435
|
-
yargs: "^17.6.2"
|
|
436
|
-
},
|
|
437
|
-
files: [
|
|
438
|
-
"dist",
|
|
439
|
-
"README.md",
|
|
440
|
-
"CHANGELOG.md"
|
|
441
|
-
]
|
|
442
|
-
};
|
|
443
|
-
|
|
444
580
|
// src/util/versions.ts
|
|
445
581
|
import { mainSymbols } from "figures";
|
|
446
582
|
var { triangleRightSmall: t } = mainSymbols;
|
|
447
583
|
var versions_default = (stepId, versions, adaptor) => {
|
|
448
|
-
let longest = "
|
|
584
|
+
let longest = "worker".length;
|
|
449
585
|
for (const v in versions) {
|
|
450
586
|
longest = Math.max(v.length, longest);
|
|
451
587
|
}
|
|
452
|
-
const { node,
|
|
588
|
+
const { node, worker, ...adaptors } = versions;
|
|
453
589
|
const prefix = (str2) => ` ${t} ${str2.padEnd(longest + 4, " ")}`;
|
|
454
590
|
let str = `Versions for step ${stepId}:
|
|
455
591
|
${prefix("node.js")}${versions.node || "unknown"}
|
|
456
|
-
${prefix("worker")}${versions.worker || "unknown"}
|
|
457
|
-
${prefix("engine")}${versions.engine || "unknown"}`;
|
|
592
|
+
${prefix("worker")}${versions.worker || "unknown"}`;
|
|
458
593
|
if (Object.keys(adaptors).length) {
|
|
459
594
|
let allAdaptors = Object.keys(adaptors);
|
|
460
595
|
if (adaptor) {
|
|
461
|
-
allAdaptors = allAdaptors.filter((
|
|
596
|
+
allAdaptors = allAdaptors.filter((name2) => adaptor.startsWith(name2));
|
|
462
597
|
}
|
|
463
598
|
str += "\n" + allAdaptors.sort().map((adaptorName) => `${prefix(adaptorName)}${adaptors[adaptorName]}`).join("\n");
|
|
464
599
|
}
|
|
@@ -469,9 +604,11 @@ ${prefix("engine")}${versions.engine || "unknown"}`;
|
|
|
469
604
|
async function onStepStart(context, event) {
|
|
470
605
|
const time = (timestamp() - BigInt(1e7)).toString();
|
|
471
606
|
const { channel, state } = context;
|
|
472
|
-
state.activeStep =
|
|
607
|
+
state.activeStep = crypto4.randomUUID();
|
|
473
608
|
state.activeJob = event.jobId;
|
|
474
|
-
const job = state.plan.
|
|
609
|
+
const job = state.plan.workflow.steps.find(
|
|
610
|
+
({ id }) => id === event.jobId
|
|
611
|
+
);
|
|
475
612
|
const input_dataclip_id = state.inputDataclips[event.jobId];
|
|
476
613
|
const versions = {
|
|
477
614
|
worker: package_default.version,
|
|
@@ -549,39 +686,12 @@ async function onRunError(context, event) {
|
|
|
549
686
|
});
|
|
550
687
|
onFinish({ reason });
|
|
551
688
|
} catch (e) {
|
|
552
|
-
logger.error("ERROR in
|
|
689
|
+
logger.error("ERROR in run-error handler:", e.message);
|
|
553
690
|
logger.error(e);
|
|
554
691
|
onFinish({});
|
|
555
692
|
}
|
|
556
693
|
}
|
|
557
694
|
|
|
558
|
-
// src/util/throttle.ts
|
|
559
|
-
var createThrottler = () => {
|
|
560
|
-
const q = [];
|
|
561
|
-
let activePromise;
|
|
562
|
-
const add = (fn) => {
|
|
563
|
-
return (...args) => new Promise((resolve, reject) => {
|
|
564
|
-
q.push({ fn, args, resolve, reject });
|
|
565
|
-
shift();
|
|
566
|
-
});
|
|
567
|
-
};
|
|
568
|
-
const shift = () => {
|
|
569
|
-
if (activePromise) {
|
|
570
|
-
return;
|
|
571
|
-
}
|
|
572
|
-
const next = q.shift();
|
|
573
|
-
if (next) {
|
|
574
|
-
const { fn, args, resolve, reject } = next;
|
|
575
|
-
activePromise = fn(...args).then(resolve).catch(reject).finally(() => {
|
|
576
|
-
activePromise = void 0;
|
|
577
|
-
shift();
|
|
578
|
-
});
|
|
579
|
-
}
|
|
580
|
-
};
|
|
581
|
-
return add;
|
|
582
|
-
};
|
|
583
|
-
var throttle_default = createThrottler;
|
|
584
|
-
|
|
585
695
|
// src/api/execute.ts
|
|
586
696
|
var enc = new TextDecoder("utf-8");
|
|
587
697
|
var eventMap = {
|
|
@@ -591,11 +701,18 @@ var eventMap = {
|
|
|
591
701
|
"workflow-log": RUN_LOG,
|
|
592
702
|
"workflow-complete": RUN_COMPLETE
|
|
593
703
|
};
|
|
594
|
-
function execute(channel, engine, logger, plan, options = {}, onFinish = (_result) => {
|
|
704
|
+
function execute(channel, engine, logger, plan, input, options = {}, onFinish = (_result) => {
|
|
595
705
|
}) {
|
|
596
706
|
logger.info("executing ", plan.id);
|
|
597
|
-
const state = create_run_state_default(plan,
|
|
598
|
-
const context = {
|
|
707
|
+
const state = create_run_state_default(plan, input);
|
|
708
|
+
const context = {
|
|
709
|
+
channel,
|
|
710
|
+
state,
|
|
711
|
+
logger,
|
|
712
|
+
engine,
|
|
713
|
+
options,
|
|
714
|
+
onFinish
|
|
715
|
+
};
|
|
599
716
|
const throttle = throttle_default();
|
|
600
717
|
const addEvent = (eventName, handler) => {
|
|
601
718
|
const wrappedFn = async (event) => {
|
|
@@ -628,17 +745,24 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
|
|
|
628
745
|
const resolvers = {
|
|
629
746
|
credential: (id) => loadCredential(channel, id)
|
|
630
747
|
};
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
748
|
+
setTimeout(async () => {
|
|
749
|
+
let loadedInput = input;
|
|
750
|
+
if (typeof input === "string") {
|
|
751
|
+
logger.debug("loading dataclip", input);
|
|
752
|
+
try {
|
|
753
|
+
loadedInput = await loadDataclip(channel, input);
|
|
754
|
+
logger.success("dataclip loaded");
|
|
755
|
+
} catch (e) {
|
|
756
|
+
return onRunError(context, {
|
|
757
|
+
workflowId: plan.id,
|
|
758
|
+
message: `Failed to load dataclip ${input}${e.message ? `: ${e.message}` : ""}`,
|
|
759
|
+
type: "DataClipError",
|
|
760
|
+
severity: "exception"
|
|
761
|
+
});
|
|
762
|
+
}
|
|
637
763
|
}
|
|
638
|
-
return plan;
|
|
639
|
-
}).then(() => {
|
|
640
764
|
try {
|
|
641
|
-
engine.execute(plan, { resolvers, ...options });
|
|
765
|
+
engine.execute(plan, loadedInput, { resolvers, ...options });
|
|
642
766
|
} catch (e) {
|
|
643
767
|
onRunError(context, {
|
|
644
768
|
workflowId: plan.id,
|
|
@@ -670,7 +794,7 @@ function onJobLog({ channel, state }, event) {
|
|
|
670
794
|
const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
|
|
671
795
|
const log = {
|
|
672
796
|
run_id: state.plan.id,
|
|
673
|
-
message: event.message,
|
|
797
|
+
message: typeof event.message === "string" ? JSON.parse(event.message) : event.message,
|
|
674
798
|
source: event.name,
|
|
675
799
|
level: event.level,
|
|
676
800
|
timestamp: timeInMicroseconds.toString()
|
|
@@ -707,9 +831,9 @@ var joinRunChannel = (socket, token, runId, logger) => {
|
|
|
707
831
|
if (!didReceiveOk) {
|
|
708
832
|
didReceiveOk = true;
|
|
709
833
|
logger.success(`connected to ${channelName}`, e);
|
|
710
|
-
const { plan, options } = await loadRun(channel);
|
|
834
|
+
const { plan, options, input } = await loadRun(channel);
|
|
711
835
|
logger.debug("converted run as execution plan:", plan);
|
|
712
|
-
resolve({ channel, plan, options });
|
|
836
|
+
resolve({ channel, plan, options, input });
|
|
713
837
|
}
|
|
714
838
|
}).receive("error", (err) => {
|
|
715
839
|
logger.error(`error connecting to ${channelName}`, err);
|
|
@@ -720,16 +844,17 @@ var joinRunChannel = (socket, token, runId, logger) => {
|
|
|
720
844
|
var run_default = joinRunChannel;
|
|
721
845
|
async function loadRun(channel) {
|
|
722
846
|
const runBody = await get_with_reply_default(channel, GET_PLAN);
|
|
723
|
-
return
|
|
847
|
+
return convert_lightning_plan_default(runBody);
|
|
724
848
|
}
|
|
725
849
|
|
|
726
850
|
// src/channels/worker-queue.ts
|
|
727
851
|
import EventEmitter from "node:events";
|
|
728
852
|
import { Socket as PhxSocket } from "phoenix";
|
|
729
853
|
import { WebSocket } from "ws";
|
|
854
|
+
import { API_VERSION } from "@openfn/lexicon/lightning";
|
|
730
855
|
|
|
731
856
|
// src/util/worker-token.ts
|
|
732
|
-
import * as
|
|
857
|
+
import * as jose2 from "jose";
|
|
733
858
|
var alg = "HS256";
|
|
734
859
|
var generateWorkerToken = async (secret, workerId, logger) => {
|
|
735
860
|
if (!secret) {
|
|
@@ -744,7 +869,7 @@ var generateWorkerToken = async (secret, workerId, logger) => {
|
|
|
744
869
|
const claims = {
|
|
745
870
|
worker_id: workerId
|
|
746
871
|
};
|
|
747
|
-
const jwt = await new
|
|
872
|
+
const jwt = await new jose2.SignJWT(claims).setProtectedHeader({ alg }).setIssuedAt().setIssuer("urn:openfn:worker").sign(encodedSecret);
|
|
748
873
|
return jwt;
|
|
749
874
|
};
|
|
750
875
|
var worker_token_default = generateWorkerToken;
|
|
@@ -752,9 +877,15 @@ var worker_token_default = generateWorkerToken;
|
|
|
752
877
|
// src/channels/worker-queue.ts
|
|
753
878
|
var connectToWorkerQueue = (endpoint, serverId, secret, logger, SocketConstructor = PhxSocket) => {
|
|
754
879
|
const events = new EventEmitter();
|
|
755
|
-
worker_token_default(secret, serverId, logger).then((token) => {
|
|
880
|
+
worker_token_default(secret, serverId, logger).then(async (token) => {
|
|
881
|
+
const pkg = await Promise.resolve().then(() => (init_package(), package_exports));
|
|
882
|
+
const params = {
|
|
883
|
+
token,
|
|
884
|
+
api_version: API_VERSION,
|
|
885
|
+
worker_version: pkg.default.version
|
|
886
|
+
};
|
|
756
887
|
const socket = new SocketConstructor(endpoint, {
|
|
757
|
-
params
|
|
888
|
+
params,
|
|
758
889
|
transport: WebSocket
|
|
759
890
|
});
|
|
760
891
|
let didOpen = false;
|
|
@@ -793,6 +924,9 @@ var MAX_BACKOFF = 1e3 * 30;
|
|
|
793
924
|
function connect(app, logger, options = {}) {
|
|
794
925
|
logger.debug("Connecting to Lightning at", options.lightning);
|
|
795
926
|
const onConnect = ({ socket, channel }) => {
|
|
927
|
+
if (app.destroyed) {
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
796
930
|
logger.success("Connected to Lightning at", options.lightning);
|
|
797
931
|
app.socket = socket;
|
|
798
932
|
app.queueChannel = channel;
|
|
@@ -827,6 +961,9 @@ function connect(app, logger, options = {}) {
|
|
|
827
961
|
}
|
|
828
962
|
};
|
|
829
963
|
const onError = (e) => {
|
|
964
|
+
if (app.destroyed) {
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
830
967
|
logger.error(
|
|
831
968
|
"CRITICAL ERROR: could not connect to lightning at",
|
|
832
969
|
options.lightning
|
|
@@ -856,13 +993,15 @@ function createServer(engine, options = {}) {
|
|
|
856
993
|
process.send?.("READY");
|
|
857
994
|
router.get("/livez", healthcheck_default);
|
|
858
995
|
router.get("/", healthcheck_default);
|
|
996
|
+
app.options = options || {};
|
|
859
997
|
app.execute = async ({ id, token }) => {
|
|
860
998
|
if (app.socket) {
|
|
861
999
|
app.workflows[id] = true;
|
|
862
1000
|
const {
|
|
863
1001
|
channel: runChannel,
|
|
864
1002
|
plan,
|
|
865
|
-
options: options2
|
|
1003
|
+
options: options2,
|
|
1004
|
+
input
|
|
866
1005
|
} = await run_default(app.socket, token, id, logger);
|
|
867
1006
|
const onFinish = () => {
|
|
868
1007
|
delete app.workflows[id];
|
|
@@ -874,6 +1013,7 @@ function createServer(engine, options = {}) {
|
|
|
874
1013
|
engine,
|
|
875
1014
|
logger,
|
|
876
1015
|
plan,
|
|
1016
|
+
input,
|
|
877
1017
|
options2,
|
|
878
1018
|
onFinish
|
|
879
1019
|
);
|
|
@@ -884,7 +1024,9 @@ function createServer(engine, options = {}) {
|
|
|
884
1024
|
};
|
|
885
1025
|
router.post("/claim", async (ctx) => {
|
|
886
1026
|
logger.info("triggering claim from POST request");
|
|
887
|
-
return claim_default(app, logger,
|
|
1027
|
+
return claim_default(app, logger, {
|
|
1028
|
+
maxWorkers: options.maxWorkflows
|
|
1029
|
+
}).then(() => {
|
|
888
1030
|
logger.info("claim complete: 1 run claimed");
|
|
889
1031
|
ctx.body = "complete";
|
|
890
1032
|
ctx.status = 200;
|