@openfn/ws-worker 1.13.6 → 1.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/dist/index.js +64 -27
- package/dist/start.js +70 -31
- package/package.json +5 -5
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# ws-worker
|
|
2
2
|
|
|
3
|
+
## 1.14.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- d765843: Fix an issue where the server can attempt to claim even while it's waiting to shut down
|
|
8
|
+
- 667e3bf: Improve logging of errors returned by lightning
|
|
9
|
+
- d765843: Fix an issue where the --backoff server argument only accepts integer values
|
|
10
|
+
|
|
11
|
+
## 1.14.0
|
|
12
|
+
|
|
13
|
+
### Minor Changes
|
|
14
|
+
|
|
15
|
+
- 9d4ece3: Add support for global functions in execution plan
|
|
16
|
+
|
|
17
|
+
### Patch Changes
|
|
18
|
+
|
|
19
|
+
- Updated dependencies [9d4ece3]
|
|
20
|
+
- @openfn/runtime@1.7.0
|
|
21
|
+
- @openfn/engine-multi@1.6.7
|
|
22
|
+
|
|
3
23
|
## 1.13.6
|
|
4
24
|
|
|
5
25
|
### Patch Changes
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { EventEmitter as EventEmitter2 } from "node:events";
|
|
3
3
|
import { promisify } from "node:util";
|
|
4
4
|
import { exec as _exec } from "node:child_process";
|
|
5
|
-
import * as
|
|
5
|
+
import * as Sentry6 from "@sentry/node";
|
|
6
6
|
import Koa from "koa";
|
|
7
7
|
import bodyParser from "koa-bodyparser";
|
|
8
8
|
import koaLogger from "koa-logger";
|
|
@@ -82,6 +82,10 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
82
82
|
await fn();
|
|
83
83
|
resolve();
|
|
84
84
|
} catch (e) {
|
|
85
|
+
if (e?.abort) {
|
|
86
|
+
cancelled = true;
|
|
87
|
+
return reject();
|
|
88
|
+
}
|
|
85
89
|
if (opts.isCancelled()) {
|
|
86
90
|
return resolve();
|
|
87
91
|
}
|
|
@@ -111,6 +115,7 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
111
115
|
var try_with_backoff_default = tryWithBackoff;
|
|
112
116
|
|
|
113
117
|
// src/api/claim.ts
|
|
118
|
+
import * as Sentry from "@sentry/node";
|
|
114
119
|
import crypto from "node:crypto";
|
|
115
120
|
import * as jose from "jose";
|
|
116
121
|
import { createMockLogger } from "@openfn/logger";
|
|
@@ -128,6 +133,13 @@ var verifyToken = async (token, publicKey) => {
|
|
|
128
133
|
};
|
|
129
134
|
var { DEPLOYED_POD_NAME, WORKER_NAME } = process.env;
|
|
130
135
|
var NAME = WORKER_NAME || DEPLOYED_POD_NAME;
|
|
136
|
+
var ClaimError = class extends Error {
|
|
137
|
+
constructor(e) {
|
|
138
|
+
super(e);
|
|
139
|
+
// This breaks the parenting backoff loop
|
|
140
|
+
this.abort = true;
|
|
141
|
+
}
|
|
142
|
+
};
|
|
131
143
|
var claim = (app, logger = mockLogger, options = {}) => {
|
|
132
144
|
return new Promise((resolve, reject) => {
|
|
133
145
|
const { maxWorkers = 5 } = options;
|
|
@@ -135,11 +147,17 @@ var claim = (app, logger = mockLogger, options = {}) => {
|
|
|
135
147
|
const activeWorkers = Object.keys(app.workflows).length;
|
|
136
148
|
if (activeWorkers >= maxWorkers) {
|
|
137
149
|
app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
|
|
138
|
-
return reject(new
|
|
150
|
+
return reject(new ClaimError("Server at capacity"));
|
|
139
151
|
}
|
|
140
152
|
if (!app.queueChannel) {
|
|
141
|
-
logger.
|
|
142
|
-
return reject(new
|
|
153
|
+
logger.warn("skipping claim attempt: websocket unavailable");
|
|
154
|
+
return reject(new ClaimError("No websocket available"));
|
|
155
|
+
}
|
|
156
|
+
if (app.queueChannel.state === "closed") {
|
|
157
|
+
const e = new ClaimError("queue closed");
|
|
158
|
+
Sentry.captureException(e);
|
|
159
|
+
logger.warn("skipping claim attempt: channel closed");
|
|
160
|
+
return reject(e);
|
|
143
161
|
}
|
|
144
162
|
logger.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
|
|
145
163
|
const start = Date.now();
|
|
@@ -202,6 +220,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
|
|
|
202
220
|
if (!cancelled) {
|
|
203
221
|
setTimeout(workLoop, minBackoff);
|
|
204
222
|
}
|
|
223
|
+
}).catch(() => {
|
|
205
224
|
});
|
|
206
225
|
}
|
|
207
226
|
};
|
|
@@ -220,7 +239,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
|
|
|
220
239
|
var workloop_default = startWorkloop;
|
|
221
240
|
|
|
222
241
|
// src/api/execute.ts
|
|
223
|
-
import * as
|
|
242
|
+
import * as Sentry3 from "@sentry/node";
|
|
224
243
|
|
|
225
244
|
// src/util/convert-lightning-plan.ts
|
|
226
245
|
import crypto2 from "node:crypto";
|
|
@@ -374,6 +393,8 @@ var convert_lightning_plan_default = (run, options = {}) => {
|
|
|
374
393
|
plan.workflow = {
|
|
375
394
|
steps: Object.values(nodes)
|
|
376
395
|
};
|
|
396
|
+
if (run.globals && typeof run.globals === "string")
|
|
397
|
+
plan.workflow.globals = run.globals;
|
|
377
398
|
if (run.name) {
|
|
378
399
|
plan.workflow.name = run.name;
|
|
379
400
|
}
|
|
@@ -433,12 +454,28 @@ var create_run_state_default = (plan, input) => {
|
|
|
433
454
|
};
|
|
434
455
|
|
|
435
456
|
// src/util/send-event.ts
|
|
436
|
-
import * as
|
|
457
|
+
import * as Sentry2 from "@sentry/node";
|
|
437
458
|
|
|
438
459
|
// src/errors.ts
|
|
460
|
+
function serializeMessage(message) {
|
|
461
|
+
if (typeof message === "string") {
|
|
462
|
+
return message;
|
|
463
|
+
}
|
|
464
|
+
if (message instanceof Error) {
|
|
465
|
+
return message.toString();
|
|
466
|
+
}
|
|
467
|
+
if (message && typeof message === "object") {
|
|
468
|
+
try {
|
|
469
|
+
return JSON.stringify(message);
|
|
470
|
+
} catch {
|
|
471
|
+
return String(message);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
return String(message);
|
|
475
|
+
}
|
|
439
476
|
var LightningSocketError = class extends Error {
|
|
440
477
|
constructor(event, message) {
|
|
441
|
-
super(`[${event}] ${message}`);
|
|
478
|
+
super(`[${event}] ${serializeMessage(message)}`);
|
|
442
479
|
this.name = "LightningSocketError";
|
|
443
480
|
this.event = "";
|
|
444
481
|
this.rejectMessage = "";
|
|
@@ -467,7 +504,7 @@ var sendEvent = (context, event, payload) => {
|
|
|
467
504
|
if (error.rejectMessage) {
|
|
468
505
|
extras.rejection_reason = error.rejectMessage;
|
|
469
506
|
}
|
|
470
|
-
|
|
507
|
+
Sentry2.captureException(error, (scope) => {
|
|
471
508
|
scope.setContext("run", context2);
|
|
472
509
|
scope.setExtras(extras);
|
|
473
510
|
return scope;
|
|
@@ -800,8 +837,8 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
|
|
|
800
837
|
options,
|
|
801
838
|
onFinish
|
|
802
839
|
};
|
|
803
|
-
|
|
804
|
-
|
|
840
|
+
Sentry3.withIsolationScope(async () => {
|
|
841
|
+
Sentry3.addBreadcrumb({
|
|
805
842
|
category: "run",
|
|
806
843
|
message: "Executing run: loading metadata",
|
|
807
844
|
level: "info",
|
|
@@ -813,7 +850,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
|
|
|
813
850
|
const addEvent = (eventName, handler) => {
|
|
814
851
|
const wrappedFn = async (event) => {
|
|
815
852
|
if (eventName !== "workflow-log") {
|
|
816
|
-
|
|
853
|
+
Sentry3.addBreadcrumb({
|
|
817
854
|
category: "event",
|
|
818
855
|
message: eventName,
|
|
819
856
|
level: "info"
|
|
@@ -825,7 +862,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
|
|
|
825
862
|
logger.info(`${plan.id} :: ${lightningEvent} :: OK`);
|
|
826
863
|
} catch (e) {
|
|
827
864
|
if (!e.reportedToSentry) {
|
|
828
|
-
|
|
865
|
+
Sentry3.captureException(e);
|
|
829
866
|
logger.error(e);
|
|
830
867
|
}
|
|
831
868
|
}
|
|
@@ -869,7 +906,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
|
|
|
869
906
|
}
|
|
870
907
|
}
|
|
871
908
|
try {
|
|
872
|
-
|
|
909
|
+
Sentry3.addBreadcrumb({
|
|
873
910
|
category: "run",
|
|
874
911
|
message: "run metadata loaded: starting run",
|
|
875
912
|
level: "info",
|
|
@@ -879,7 +916,7 @@ function execute(channel, engine, logger, plan, input, options = {}, onFinish =
|
|
|
879
916
|
});
|
|
880
917
|
engine.execute(plan, loadedInput, { resolvers, ...options });
|
|
881
918
|
} catch (e) {
|
|
882
|
-
|
|
919
|
+
Sentry3.addBreadcrumb({
|
|
883
920
|
category: "run",
|
|
884
921
|
message: "exception in run",
|
|
885
922
|
level: "info",
|
|
@@ -946,7 +983,7 @@ var healthcheck_default = (ctx) => {
|
|
|
946
983
|
};
|
|
947
984
|
|
|
948
985
|
// src/channels/run.ts
|
|
949
|
-
import * as
|
|
986
|
+
import * as Sentry4 from "@sentry/node";
|
|
950
987
|
var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
|
|
951
988
|
return new Promise((resolve, reject) => {
|
|
952
989
|
let didReceiveOk = false;
|
|
@@ -965,12 +1002,12 @@ var joinRunChannel = (socket, token, runId, logger, timeout = 30) => {
|
|
|
965
1002
|
resolve({ channel, run });
|
|
966
1003
|
}
|
|
967
1004
|
}).receive("error", (err) => {
|
|
968
|
-
|
|
1005
|
+
Sentry4.captureException(err);
|
|
969
1006
|
logger.error(`error connecting to ${channelName}`, err);
|
|
970
1007
|
channel?.leave();
|
|
971
1008
|
reject(err);
|
|
972
1009
|
}).receive("timeout", (err) => {
|
|
973
|
-
|
|
1010
|
+
Sentry4.captureException(err);
|
|
974
1011
|
logger.error(`Timeout for ${channelName}`, err);
|
|
975
1012
|
channel?.leave();
|
|
976
1013
|
reject(err);
|
|
@@ -988,7 +1025,7 @@ var run_default = joinRunChannel;
|
|
|
988
1025
|
|
|
989
1026
|
// src/channels/worker-queue.ts
|
|
990
1027
|
import EventEmitter from "node:events";
|
|
991
|
-
import * as
|
|
1028
|
+
import * as Sentry5 from "@sentry/node";
|
|
992
1029
|
import { Socket as PhxSocket } from "phoenix";
|
|
993
1030
|
import { WebSocket } from "ws";
|
|
994
1031
|
import { API_VERSION } from "@openfn/lexicon/lightning";
|
|
@@ -1017,13 +1054,13 @@ var worker_token_default = generateWorkerToken;
|
|
|
1017
1054
|
// src/channels/worker-queue.ts
|
|
1018
1055
|
var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, SocketConstructor = PhxSocket) => {
|
|
1019
1056
|
const events = new EventEmitter();
|
|
1020
|
-
|
|
1057
|
+
Sentry5.addBreadcrumb({
|
|
1021
1058
|
category: "lifecycle",
|
|
1022
1059
|
message: "Connecting to worker queue",
|
|
1023
1060
|
level: "info"
|
|
1024
1061
|
});
|
|
1025
1062
|
worker_token_default(secret, serverId, logger).then(async (token) => {
|
|
1026
|
-
|
|
1063
|
+
Sentry5.addBreadcrumb({
|
|
1027
1064
|
category: "lifecycle",
|
|
1028
1065
|
message: "Worker token generated",
|
|
1029
1066
|
level: "info"
|
|
@@ -1042,7 +1079,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
|
|
|
1042
1079
|
let didOpen = false;
|
|
1043
1080
|
let shouldReportConnectionError = true;
|
|
1044
1081
|
socket.onOpen(() => {
|
|
1045
|
-
|
|
1082
|
+
Sentry5.addBreadcrumb({
|
|
1046
1083
|
category: "lifecycle",
|
|
1047
1084
|
message: "Web socket connected",
|
|
1048
1085
|
level: "info"
|
|
@@ -1068,7 +1105,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
|
|
|
1068
1105
|
events.emit("disconnect");
|
|
1069
1106
|
});
|
|
1070
1107
|
socket.onError((e) => {
|
|
1071
|
-
|
|
1108
|
+
Sentry5.addBreadcrumb({
|
|
1072
1109
|
category: "lifecycle",
|
|
1073
1110
|
message: "Error in web socket connection",
|
|
1074
1111
|
level: "info"
|
|
@@ -1076,7 +1113,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger, So
|
|
|
1076
1113
|
if (shouldReportConnectionError) {
|
|
1077
1114
|
logger.debug("Reporting connection error to sentry");
|
|
1078
1115
|
shouldReportConnectionError = false;
|
|
1079
|
-
|
|
1116
|
+
Sentry5.captureException(e);
|
|
1080
1117
|
}
|
|
1081
1118
|
if (!didOpen) {
|
|
1082
1119
|
events.emit("error", e.message);
|
|
@@ -1183,11 +1220,11 @@ function createServer(engine, options = {}) {
|
|
|
1183
1220
|
app.events = new EventEmitter2();
|
|
1184
1221
|
app.engine = engine;
|
|
1185
1222
|
if (options.sentryDsn) {
|
|
1186
|
-
|
|
1223
|
+
Sentry6.init({
|
|
1187
1224
|
environment: options.sentryEnv,
|
|
1188
1225
|
dsn: options.sentryDsn
|
|
1189
1226
|
});
|
|
1190
|
-
|
|
1227
|
+
Sentry6.setupKoaErrorHandler(app);
|
|
1191
1228
|
}
|
|
1192
1229
|
app.use(bodyParser());
|
|
1193
1230
|
app.use(
|
|
@@ -1204,7 +1241,7 @@ function createServer(engine, options = {}) {
|
|
|
1204
1241
|
router.get("/", healthcheck_default);
|
|
1205
1242
|
app.options = options;
|
|
1206
1243
|
app.resumeWorkloop = () => {
|
|
1207
|
-
if (options.noLoop) {
|
|
1244
|
+
if (options.noLoop || app.destroyed) {
|
|
1208
1245
|
return;
|
|
1209
1246
|
}
|
|
1210
1247
|
if (!app.workloop || app.workloop?.isStopped()) {
|
|
@@ -1304,7 +1341,7 @@ function createServer(engine, options = {}) {
|
|
|
1304
1341
|
shutdown = true;
|
|
1305
1342
|
logger.always(`${signal} RECEIVED: CLOSING SERVER`);
|
|
1306
1343
|
await app.destroy();
|
|
1307
|
-
process.exit();
|
|
1344
|
+
process.exit(0);
|
|
1308
1345
|
}
|
|
1309
1346
|
};
|
|
1310
1347
|
process.on("SIGINT", () => exit("SIGINT"));
|
package/dist/start.js
CHANGED
|
@@ -142,7 +142,7 @@ var runtime_engine_default = createMock;
|
|
|
142
142
|
import { EventEmitter as EventEmitter3 } from "node:events";
|
|
143
143
|
import { promisify } from "node:util";
|
|
144
144
|
import { exec as _exec } from "node:child_process";
|
|
145
|
-
import * as
|
|
145
|
+
import * as Sentry6 from "@sentry/node";
|
|
146
146
|
import Koa from "koa";
|
|
147
147
|
import bodyParser from "koa-bodyparser";
|
|
148
148
|
import koaLogger from "koa-logger";
|
|
@@ -222,6 +222,10 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
222
222
|
await fn();
|
|
223
223
|
resolve5();
|
|
224
224
|
} catch (e) {
|
|
225
|
+
if (e?.abort) {
|
|
226
|
+
cancelled = true;
|
|
227
|
+
return reject();
|
|
228
|
+
}
|
|
225
229
|
if (opts.isCancelled()) {
|
|
226
230
|
return resolve5();
|
|
227
231
|
}
|
|
@@ -251,6 +255,7 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
251
255
|
var try_with_backoff_default = tryWithBackoff;
|
|
252
256
|
|
|
253
257
|
// src/api/claim.ts
|
|
258
|
+
import * as Sentry from "@sentry/node";
|
|
254
259
|
import crypto2 from "node:crypto";
|
|
255
260
|
import * as jose from "jose";
|
|
256
261
|
import { createMockLogger } from "@openfn/logger";
|
|
@@ -268,6 +273,13 @@ var verifyToken = async (token, publicKey) => {
|
|
|
268
273
|
};
|
|
269
274
|
var { DEPLOYED_POD_NAME, WORKER_NAME } = process.env;
|
|
270
275
|
var NAME = WORKER_NAME || DEPLOYED_POD_NAME;
|
|
276
|
+
var ClaimError = class extends Error {
|
|
277
|
+
constructor(e) {
|
|
278
|
+
super(e);
|
|
279
|
+
// This breaks the parenting backoff loop
|
|
280
|
+
this.abort = true;
|
|
281
|
+
}
|
|
282
|
+
};
|
|
271
283
|
var claim = (app, logger2 = mockLogger, options = {}) => {
|
|
272
284
|
return new Promise((resolve5, reject) => {
|
|
273
285
|
const { maxWorkers = 5 } = options;
|
|
@@ -275,11 +287,17 @@ var claim = (app, logger2 = mockLogger, options = {}) => {
|
|
|
275
287
|
const activeWorkers = Object.keys(app.workflows).length;
|
|
276
288
|
if (activeWorkers >= maxWorkers) {
|
|
277
289
|
app.workloop?.stop(`server at capacity (${activeWorkers}/${maxWorkers})`);
|
|
278
|
-
return reject(new
|
|
290
|
+
return reject(new ClaimError("Server at capacity"));
|
|
279
291
|
}
|
|
280
292
|
if (!app.queueChannel) {
|
|
281
|
-
logger2.
|
|
282
|
-
return reject(new
|
|
293
|
+
logger2.warn("skipping claim attempt: websocket unavailable");
|
|
294
|
+
return reject(new ClaimError("No websocket available"));
|
|
295
|
+
}
|
|
296
|
+
if (app.queueChannel.state === "closed") {
|
|
297
|
+
const e = new ClaimError("queue closed");
|
|
298
|
+
Sentry.captureException(e);
|
|
299
|
+
logger2.warn("skipping claim attempt: channel closed");
|
|
300
|
+
return reject(e);
|
|
283
301
|
}
|
|
284
302
|
logger2.debug(`requesting run (capacity ${activeWorkers}/${maxWorkers})`);
|
|
285
303
|
const start = Date.now();
|
|
@@ -342,6 +360,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
|
|
|
342
360
|
if (!cancelled) {
|
|
343
361
|
setTimeout(workLoop, minBackoff2);
|
|
344
362
|
}
|
|
363
|
+
}).catch(() => {
|
|
345
364
|
});
|
|
346
365
|
}
|
|
347
366
|
};
|
|
@@ -360,7 +379,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
|
|
|
360
379
|
var workloop_default = startWorkloop;
|
|
361
380
|
|
|
362
381
|
// src/api/execute.ts
|
|
363
|
-
import * as
|
|
382
|
+
import * as Sentry3 from "@sentry/node";
|
|
364
383
|
|
|
365
384
|
// src/util/convert-lightning-plan.ts
|
|
366
385
|
import crypto3 from "node:crypto";
|
|
@@ -514,6 +533,8 @@ var convert_lightning_plan_default = (run2, options = {}) => {
|
|
|
514
533
|
plan.workflow = {
|
|
515
534
|
steps: Object.values(nodes)
|
|
516
535
|
};
|
|
536
|
+
if (run2.globals && typeof run2.globals === "string")
|
|
537
|
+
plan.workflow.globals = run2.globals;
|
|
517
538
|
if (run2.name) {
|
|
518
539
|
plan.workflow.name = run2.name;
|
|
519
540
|
}
|
|
@@ -573,12 +594,28 @@ var create_run_state_default = (plan, input) => {
|
|
|
573
594
|
};
|
|
574
595
|
|
|
575
596
|
// src/util/send-event.ts
|
|
576
|
-
import * as
|
|
597
|
+
import * as Sentry2 from "@sentry/node";
|
|
577
598
|
|
|
578
599
|
// src/errors.ts
|
|
600
|
+
function serializeMessage(message) {
|
|
601
|
+
if (typeof message === "string") {
|
|
602
|
+
return message;
|
|
603
|
+
}
|
|
604
|
+
if (message instanceof Error) {
|
|
605
|
+
return message.toString();
|
|
606
|
+
}
|
|
607
|
+
if (message && typeof message === "object") {
|
|
608
|
+
try {
|
|
609
|
+
return JSON.stringify(message);
|
|
610
|
+
} catch {
|
|
611
|
+
return String(message);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
return String(message);
|
|
615
|
+
}
|
|
579
616
|
var LightningSocketError = class extends Error {
|
|
580
617
|
constructor(event, message) {
|
|
581
|
-
super(`[${event}] ${message}`);
|
|
618
|
+
super(`[${event}] ${serializeMessage(message)}`);
|
|
582
619
|
this.name = "LightningSocketError";
|
|
583
620
|
this.event = "";
|
|
584
621
|
this.rejectMessage = "";
|
|
@@ -607,7 +644,7 @@ var sendEvent = (context, event, payload) => {
|
|
|
607
644
|
if (error.rejectMessage) {
|
|
608
645
|
extras.rejection_reason = error.rejectMessage;
|
|
609
646
|
}
|
|
610
|
-
|
|
647
|
+
Sentry2.captureException(error, (scope) => {
|
|
611
648
|
scope.setContext("run", context2);
|
|
612
649
|
scope.setExtras(extras);
|
|
613
650
|
return scope;
|
|
@@ -940,8 +977,8 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
|
|
|
940
977
|
options,
|
|
941
978
|
onFinish
|
|
942
979
|
};
|
|
943
|
-
|
|
944
|
-
|
|
980
|
+
Sentry3.withIsolationScope(async () => {
|
|
981
|
+
Sentry3.addBreadcrumb({
|
|
945
982
|
category: "run",
|
|
946
983
|
message: "Executing run: loading metadata",
|
|
947
984
|
level: "info",
|
|
@@ -953,7 +990,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
|
|
|
953
990
|
const addEvent = (eventName, handler) => {
|
|
954
991
|
const wrappedFn = async (event) => {
|
|
955
992
|
if (eventName !== "workflow-log") {
|
|
956
|
-
|
|
993
|
+
Sentry3.addBreadcrumb({
|
|
957
994
|
category: "event",
|
|
958
995
|
message: eventName,
|
|
959
996
|
level: "info"
|
|
@@ -965,7 +1002,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
|
|
|
965
1002
|
logger2.info(`${plan.id} :: ${lightningEvent} :: OK`);
|
|
966
1003
|
} catch (e) {
|
|
967
1004
|
if (!e.reportedToSentry) {
|
|
968
|
-
|
|
1005
|
+
Sentry3.captureException(e);
|
|
969
1006
|
logger2.error(e);
|
|
970
1007
|
}
|
|
971
1008
|
}
|
|
@@ -1009,7 +1046,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
|
|
|
1009
1046
|
}
|
|
1010
1047
|
}
|
|
1011
1048
|
try {
|
|
1012
|
-
|
|
1049
|
+
Sentry3.addBreadcrumb({
|
|
1013
1050
|
category: "run",
|
|
1014
1051
|
message: "run metadata loaded: starting run",
|
|
1015
1052
|
level: "info",
|
|
@@ -1019,7 +1056,7 @@ function execute(channel, engine, logger2, plan, input, options = {}, onFinish =
|
|
|
1019
1056
|
});
|
|
1020
1057
|
engine.execute(plan, loadedInput, { resolvers, ...options });
|
|
1021
1058
|
} catch (e) {
|
|
1022
|
-
|
|
1059
|
+
Sentry3.addBreadcrumb({
|
|
1023
1060
|
category: "run",
|
|
1024
1061
|
message: "exception in run",
|
|
1025
1062
|
level: "info",
|
|
@@ -1086,7 +1123,7 @@ var healthcheck_default = (ctx) => {
|
|
|
1086
1123
|
};
|
|
1087
1124
|
|
|
1088
1125
|
// src/channels/run.ts
|
|
1089
|
-
import * as
|
|
1126
|
+
import * as Sentry4 from "@sentry/node";
|
|
1090
1127
|
var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
|
|
1091
1128
|
return new Promise((resolve5, reject) => {
|
|
1092
1129
|
let didReceiveOk = false;
|
|
@@ -1105,12 +1142,12 @@ var joinRunChannel = (socket, token, runId, logger2, timeout = 30) => {
|
|
|
1105
1142
|
resolve5({ channel, run: run2 });
|
|
1106
1143
|
}
|
|
1107
1144
|
}).receive("error", (err) => {
|
|
1108
|
-
|
|
1145
|
+
Sentry4.captureException(err);
|
|
1109
1146
|
logger2.error(`error connecting to ${channelName}`, err);
|
|
1110
1147
|
channel?.leave();
|
|
1111
1148
|
reject(err);
|
|
1112
1149
|
}).receive("timeout", (err) => {
|
|
1113
|
-
|
|
1150
|
+
Sentry4.captureException(err);
|
|
1114
1151
|
logger2.error(`Timeout for ${channelName}`, err);
|
|
1115
1152
|
channel?.leave();
|
|
1116
1153
|
reject(err);
|
|
@@ -1128,7 +1165,7 @@ var run_default = joinRunChannel;
|
|
|
1128
1165
|
|
|
1129
1166
|
// src/channels/worker-queue.ts
|
|
1130
1167
|
import EventEmitter2 from "node:events";
|
|
1131
|
-
import * as
|
|
1168
|
+
import * as Sentry5 from "@sentry/node";
|
|
1132
1169
|
import { Socket as PhxSocket } from "phoenix";
|
|
1133
1170
|
import { WebSocket } from "ws";
|
|
1134
1171
|
import { API_VERSION } from "@openfn/lexicon/lightning";
|
|
@@ -1157,13 +1194,13 @@ var worker_token_default = generateWorkerToken;
|
|
|
1157
1194
|
// src/channels/worker-queue.ts
|
|
1158
1195
|
var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, SocketConstructor = PhxSocket) => {
|
|
1159
1196
|
const events = new EventEmitter2();
|
|
1160
|
-
|
|
1197
|
+
Sentry5.addBreadcrumb({
|
|
1161
1198
|
category: "lifecycle",
|
|
1162
1199
|
message: "Connecting to worker queue",
|
|
1163
1200
|
level: "info"
|
|
1164
1201
|
});
|
|
1165
1202
|
worker_token_default(secret, serverId, logger2).then(async (token) => {
|
|
1166
|
-
|
|
1203
|
+
Sentry5.addBreadcrumb({
|
|
1167
1204
|
category: "lifecycle",
|
|
1168
1205
|
message: "Worker token generated",
|
|
1169
1206
|
level: "info"
|
|
@@ -1182,7 +1219,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
|
|
|
1182
1219
|
let didOpen = false;
|
|
1183
1220
|
let shouldReportConnectionError = true;
|
|
1184
1221
|
socket.onOpen(() => {
|
|
1185
|
-
|
|
1222
|
+
Sentry5.addBreadcrumb({
|
|
1186
1223
|
category: "lifecycle",
|
|
1187
1224
|
message: "Web socket connected",
|
|
1188
1225
|
level: "info"
|
|
@@ -1208,7 +1245,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
|
|
|
1208
1245
|
events.emit("disconnect");
|
|
1209
1246
|
});
|
|
1210
1247
|
socket.onError((e) => {
|
|
1211
|
-
|
|
1248
|
+
Sentry5.addBreadcrumb({
|
|
1212
1249
|
category: "lifecycle",
|
|
1213
1250
|
message: "Error in web socket connection",
|
|
1214
1251
|
level: "info"
|
|
@@ -1216,7 +1253,7 @@ var connectToWorkerQueue = (endpoint, serverId, secret, timeout = 10, logger2, S
|
|
|
1216
1253
|
if (shouldReportConnectionError) {
|
|
1217
1254
|
logger2.debug("Reporting connection error to sentry");
|
|
1218
1255
|
shouldReportConnectionError = false;
|
|
1219
|
-
|
|
1256
|
+
Sentry5.captureException(e);
|
|
1220
1257
|
}
|
|
1221
1258
|
if (!didOpen) {
|
|
1222
1259
|
events.emit("error", e.message);
|
|
@@ -1323,11 +1360,11 @@ function createServer(engine, options = {}) {
|
|
|
1323
1360
|
app.events = new EventEmitter3();
|
|
1324
1361
|
app.engine = engine;
|
|
1325
1362
|
if (options.sentryDsn) {
|
|
1326
|
-
|
|
1363
|
+
Sentry6.init({
|
|
1327
1364
|
environment: options.sentryEnv,
|
|
1328
1365
|
dsn: options.sentryDsn
|
|
1329
1366
|
});
|
|
1330
|
-
|
|
1367
|
+
Sentry6.setupKoaErrorHandler(app);
|
|
1331
1368
|
}
|
|
1332
1369
|
app.use(bodyParser());
|
|
1333
1370
|
app.use(
|
|
@@ -1344,7 +1381,7 @@ function createServer(engine, options = {}) {
|
|
|
1344
1381
|
router.get("/", healthcheck_default);
|
|
1345
1382
|
app.options = options;
|
|
1346
1383
|
app.resumeWorkloop = () => {
|
|
1347
|
-
if (options.noLoop) {
|
|
1384
|
+
if (options.noLoop || app.destroyed) {
|
|
1348
1385
|
return;
|
|
1349
1386
|
}
|
|
1350
1387
|
if (!app.workloop || app.workloop?.isStopped()) {
|
|
@@ -1444,7 +1481,7 @@ function createServer(engine, options = {}) {
|
|
|
1444
1481
|
shutdown = true;
|
|
1445
1482
|
logger2.always(`${signal} RECEIVED: CLOSING SERVER`);
|
|
1446
1483
|
await app.destroy();
|
|
1447
|
-
process.exit();
|
|
1484
|
+
process.exit(0);
|
|
1448
1485
|
}
|
|
1449
1486
|
};
|
|
1450
1487
|
process.on("SIGINT", () => exit("SIGINT"));
|
|
@@ -6348,7 +6385,10 @@ function parseArgs(argv) {
|
|
|
6348
6385
|
WORKER_SOCKET_TIMEOUT_SECONDS,
|
|
6349
6386
|
WORKER_STATE_PROPS_TO_REMOVE
|
|
6350
6387
|
} = process.env;
|
|
6351
|
-
const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("
|
|
6388
|
+
const parser2 = yargs_default(hideBin(argv)).command("server", "Start a ws-worker server").option("debug", {
|
|
6389
|
+
hidden: true,
|
|
6390
|
+
type: "boolean"
|
|
6391
|
+
}).option("port", {
|
|
6352
6392
|
alias: "p",
|
|
6353
6393
|
description: `Port to run the server on. Default ${DEFAULT_PORT2}. Env: WORKER_PORT`,
|
|
6354
6394
|
type: "number"
|
|
@@ -6470,11 +6510,11 @@ if (args.lightning === "mock") {
|
|
|
6470
6510
|
if (!args.secret) {
|
|
6471
6511
|
args.secret = "abdefg";
|
|
6472
6512
|
}
|
|
6473
|
-
} else if (!args.secret) {
|
|
6513
|
+
} else if (!args.debug && !args.secret) {
|
|
6474
6514
|
logger.error("WORKER_SECRET is not set");
|
|
6475
6515
|
process.exit(1);
|
|
6476
6516
|
}
|
|
6477
|
-
var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) =>
|
|
6517
|
+
var [minBackoff, maxBackoff] = args.backoff.split("/").map((n) => parseFloat(n) * 1e3);
|
|
6478
6518
|
function engineReady(engine) {
|
|
6479
6519
|
logger.debug("Creating worker instance");
|
|
6480
6520
|
const workerOptions = {
|
|
@@ -6485,7 +6525,6 @@ function engineReady(engine) {
|
|
|
6485
6525
|
sentryDsn: args.sentryDsn,
|
|
6486
6526
|
sentryEnv: args.sentryEnv,
|
|
6487
6527
|
noLoop: !args.loop,
|
|
6488
|
-
// TODO need to feed this through properly
|
|
6489
6528
|
backoff: {
|
|
6490
6529
|
min: minBackoff,
|
|
6491
6530
|
max: maxBackoff
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openfn/ws-worker",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.14.1",
|
|
4
4
|
"description": "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -23,10 +23,10 @@
|
|
|
23
23
|
"koa-logger": "^3.2.1",
|
|
24
24
|
"phoenix": "1.7.10",
|
|
25
25
|
"ws": "^8.18.0",
|
|
26
|
-
"@openfn/engine-multi": "1.6.
|
|
26
|
+
"@openfn/engine-multi": "1.6.7",
|
|
27
27
|
"@openfn/lexicon": "^1.2.2",
|
|
28
|
-
"@openfn/
|
|
29
|
-
"@openfn/
|
|
28
|
+
"@openfn/runtime": "1.7.0",
|
|
29
|
+
"@openfn/logger": "1.0.5"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"@types/koa": "^2.13.5",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"tsup": "^6.2.3",
|
|
44
44
|
"typescript": "^4.6.4",
|
|
45
45
|
"yargs": "^17.6.2",
|
|
46
|
-
"@openfn/lightning-mock": "2.1
|
|
46
|
+
"@openfn/lightning-mock": "2.2.1"
|
|
47
47
|
},
|
|
48
48
|
"files": [
|
|
49
49
|
"dist",
|