@openfn/ws-worker 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +1 -1
- package/dist/index.d.ts +29 -39
- package/dist/index.js +81 -86
- package/dist/start.js +78 -83
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -68,7 +68,7 @@ curl -X POST http://localhost:2222/claim
|
|
|
68
68
|
|
|
69
69
|
## Architecture
|
|
70
70
|
|
|
71
|
-
Lightning is expected to maintain a queue of
|
|
71
|
+
Lightning is expected to maintain a queue of runs. The Worker pulls those runs from the queue, via websocket, and sends them off to the Engine for execution.
|
|
72
72
|
|
|
73
73
|
While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements.
|
|
74
74
|
|
package/dist/index.d.ts
CHANGED
|
@@ -42,8 +42,8 @@ interface Edge {
|
|
|
42
42
|
enabled?: boolean;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
// An
|
|
46
|
-
type
|
|
45
|
+
// An run object returned by Lightning
|
|
46
|
+
type Run = {
|
|
47
47
|
id: string;
|
|
48
48
|
dataclip_id: string;
|
|
49
49
|
starting_node_id: string;
|
|
@@ -52,31 +52,21 @@ type Attempt = {
|
|
|
52
52
|
jobs: Node[];
|
|
53
53
|
edges: Edge[];
|
|
54
54
|
|
|
55
|
-
options?:
|
|
55
|
+
options?: RunOptions;
|
|
56
56
|
};
|
|
57
57
|
|
|
58
|
-
type
|
|
59
|
-
|
|
60
|
-
// Note that this is the NEW terminology, so it's the timeout for the whole "attempt"
|
|
61
|
-
runTimeout?: number;
|
|
62
|
-
|
|
63
|
-
// this is the internal old terminology, which will be deprecated soon
|
|
64
|
-
attemptTimeoutMs?: number;
|
|
65
|
-
|
|
66
|
-
attemptTimeout?: number; // deprecated
|
|
67
|
-
|
|
68
|
-
// deprecated alias for timeout. Maps to "attemptTimeout" internally
|
|
69
|
-
timeout?: number;
|
|
58
|
+
type RunOptions = {
|
|
59
|
+
runTimeoutMs?: number;
|
|
70
60
|
|
|
71
61
|
sanitize?: SanitizePolicies;
|
|
72
62
|
};
|
|
73
63
|
|
|
74
|
-
// Internal server state for each
|
|
75
|
-
type
|
|
64
|
+
// Internal server state for each run
|
|
65
|
+
type RunState = {
|
|
76
66
|
activeStep?: string;
|
|
77
67
|
activeJob?: string;
|
|
78
68
|
plan: ExecutionPlan;
|
|
79
|
-
options:
|
|
69
|
+
options: RunOptions;
|
|
80
70
|
dataclips: Record<string, any>;
|
|
81
71
|
// For each run, map the input ids
|
|
82
72
|
// TODO better name maybe?
|
|
@@ -107,15 +97,15 @@ declare type ClaimPayload = {
|
|
|
107
97
|
demand?: number;
|
|
108
98
|
};
|
|
109
99
|
declare type ClaimReply = {
|
|
110
|
-
|
|
100
|
+
runs: Array<ClaimRun>;
|
|
111
101
|
};
|
|
112
|
-
declare type
|
|
102
|
+
declare type ClaimRun = {
|
|
113
103
|
id: string;
|
|
114
104
|
token: string;
|
|
115
105
|
};
|
|
116
|
-
declare const
|
|
117
|
-
declare type
|
|
118
|
-
declare type
|
|
106
|
+
declare const GET_PLAN = "fetch:plan";
|
|
107
|
+
declare type GetPlanPayload = void;
|
|
108
|
+
declare type GetPlanReply = Run;
|
|
119
109
|
declare const GET_CREDENTIAL = "fetch:credential";
|
|
120
110
|
declare type GetCredentialPayload = {
|
|
121
111
|
id: string;
|
|
@@ -126,48 +116,48 @@ declare type GetDataclipPayload = {
|
|
|
126
116
|
id: string;
|
|
127
117
|
};
|
|
128
118
|
declare type GetDataClipReply = Uint8Array;
|
|
129
|
-
declare const
|
|
130
|
-
declare type
|
|
131
|
-
declare type
|
|
132
|
-
declare const
|
|
133
|
-
declare type
|
|
119
|
+
declare const RUN_START = "run:start";
|
|
120
|
+
declare type RunStartPayload = void;
|
|
121
|
+
declare type RunStartReply = {};
|
|
122
|
+
declare const RUN_COMPLETE = "run:complete";
|
|
123
|
+
declare type RunCompletePayload = ExitReason & {
|
|
134
124
|
final_dataclip_id?: string;
|
|
135
125
|
};
|
|
136
|
-
declare type
|
|
137
|
-
declare const
|
|
138
|
-
declare type
|
|
126
|
+
declare type RunCompleteReply = undefined;
|
|
127
|
+
declare const RUN_LOG = "run:log";
|
|
128
|
+
declare type RunLogPayload = {
|
|
139
129
|
message: Array<string | object>;
|
|
140
130
|
timestamp: string;
|
|
141
|
-
|
|
131
|
+
run_id: string;
|
|
142
132
|
level?: string;
|
|
143
133
|
source?: string;
|
|
144
134
|
job_id?: string;
|
|
145
135
|
step_id?: string;
|
|
146
136
|
};
|
|
147
|
-
declare type
|
|
137
|
+
declare type RunLogReply = void;
|
|
148
138
|
declare const STEP_START = "step:start";
|
|
149
139
|
declare type StepStartPayload = {
|
|
150
140
|
job_id: string;
|
|
151
141
|
step_id: string;
|
|
152
|
-
|
|
142
|
+
run_id?: string;
|
|
153
143
|
input_dataclip_id?: string;
|
|
154
144
|
versions: Record<string, string>;
|
|
155
145
|
};
|
|
156
146
|
declare type StepStartReply = void;
|
|
157
147
|
declare const STEP_COMPLETE = "step:complete";
|
|
158
148
|
declare type StepCompletePayload = ExitReason & {
|
|
159
|
-
|
|
149
|
+
run_id?: string;
|
|
160
150
|
job_id: string;
|
|
161
151
|
step_id: string;
|
|
162
152
|
output_dataclip?: string;
|
|
163
153
|
output_dataclip_id?: string;
|
|
164
154
|
};
|
|
165
155
|
declare type StepCompleteReply = void;
|
|
166
|
-
declare const
|
|
156
|
+
declare const INTERNAL_RUN_COMPLETE = "server:run-complete";
|
|
167
157
|
|
|
168
158
|
declare type Context = {
|
|
169
159
|
channel: Channel;
|
|
170
|
-
state:
|
|
160
|
+
state: RunState;
|
|
171
161
|
logger: Logger;
|
|
172
162
|
engine: RuntimeEngine;
|
|
173
163
|
onFinish: (result: any) => void;
|
|
@@ -194,10 +184,10 @@ interface ServerApp extends Koa {
|
|
|
194
184
|
events: EventEmitter;
|
|
195
185
|
server: Server;
|
|
196
186
|
engine: RuntimeEngine;
|
|
197
|
-
execute: ({ id, token }:
|
|
187
|
+
execute: ({ id, token }: ClaimRun) => Promise<void>;
|
|
198
188
|
destroy: () => void;
|
|
199
189
|
killWorkloop?: () => void;
|
|
200
190
|
}
|
|
201
191
|
declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
|
|
202
192
|
|
|
203
|
-
export {
|
|
193
|
+
export { CLAIM, ClaimPayload, ClaimReply, ClaimRun, GET_CREDENTIAL, GET_DATACLIP, GET_PLAN, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, GetPlanPayload, GetPlanReply, INTERNAL_RUN_COMPLETE, RUN_COMPLETE, RUN_LOG, RUN_START, RunCompletePayload, RunCompleteReply, RunLogPayload, RunLogReply, RunStartPayload, RunStartReply, STEP_COMPLETE, STEP_START, StepCompletePayload, StepCompleteReply, StepStartPayload, StepStartReply, createServer as default };
|
package/dist/index.js
CHANGED
|
@@ -9,15 +9,15 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
|
|
|
9
9
|
|
|
10
10
|
// src/events.ts
|
|
11
11
|
var CLAIM = "claim";
|
|
12
|
-
var
|
|
12
|
+
var GET_PLAN = "fetch:plan";
|
|
13
13
|
var GET_CREDENTIAL = "fetch:credential";
|
|
14
14
|
var GET_DATACLIP = "fetch:dataclip";
|
|
15
|
-
var
|
|
16
|
-
var
|
|
17
|
-
var
|
|
15
|
+
var RUN_START = "run:start";
|
|
16
|
+
var RUN_COMPLETE = "run:complete";
|
|
17
|
+
var RUN_LOG = "run:log";
|
|
18
18
|
var STEP_START = "step:start";
|
|
19
19
|
var STEP_COMPLETE = "step:complete";
|
|
20
|
-
var
|
|
20
|
+
var INTERNAL_RUN_COMPLETE = "server:run-complete";
|
|
21
21
|
|
|
22
22
|
// src/api/destroy.ts
|
|
23
23
|
var destroy = async (app, logger) => {
|
|
@@ -32,7 +32,7 @@ var destroy = async (app, logger) => {
|
|
|
32
32
|
});
|
|
33
33
|
}),
|
|
34
34
|
new Promise(async (resolve) => {
|
|
35
|
-
await
|
|
35
|
+
await waitForRuns(app, logger);
|
|
36
36
|
await app.engine.destroy();
|
|
37
37
|
app.socket?.disconnect();
|
|
38
38
|
resolve();
|
|
@@ -40,16 +40,16 @@ var destroy = async (app, logger) => {
|
|
|
40
40
|
]);
|
|
41
41
|
logger.success("Server closed");
|
|
42
42
|
};
|
|
43
|
-
var
|
|
43
|
+
var waitForRuns = (app, logger) => new Promise((resolve) => {
|
|
44
44
|
const log = () => {
|
|
45
45
|
logger.debug(
|
|
46
|
-
`Waiting for ${Object.keys(app.workflows).length}
|
|
46
|
+
`Waiting for ${Object.keys(app.workflows).length} runs to complete...`
|
|
47
47
|
);
|
|
48
48
|
};
|
|
49
|
-
const
|
|
49
|
+
const onRunComplete = () => {
|
|
50
50
|
if (Object.keys(app.workflows).length === 0) {
|
|
51
|
-
logger.debug("All
|
|
52
|
-
app.events.off(
|
|
51
|
+
logger.debug("All runs completed!");
|
|
52
|
+
app.events.off(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
53
53
|
resolve();
|
|
54
54
|
} else {
|
|
55
55
|
log();
|
|
@@ -57,7 +57,7 @@ var waitForAttempts = (app, logger) => new Promise((resolve) => {
|
|
|
57
57
|
};
|
|
58
58
|
if (Object.keys(app.workflows).length) {
|
|
59
59
|
log();
|
|
60
|
-
app.events.on(
|
|
60
|
+
app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
61
61
|
} else {
|
|
62
62
|
resolve();
|
|
63
63
|
}
|
|
@@ -67,7 +67,7 @@ var destroy_default = destroy;
|
|
|
67
67
|
// src/util/try-with-backoff.ts
|
|
68
68
|
var BACKOFF_MULTIPLIER = 1.15;
|
|
69
69
|
var tryWithBackoff = (fn, opts = {}) => {
|
|
70
|
-
const { min = 1e3, max = 1e4,
|
|
70
|
+
const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
|
|
71
71
|
let cancelled = false;
|
|
72
72
|
if (!opts.isCancelled) {
|
|
73
73
|
opts.isCancelled = () => cancelled;
|
|
@@ -80,16 +80,16 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
80
80
|
if (opts.isCancelled()) {
|
|
81
81
|
return resolve();
|
|
82
82
|
}
|
|
83
|
-
if (!isNaN(
|
|
84
|
-
return reject(new Error("max
|
|
83
|
+
if (!isNaN(maxRuns) && runs >= maxRuns) {
|
|
84
|
+
return reject(new Error("max runs exceeded"));
|
|
85
85
|
}
|
|
86
86
|
setTimeout(() => {
|
|
87
87
|
if (opts.isCancelled()) {
|
|
88
88
|
return resolve();
|
|
89
89
|
}
|
|
90
90
|
const nextOpts = {
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
maxRuns,
|
|
92
|
+
runs: runs + 1,
|
|
93
93
|
min: Math.min(max, min * BACKOFF_MULTIPLIER),
|
|
94
94
|
max,
|
|
95
95
|
isCancelled: opts.isCancelled
|
|
@@ -117,15 +117,15 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
|
|
|
117
117
|
if (!app.queueChannel) {
|
|
118
118
|
return reject(new Error("No websocket available"));
|
|
119
119
|
}
|
|
120
|
-
logger.debug("requesting
|
|
121
|
-
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({
|
|
122
|
-
logger.debug(`pulled ${
|
|
123
|
-
if (!
|
|
124
|
-
return reject(new Error("No
|
|
120
|
+
logger.debug("requesting run...");
|
|
121
|
+
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
|
|
122
|
+
logger.debug(`pulled ${runs.length} runs`);
|
|
123
|
+
if (!runs?.length) {
|
|
124
|
+
return reject(new Error("No runs returned"));
|
|
125
125
|
}
|
|
126
|
-
|
|
127
|
-
logger.debug("starting
|
|
128
|
-
app.execute(
|
|
126
|
+
runs.forEach((run) => {
|
|
127
|
+
logger.debug("starting run", run.id);
|
|
128
|
+
app.execute(run);
|
|
129
129
|
resolve();
|
|
130
130
|
});
|
|
131
131
|
}).receive("error", () => {
|
|
@@ -165,7 +165,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
|
|
|
165
165
|
};
|
|
166
166
|
var workloop_default = startWorkloop;
|
|
167
167
|
|
|
168
|
-
// src/util/convert-
|
|
168
|
+
// src/util/convert-run.ts
|
|
169
169
|
import crypto from "node:crypto";
|
|
170
170
|
var conditions = {
|
|
171
171
|
on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
|
|
@@ -187,28 +187,23 @@ var mapTriggerEdgeCondition = (edge) => {
|
|
|
187
187
|
return condition;
|
|
188
188
|
};
|
|
189
189
|
var mapOptions = (options) => {
|
|
190
|
-
|
|
191
|
-
const to = runTimeout || attemptTimeout || timeout;
|
|
192
|
-
if (to) {
|
|
193
|
-
opts.attemptTimeoutMs = to;
|
|
194
|
-
}
|
|
195
|
-
return opts;
|
|
190
|
+
return options;
|
|
196
191
|
};
|
|
197
|
-
var
|
|
198
|
-
const options =
|
|
192
|
+
var convert_run_default = (run) => {
|
|
193
|
+
const options = run.options || {};
|
|
199
194
|
const plan = {
|
|
200
|
-
id:
|
|
195
|
+
id: run.id
|
|
201
196
|
};
|
|
202
|
-
if (
|
|
203
|
-
plan.initialState =
|
|
197
|
+
if (run.dataclip_id) {
|
|
198
|
+
plan.initialState = run.dataclip_id;
|
|
204
199
|
}
|
|
205
|
-
if (
|
|
206
|
-
plan.start =
|
|
200
|
+
if (run.starting_node_id) {
|
|
201
|
+
plan.start = run.starting_node_id;
|
|
207
202
|
}
|
|
208
203
|
const nodes = {};
|
|
209
|
-
const edges =
|
|
210
|
-
if (
|
|
211
|
-
|
|
204
|
+
const edges = run.edges ?? [];
|
|
205
|
+
if (run.triggers?.length) {
|
|
206
|
+
run.triggers.forEach((trigger) => {
|
|
212
207
|
const id = trigger.id || "trigger";
|
|
213
208
|
nodes[id] = {
|
|
214
209
|
id
|
|
@@ -225,8 +220,8 @@ var convert_attempt_default = (attempt) => {
|
|
|
225
220
|
}
|
|
226
221
|
});
|
|
227
222
|
}
|
|
228
|
-
if (
|
|
229
|
-
|
|
223
|
+
if (run.jobs?.length) {
|
|
224
|
+
run.jobs.forEach((job) => {
|
|
230
225
|
const id = job.id || crypto.randomUUID();
|
|
231
226
|
nodes[id] = {
|
|
232
227
|
id,
|
|
@@ -277,8 +272,8 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
|
|
|
277
272
|
return value;
|
|
278
273
|
});
|
|
279
274
|
|
|
280
|
-
// src/util/create-
|
|
281
|
-
var
|
|
275
|
+
// src/util/create-run-state.ts
|
|
276
|
+
var create_run_state_default = (plan, options = {}) => {
|
|
282
277
|
const state = {
|
|
283
278
|
plan,
|
|
284
279
|
lastDataclipId: "",
|
|
@@ -331,7 +326,7 @@ var isLeafNode = (state, job) => {
|
|
|
331
326
|
const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
|
|
332
327
|
return !hasDownstream;
|
|
333
328
|
};
|
|
334
|
-
var
|
|
329
|
+
var calculateRunExitReason = (state) => {
|
|
335
330
|
if (state.plan && state.reasons) {
|
|
336
331
|
const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
|
|
337
332
|
const fail = leafJobReasons.find((r) => r && r.reason === "fail");
|
|
@@ -386,7 +381,7 @@ import { timestamp } from "@openfn/logger";
|
|
|
386
381
|
// package.json
|
|
387
382
|
var package_default = {
|
|
388
383
|
name: "@openfn/ws-worker",
|
|
389
|
-
version: "0.
|
|
384
|
+
version: "0.8.0",
|
|
390
385
|
description: "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
391
386
|
main: "dist/index.js",
|
|
392
387
|
type: "module",
|
|
@@ -526,21 +521,21 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
|
|
|
526
521
|
});
|
|
527
522
|
};
|
|
528
523
|
|
|
529
|
-
// src/events/
|
|
524
|
+
// src/events/run-complete.ts
|
|
530
525
|
async function onWorkflowComplete(context, _event) {
|
|
531
526
|
const { state, channel, onFinish } = context;
|
|
532
527
|
const result = state.dataclips[state.lastDataclipId];
|
|
533
|
-
const reason =
|
|
528
|
+
const reason = calculateRunExitReason(state);
|
|
534
529
|
await log_final_reason_default(context, reason);
|
|
535
|
-
await sendEvent(channel,
|
|
530
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
536
531
|
final_dataclip_id: state.lastDataclipId,
|
|
537
532
|
...reason
|
|
538
533
|
});
|
|
539
534
|
onFinish({ reason, state: result });
|
|
540
535
|
}
|
|
541
536
|
|
|
542
|
-
// src/events/
|
|
543
|
-
async function
|
|
537
|
+
// src/events/run-error.ts
|
|
538
|
+
async function onRunError(context, event) {
|
|
544
539
|
const { state, channel, logger, onFinish } = context;
|
|
545
540
|
try {
|
|
546
541
|
const reason = calculateJobExitReason("", { data: {} }, event);
|
|
@@ -548,7 +543,7 @@ async function onAttemptError(context, event) {
|
|
|
548
543
|
await onJobError(context, { error: event });
|
|
549
544
|
}
|
|
550
545
|
await log_final_reason_default(context, reason);
|
|
551
|
-
await sendEvent(channel,
|
|
546
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
552
547
|
final_dataclip_id: state.lastDataclipId,
|
|
553
548
|
...reason
|
|
554
549
|
});
|
|
@@ -590,16 +585,16 @@ var throttle_default = createThrottler;
|
|
|
590
585
|
// src/api/execute.ts
|
|
591
586
|
var enc = new TextDecoder("utf-8");
|
|
592
587
|
var eventMap = {
|
|
593
|
-
"workflow-start":
|
|
588
|
+
"workflow-start": RUN_START,
|
|
594
589
|
"job-start": STEP_START,
|
|
595
590
|
"job-complete": STEP_COMPLETE,
|
|
596
|
-
"workflow-log":
|
|
597
|
-
"workflow-complete":
|
|
591
|
+
"workflow-log": RUN_LOG,
|
|
592
|
+
"workflow-complete": RUN_COMPLETE
|
|
598
593
|
};
|
|
599
594
|
function execute(channel, engine, logger, plan, options = {}, onFinish = (_result) => {
|
|
600
595
|
}) {
|
|
601
596
|
logger.info("executing ", plan.id);
|
|
602
|
-
const state =
|
|
597
|
+
const state = create_run_state_default(plan, options);
|
|
603
598
|
const context = { channel, state, logger, engine, onFinish };
|
|
604
599
|
const throttle = throttle_default();
|
|
605
600
|
const addEvent = (eventName, handler) => {
|
|
@@ -627,7 +622,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
|
|
|
627
622
|
addEvent("job-error", throttle(onJobError)),
|
|
628
623
|
addEvent("workflow-log", throttle(onJobLog)),
|
|
629
624
|
addEvent("workflow-complete", throttle(onWorkflowComplete)),
|
|
630
|
-
addEvent("workflow-error", throttle(
|
|
625
|
+
addEvent("workflow-error", throttle(onRunError))
|
|
631
626
|
);
|
|
632
627
|
engine.listen(plan.id, listeners);
|
|
633
628
|
const resolvers = {
|
|
@@ -645,7 +640,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
|
|
|
645
640
|
try {
|
|
646
641
|
engine.execute(plan, { resolvers, ...options });
|
|
647
642
|
} catch (e) {
|
|
648
|
-
|
|
643
|
+
onRunError(context, {
|
|
649
644
|
workflowId: plan.id,
|
|
650
645
|
message: e.message,
|
|
651
646
|
type: e.type,
|
|
@@ -669,12 +664,12 @@ function onJobError(context, event) {
|
|
|
669
664
|
}
|
|
670
665
|
}
|
|
671
666
|
function onWorkflowStart({ channel }, _event) {
|
|
672
|
-
return sendEvent(channel,
|
|
667
|
+
return sendEvent(channel, RUN_START);
|
|
673
668
|
}
|
|
674
669
|
function onJobLog({ channel, state }, event) {
|
|
675
670
|
const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
|
|
676
671
|
const log = {
|
|
677
|
-
|
|
672
|
+
run_id: state.plan.id,
|
|
678
673
|
message: event.message,
|
|
679
674
|
source: event.name,
|
|
680
675
|
level: event.level,
|
|
@@ -683,7 +678,7 @@ function onJobLog({ channel, state }, event) {
|
|
|
683
678
|
if (state.activeStep) {
|
|
684
679
|
log.step_id = state.activeStep;
|
|
685
680
|
}
|
|
686
|
-
return sendEvent(channel,
|
|
681
|
+
return sendEvent(channel, RUN_LOG, log);
|
|
687
682
|
}
|
|
688
683
|
async function loadDataclip(channel, stateId) {
|
|
689
684
|
const result = await get_with_reply_default(channel, GET_DATACLIP, {
|
|
@@ -701,19 +696,19 @@ var healthcheck_default = (ctx) => {
|
|
|
701
696
|
ctx.status = 200;
|
|
702
697
|
};
|
|
703
698
|
|
|
704
|
-
// src/channels/
|
|
705
|
-
var
|
|
699
|
+
// src/channels/run.ts
|
|
700
|
+
var joinRunChannel = (socket, token, runId, logger) => {
|
|
706
701
|
return new Promise((resolve, reject) => {
|
|
707
702
|
let didReceiveOk = false;
|
|
708
|
-
const channelName = `
|
|
703
|
+
const channelName = `run:${runId}`;
|
|
709
704
|
logger.debug("connecting to ", channelName);
|
|
710
705
|
const channel = socket.channel(channelName, { token });
|
|
711
706
|
channel.join().receive("ok", async (e) => {
|
|
712
707
|
if (!didReceiveOk) {
|
|
713
708
|
didReceiveOk = true;
|
|
714
709
|
logger.success(`connected to ${channelName}`, e);
|
|
715
|
-
const { plan, options } = await
|
|
716
|
-
logger.debug("converted
|
|
710
|
+
const { plan, options } = await loadRun(channel);
|
|
711
|
+
logger.debug("converted run as execution plan:", plan);
|
|
717
712
|
resolve({ channel, plan, options });
|
|
718
713
|
}
|
|
719
714
|
}).receive("error", (err) => {
|
|
@@ -722,10 +717,10 @@ var joinAttemptChannel = (socket, token, attemptId, logger) => {
|
|
|
722
717
|
});
|
|
723
718
|
});
|
|
724
719
|
};
|
|
725
|
-
var
|
|
726
|
-
async function
|
|
727
|
-
const
|
|
728
|
-
return
|
|
720
|
+
var run_default = joinRunChannel;
|
|
721
|
+
async function loadRun(channel) {
|
|
722
|
+
const runBody = await get_with_reply_default(channel, GET_PLAN);
|
|
723
|
+
return convert_run_default(runBody);
|
|
729
724
|
}
|
|
730
725
|
|
|
731
726
|
// src/channels/worker-queue.ts
|
|
@@ -741,7 +736,7 @@ var generateWorkerToken = async (secret, workerId, logger) => {
|
|
|
741
736
|
logger.warn();
|
|
742
737
|
logger.warn("WARNING: Worker Secret not provided!");
|
|
743
738
|
logger.warn(
|
|
744
|
-
"This worker will
|
|
739
|
+
"This worker will try to connect to Lightning with default secret"
|
|
745
740
|
);
|
|
746
741
|
logger.warn();
|
|
747
742
|
}
|
|
@@ -865,17 +860,17 @@ function createServer(engine, options = {}) {
|
|
|
865
860
|
if (app.socket) {
|
|
866
861
|
app.workflows[id] = true;
|
|
867
862
|
const {
|
|
868
|
-
channel:
|
|
863
|
+
channel: runChannel,
|
|
869
864
|
plan,
|
|
870
865
|
options: options2
|
|
871
|
-
} = await
|
|
866
|
+
} = await run_default(app.socket, token, id, logger);
|
|
872
867
|
const onFinish = () => {
|
|
873
868
|
delete app.workflows[id];
|
|
874
|
-
|
|
875
|
-
app.events.emit(
|
|
869
|
+
runChannel.leave();
|
|
870
|
+
app.events.emit(INTERNAL_RUN_COMPLETE);
|
|
876
871
|
};
|
|
877
872
|
const context = execute(
|
|
878
|
-
|
|
873
|
+
runChannel,
|
|
879
874
|
engine,
|
|
880
875
|
logger,
|
|
881
876
|
plan,
|
|
@@ -890,12 +885,12 @@ function createServer(engine, options = {}) {
|
|
|
890
885
|
router.post("/claim", async (ctx) => {
|
|
891
886
|
logger.info("triggering claim from POST request");
|
|
892
887
|
return claim_default(app, logger, options.maxWorkflows).then(() => {
|
|
893
|
-
logger.info("claim complete: 1
|
|
888
|
+
logger.info("claim complete: 1 run claimed");
|
|
894
889
|
ctx.body = "complete";
|
|
895
890
|
ctx.status = 200;
|
|
896
891
|
}).catch(() => {
|
|
897
|
-
logger.info("claim complete: no
|
|
898
|
-
ctx.body = "no
|
|
892
|
+
logger.info("claim complete: no runs");
|
|
893
|
+
ctx.body = "no runs";
|
|
899
894
|
ctx.status = 204;
|
|
900
895
|
});
|
|
901
896
|
});
|
|
@@ -927,14 +922,14 @@ var server_default = createServer;
|
|
|
927
922
|
// src/index.ts
|
|
928
923
|
var src_default = server_default;
|
|
929
924
|
export {
|
|
930
|
-
ATTEMPT_COMPLETE,
|
|
931
|
-
ATTEMPT_LOG,
|
|
932
|
-
ATTEMPT_START,
|
|
933
925
|
CLAIM,
|
|
934
|
-
GET_ATTEMPT,
|
|
935
926
|
GET_CREDENTIAL,
|
|
936
927
|
GET_DATACLIP,
|
|
937
|
-
|
|
928
|
+
GET_PLAN,
|
|
929
|
+
INTERNAL_RUN_COMPLETE,
|
|
930
|
+
RUN_COMPLETE,
|
|
931
|
+
RUN_LOG,
|
|
932
|
+
RUN_START,
|
|
938
933
|
STEP_COMPLETE,
|
|
939
934
|
STEP_START,
|
|
940
935
|
src_default as default
|
package/dist/start.js
CHANGED
|
@@ -4998,15 +4998,15 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
|
|
|
4998
4998
|
|
|
4999
4999
|
// src/events.ts
|
|
5000
5000
|
var CLAIM = "claim";
|
|
5001
|
-
var
|
|
5001
|
+
var GET_PLAN = "fetch:plan";
|
|
5002
5002
|
var GET_CREDENTIAL = "fetch:credential";
|
|
5003
5003
|
var GET_DATACLIP = "fetch:dataclip";
|
|
5004
|
-
var
|
|
5005
|
-
var
|
|
5006
|
-
var
|
|
5004
|
+
var RUN_START = "run:start";
|
|
5005
|
+
var RUN_COMPLETE = "run:complete";
|
|
5006
|
+
var RUN_LOG = "run:log";
|
|
5007
5007
|
var STEP_START = "step:start";
|
|
5008
5008
|
var STEP_COMPLETE = "step:complete";
|
|
5009
|
-
var
|
|
5009
|
+
var INTERNAL_RUN_COMPLETE = "server:run-complete";
|
|
5010
5010
|
|
|
5011
5011
|
// src/api/destroy.ts
|
|
5012
5012
|
var destroy = async (app, logger2) => {
|
|
@@ -5021,7 +5021,7 @@ var destroy = async (app, logger2) => {
|
|
|
5021
5021
|
});
|
|
5022
5022
|
}),
|
|
5023
5023
|
new Promise(async (resolve5) => {
|
|
5024
|
-
await
|
|
5024
|
+
await waitForRuns(app, logger2);
|
|
5025
5025
|
await app.engine.destroy();
|
|
5026
5026
|
app.socket?.disconnect();
|
|
5027
5027
|
resolve5();
|
|
@@ -5029,16 +5029,16 @@ var destroy = async (app, logger2) => {
|
|
|
5029
5029
|
]);
|
|
5030
5030
|
logger2.success("Server closed");
|
|
5031
5031
|
};
|
|
5032
|
-
var
|
|
5032
|
+
var waitForRuns = (app, logger2) => new Promise((resolve5) => {
|
|
5033
5033
|
const log = () => {
|
|
5034
5034
|
logger2.debug(
|
|
5035
|
-
`Waiting for ${Object.keys(app.workflows).length}
|
|
5035
|
+
`Waiting for ${Object.keys(app.workflows).length} runs to complete...`
|
|
5036
5036
|
);
|
|
5037
5037
|
};
|
|
5038
|
-
const
|
|
5038
|
+
const onRunComplete = () => {
|
|
5039
5039
|
if (Object.keys(app.workflows).length === 0) {
|
|
5040
|
-
logger2.debug("All
|
|
5041
|
-
app.events.off(
|
|
5040
|
+
logger2.debug("All runs completed!");
|
|
5041
|
+
app.events.off(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
5042
5042
|
resolve5();
|
|
5043
5043
|
} else {
|
|
5044
5044
|
log();
|
|
@@ -5046,7 +5046,7 @@ var waitForAttempts = (app, logger2) => new Promise((resolve5) => {
|
|
|
5046
5046
|
};
|
|
5047
5047
|
if (Object.keys(app.workflows).length) {
|
|
5048
5048
|
log();
|
|
5049
|
-
app.events.on(
|
|
5049
|
+
app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
5050
5050
|
} else {
|
|
5051
5051
|
resolve5();
|
|
5052
5052
|
}
|
|
@@ -5056,7 +5056,7 @@ var destroy_default = destroy;
|
|
|
5056
5056
|
// src/util/try-with-backoff.ts
|
|
5057
5057
|
var BACKOFF_MULTIPLIER = 1.15;
|
|
5058
5058
|
var tryWithBackoff = (fn, opts = {}) => {
|
|
5059
|
-
const { min = 1e3, max = 1e4,
|
|
5059
|
+
const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
|
|
5060
5060
|
let cancelled = false;
|
|
5061
5061
|
if (!opts.isCancelled) {
|
|
5062
5062
|
opts.isCancelled = () => cancelled;
|
|
@@ -5069,16 +5069,16 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
5069
5069
|
if (opts.isCancelled()) {
|
|
5070
5070
|
return resolve5();
|
|
5071
5071
|
}
|
|
5072
|
-
if (!isNaN(
|
|
5073
|
-
return reject(new Error("max
|
|
5072
|
+
if (!isNaN(maxRuns) && runs >= maxRuns) {
|
|
5073
|
+
return reject(new Error("max runs exceeded"));
|
|
5074
5074
|
}
|
|
5075
5075
|
setTimeout(() => {
|
|
5076
5076
|
if (opts.isCancelled()) {
|
|
5077
5077
|
return resolve5();
|
|
5078
5078
|
}
|
|
5079
5079
|
const nextOpts = {
|
|
5080
|
-
|
|
5081
|
-
|
|
5080
|
+
maxRuns,
|
|
5081
|
+
runs: runs + 1,
|
|
5082
5082
|
min: Math.min(max, min * BACKOFF_MULTIPLIER),
|
|
5083
5083
|
max,
|
|
5084
5084
|
isCancelled: opts.isCancelled
|
|
@@ -5106,15 +5106,15 @@ var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
|
|
|
5106
5106
|
if (!app.queueChannel) {
|
|
5107
5107
|
return reject(new Error("No websocket available"));
|
|
5108
5108
|
}
|
|
5109
|
-
logger2.debug("requesting
|
|
5110
|
-
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({
|
|
5111
|
-
logger2.debug(`pulled ${
|
|
5112
|
-
if (!
|
|
5113
|
-
return reject(new Error("No
|
|
5109
|
+
logger2.debug("requesting run...");
|
|
5110
|
+
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
|
|
5111
|
+
logger2.debug(`pulled ${runs.length} runs`);
|
|
5112
|
+
if (!runs?.length) {
|
|
5113
|
+
return reject(new Error("No runs returned"));
|
|
5114
5114
|
}
|
|
5115
|
-
|
|
5116
|
-
logger2.debug("starting
|
|
5117
|
-
app.execute(
|
|
5115
|
+
runs.forEach((run2) => {
|
|
5116
|
+
logger2.debug("starting run", run2.id);
|
|
5117
|
+
app.execute(run2);
|
|
5118
5118
|
resolve5();
|
|
5119
5119
|
});
|
|
5120
5120
|
}).receive("error", () => {
|
|
@@ -5154,7 +5154,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
|
|
|
5154
5154
|
};
|
|
5155
5155
|
var workloop_default = startWorkloop;
|
|
5156
5156
|
|
|
5157
|
-
// src/util/convert-
|
|
5157
|
+
// src/util/convert-run.ts
|
|
5158
5158
|
import crypto2 from "node:crypto";
|
|
5159
5159
|
var conditions = {
|
|
5160
5160
|
on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
|
|
@@ -5176,28 +5176,23 @@ var mapTriggerEdgeCondition = (edge) => {
|
|
|
5176
5176
|
return condition;
|
|
5177
5177
|
};
|
|
5178
5178
|
var mapOptions = (options) => {
|
|
5179
|
-
|
|
5180
|
-
const to = runTimeout || attemptTimeout || timeout;
|
|
5181
|
-
if (to) {
|
|
5182
|
-
opts.attemptTimeoutMs = to;
|
|
5183
|
-
}
|
|
5184
|
-
return opts;
|
|
5179
|
+
return options;
|
|
5185
5180
|
};
|
|
5186
|
-
var
|
|
5187
|
-
const options =
|
|
5181
|
+
var convert_run_default = (run2) => {
|
|
5182
|
+
const options = run2.options || {};
|
|
5188
5183
|
const plan = {
|
|
5189
|
-
id:
|
|
5184
|
+
id: run2.id
|
|
5190
5185
|
};
|
|
5191
|
-
if (
|
|
5192
|
-
plan.initialState =
|
|
5186
|
+
if (run2.dataclip_id) {
|
|
5187
|
+
plan.initialState = run2.dataclip_id;
|
|
5193
5188
|
}
|
|
5194
|
-
if (
|
|
5195
|
-
plan.start =
|
|
5189
|
+
if (run2.starting_node_id) {
|
|
5190
|
+
plan.start = run2.starting_node_id;
|
|
5196
5191
|
}
|
|
5197
5192
|
const nodes = {};
|
|
5198
|
-
const edges =
|
|
5199
|
-
if (
|
|
5200
|
-
|
|
5193
|
+
const edges = run2.edges ?? [];
|
|
5194
|
+
if (run2.triggers?.length) {
|
|
5195
|
+
run2.triggers.forEach((trigger) => {
|
|
5201
5196
|
const id = trigger.id || "trigger";
|
|
5202
5197
|
nodes[id] = {
|
|
5203
5198
|
id
|
|
@@ -5214,8 +5209,8 @@ var convert_attempt_default = (attempt) => {
|
|
|
5214
5209
|
}
|
|
5215
5210
|
});
|
|
5216
5211
|
}
|
|
5217
|
-
if (
|
|
5218
|
-
|
|
5212
|
+
if (run2.jobs?.length) {
|
|
5213
|
+
run2.jobs.forEach((job) => {
|
|
5219
5214
|
const id = job.id || crypto2.randomUUID();
|
|
5220
5215
|
nodes[id] = {
|
|
5221
5216
|
id,
|
|
@@ -5266,8 +5261,8 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
|
|
|
5266
5261
|
return value;
|
|
5267
5262
|
});
|
|
5268
5263
|
|
|
5269
|
-
// src/util/create-
|
|
5270
|
-
var
|
|
5264
|
+
// src/util/create-run-state.ts
|
|
5265
|
+
var create_run_state_default = (plan, options = {}) => {
|
|
5271
5266
|
const state = {
|
|
5272
5267
|
plan,
|
|
5273
5268
|
lastDataclipId: "",
|
|
@@ -5320,7 +5315,7 @@ var isLeafNode = (state, job) => {
|
|
|
5320
5315
|
const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
|
|
5321
5316
|
return !hasDownstream;
|
|
5322
5317
|
};
|
|
5323
|
-
var
|
|
5318
|
+
var calculateRunExitReason = (state) => {
|
|
5324
5319
|
if (state.plan && state.reasons) {
|
|
5325
5320
|
const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
|
|
5326
5321
|
const fail = leafJobReasons.find((r) => r && r.reason === "fail");
|
|
@@ -5375,7 +5370,7 @@ import { timestamp } from "@openfn/logger";
|
|
|
5375
5370
|
// package.json
|
|
5376
5371
|
var package_default = {
|
|
5377
5372
|
name: "@openfn/ws-worker",
|
|
5378
|
-
version: "0.
|
|
5373
|
+
version: "0.8.0",
|
|
5379
5374
|
description: "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
5380
5375
|
main: "dist/index.js",
|
|
5381
5376
|
type: "module",
|
|
@@ -5515,21 +5510,21 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
|
|
|
5515
5510
|
});
|
|
5516
5511
|
};
|
|
5517
5512
|
|
|
5518
|
-
// src/events/
|
|
5513
|
+
// src/events/run-complete.ts
|
|
5519
5514
|
async function onWorkflowComplete(context, _event) {
|
|
5520
5515
|
const { state, channel, onFinish } = context;
|
|
5521
5516
|
const result = state.dataclips[state.lastDataclipId];
|
|
5522
|
-
const reason =
|
|
5517
|
+
const reason = calculateRunExitReason(state);
|
|
5523
5518
|
await log_final_reason_default(context, reason);
|
|
5524
|
-
await sendEvent(channel,
|
|
5519
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
5525
5520
|
final_dataclip_id: state.lastDataclipId,
|
|
5526
5521
|
...reason
|
|
5527
5522
|
});
|
|
5528
5523
|
onFinish({ reason, state: result });
|
|
5529
5524
|
}
|
|
5530
5525
|
|
|
5531
|
-
// src/events/
|
|
5532
|
-
async function
|
|
5526
|
+
// src/events/run-error.ts
|
|
5527
|
+
async function onRunError(context, event) {
|
|
5533
5528
|
const { state, channel, logger: logger2, onFinish } = context;
|
|
5534
5529
|
try {
|
|
5535
5530
|
const reason = calculateJobExitReason("", { data: {} }, event);
|
|
@@ -5537,7 +5532,7 @@ async function onAttemptError(context, event) {
|
|
|
5537
5532
|
await onJobError(context, { error: event });
|
|
5538
5533
|
}
|
|
5539
5534
|
await log_final_reason_default(context, reason);
|
|
5540
|
-
await sendEvent(channel,
|
|
5535
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
5541
5536
|
final_dataclip_id: state.lastDataclipId,
|
|
5542
5537
|
...reason
|
|
5543
5538
|
});
|
|
@@ -5579,16 +5574,16 @@ var throttle_default = createThrottler;
|
|
|
5579
5574
|
// src/api/execute.ts
|
|
5580
5575
|
var enc = new TextDecoder("utf-8");
|
|
5581
5576
|
var eventMap = {
|
|
5582
|
-
"workflow-start":
|
|
5577
|
+
"workflow-start": RUN_START,
|
|
5583
5578
|
"job-start": STEP_START,
|
|
5584
5579
|
"job-complete": STEP_COMPLETE,
|
|
5585
|
-
"workflow-log":
|
|
5586
|
-
"workflow-complete":
|
|
5580
|
+
"workflow-log": RUN_LOG,
|
|
5581
|
+
"workflow-complete": RUN_COMPLETE
|
|
5587
5582
|
};
|
|
5588
5583
|
function execute(channel, engine, logger2, plan, options = {}, onFinish = (_result) => {
|
|
5589
5584
|
}) {
|
|
5590
5585
|
logger2.info("executing ", plan.id);
|
|
5591
|
-
const state =
|
|
5586
|
+
const state = create_run_state_default(plan, options);
|
|
5592
5587
|
const context = { channel, state, logger: logger2, engine, onFinish };
|
|
5593
5588
|
const throttle = throttle_default();
|
|
5594
5589
|
const addEvent = (eventName, handler) => {
|
|
@@ -5616,7 +5611,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
|
|
|
5616
5611
|
addEvent("job-error", throttle(onJobError)),
|
|
5617
5612
|
addEvent("workflow-log", throttle(onJobLog)),
|
|
5618
5613
|
addEvent("workflow-complete", throttle(onWorkflowComplete)),
|
|
5619
|
-
addEvent("workflow-error", throttle(
|
|
5614
|
+
addEvent("workflow-error", throttle(onRunError))
|
|
5620
5615
|
);
|
|
5621
5616
|
engine.listen(plan.id, listeners);
|
|
5622
5617
|
const resolvers = {
|
|
@@ -5634,7 +5629,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
|
|
|
5634
5629
|
try {
|
|
5635
5630
|
engine.execute(plan, { resolvers, ...options });
|
|
5636
5631
|
} catch (e) {
|
|
5637
|
-
|
|
5632
|
+
onRunError(context, {
|
|
5638
5633
|
workflowId: plan.id,
|
|
5639
5634
|
message: e.message,
|
|
5640
5635
|
type: e.type,
|
|
@@ -5658,12 +5653,12 @@ function onJobError(context, event) {
|
|
|
5658
5653
|
}
|
|
5659
5654
|
}
|
|
5660
5655
|
function onWorkflowStart({ channel }, _event) {
|
|
5661
|
-
return sendEvent(channel,
|
|
5656
|
+
return sendEvent(channel, RUN_START);
|
|
5662
5657
|
}
|
|
5663
5658
|
function onJobLog({ channel, state }, event) {
|
|
5664
5659
|
const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
|
|
5665
5660
|
const log = {
|
|
5666
|
-
|
|
5661
|
+
run_id: state.plan.id,
|
|
5667
5662
|
message: event.message,
|
|
5668
5663
|
source: event.name,
|
|
5669
5664
|
level: event.level,
|
|
@@ -5672,7 +5667,7 @@ function onJobLog({ channel, state }, event) {
|
|
|
5672
5667
|
if (state.activeStep) {
|
|
5673
5668
|
log.step_id = state.activeStep;
|
|
5674
5669
|
}
|
|
5675
|
-
return sendEvent(channel,
|
|
5670
|
+
return sendEvent(channel, RUN_LOG, log);
|
|
5676
5671
|
}
|
|
5677
5672
|
async function loadDataclip(channel, stateId) {
|
|
5678
5673
|
const result = await get_with_reply_default(channel, GET_DATACLIP, {
|
|
@@ -5690,19 +5685,19 @@ var healthcheck_default = (ctx) => {
|
|
|
5690
5685
|
ctx.status = 200;
|
|
5691
5686
|
};
|
|
5692
5687
|
|
|
5693
|
-
// src/channels/
|
|
5694
|
-
var
|
|
5688
|
+
// src/channels/run.ts
|
|
5689
|
+
var joinRunChannel = (socket, token, runId, logger2) => {
|
|
5695
5690
|
return new Promise((resolve5, reject) => {
|
|
5696
5691
|
let didReceiveOk = false;
|
|
5697
|
-
const channelName = `
|
|
5692
|
+
const channelName = `run:${runId}`;
|
|
5698
5693
|
logger2.debug("connecting to ", channelName);
|
|
5699
5694
|
const channel = socket.channel(channelName, { token });
|
|
5700
5695
|
channel.join().receive("ok", async (e) => {
|
|
5701
5696
|
if (!didReceiveOk) {
|
|
5702
5697
|
didReceiveOk = true;
|
|
5703
5698
|
logger2.success(`connected to ${channelName}`, e);
|
|
5704
|
-
const { plan, options } = await
|
|
5705
|
-
logger2.debug("converted
|
|
5699
|
+
const { plan, options } = await loadRun(channel);
|
|
5700
|
+
logger2.debug("converted run as execution plan:", plan);
|
|
5706
5701
|
resolve5({ channel, plan, options });
|
|
5707
5702
|
}
|
|
5708
5703
|
}).receive("error", (err) => {
|
|
@@ -5711,10 +5706,10 @@ var joinAttemptChannel = (socket, token, attemptId, logger2) => {
|
|
|
5711
5706
|
});
|
|
5712
5707
|
});
|
|
5713
5708
|
};
|
|
5714
|
-
var
|
|
5715
|
-
async function
|
|
5716
|
-
const
|
|
5717
|
-
return
|
|
5709
|
+
var run_default = joinRunChannel;
|
|
5710
|
+
async function loadRun(channel) {
|
|
5711
|
+
const runBody = await get_with_reply_default(channel, GET_PLAN);
|
|
5712
|
+
return convert_run_default(runBody);
|
|
5718
5713
|
}
|
|
5719
5714
|
|
|
5720
5715
|
// src/channels/worker-queue.ts
|
|
@@ -5730,7 +5725,7 @@ var generateWorkerToken = async (secret, workerId, logger2) => {
|
|
|
5730
5725
|
logger2.warn();
|
|
5731
5726
|
logger2.warn("WARNING: Worker Secret not provided!");
|
|
5732
5727
|
logger2.warn(
|
|
5733
|
-
"This worker will
|
|
5728
|
+
"This worker will try to connect to Lightning with default secret"
|
|
5734
5729
|
);
|
|
5735
5730
|
logger2.warn();
|
|
5736
5731
|
}
|
|
@@ -5854,17 +5849,17 @@ function createServer(engine, options = {}) {
|
|
|
5854
5849
|
if (app.socket) {
|
|
5855
5850
|
app.workflows[id] = true;
|
|
5856
5851
|
const {
|
|
5857
|
-
channel:
|
|
5852
|
+
channel: runChannel,
|
|
5858
5853
|
plan,
|
|
5859
5854
|
options: options2
|
|
5860
|
-
} = await
|
|
5855
|
+
} = await run_default(app.socket, token, id, logger2);
|
|
5861
5856
|
const onFinish = () => {
|
|
5862
5857
|
delete app.workflows[id];
|
|
5863
|
-
|
|
5864
|
-
app.events.emit(
|
|
5858
|
+
runChannel.leave();
|
|
5859
|
+
app.events.emit(INTERNAL_RUN_COMPLETE);
|
|
5865
5860
|
};
|
|
5866
5861
|
const context = execute(
|
|
5867
|
-
|
|
5862
|
+
runChannel,
|
|
5868
5863
|
engine,
|
|
5869
5864
|
logger2,
|
|
5870
5865
|
plan,
|
|
@@ -5879,12 +5874,12 @@ function createServer(engine, options = {}) {
|
|
|
5879
5874
|
router.post("/claim", async (ctx) => {
|
|
5880
5875
|
logger2.info("triggering claim from POST request");
|
|
5881
5876
|
return claim_default(app, logger2, options.maxWorkflows).then(() => {
|
|
5882
|
-
logger2.info("claim complete: 1
|
|
5877
|
+
logger2.info("claim complete: 1 run claimed");
|
|
5883
5878
|
ctx.body = "complete";
|
|
5884
5879
|
ctx.status = 200;
|
|
5885
5880
|
}).catch(() => {
|
|
5886
|
-
logger2.info("claim complete: no
|
|
5887
|
-
ctx.body = "no
|
|
5881
|
+
logger2.info("claim complete: no runs");
|
|
5882
|
+
ctx.body = "no runs";
|
|
5888
5883
|
ctx.status = 204;
|
|
5889
5884
|
});
|
|
5890
5885
|
});
|
|
@@ -5972,7 +5967,7 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
|
|
|
5972
5967
|
default: WORKER_MAX_RUN_MEMORY_MB ? parseInt(WORKER_MAX_RUN_MEMORY_MB) : 500
|
|
5973
5968
|
}).option("max-run-duration-seconds", {
|
|
5974
5969
|
alias: "t",
|
|
5975
|
-
description: "Default
|
|
5970
|
+
description: "Default run timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
|
|
5976
5971
|
type: "number",
|
|
5977
5972
|
default: WORKER_MAX_RUN_DURATION_SECONDS || 60 * 5
|
|
5978
5973
|
}).parse();
|
|
@@ -6016,7 +6011,7 @@ if (args.mock) {
|
|
|
6016
6011
|
memoryLimitMb: args.runMemory,
|
|
6017
6012
|
maxWorkers: args.capacity,
|
|
6018
6013
|
statePropsToRemove: args.statePropsToRemove,
|
|
6019
|
-
|
|
6014
|
+
runTimeoutMs: args.maxRunDurationSeconds * 1e3
|
|
6020
6015
|
};
|
|
6021
6016
|
logger.debug("Creating runtime engine...");
|
|
6022
6017
|
logger.debug("Engine options:", engineOptions);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openfn/ws-worker",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"koa-logger": "^3.2.1",
|
|
23
23
|
"phoenix": "^1.7.7",
|
|
24
24
|
"ws": "^8.14.1",
|
|
25
|
-
"@openfn/engine-multi": "0.
|
|
25
|
+
"@openfn/engine-multi": "0.4.0",
|
|
26
26
|
"@openfn/logger": "0.0.19",
|
|
27
27
|
"@openfn/runtime": "0.2.5"
|
|
28
28
|
},
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"tsup": "^6.2.3",
|
|
42
42
|
"typescript": "^4.6.4",
|
|
43
43
|
"yargs": "^17.6.2",
|
|
44
|
-
"@openfn/lightning-mock": "1.
|
|
44
|
+
"@openfn/lightning-mock": "1.2.0"
|
|
45
45
|
},
|
|
46
46
|
"files": [
|
|
47
47
|
"dist",
|