@openfn/ws-worker 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +1 -1
- package/dist/index.d.ts +39 -49
- package/dist/index.js +106 -111
- package/dist/start.js +103 -108
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# ws-worker
|
|
2
2
|
|
|
3
|
+
## 0.8.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 7e4c159: Rename attempts to runs
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- Updated dependencies [7e4c159]
|
|
12
|
+
- @openfn/engine-multi@0.4.0
|
|
13
|
+
|
|
14
|
+
## 0.7.0
|
|
15
|
+
|
|
16
|
+
### Minor Changes
|
|
17
|
+
|
|
18
|
+
- 39af8e1: Ensure that we refer to the child of a 'run' (aka attempt) as a 'step'
|
|
19
|
+
|
|
3
20
|
## 0.6.0
|
|
4
21
|
|
|
5
22
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -68,7 +68,7 @@ curl -X POST http://localhost:2222/claim
|
|
|
68
68
|
|
|
69
69
|
## Architecture
|
|
70
70
|
|
|
71
|
-
Lightning is expected to maintain a queue of
|
|
71
|
+
Lightning is expected to maintain a queue of runs. The Worker pulls those runs from the queue, via websocket, and sends them off to the Engine for execution.
|
|
72
72
|
|
|
73
73
|
While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements.
|
|
74
74
|
|
package/dist/index.d.ts
CHANGED
|
@@ -42,8 +42,8 @@ interface Edge {
|
|
|
42
42
|
enabled?: boolean;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
// An
|
|
46
|
-
type
|
|
45
|
+
// An run object returned by Lightning
|
|
46
|
+
type Run = {
|
|
47
47
|
id: string;
|
|
48
48
|
dataclip_id: string;
|
|
49
49
|
starting_node_id: string;
|
|
@@ -52,31 +52,21 @@ type Attempt = {
|
|
|
52
52
|
jobs: Node[];
|
|
53
53
|
edges: Edge[];
|
|
54
54
|
|
|
55
|
-
options?:
|
|
55
|
+
options?: RunOptions;
|
|
56
56
|
};
|
|
57
57
|
|
|
58
|
-
type
|
|
59
|
-
|
|
60
|
-
// Note that this is the NEW terminology, so it's the timeout for the whole "attempt"
|
|
61
|
-
runTimeout?: number;
|
|
62
|
-
|
|
63
|
-
// this is the internal old terminology, which will be deprecated soon
|
|
64
|
-
attemptTimeoutMs?: number;
|
|
65
|
-
|
|
66
|
-
attemptTimeout?: number; // deprecated
|
|
67
|
-
|
|
68
|
-
// deprecated alias for timeout. Maps to "attemptTimeout" internally
|
|
69
|
-
timeout?: number;
|
|
58
|
+
type RunOptions = {
|
|
59
|
+
runTimeoutMs?: number;
|
|
70
60
|
|
|
71
61
|
sanitize?: SanitizePolicies;
|
|
72
62
|
};
|
|
73
63
|
|
|
74
|
-
// Internal server state for each
|
|
75
|
-
type
|
|
76
|
-
|
|
64
|
+
// Internal server state for each run
|
|
65
|
+
type RunState = {
|
|
66
|
+
activeStep?: string;
|
|
77
67
|
activeJob?: string;
|
|
78
68
|
plan: ExecutionPlan;
|
|
79
|
-
options:
|
|
69
|
+
options: RunOptions;
|
|
80
70
|
dataclips: Record<string, any>;
|
|
81
71
|
// For each run, map the input ids
|
|
82
72
|
// TODO better name maybe?
|
|
@@ -107,15 +97,15 @@ declare type ClaimPayload = {
|
|
|
107
97
|
demand?: number;
|
|
108
98
|
};
|
|
109
99
|
declare type ClaimReply = {
|
|
110
|
-
|
|
100
|
+
runs: Array<ClaimRun>;
|
|
111
101
|
};
|
|
112
|
-
declare type
|
|
102
|
+
declare type ClaimRun = {
|
|
113
103
|
id: string;
|
|
114
104
|
token: string;
|
|
115
105
|
};
|
|
116
|
-
declare const
|
|
117
|
-
declare type
|
|
118
|
-
declare type
|
|
106
|
+
declare const GET_PLAN = "fetch:plan";
|
|
107
|
+
declare type GetPlanPayload = void;
|
|
108
|
+
declare type GetPlanReply = Run;
|
|
119
109
|
declare const GET_CREDENTIAL = "fetch:credential";
|
|
120
110
|
declare type GetCredentialPayload = {
|
|
121
111
|
id: string;
|
|
@@ -126,48 +116,48 @@ declare type GetDataclipPayload = {
|
|
|
126
116
|
id: string;
|
|
127
117
|
};
|
|
128
118
|
declare type GetDataClipReply = Uint8Array;
|
|
129
|
-
declare const
|
|
130
|
-
declare type
|
|
131
|
-
declare type
|
|
132
|
-
declare const
|
|
133
|
-
declare type
|
|
119
|
+
declare const RUN_START = "run:start";
|
|
120
|
+
declare type RunStartPayload = void;
|
|
121
|
+
declare type RunStartReply = {};
|
|
122
|
+
declare const RUN_COMPLETE = "run:complete";
|
|
123
|
+
declare type RunCompletePayload = ExitReason & {
|
|
134
124
|
final_dataclip_id?: string;
|
|
135
125
|
};
|
|
136
|
-
declare type
|
|
137
|
-
declare const
|
|
138
|
-
declare type
|
|
126
|
+
declare type RunCompleteReply = undefined;
|
|
127
|
+
declare const RUN_LOG = "run:log";
|
|
128
|
+
declare type RunLogPayload = {
|
|
139
129
|
message: Array<string | object>;
|
|
140
130
|
timestamp: string;
|
|
141
|
-
|
|
131
|
+
run_id: string;
|
|
142
132
|
level?: string;
|
|
143
133
|
source?: string;
|
|
144
134
|
job_id?: string;
|
|
145
|
-
|
|
135
|
+
step_id?: string;
|
|
146
136
|
};
|
|
147
|
-
declare type
|
|
148
|
-
declare const
|
|
149
|
-
declare type
|
|
137
|
+
declare type RunLogReply = void;
|
|
138
|
+
declare const STEP_START = "step:start";
|
|
139
|
+
declare type StepStartPayload = {
|
|
150
140
|
job_id: string;
|
|
151
|
-
|
|
152
|
-
|
|
141
|
+
step_id: string;
|
|
142
|
+
run_id?: string;
|
|
153
143
|
input_dataclip_id?: string;
|
|
154
144
|
versions: Record<string, string>;
|
|
155
145
|
};
|
|
156
|
-
declare type
|
|
157
|
-
declare const
|
|
158
|
-
declare type
|
|
159
|
-
|
|
146
|
+
declare type StepStartReply = void;
|
|
147
|
+
declare const STEP_COMPLETE = "step:complete";
|
|
148
|
+
declare type StepCompletePayload = ExitReason & {
|
|
149
|
+
run_id?: string;
|
|
160
150
|
job_id: string;
|
|
161
|
-
|
|
151
|
+
step_id: string;
|
|
162
152
|
output_dataclip?: string;
|
|
163
153
|
output_dataclip_id?: string;
|
|
164
154
|
};
|
|
165
|
-
declare type
|
|
166
|
-
declare const
|
|
155
|
+
declare type StepCompleteReply = void;
|
|
156
|
+
declare const INTERNAL_RUN_COMPLETE = "server:run-complete";
|
|
167
157
|
|
|
168
158
|
declare type Context = {
|
|
169
159
|
channel: Channel;
|
|
170
|
-
state:
|
|
160
|
+
state: RunState;
|
|
171
161
|
logger: Logger;
|
|
172
162
|
engine: RuntimeEngine;
|
|
173
163
|
onFinish: (result: any) => void;
|
|
@@ -194,10 +184,10 @@ interface ServerApp extends Koa {
|
|
|
194
184
|
events: EventEmitter;
|
|
195
185
|
server: Server;
|
|
196
186
|
engine: RuntimeEngine;
|
|
197
|
-
execute: ({ id, token }:
|
|
187
|
+
execute: ({ id, token }: ClaimRun) => Promise<void>;
|
|
198
188
|
destroy: () => void;
|
|
199
189
|
killWorkloop?: () => void;
|
|
200
190
|
}
|
|
201
191
|
declare function createServer(engine: RuntimeEngine, options?: ServerOptions): ServerApp;
|
|
202
192
|
|
|
203
|
-
export {
|
|
193
|
+
export { CLAIM, ClaimPayload, ClaimReply, ClaimRun, GET_CREDENTIAL, GET_DATACLIP, GET_PLAN, GetCredentialPayload, GetCredentialReply, GetDataClipReply, GetDataclipPayload, GetPlanPayload, GetPlanReply, INTERNAL_RUN_COMPLETE, RUN_COMPLETE, RUN_LOG, RUN_START, RunCompletePayload, RunCompleteReply, RunLogPayload, RunLogReply, RunStartPayload, RunStartReply, STEP_COMPLETE, STEP_START, StepCompletePayload, StepCompleteReply, StepStartPayload, StepStartReply, createServer as default };
|
package/dist/index.js
CHANGED
|
@@ -9,15 +9,15 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
|
|
|
9
9
|
|
|
10
10
|
// src/events.ts
|
|
11
11
|
var CLAIM = "claim";
|
|
12
|
-
var
|
|
12
|
+
var GET_PLAN = "fetch:plan";
|
|
13
13
|
var GET_CREDENTIAL = "fetch:credential";
|
|
14
14
|
var GET_DATACLIP = "fetch:dataclip";
|
|
15
|
-
var ATTEMPT_START = "attempt:start";
|
|
16
|
-
var ATTEMPT_COMPLETE = "attempt:complete";
|
|
17
|
-
var ATTEMPT_LOG = "attempt:log";
|
|
18
15
|
var RUN_START = "run:start";
|
|
19
16
|
var RUN_COMPLETE = "run:complete";
|
|
20
|
-
var
|
|
17
|
+
var RUN_LOG = "run:log";
|
|
18
|
+
var STEP_START = "step:start";
|
|
19
|
+
var STEP_COMPLETE = "step:complete";
|
|
20
|
+
var INTERNAL_RUN_COMPLETE = "server:run-complete";
|
|
21
21
|
|
|
22
22
|
// src/api/destroy.ts
|
|
23
23
|
var destroy = async (app, logger) => {
|
|
@@ -32,7 +32,7 @@ var destroy = async (app, logger) => {
|
|
|
32
32
|
});
|
|
33
33
|
}),
|
|
34
34
|
new Promise(async (resolve) => {
|
|
35
|
-
await
|
|
35
|
+
await waitForRuns(app, logger);
|
|
36
36
|
await app.engine.destroy();
|
|
37
37
|
app.socket?.disconnect();
|
|
38
38
|
resolve();
|
|
@@ -40,16 +40,16 @@ var destroy = async (app, logger) => {
|
|
|
40
40
|
]);
|
|
41
41
|
logger.success("Server closed");
|
|
42
42
|
};
|
|
43
|
-
var
|
|
43
|
+
var waitForRuns = (app, logger) => new Promise((resolve) => {
|
|
44
44
|
const log = () => {
|
|
45
45
|
logger.debug(
|
|
46
|
-
`Waiting for ${Object.keys(app.workflows).length}
|
|
46
|
+
`Waiting for ${Object.keys(app.workflows).length} runs to complete...`
|
|
47
47
|
);
|
|
48
48
|
};
|
|
49
|
-
const
|
|
49
|
+
const onRunComplete = () => {
|
|
50
50
|
if (Object.keys(app.workflows).length === 0) {
|
|
51
|
-
logger.debug("All
|
|
52
|
-
app.events.off(
|
|
51
|
+
logger.debug("All runs completed!");
|
|
52
|
+
app.events.off(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
53
53
|
resolve();
|
|
54
54
|
} else {
|
|
55
55
|
log();
|
|
@@ -57,7 +57,7 @@ var waitForAttempts = (app, logger) => new Promise((resolve) => {
|
|
|
57
57
|
};
|
|
58
58
|
if (Object.keys(app.workflows).length) {
|
|
59
59
|
log();
|
|
60
|
-
app.events.on(
|
|
60
|
+
app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
61
61
|
} else {
|
|
62
62
|
resolve();
|
|
63
63
|
}
|
|
@@ -67,7 +67,7 @@ var destroy_default = destroy;
|
|
|
67
67
|
// src/util/try-with-backoff.ts
|
|
68
68
|
var BACKOFF_MULTIPLIER = 1.15;
|
|
69
69
|
var tryWithBackoff = (fn, opts = {}) => {
|
|
70
|
-
const { min = 1e3, max = 1e4,
|
|
70
|
+
const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
|
|
71
71
|
let cancelled = false;
|
|
72
72
|
if (!opts.isCancelled) {
|
|
73
73
|
opts.isCancelled = () => cancelled;
|
|
@@ -80,16 +80,16 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
80
80
|
if (opts.isCancelled()) {
|
|
81
81
|
return resolve();
|
|
82
82
|
}
|
|
83
|
-
if (!isNaN(
|
|
84
|
-
return reject(new Error("max
|
|
83
|
+
if (!isNaN(maxRuns) && runs >= maxRuns) {
|
|
84
|
+
return reject(new Error("max runs exceeded"));
|
|
85
85
|
}
|
|
86
86
|
setTimeout(() => {
|
|
87
87
|
if (opts.isCancelled()) {
|
|
88
88
|
return resolve();
|
|
89
89
|
}
|
|
90
90
|
const nextOpts = {
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
maxRuns,
|
|
92
|
+
runs: runs + 1,
|
|
93
93
|
min: Math.min(max, min * BACKOFF_MULTIPLIER),
|
|
94
94
|
max,
|
|
95
95
|
isCancelled: opts.isCancelled
|
|
@@ -117,15 +117,15 @@ var claim = (app, logger = mockLogger, maxWorkers = 5) => {
|
|
|
117
117
|
if (!app.queueChannel) {
|
|
118
118
|
return reject(new Error("No websocket available"));
|
|
119
119
|
}
|
|
120
|
-
logger.debug("requesting
|
|
121
|
-
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({
|
|
122
|
-
logger.debug(`pulled ${
|
|
123
|
-
if (!
|
|
124
|
-
return reject(new Error("No
|
|
120
|
+
logger.debug("requesting run...");
|
|
121
|
+
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
|
|
122
|
+
logger.debug(`pulled ${runs.length} runs`);
|
|
123
|
+
if (!runs?.length) {
|
|
124
|
+
return reject(new Error("No runs returned"));
|
|
125
125
|
}
|
|
126
|
-
|
|
127
|
-
logger.debug("starting
|
|
128
|
-
app.execute(
|
|
126
|
+
runs.forEach((run) => {
|
|
127
|
+
logger.debug("starting run", run.id);
|
|
128
|
+
app.execute(run);
|
|
129
129
|
resolve();
|
|
130
130
|
});
|
|
131
131
|
}).receive("error", () => {
|
|
@@ -165,7 +165,7 @@ var startWorkloop = (app, logger, minBackoff, maxBackoff, maxWorkers) => {
|
|
|
165
165
|
};
|
|
166
166
|
var workloop_default = startWorkloop;
|
|
167
167
|
|
|
168
|
-
// src/util/convert-
|
|
168
|
+
// src/util/convert-run.ts
|
|
169
169
|
import crypto from "node:crypto";
|
|
170
170
|
var conditions = {
|
|
171
171
|
on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
|
|
@@ -187,28 +187,23 @@ var mapTriggerEdgeCondition = (edge) => {
|
|
|
187
187
|
return condition;
|
|
188
188
|
};
|
|
189
189
|
var mapOptions = (options) => {
|
|
190
|
-
|
|
191
|
-
const to = runTimeout || attemptTimeout || timeout;
|
|
192
|
-
if (to) {
|
|
193
|
-
opts.attemptTimeoutMs = to;
|
|
194
|
-
}
|
|
195
|
-
return opts;
|
|
190
|
+
return options;
|
|
196
191
|
};
|
|
197
|
-
var
|
|
198
|
-
const options =
|
|
192
|
+
var convert_run_default = (run) => {
|
|
193
|
+
const options = run.options || {};
|
|
199
194
|
const plan = {
|
|
200
|
-
id:
|
|
195
|
+
id: run.id
|
|
201
196
|
};
|
|
202
|
-
if (
|
|
203
|
-
plan.initialState =
|
|
197
|
+
if (run.dataclip_id) {
|
|
198
|
+
plan.initialState = run.dataclip_id;
|
|
204
199
|
}
|
|
205
|
-
if (
|
|
206
|
-
plan.start =
|
|
200
|
+
if (run.starting_node_id) {
|
|
201
|
+
plan.start = run.starting_node_id;
|
|
207
202
|
}
|
|
208
203
|
const nodes = {};
|
|
209
|
-
const edges =
|
|
210
|
-
if (
|
|
211
|
-
|
|
204
|
+
const edges = run.edges ?? [];
|
|
205
|
+
if (run.triggers?.length) {
|
|
206
|
+
run.triggers.forEach((trigger) => {
|
|
212
207
|
const id = trigger.id || "trigger";
|
|
213
208
|
nodes[id] = {
|
|
214
209
|
id
|
|
@@ -225,8 +220,8 @@ var convert_attempt_default = (attempt) => {
|
|
|
225
220
|
}
|
|
226
221
|
});
|
|
227
222
|
}
|
|
228
|
-
if (
|
|
229
|
-
|
|
223
|
+
if (run.jobs?.length) {
|
|
224
|
+
run.jobs.forEach((job) => {
|
|
230
225
|
const id = job.id || crypto.randomUUID();
|
|
231
226
|
nodes[id] = {
|
|
232
227
|
id,
|
|
@@ -277,8 +272,8 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
|
|
|
277
272
|
return value;
|
|
278
273
|
});
|
|
279
274
|
|
|
280
|
-
// src/util/create-
|
|
281
|
-
var
|
|
275
|
+
// src/util/create-run-state.ts
|
|
276
|
+
var create_run_state_default = (plan, options = {}) => {
|
|
282
277
|
const state = {
|
|
283
278
|
plan,
|
|
284
279
|
lastDataclipId: "",
|
|
@@ -306,7 +301,7 @@ var create_attempt_state_default = (plan, options = {}) => {
|
|
|
306
301
|
return state;
|
|
307
302
|
};
|
|
308
303
|
|
|
309
|
-
// src/events/
|
|
304
|
+
// src/events/step-complete.ts
|
|
310
305
|
import crypto2 from "node:crypto";
|
|
311
306
|
|
|
312
307
|
// src/api/reasons.ts
|
|
@@ -331,7 +326,7 @@ var isLeafNode = (state, job) => {
|
|
|
331
326
|
const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
|
|
332
327
|
return !hasDownstream;
|
|
333
328
|
};
|
|
334
|
-
var
|
|
329
|
+
var calculateRunExitReason = (state) => {
|
|
335
330
|
if (state.plan && state.reasons) {
|
|
336
331
|
const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
|
|
337
332
|
const fail = leafJobReasons.find((r) => r && r.reason === "fail");
|
|
@@ -342,17 +337,17 @@ var calculateAttemptExitReason = (state) => {
|
|
|
342
337
|
return { reason: "success", error_type: null, error_message: null };
|
|
343
338
|
};
|
|
344
339
|
|
|
345
|
-
// src/events/
|
|
346
|
-
function
|
|
340
|
+
// src/events/step-complete.ts
|
|
341
|
+
function onStepComplete({ channel, state }, event, error) {
|
|
347
342
|
const dataclipId = crypto2.randomUUID();
|
|
348
|
-
const
|
|
343
|
+
const step_id = state.activeStep;
|
|
349
344
|
const job_id = state.activeJob;
|
|
350
345
|
if (!state.dataclips) {
|
|
351
346
|
state.dataclips = {};
|
|
352
347
|
}
|
|
353
348
|
const outputState = event.state || {};
|
|
354
349
|
state.dataclips[dataclipId] = event.state;
|
|
355
|
-
delete state.
|
|
350
|
+
delete state.activeStep;
|
|
356
351
|
delete state.activeJob;
|
|
357
352
|
state.lastDataclipId = dataclipId;
|
|
358
353
|
event.next?.forEach((nextJobId) => {
|
|
@@ -365,7 +360,7 @@ function onRunComplete({ channel, state }, event, error) {
|
|
|
365
360
|
);
|
|
366
361
|
state.reasons[job_id] = { reason, error_message, error_type };
|
|
367
362
|
const evt = {
|
|
368
|
-
|
|
363
|
+
step_id,
|
|
369
364
|
job_id,
|
|
370
365
|
output_dataclip_id: dataclipId,
|
|
371
366
|
output_dataclip: stringify_default(outputState),
|
|
@@ -376,17 +371,17 @@ function onRunComplete({ channel, state }, event, error) {
|
|
|
376
371
|
duration: event.duration,
|
|
377
372
|
thread_id: event.threadId
|
|
378
373
|
};
|
|
379
|
-
return sendEvent(channel,
|
|
374
|
+
return sendEvent(channel, STEP_COMPLETE, evt);
|
|
380
375
|
}
|
|
381
376
|
|
|
382
|
-
// src/events/
|
|
377
|
+
// src/events/step-start.ts
|
|
383
378
|
import crypto3 from "node:crypto";
|
|
384
379
|
import { timestamp } from "@openfn/logger";
|
|
385
380
|
|
|
386
381
|
// package.json
|
|
387
382
|
var package_default = {
|
|
388
383
|
name: "@openfn/ws-worker",
|
|
389
|
-
version: "0.
|
|
384
|
+
version: "0.8.0",
|
|
390
385
|
description: "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
391
386
|
main: "dist/index.js",
|
|
392
387
|
type: "module",
|
|
@@ -449,14 +444,14 @@ var package_default = {
|
|
|
449
444
|
// src/util/versions.ts
|
|
450
445
|
import { mainSymbols } from "figures";
|
|
451
446
|
var { triangleRightSmall: t } = mainSymbols;
|
|
452
|
-
var versions_default = (
|
|
447
|
+
var versions_default = (stepId, versions, adaptor) => {
|
|
453
448
|
let longest = "compiler".length;
|
|
454
449
|
for (const v in versions) {
|
|
455
450
|
longest = Math.max(v.length, longest);
|
|
456
451
|
}
|
|
457
452
|
const { node, compiler, engine, worker, runtime, ...adaptors } = versions;
|
|
458
453
|
const prefix = (str2) => ` ${t} ${str2.padEnd(longest + 4, " ")}`;
|
|
459
|
-
let str = `Versions for
|
|
454
|
+
let str = `Versions for step ${stepId}:
|
|
460
455
|
${prefix("node.js")}${versions.node || "unknown"}
|
|
461
456
|
${prefix("worker")}${versions.worker || "unknown"}
|
|
462
457
|
${prefix("engine")}${versions.engine || "unknown"}`;
|
|
@@ -470,11 +465,11 @@ ${prefix("engine")}${versions.engine || "unknown"}`;
|
|
|
470
465
|
return str;
|
|
471
466
|
};
|
|
472
467
|
|
|
473
|
-
// src/events/
|
|
474
|
-
async function
|
|
468
|
+
// src/events/step-start.ts
|
|
469
|
+
async function onStepStart(context, event) {
|
|
475
470
|
const time = (timestamp() - BigInt(1e7)).toString();
|
|
476
471
|
const { channel, state } = context;
|
|
477
|
-
state.
|
|
472
|
+
state.activeStep = crypto3.randomUUID();
|
|
478
473
|
state.activeJob = event.jobId;
|
|
479
474
|
const job = state.plan.jobs.find(({ id }) => id === event.jobId);
|
|
480
475
|
const input_dataclip_id = state.inputDataclips[event.jobId];
|
|
@@ -486,17 +481,17 @@ async function onRunStart(context, event) {
|
|
|
486
481
|
...context,
|
|
487
482
|
state: {
|
|
488
483
|
...state,
|
|
489
|
-
|
|
484
|
+
activeStep: state.activeStep
|
|
490
485
|
}
|
|
491
486
|
};
|
|
492
|
-
await sendEvent(channel,
|
|
493
|
-
|
|
487
|
+
await sendEvent(channel, STEP_START, {
|
|
488
|
+
step_id: state.activeStep,
|
|
494
489
|
job_id: state.activeJob,
|
|
495
490
|
input_dataclip_id,
|
|
496
491
|
versions
|
|
497
492
|
});
|
|
498
493
|
const versionMessage = versions_default(
|
|
499
|
-
versionLogContext.state.
|
|
494
|
+
versionLogContext.state.activeStep,
|
|
500
495
|
versions,
|
|
501
496
|
job?.adaptor
|
|
502
497
|
);
|
|
@@ -526,21 +521,21 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
|
|
|
526
521
|
});
|
|
527
522
|
};
|
|
528
523
|
|
|
529
|
-
// src/events/
|
|
524
|
+
// src/events/run-complete.ts
|
|
530
525
|
async function onWorkflowComplete(context, _event) {
|
|
531
526
|
const { state, channel, onFinish } = context;
|
|
532
527
|
const result = state.dataclips[state.lastDataclipId];
|
|
533
|
-
const reason =
|
|
528
|
+
const reason = calculateRunExitReason(state);
|
|
534
529
|
await log_final_reason_default(context, reason);
|
|
535
|
-
await sendEvent(channel,
|
|
530
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
536
531
|
final_dataclip_id: state.lastDataclipId,
|
|
537
532
|
...reason
|
|
538
533
|
});
|
|
539
534
|
onFinish({ reason, state: result });
|
|
540
535
|
}
|
|
541
536
|
|
|
542
|
-
// src/events/
|
|
543
|
-
async function
|
|
537
|
+
// src/events/run-error.ts
|
|
538
|
+
async function onRunError(context, event) {
|
|
544
539
|
const { state, channel, logger, onFinish } = context;
|
|
545
540
|
try {
|
|
546
541
|
const reason = calculateJobExitReason("", { data: {} }, event);
|
|
@@ -548,7 +543,7 @@ async function onAttemptError(context, event) {
|
|
|
548
543
|
await onJobError(context, { error: event });
|
|
549
544
|
}
|
|
550
545
|
await log_final_reason_default(context, reason);
|
|
551
|
-
await sendEvent(channel,
|
|
546
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
552
547
|
final_dataclip_id: state.lastDataclipId,
|
|
553
548
|
...reason
|
|
554
549
|
});
|
|
@@ -590,16 +585,16 @@ var throttle_default = createThrottler;
|
|
|
590
585
|
// src/api/execute.ts
|
|
591
586
|
var enc = new TextDecoder("utf-8");
|
|
592
587
|
var eventMap = {
|
|
593
|
-
"workflow-start":
|
|
594
|
-
"job-start":
|
|
595
|
-
"job-complete":
|
|
596
|
-
"workflow-log":
|
|
597
|
-
"workflow-complete":
|
|
588
|
+
"workflow-start": RUN_START,
|
|
589
|
+
"job-start": STEP_START,
|
|
590
|
+
"job-complete": STEP_COMPLETE,
|
|
591
|
+
"workflow-log": RUN_LOG,
|
|
592
|
+
"workflow-complete": RUN_COMPLETE
|
|
598
593
|
};
|
|
599
594
|
function execute(channel, engine, logger, plan, options = {}, onFinish = (_result) => {
|
|
600
595
|
}) {
|
|
601
596
|
logger.info("executing ", plan.id);
|
|
602
|
-
const state =
|
|
597
|
+
const state = create_run_state_default(plan, options);
|
|
603
598
|
const context = { channel, state, logger, engine, onFinish };
|
|
604
599
|
const throttle = throttle_default();
|
|
605
600
|
const addEvent = (eventName, handler) => {
|
|
@@ -622,12 +617,12 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
|
|
|
622
617
|
const listeners = Object.assign(
|
|
623
618
|
{},
|
|
624
619
|
addEvent("workflow-start", throttle(onWorkflowStart)),
|
|
625
|
-
addEvent("job-start", throttle(
|
|
626
|
-
addEvent("job-complete", throttle(
|
|
620
|
+
addEvent("job-start", throttle(onStepStart)),
|
|
621
|
+
addEvent("job-complete", throttle(onStepComplete)),
|
|
627
622
|
addEvent("job-error", throttle(onJobError)),
|
|
628
623
|
addEvent("workflow-log", throttle(onJobLog)),
|
|
629
624
|
addEvent("workflow-complete", throttle(onWorkflowComplete)),
|
|
630
|
-
addEvent("workflow-error", throttle(
|
|
625
|
+
addEvent("workflow-error", throttle(onRunError))
|
|
631
626
|
);
|
|
632
627
|
engine.listen(plan.id, listeners);
|
|
633
628
|
const resolvers = {
|
|
@@ -645,7 +640,7 @@ function execute(channel, engine, logger, plan, options = {}, onFinish = (_resul
|
|
|
645
640
|
try {
|
|
646
641
|
engine.execute(plan, { resolvers, ...options });
|
|
647
642
|
} catch (e) {
|
|
648
|
-
|
|
643
|
+
onRunError(context, {
|
|
649
644
|
workflowId: plan.id,
|
|
650
645
|
message: e.message,
|
|
651
646
|
type: e.type,
|
|
@@ -663,27 +658,27 @@ var sendEvent = (channel, event, payload) => new Promise((resolve, reject) => {
|
|
|
663
658
|
function onJobError(context, event) {
|
|
664
659
|
const { state, error, jobId } = event;
|
|
665
660
|
if (state?.errors?.[jobId]?.message === error.message) {
|
|
666
|
-
return
|
|
661
|
+
return onStepComplete(context, event);
|
|
667
662
|
} else {
|
|
668
|
-
return
|
|
663
|
+
return onStepComplete(context, event, event.error);
|
|
669
664
|
}
|
|
670
665
|
}
|
|
671
666
|
function onWorkflowStart({ channel }, _event) {
|
|
672
|
-
return sendEvent(channel,
|
|
667
|
+
return sendEvent(channel, RUN_START);
|
|
673
668
|
}
|
|
674
669
|
function onJobLog({ channel, state }, event) {
|
|
675
670
|
const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
|
|
676
671
|
const log = {
|
|
677
|
-
|
|
672
|
+
run_id: state.plan.id,
|
|
678
673
|
message: event.message,
|
|
679
674
|
source: event.name,
|
|
680
675
|
level: event.level,
|
|
681
676
|
timestamp: timeInMicroseconds.toString()
|
|
682
677
|
};
|
|
683
|
-
if (state.
|
|
684
|
-
log.
|
|
678
|
+
if (state.activeStep) {
|
|
679
|
+
log.step_id = state.activeStep;
|
|
685
680
|
}
|
|
686
|
-
return sendEvent(channel,
|
|
681
|
+
return sendEvent(channel, RUN_LOG, log);
|
|
687
682
|
}
|
|
688
683
|
async function loadDataclip(channel, stateId) {
|
|
689
684
|
const result = await get_with_reply_default(channel, GET_DATACLIP, {
|
|
@@ -701,19 +696,19 @@ var healthcheck_default = (ctx) => {
|
|
|
701
696
|
ctx.status = 200;
|
|
702
697
|
};
|
|
703
698
|
|
|
704
|
-
// src/channels/
|
|
705
|
-
var
|
|
699
|
+
// src/channels/run.ts
|
|
700
|
+
var joinRunChannel = (socket, token, runId, logger) => {
|
|
706
701
|
return new Promise((resolve, reject) => {
|
|
707
702
|
let didReceiveOk = false;
|
|
708
|
-
const channelName = `
|
|
703
|
+
const channelName = `run:${runId}`;
|
|
709
704
|
logger.debug("connecting to ", channelName);
|
|
710
705
|
const channel = socket.channel(channelName, { token });
|
|
711
706
|
channel.join().receive("ok", async (e) => {
|
|
712
707
|
if (!didReceiveOk) {
|
|
713
708
|
didReceiveOk = true;
|
|
714
709
|
logger.success(`connected to ${channelName}`, e);
|
|
715
|
-
const { plan, options } = await
|
|
716
|
-
logger.debug("converted
|
|
710
|
+
const { plan, options } = await loadRun(channel);
|
|
711
|
+
logger.debug("converted run as execution plan:", plan);
|
|
717
712
|
resolve({ channel, plan, options });
|
|
718
713
|
}
|
|
719
714
|
}).receive("error", (err) => {
|
|
@@ -722,10 +717,10 @@ var joinAttemptChannel = (socket, token, attemptId, logger) => {
|
|
|
722
717
|
});
|
|
723
718
|
});
|
|
724
719
|
};
|
|
725
|
-
var
|
|
726
|
-
async function
|
|
727
|
-
const
|
|
728
|
-
return
|
|
720
|
+
var run_default = joinRunChannel;
|
|
721
|
+
async function loadRun(channel) {
|
|
722
|
+
const runBody = await get_with_reply_default(channel, GET_PLAN);
|
|
723
|
+
return convert_run_default(runBody);
|
|
729
724
|
}
|
|
730
725
|
|
|
731
726
|
// src/channels/worker-queue.ts
|
|
@@ -741,7 +736,7 @@ var generateWorkerToken = async (secret, workerId, logger) => {
|
|
|
741
736
|
logger.warn();
|
|
742
737
|
logger.warn("WARNING: Worker Secret not provided!");
|
|
743
738
|
logger.warn(
|
|
744
|
-
"This worker will
|
|
739
|
+
"This worker will try to connect to Lightning with default secret"
|
|
745
740
|
);
|
|
746
741
|
logger.warn();
|
|
747
742
|
}
|
|
@@ -865,17 +860,17 @@ function createServer(engine, options = {}) {
|
|
|
865
860
|
if (app.socket) {
|
|
866
861
|
app.workflows[id] = true;
|
|
867
862
|
const {
|
|
868
|
-
channel:
|
|
863
|
+
channel: runChannel,
|
|
869
864
|
plan,
|
|
870
865
|
options: options2
|
|
871
|
-
} = await
|
|
866
|
+
} = await run_default(app.socket, token, id, logger);
|
|
872
867
|
const onFinish = () => {
|
|
873
868
|
delete app.workflows[id];
|
|
874
|
-
|
|
875
|
-
app.events.emit(
|
|
869
|
+
runChannel.leave();
|
|
870
|
+
app.events.emit(INTERNAL_RUN_COMPLETE);
|
|
876
871
|
};
|
|
877
872
|
const context = execute(
|
|
878
|
-
|
|
873
|
+
runChannel,
|
|
879
874
|
engine,
|
|
880
875
|
logger,
|
|
881
876
|
plan,
|
|
@@ -890,12 +885,12 @@ function createServer(engine, options = {}) {
|
|
|
890
885
|
router.post("/claim", async (ctx) => {
|
|
891
886
|
logger.info("triggering claim from POST request");
|
|
892
887
|
return claim_default(app, logger, options.maxWorkflows).then(() => {
|
|
893
|
-
logger.info("claim complete: 1
|
|
888
|
+
logger.info("claim complete: 1 run claimed");
|
|
894
889
|
ctx.body = "complete";
|
|
895
890
|
ctx.status = 200;
|
|
896
891
|
}).catch(() => {
|
|
897
|
-
logger.info("claim complete: no
|
|
898
|
-
ctx.body = "no
|
|
892
|
+
logger.info("claim complete: no runs");
|
|
893
|
+
ctx.body = "no runs";
|
|
899
894
|
ctx.status = 204;
|
|
900
895
|
});
|
|
901
896
|
});
|
|
@@ -927,15 +922,15 @@ var server_default = createServer;
|
|
|
927
922
|
// src/index.ts
|
|
928
923
|
var src_default = server_default;
|
|
929
924
|
export {
|
|
930
|
-
ATTEMPT_COMPLETE,
|
|
931
|
-
ATTEMPT_LOG,
|
|
932
|
-
ATTEMPT_START,
|
|
933
925
|
CLAIM,
|
|
934
|
-
GET_ATTEMPT,
|
|
935
926
|
GET_CREDENTIAL,
|
|
936
927
|
GET_DATACLIP,
|
|
937
|
-
|
|
928
|
+
GET_PLAN,
|
|
929
|
+
INTERNAL_RUN_COMPLETE,
|
|
938
930
|
RUN_COMPLETE,
|
|
931
|
+
RUN_LOG,
|
|
939
932
|
RUN_START,
|
|
933
|
+
STEP_COMPLETE,
|
|
934
|
+
STEP_START,
|
|
940
935
|
src_default as default
|
|
941
936
|
};
|
package/dist/start.js
CHANGED
|
@@ -4998,15 +4998,15 @@ import { createMockLogger as createMockLogger2 } from "@openfn/logger";
|
|
|
4998
4998
|
|
|
4999
4999
|
// src/events.ts
|
|
5000
5000
|
var CLAIM = "claim";
|
|
5001
|
-
var
|
|
5001
|
+
var GET_PLAN = "fetch:plan";
|
|
5002
5002
|
var GET_CREDENTIAL = "fetch:credential";
|
|
5003
5003
|
var GET_DATACLIP = "fetch:dataclip";
|
|
5004
|
-
var ATTEMPT_START = "attempt:start";
|
|
5005
|
-
var ATTEMPT_COMPLETE = "attempt:complete";
|
|
5006
|
-
var ATTEMPT_LOG = "attempt:log";
|
|
5007
5004
|
var RUN_START = "run:start";
|
|
5008
5005
|
var RUN_COMPLETE = "run:complete";
|
|
5009
|
-
var
|
|
5006
|
+
var RUN_LOG = "run:log";
|
|
5007
|
+
var STEP_START = "step:start";
|
|
5008
|
+
var STEP_COMPLETE = "step:complete";
|
|
5009
|
+
var INTERNAL_RUN_COMPLETE = "server:run-complete";
|
|
5010
5010
|
|
|
5011
5011
|
// src/api/destroy.ts
|
|
5012
5012
|
var destroy = async (app, logger2) => {
|
|
@@ -5021,7 +5021,7 @@ var destroy = async (app, logger2) => {
|
|
|
5021
5021
|
});
|
|
5022
5022
|
}),
|
|
5023
5023
|
new Promise(async (resolve5) => {
|
|
5024
|
-
await
|
|
5024
|
+
await waitForRuns(app, logger2);
|
|
5025
5025
|
await app.engine.destroy();
|
|
5026
5026
|
app.socket?.disconnect();
|
|
5027
5027
|
resolve5();
|
|
@@ -5029,16 +5029,16 @@ var destroy = async (app, logger2) => {
|
|
|
5029
5029
|
]);
|
|
5030
5030
|
logger2.success("Server closed");
|
|
5031
5031
|
};
|
|
5032
|
-
var
|
|
5032
|
+
var waitForRuns = (app, logger2) => new Promise((resolve5) => {
|
|
5033
5033
|
const log = () => {
|
|
5034
5034
|
logger2.debug(
|
|
5035
|
-
`Waiting for ${Object.keys(app.workflows).length}
|
|
5035
|
+
`Waiting for ${Object.keys(app.workflows).length} runs to complete...`
|
|
5036
5036
|
);
|
|
5037
5037
|
};
|
|
5038
|
-
const
|
|
5038
|
+
const onRunComplete = () => {
|
|
5039
5039
|
if (Object.keys(app.workflows).length === 0) {
|
|
5040
|
-
logger2.debug("All
|
|
5041
|
-
app.events.off(
|
|
5040
|
+
logger2.debug("All runs completed!");
|
|
5041
|
+
app.events.off(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
5042
5042
|
resolve5();
|
|
5043
5043
|
} else {
|
|
5044
5044
|
log();
|
|
@@ -5046,7 +5046,7 @@ var waitForAttempts = (app, logger2) => new Promise((resolve5) => {
|
|
|
5046
5046
|
};
|
|
5047
5047
|
if (Object.keys(app.workflows).length) {
|
|
5048
5048
|
log();
|
|
5049
|
-
app.events.on(
|
|
5049
|
+
app.events.on(INTERNAL_RUN_COMPLETE, onRunComplete);
|
|
5050
5050
|
} else {
|
|
5051
5051
|
resolve5();
|
|
5052
5052
|
}
|
|
@@ -5056,7 +5056,7 @@ var destroy_default = destroy;
|
|
|
5056
5056
|
// src/util/try-with-backoff.ts
|
|
5057
5057
|
var BACKOFF_MULTIPLIER = 1.15;
|
|
5058
5058
|
var tryWithBackoff = (fn, opts = {}) => {
|
|
5059
|
-
const { min = 1e3, max = 1e4,
|
|
5059
|
+
const { min = 1e3, max = 1e4, maxRuns, runs = 1 } = opts;
|
|
5060
5060
|
let cancelled = false;
|
|
5061
5061
|
if (!opts.isCancelled) {
|
|
5062
5062
|
opts.isCancelled = () => cancelled;
|
|
@@ -5069,16 +5069,16 @@ var tryWithBackoff = (fn, opts = {}) => {
|
|
|
5069
5069
|
if (opts.isCancelled()) {
|
|
5070
5070
|
return resolve5();
|
|
5071
5071
|
}
|
|
5072
|
-
if (!isNaN(
|
|
5073
|
-
return reject(new Error("max
|
|
5072
|
+
if (!isNaN(maxRuns) && runs >= maxRuns) {
|
|
5073
|
+
return reject(new Error("max runs exceeded"));
|
|
5074
5074
|
}
|
|
5075
5075
|
setTimeout(() => {
|
|
5076
5076
|
if (opts.isCancelled()) {
|
|
5077
5077
|
return resolve5();
|
|
5078
5078
|
}
|
|
5079
5079
|
const nextOpts = {
|
|
5080
|
-
|
|
5081
|
-
|
|
5080
|
+
maxRuns,
|
|
5081
|
+
runs: runs + 1,
|
|
5082
5082
|
min: Math.min(max, min * BACKOFF_MULTIPLIER),
|
|
5083
5083
|
max,
|
|
5084
5084
|
isCancelled: opts.isCancelled
|
|
@@ -5106,15 +5106,15 @@ var claim = (app, logger2 = mockLogger, maxWorkers = 5) => {
|
|
|
5106
5106
|
if (!app.queueChannel) {
|
|
5107
5107
|
return reject(new Error("No websocket available"));
|
|
5108
5108
|
}
|
|
5109
|
-
logger2.debug("requesting
|
|
5110
|
-
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({
|
|
5111
|
-
logger2.debug(`pulled ${
|
|
5112
|
-
if (!
|
|
5113
|
-
return reject(new Error("No
|
|
5109
|
+
logger2.debug("requesting run...");
|
|
5110
|
+
app.queueChannel.push(CLAIM, { demand: 1 }).receive("ok", ({ runs }) => {
|
|
5111
|
+
logger2.debug(`pulled ${runs.length} runs`);
|
|
5112
|
+
if (!runs?.length) {
|
|
5113
|
+
return reject(new Error("No runs returned"));
|
|
5114
5114
|
}
|
|
5115
|
-
|
|
5116
|
-
logger2.debug("starting
|
|
5117
|
-
app.execute(
|
|
5115
|
+
runs.forEach((run2) => {
|
|
5116
|
+
logger2.debug("starting run", run2.id);
|
|
5117
|
+
app.execute(run2);
|
|
5118
5118
|
resolve5();
|
|
5119
5119
|
});
|
|
5120
5120
|
}).receive("error", () => {
|
|
@@ -5154,7 +5154,7 @@ var startWorkloop = (app, logger2, minBackoff2, maxBackoff2, maxWorkers) => {
|
|
|
5154
5154
|
};
|
|
5155
5155
|
var workloop_default = startWorkloop;
|
|
5156
5156
|
|
|
5157
|
-
// src/util/convert-
|
|
5157
|
+
// src/util/convert-run.ts
|
|
5158
5158
|
import crypto2 from "node:crypto";
|
|
5159
5159
|
var conditions = {
|
|
5160
5160
|
on_job_success: (upstreamId) => `Boolean(!state?.errors?.["${upstreamId}"] ?? true)`,
|
|
@@ -5176,28 +5176,23 @@ var mapTriggerEdgeCondition = (edge) => {
|
|
|
5176
5176
|
return condition;
|
|
5177
5177
|
};
|
|
5178
5178
|
var mapOptions = (options) => {
|
|
5179
|
-
|
|
5180
|
-
const to = runTimeout || attemptTimeout || timeout;
|
|
5181
|
-
if (to) {
|
|
5182
|
-
opts.attemptTimeoutMs = to;
|
|
5183
|
-
}
|
|
5184
|
-
return opts;
|
|
5179
|
+
return options;
|
|
5185
5180
|
};
|
|
5186
|
-
var
|
|
5187
|
-
const options =
|
|
5181
|
+
var convert_run_default = (run2) => {
|
|
5182
|
+
const options = run2.options || {};
|
|
5188
5183
|
const plan = {
|
|
5189
|
-
id:
|
|
5184
|
+
id: run2.id
|
|
5190
5185
|
};
|
|
5191
|
-
if (
|
|
5192
|
-
plan.initialState =
|
|
5186
|
+
if (run2.dataclip_id) {
|
|
5187
|
+
plan.initialState = run2.dataclip_id;
|
|
5193
5188
|
}
|
|
5194
|
-
if (
|
|
5195
|
-
plan.start =
|
|
5189
|
+
if (run2.starting_node_id) {
|
|
5190
|
+
plan.start = run2.starting_node_id;
|
|
5196
5191
|
}
|
|
5197
5192
|
const nodes = {};
|
|
5198
|
-
const edges =
|
|
5199
|
-
if (
|
|
5200
|
-
|
|
5193
|
+
const edges = run2.edges ?? [];
|
|
5194
|
+
if (run2.triggers?.length) {
|
|
5195
|
+
run2.triggers.forEach((trigger) => {
|
|
5201
5196
|
const id = trigger.id || "trigger";
|
|
5202
5197
|
nodes[id] = {
|
|
5203
5198
|
id
|
|
@@ -5214,8 +5209,8 @@ var convert_attempt_default = (attempt) => {
|
|
|
5214
5209
|
}
|
|
5215
5210
|
});
|
|
5216
5211
|
}
|
|
5217
|
-
if (
|
|
5218
|
-
|
|
5212
|
+
if (run2.jobs?.length) {
|
|
5213
|
+
run2.jobs.forEach((job) => {
|
|
5219
5214
|
const id = job.id || crypto2.randomUUID();
|
|
5220
5215
|
nodes[id] = {
|
|
5221
5216
|
id,
|
|
@@ -5266,8 +5261,8 @@ var stringify_default = (obj) => stringify(obj, (_key, value) => {
|
|
|
5266
5261
|
return value;
|
|
5267
5262
|
});
|
|
5268
5263
|
|
|
5269
|
-
// src/util/create-
|
|
5270
|
-
var
|
|
5264
|
+
// src/util/create-run-state.ts
|
|
5265
|
+
var create_run_state_default = (plan, options = {}) => {
|
|
5271
5266
|
const state = {
|
|
5272
5267
|
plan,
|
|
5273
5268
|
lastDataclipId: "",
|
|
@@ -5295,7 +5290,7 @@ var create_attempt_state_default = (plan, options = {}) => {
|
|
|
5295
5290
|
return state;
|
|
5296
5291
|
};
|
|
5297
5292
|
|
|
5298
|
-
// src/events/
|
|
5293
|
+
// src/events/step-complete.ts
|
|
5299
5294
|
import crypto3 from "node:crypto";
|
|
5300
5295
|
|
|
5301
5296
|
// src/api/reasons.ts
|
|
@@ -5320,7 +5315,7 @@ var isLeafNode = (state, job) => {
|
|
|
5320
5315
|
const hasDownstream = Object.keys(job.next).find((id) => state.reasons[id]);
|
|
5321
5316
|
return !hasDownstream;
|
|
5322
5317
|
};
|
|
5323
|
-
var
|
|
5318
|
+
var calculateRunExitReason = (state) => {
|
|
5324
5319
|
if (state.plan && state.reasons) {
|
|
5325
5320
|
const leafJobReasons = state.plan.jobs.filter((job) => isLeafNode(state, job)).map(({ id }) => state.reasons[id]);
|
|
5326
5321
|
const fail = leafJobReasons.find((r) => r && r.reason === "fail");
|
|
@@ -5331,17 +5326,17 @@ var calculateAttemptExitReason = (state) => {
|
|
|
5331
5326
|
return { reason: "success", error_type: null, error_message: null };
|
|
5332
5327
|
};
|
|
5333
5328
|
|
|
5334
|
-
// src/events/
|
|
5335
|
-
function
|
|
5329
|
+
// src/events/step-complete.ts
|
|
5330
|
+
function onStepComplete({ channel, state }, event, error) {
|
|
5336
5331
|
const dataclipId = crypto3.randomUUID();
|
|
5337
|
-
const
|
|
5332
|
+
const step_id = state.activeStep;
|
|
5338
5333
|
const job_id = state.activeJob;
|
|
5339
5334
|
if (!state.dataclips) {
|
|
5340
5335
|
state.dataclips = {};
|
|
5341
5336
|
}
|
|
5342
5337
|
const outputState = event.state || {};
|
|
5343
5338
|
state.dataclips[dataclipId] = event.state;
|
|
5344
|
-
delete state.
|
|
5339
|
+
delete state.activeStep;
|
|
5345
5340
|
delete state.activeJob;
|
|
5346
5341
|
state.lastDataclipId = dataclipId;
|
|
5347
5342
|
event.next?.forEach((nextJobId) => {
|
|
@@ -5354,7 +5349,7 @@ function onRunComplete({ channel, state }, event, error) {
|
|
|
5354
5349
|
);
|
|
5355
5350
|
state.reasons[job_id] = { reason, error_message, error_type };
|
|
5356
5351
|
const evt = {
|
|
5357
|
-
|
|
5352
|
+
step_id,
|
|
5358
5353
|
job_id,
|
|
5359
5354
|
output_dataclip_id: dataclipId,
|
|
5360
5355
|
output_dataclip: stringify_default(outputState),
|
|
@@ -5365,17 +5360,17 @@ function onRunComplete({ channel, state }, event, error) {
|
|
|
5365
5360
|
duration: event.duration,
|
|
5366
5361
|
thread_id: event.threadId
|
|
5367
5362
|
};
|
|
5368
|
-
return sendEvent(channel,
|
|
5363
|
+
return sendEvent(channel, STEP_COMPLETE, evt);
|
|
5369
5364
|
}
|
|
5370
5365
|
|
|
5371
|
-
// src/events/
|
|
5366
|
+
// src/events/step-start.ts
|
|
5372
5367
|
import crypto4 from "node:crypto";
|
|
5373
5368
|
import { timestamp } from "@openfn/logger";
|
|
5374
5369
|
|
|
5375
5370
|
// package.json
|
|
5376
5371
|
var package_default = {
|
|
5377
5372
|
name: "@openfn/ws-worker",
|
|
5378
|
-
version: "0.
|
|
5373
|
+
version: "0.8.0",
|
|
5379
5374
|
description: "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
5380
5375
|
main: "dist/index.js",
|
|
5381
5376
|
type: "module",
|
|
@@ -5438,14 +5433,14 @@ var package_default = {
|
|
|
5438
5433
|
// src/util/versions.ts
|
|
5439
5434
|
import { mainSymbols } from "figures";
|
|
5440
5435
|
var { triangleRightSmall: t } = mainSymbols;
|
|
5441
|
-
var versions_default = (
|
|
5436
|
+
var versions_default = (stepId, versions, adaptor) => {
|
|
5442
5437
|
let longest = "compiler".length;
|
|
5443
5438
|
for (const v in versions) {
|
|
5444
5439
|
longest = Math.max(v.length, longest);
|
|
5445
5440
|
}
|
|
5446
5441
|
const { node, compiler, engine, worker, runtime, ...adaptors } = versions;
|
|
5447
5442
|
const prefix = (str2) => ` ${t} ${str2.padEnd(longest + 4, " ")}`;
|
|
5448
|
-
let str = `Versions for
|
|
5443
|
+
let str = `Versions for step ${stepId}:
|
|
5449
5444
|
${prefix("node.js")}${versions.node || "unknown"}
|
|
5450
5445
|
${prefix("worker")}${versions.worker || "unknown"}
|
|
5451
5446
|
${prefix("engine")}${versions.engine || "unknown"}`;
|
|
@@ -5459,11 +5454,11 @@ ${prefix("engine")}${versions.engine || "unknown"}`;
|
|
|
5459
5454
|
return str;
|
|
5460
5455
|
};
|
|
5461
5456
|
|
|
5462
|
-
// src/events/
|
|
5463
|
-
async function
|
|
5457
|
+
// src/events/step-start.ts
|
|
5458
|
+
async function onStepStart(context, event) {
|
|
5464
5459
|
const time = (timestamp() - BigInt(1e7)).toString();
|
|
5465
5460
|
const { channel, state } = context;
|
|
5466
|
-
state.
|
|
5461
|
+
state.activeStep = crypto4.randomUUID();
|
|
5467
5462
|
state.activeJob = event.jobId;
|
|
5468
5463
|
const job = state.plan.jobs.find(({ id }) => id === event.jobId);
|
|
5469
5464
|
const input_dataclip_id = state.inputDataclips[event.jobId];
|
|
@@ -5475,17 +5470,17 @@ async function onRunStart(context, event) {
|
|
|
5475
5470
|
...context,
|
|
5476
5471
|
state: {
|
|
5477
5472
|
...state,
|
|
5478
|
-
|
|
5473
|
+
activeStep: state.activeStep
|
|
5479
5474
|
}
|
|
5480
5475
|
};
|
|
5481
|
-
await sendEvent(channel,
|
|
5482
|
-
|
|
5476
|
+
await sendEvent(channel, STEP_START, {
|
|
5477
|
+
step_id: state.activeStep,
|
|
5483
5478
|
job_id: state.activeJob,
|
|
5484
5479
|
input_dataclip_id,
|
|
5485
5480
|
versions
|
|
5486
5481
|
});
|
|
5487
5482
|
const versionMessage = versions_default(
|
|
5488
|
-
versionLogContext.state.
|
|
5483
|
+
versionLogContext.state.activeStep,
|
|
5489
5484
|
versions,
|
|
5490
5485
|
job?.adaptor
|
|
5491
5486
|
);
|
|
@@ -5515,21 +5510,21 @@ ${reason.error_type}: ${reason.error_message || "unknown"}`;
|
|
|
5515
5510
|
});
|
|
5516
5511
|
};
|
|
5517
5512
|
|
|
5518
|
-
// src/events/
|
|
5513
|
+
// src/events/run-complete.ts
|
|
5519
5514
|
async function onWorkflowComplete(context, _event) {
|
|
5520
5515
|
const { state, channel, onFinish } = context;
|
|
5521
5516
|
const result = state.dataclips[state.lastDataclipId];
|
|
5522
|
-
const reason =
|
|
5517
|
+
const reason = calculateRunExitReason(state);
|
|
5523
5518
|
await log_final_reason_default(context, reason);
|
|
5524
|
-
await sendEvent(channel,
|
|
5519
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
5525
5520
|
final_dataclip_id: state.lastDataclipId,
|
|
5526
5521
|
...reason
|
|
5527
5522
|
});
|
|
5528
5523
|
onFinish({ reason, state: result });
|
|
5529
5524
|
}
|
|
5530
5525
|
|
|
5531
|
-
// src/events/
|
|
5532
|
-
async function
|
|
5526
|
+
// src/events/run-error.ts
|
|
5527
|
+
async function onRunError(context, event) {
|
|
5533
5528
|
const { state, channel, logger: logger2, onFinish } = context;
|
|
5534
5529
|
try {
|
|
5535
5530
|
const reason = calculateJobExitReason("", { data: {} }, event);
|
|
@@ -5537,7 +5532,7 @@ async function onAttemptError(context, event) {
|
|
|
5537
5532
|
await onJobError(context, { error: event });
|
|
5538
5533
|
}
|
|
5539
5534
|
await log_final_reason_default(context, reason);
|
|
5540
|
-
await sendEvent(channel,
|
|
5535
|
+
await sendEvent(channel, RUN_COMPLETE, {
|
|
5541
5536
|
final_dataclip_id: state.lastDataclipId,
|
|
5542
5537
|
...reason
|
|
5543
5538
|
});
|
|
@@ -5579,16 +5574,16 @@ var throttle_default = createThrottler;
|
|
|
5579
5574
|
// src/api/execute.ts
|
|
5580
5575
|
var enc = new TextDecoder("utf-8");
|
|
5581
5576
|
var eventMap = {
|
|
5582
|
-
"workflow-start":
|
|
5583
|
-
"job-start":
|
|
5584
|
-
"job-complete":
|
|
5585
|
-
"workflow-log":
|
|
5586
|
-
"workflow-complete":
|
|
5577
|
+
"workflow-start": RUN_START,
|
|
5578
|
+
"job-start": STEP_START,
|
|
5579
|
+
"job-complete": STEP_COMPLETE,
|
|
5580
|
+
"workflow-log": RUN_LOG,
|
|
5581
|
+
"workflow-complete": RUN_COMPLETE
|
|
5587
5582
|
};
|
|
5588
5583
|
function execute(channel, engine, logger2, plan, options = {}, onFinish = (_result) => {
|
|
5589
5584
|
}) {
|
|
5590
5585
|
logger2.info("executing ", plan.id);
|
|
5591
|
-
const state =
|
|
5586
|
+
const state = create_run_state_default(plan, options);
|
|
5592
5587
|
const context = { channel, state, logger: logger2, engine, onFinish };
|
|
5593
5588
|
const throttle = throttle_default();
|
|
5594
5589
|
const addEvent = (eventName, handler) => {
|
|
@@ -5611,12 +5606,12 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
|
|
|
5611
5606
|
const listeners = Object.assign(
|
|
5612
5607
|
{},
|
|
5613
5608
|
addEvent("workflow-start", throttle(onWorkflowStart)),
|
|
5614
|
-
addEvent("job-start", throttle(
|
|
5615
|
-
addEvent("job-complete", throttle(
|
|
5609
|
+
addEvent("job-start", throttle(onStepStart)),
|
|
5610
|
+
addEvent("job-complete", throttle(onStepComplete)),
|
|
5616
5611
|
addEvent("job-error", throttle(onJobError)),
|
|
5617
5612
|
addEvent("workflow-log", throttle(onJobLog)),
|
|
5618
5613
|
addEvent("workflow-complete", throttle(onWorkflowComplete)),
|
|
5619
|
-
addEvent("workflow-error", throttle(
|
|
5614
|
+
addEvent("workflow-error", throttle(onRunError))
|
|
5620
5615
|
);
|
|
5621
5616
|
engine.listen(plan.id, listeners);
|
|
5622
5617
|
const resolvers = {
|
|
@@ -5634,7 +5629,7 @@ function execute(channel, engine, logger2, plan, options = {}, onFinish = (_resu
|
|
|
5634
5629
|
try {
|
|
5635
5630
|
engine.execute(plan, { resolvers, ...options });
|
|
5636
5631
|
} catch (e) {
|
|
5637
|
-
|
|
5632
|
+
onRunError(context, {
|
|
5638
5633
|
workflowId: plan.id,
|
|
5639
5634
|
message: e.message,
|
|
5640
5635
|
type: e.type,
|
|
@@ -5652,27 +5647,27 @@ var sendEvent = (channel, event, payload) => new Promise((resolve5, reject) => {
|
|
|
5652
5647
|
function onJobError(context, event) {
|
|
5653
5648
|
const { state, error, jobId } = event;
|
|
5654
5649
|
if (state?.errors?.[jobId]?.message === error.message) {
|
|
5655
|
-
return
|
|
5650
|
+
return onStepComplete(context, event);
|
|
5656
5651
|
} else {
|
|
5657
|
-
return
|
|
5652
|
+
return onStepComplete(context, event, event.error);
|
|
5658
5653
|
}
|
|
5659
5654
|
}
|
|
5660
5655
|
function onWorkflowStart({ channel }, _event) {
|
|
5661
|
-
return sendEvent(channel,
|
|
5656
|
+
return sendEvent(channel, RUN_START);
|
|
5662
5657
|
}
|
|
5663
5658
|
function onJobLog({ channel, state }, event) {
|
|
5664
5659
|
const timeInMicroseconds = BigInt(event.time) / BigInt(1e3);
|
|
5665
5660
|
const log = {
|
|
5666
|
-
|
|
5661
|
+
run_id: state.plan.id,
|
|
5667
5662
|
message: event.message,
|
|
5668
5663
|
source: event.name,
|
|
5669
5664
|
level: event.level,
|
|
5670
5665
|
timestamp: timeInMicroseconds.toString()
|
|
5671
5666
|
};
|
|
5672
|
-
if (state.
|
|
5673
|
-
log.
|
|
5667
|
+
if (state.activeStep) {
|
|
5668
|
+
log.step_id = state.activeStep;
|
|
5674
5669
|
}
|
|
5675
|
-
return sendEvent(channel,
|
|
5670
|
+
return sendEvent(channel, RUN_LOG, log);
|
|
5676
5671
|
}
|
|
5677
5672
|
async function loadDataclip(channel, stateId) {
|
|
5678
5673
|
const result = await get_with_reply_default(channel, GET_DATACLIP, {
|
|
@@ -5690,19 +5685,19 @@ var healthcheck_default = (ctx) => {
|
|
|
5690
5685
|
ctx.status = 200;
|
|
5691
5686
|
};
|
|
5692
5687
|
|
|
5693
|
-
// src/channels/
|
|
5694
|
-
var
|
|
5688
|
+
// src/channels/run.ts
|
|
5689
|
+
var joinRunChannel = (socket, token, runId, logger2) => {
|
|
5695
5690
|
return new Promise((resolve5, reject) => {
|
|
5696
5691
|
let didReceiveOk = false;
|
|
5697
|
-
const channelName = `
|
|
5692
|
+
const channelName = `run:${runId}`;
|
|
5698
5693
|
logger2.debug("connecting to ", channelName);
|
|
5699
5694
|
const channel = socket.channel(channelName, { token });
|
|
5700
5695
|
channel.join().receive("ok", async (e) => {
|
|
5701
5696
|
if (!didReceiveOk) {
|
|
5702
5697
|
didReceiveOk = true;
|
|
5703
5698
|
logger2.success(`connected to ${channelName}`, e);
|
|
5704
|
-
const { plan, options } = await
|
|
5705
|
-
logger2.debug("converted
|
|
5699
|
+
const { plan, options } = await loadRun(channel);
|
|
5700
|
+
logger2.debug("converted run as execution plan:", plan);
|
|
5706
5701
|
resolve5({ channel, plan, options });
|
|
5707
5702
|
}
|
|
5708
5703
|
}).receive("error", (err) => {
|
|
@@ -5711,10 +5706,10 @@ var joinAttemptChannel = (socket, token, attemptId, logger2) => {
|
|
|
5711
5706
|
});
|
|
5712
5707
|
});
|
|
5713
5708
|
};
|
|
5714
|
-
var
|
|
5715
|
-
async function
|
|
5716
|
-
const
|
|
5717
|
-
return
|
|
5709
|
+
var run_default = joinRunChannel;
|
|
5710
|
+
async function loadRun(channel) {
|
|
5711
|
+
const runBody = await get_with_reply_default(channel, GET_PLAN);
|
|
5712
|
+
return convert_run_default(runBody);
|
|
5718
5713
|
}
|
|
5719
5714
|
|
|
5720
5715
|
// src/channels/worker-queue.ts
|
|
@@ -5730,7 +5725,7 @@ var generateWorkerToken = async (secret, workerId, logger2) => {
|
|
|
5730
5725
|
logger2.warn();
|
|
5731
5726
|
logger2.warn("WARNING: Worker Secret not provided!");
|
|
5732
5727
|
logger2.warn(
|
|
5733
|
-
"This worker will
|
|
5728
|
+
"This worker will try to connect to Lightning with default secret"
|
|
5734
5729
|
);
|
|
5735
5730
|
logger2.warn();
|
|
5736
5731
|
}
|
|
@@ -5854,17 +5849,17 @@ function createServer(engine, options = {}) {
|
|
|
5854
5849
|
if (app.socket) {
|
|
5855
5850
|
app.workflows[id] = true;
|
|
5856
5851
|
const {
|
|
5857
|
-
channel:
|
|
5852
|
+
channel: runChannel,
|
|
5858
5853
|
plan,
|
|
5859
5854
|
options: options2
|
|
5860
|
-
} = await
|
|
5855
|
+
} = await run_default(app.socket, token, id, logger2);
|
|
5861
5856
|
const onFinish = () => {
|
|
5862
5857
|
delete app.workflows[id];
|
|
5863
|
-
|
|
5864
|
-
app.events.emit(
|
|
5858
|
+
runChannel.leave();
|
|
5859
|
+
app.events.emit(INTERNAL_RUN_COMPLETE);
|
|
5865
5860
|
};
|
|
5866
5861
|
const context = execute(
|
|
5867
|
-
|
|
5862
|
+
runChannel,
|
|
5868
5863
|
engine,
|
|
5869
5864
|
logger2,
|
|
5870
5865
|
plan,
|
|
@@ -5879,12 +5874,12 @@ function createServer(engine, options = {}) {
|
|
|
5879
5874
|
router.post("/claim", async (ctx) => {
|
|
5880
5875
|
logger2.info("triggering claim from POST request");
|
|
5881
5876
|
return claim_default(app, logger2, options.maxWorkflows).then(() => {
|
|
5882
|
-
logger2.info("claim complete: 1
|
|
5877
|
+
logger2.info("claim complete: 1 run claimed");
|
|
5883
5878
|
ctx.body = "complete";
|
|
5884
5879
|
ctx.status = 200;
|
|
5885
5880
|
}).catch(() => {
|
|
5886
|
-
logger2.info("claim complete: no
|
|
5887
|
-
ctx.body = "no
|
|
5881
|
+
logger2.info("claim complete: no runs");
|
|
5882
|
+
ctx.body = "no runs";
|
|
5888
5883
|
ctx.status = 204;
|
|
5889
5884
|
});
|
|
5890
5885
|
});
|
|
@@ -5972,7 +5967,7 @@ var args = yargs_default(hideBin(process.argv)).command("server", "Start a ws-wo
|
|
|
5972
5967
|
default: WORKER_MAX_RUN_MEMORY_MB ? parseInt(WORKER_MAX_RUN_MEMORY_MB) : 500
|
|
5973
5968
|
}).option("max-run-duration-seconds", {
|
|
5974
5969
|
alias: "t",
|
|
5975
|
-
description: "Default
|
|
5970
|
+
description: "Default run timeout for the server, in seconds. Env: WORKER_MAX_RUN_DURATION_SECONDS",
|
|
5976
5971
|
type: "number",
|
|
5977
5972
|
default: WORKER_MAX_RUN_DURATION_SECONDS || 60 * 5
|
|
5978
5973
|
}).parse();
|
|
@@ -6016,7 +6011,7 @@ if (args.mock) {
|
|
|
6016
6011
|
memoryLimitMb: args.runMemory,
|
|
6017
6012
|
maxWorkers: args.capacity,
|
|
6018
6013
|
statePropsToRemove: args.statePropsToRemove,
|
|
6019
|
-
|
|
6014
|
+
runTimeoutMs: args.maxRunDurationSeconds * 1e3
|
|
6020
6015
|
};
|
|
6021
6016
|
logger.debug("Creating runtime engine...");
|
|
6022
6017
|
logger.debug("Engine options:", engineOptions);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openfn/ws-worker",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "A Websocket Worker to connect Lightning to a Runtime Engine",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"koa-logger": "^3.2.1",
|
|
23
23
|
"phoenix": "^1.7.7",
|
|
24
24
|
"ws": "^8.14.1",
|
|
25
|
-
"@openfn/engine-multi": "0.
|
|
25
|
+
"@openfn/engine-multi": "0.4.0",
|
|
26
26
|
"@openfn/logger": "0.0.19",
|
|
27
27
|
"@openfn/runtime": "0.2.5"
|
|
28
28
|
},
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"tsup": "^6.2.3",
|
|
42
42
|
"typescript": "^4.6.4",
|
|
43
43
|
"yargs": "^17.6.2",
|
|
44
|
-
"@openfn/lightning-mock": "1.
|
|
44
|
+
"@openfn/lightning-mock": "1.2.0"
|
|
45
45
|
},
|
|
46
46
|
"files": [
|
|
47
47
|
"dist",
|