@decaf-ts/core 0.8.53 → 0.8.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/core.cjs +1 -1
- package/dist/core.cjs.map +1 -1
- package/dist/core.js +1 -1
- package/dist/core.js.map +1 -1
- package/lib/esm/fs/FilesystemAdapter.d.ts +4 -2
- package/lib/esm/fs/FilesystemAdapter.js +37 -15
- package/lib/esm/fs/FilesystemAdapter.js.map +1 -1
- package/lib/esm/index.d.ts +1 -1
- package/lib/esm/index.js +1 -1
- package/lib/esm/tasks/TaskEngine.d.ts +28 -46
- package/lib/esm/tasks/TaskEngine.js +16 -358
- package/lib/esm/tasks/TaskEngine.js.map +1 -1
- package/lib/esm/tasks/TaskService.d.ts +4 -1
- package/lib/esm/tasks/TaskService.js +1 -4
- package/lib/esm/tasks/TaskService.js.map +1 -1
- package/lib/esm/tasks/TaskTracker.d.ts +1 -1
- package/lib/esm/tasks/TaskTracker.js.map +1 -1
- package/lib/esm/tasks/constants.js +0 -1
- package/lib/esm/tasks/constants.js.map +1 -1
- package/lib/esm/tasks/types.d.ts +0 -12
- package/lib/esm/workers/TaskEngine.d.ts +40 -0
- package/lib/esm/workers/TaskEngine.js +499 -0
- package/lib/esm/workers/TaskEngine.js.map +1 -0
- package/lib/esm/{tasks/workers → workers}/WorkThreadEnvironment.d.ts +1 -0
- package/lib/esm/workers/WorkThreadEnvironment.js.map +1 -0
- package/lib/esm/workers/index.d.ts +4 -0
- package/lib/esm/workers/index.js +5 -0
- package/lib/esm/workers/index.js.map +1 -0
- package/lib/{tasks → esm}/workers/messages.d.ts +1 -1
- package/lib/{tasks → esm}/workers/messages.js.map +1 -1
- package/lib/esm/workers/types.d.ts +41 -0
- package/lib/esm/workers/types.js +2 -0
- package/lib/esm/workers/types.js.map +1 -0
- package/lib/esm/workers/workerThread.d.ts +1 -0
- package/lib/esm/{tasks/workers → workers}/workerThread.js +8 -8
- package/lib/esm/workers/workerThread.js.map +1 -0
- package/lib/fs/FilesystemAdapter.cjs +34 -12
- package/lib/fs/FilesystemAdapter.d.ts +4 -2
- package/lib/fs/FilesystemAdapter.js.map +1 -1
- package/lib/index.cjs +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/tasks/TaskEngine.cjs +15 -357
- package/lib/tasks/TaskEngine.d.ts +28 -46
- package/lib/tasks/TaskEngine.js.map +1 -1
- package/lib/tasks/TaskService.cjs +1 -4
- package/lib/tasks/TaskService.d.ts +4 -1
- package/lib/tasks/TaskService.js.map +1 -1
- package/lib/tasks/TaskTracker.d.ts +1 -1
- package/lib/tasks/TaskTracker.js.map +1 -1
- package/lib/tasks/constants.cjs +0 -1
- package/lib/tasks/constants.js.map +1 -1
- package/lib/tasks/types.d.ts +0 -12
- package/lib/workers/TaskEngine.cjs +503 -0
- package/lib/workers/TaskEngine.d.ts +40 -0
- package/lib/workers/TaskEngine.js.map +1 -0
- package/lib/{tasks/workers → workers}/WorkThreadEnvironment.d.ts +1 -0
- package/lib/workers/WorkThreadEnvironment.js.map +1 -0
- package/lib/workers/index.cjs +21 -0
- package/lib/workers/index.d.ts +4 -0
- package/lib/workers/index.js.map +1 -0
- package/lib/{esm/tasks/workers → workers}/messages.d.ts +1 -1
- package/lib/workers/messages.js.map +1 -0
- package/lib/workers/types.cjs +3 -0
- package/lib/workers/types.d.ts +41 -0
- package/lib/workers/types.js.map +1 -0
- package/lib/{tasks/workers → workers}/workerThread.cjs +17 -17
- package/lib/workers/workerThread.d.ts +1 -0
- package/lib/workers/workerThread.js.map +1 -0
- package/package.json +6 -1
- package/lib/esm/tasks/workers/WorkThreadEnvironment.js.map +0 -1
- package/lib/esm/tasks/workers/messages.js.map +0 -1
- package/lib/esm/tasks/workers/workerThread.d.ts +0 -1
- package/lib/esm/tasks/workers/workerThread.js.map +0 -1
- package/lib/tasks/workers/WorkThreadEnvironment.js.map +0 -1
- package/lib/tasks/workers/workerThread.d.ts +0 -1
- package/lib/tasks/workers/workerThread.js.map +0 -1
- /package/lib/esm/{tasks/workers → workers}/WorkThreadEnvironment.js +0 -0
- /package/lib/esm/{tasks/workers → workers}/messages.js +0 -0
- /package/lib/{tasks/workers → workers}/WorkThreadEnvironment.cjs +0 -0
- /package/lib/{tasks/workers → workers}/messages.cjs +0 -0
package/lib/tasks/TaskEngine.cjs
CHANGED
|
@@ -11,7 +11,6 @@ const TaskStepResultModel_1 = require("./models/TaskStepResultModel.cjs");
|
|
|
11
11
|
const TaskLogEntryModel_1 = require("./models/TaskLogEntryModel.cjs");
|
|
12
12
|
const TaskBackoffModel_1 = require("./models/TaskBackoffModel.cjs");
|
|
13
13
|
const TaskStepSpecModel_1 = require("./models/TaskStepSpecModel.cjs");
|
|
14
|
-
const worker_threads_1 = require("worker_threads");
|
|
15
14
|
const constants_1 = require("./constants.cjs");
|
|
16
15
|
const logging_1 = require("@decaf-ts/logging");
|
|
17
16
|
const ContextualLoggedClass_1 = require("./../utils/ContextualLoggedClass.cjs");
|
|
@@ -24,7 +23,6 @@ const TaskErrorModel_1 = require("./models/TaskErrorModel.cjs");
|
|
|
24
23
|
const TaskTracker_1 = require("./TaskTracker.cjs");
|
|
25
24
|
const transactional_decorators_1 = require("@decaf-ts/transactional-decorators");
|
|
26
25
|
const index_1 = require("./../persistence/index.cjs");
|
|
27
|
-
const WorkThreadEnvironment_1 = require("./workers/WorkThreadEnvironment.cjs");
|
|
28
26
|
class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
29
27
|
get Context() {
|
|
30
28
|
return TaskContext_1.TaskContext;
|
|
@@ -42,16 +40,12 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
42
40
|
if (this._tasks)
|
|
43
41
|
return this._tasks;
|
|
44
42
|
this._tasks = Repository_1.Repository.forModel(TaskModel_1.TaskModel, this.adapter.alias);
|
|
45
|
-
if (this.config.overrides)
|
|
46
|
-
this._tasks = this._tasks.override(this.config.overrides);
|
|
47
43
|
return this._tasks;
|
|
48
44
|
}
|
|
49
45
|
get events() {
|
|
50
46
|
if (this._events)
|
|
51
47
|
return this._events;
|
|
52
48
|
this._events = Repository_1.Repository.forModel(TaskEventModel_1.TaskEventModel, this.config.adapter.alias);
|
|
53
|
-
if (this.config.overrides)
|
|
54
|
-
this._events = this._events.override(this.config.overrides);
|
|
55
49
|
return this._events;
|
|
56
50
|
}
|
|
57
51
|
static createTaskContext(base, overrides) {
|
|
@@ -64,77 +58,12 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
64
58
|
constructor(config) {
|
|
65
59
|
super();
|
|
66
60
|
this.config = config;
|
|
67
|
-
this.workerThreads = [];
|
|
68
|
-
this.workerJobQueue = [];
|
|
69
|
-
this.workerJobs = new Map();
|
|
70
|
-
this.workerCounter = 0;
|
|
71
|
-
this.workerThreadCapacity = 1;
|
|
72
61
|
this.lock = new transactional_decorators_1.Lock();
|
|
73
62
|
this.running = false;
|
|
74
|
-
if (config.workerPool && !config.workerAdapter) {
|
|
75
|
-
throw new db_decorators_1.InternalError("Worker pool requires workerAdapter descriptor in TaskEngineConfig");
|
|
76
|
-
}
|
|
77
63
|
this.config = Object.assign({}, constants_1.DefaultTaskEngineConfig, config, {
|
|
78
64
|
bus: config.bus || new TaskEventBus_1.TaskEventBus(),
|
|
79
65
|
registry: config.registry || new TaskHandlerRegistry_1.TaskHandlerRegistry(),
|
|
80
66
|
});
|
|
81
|
-
this.workerThreadCapacity = Math.max(1, this.config.workerConcurrency ?? 1);
|
|
82
|
-
this.workerPoolConfig = this.normalizeWorkerPoolConfig(this.config.workerPool);
|
|
83
|
-
}
|
|
84
|
-
normalizeWorkerPoolConfig(pool) {
|
|
85
|
-
if (!pool)
|
|
86
|
-
return undefined;
|
|
87
|
-
if (!pool.entry) {
|
|
88
|
-
throw new db_decorators_1.InternalError("Worker pool configuration requires an explicit entry file path");
|
|
89
|
-
}
|
|
90
|
-
if (pool.size != null && pool.size !== this.config.concurrency) {
|
|
91
|
-
throw new db_decorators_1.InternalError("TaskEngine concurrency must match workerPool.size when worker pool is enabled");
|
|
92
|
-
}
|
|
93
|
-
return Object.assign({}, pool, {
|
|
94
|
-
size: this.config.concurrency,
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
hasWorkerPool() {
|
|
98
|
-
return !!this.workerPoolConfig && (this.workerPoolConfig.size ?? 0) > 0;
|
|
99
|
-
}
|
|
100
|
-
getWorkerCount() {
|
|
101
|
-
if (!this.workerPoolConfig)
|
|
102
|
-
return 0;
|
|
103
|
-
return this.workerPoolConfig.size ?? 0;
|
|
104
|
-
}
|
|
105
|
-
getWorkerExecutionSlots() {
|
|
106
|
-
return this.getWorkerCount() * this.workerThreadCapacity;
|
|
107
|
-
}
|
|
108
|
-
canDispatchToWorkers() {
|
|
109
|
-
return this.hasWorkerPool();
|
|
110
|
-
}
|
|
111
|
-
getExecutionConcurrency() {
|
|
112
|
-
if (this.hasWorkerPool()) {
|
|
113
|
-
return Math.max(1, this.getWorkerExecutionSlots());
|
|
114
|
-
}
|
|
115
|
-
return this.config.concurrency;
|
|
116
|
-
}
|
|
117
|
-
computeWorkerModules() {
|
|
118
|
-
const adapterDescriptor = this.config.workerAdapter ?? WorkThreadEnvironment_1.DefaultWorkThreadEnvironment.persistence;
|
|
119
|
-
if (!adapterDescriptor?.adapterModule) {
|
|
120
|
-
throw new db_decorators_1.InternalError("Worker adapter descriptor must include adapterModule");
|
|
121
|
-
}
|
|
122
|
-
const configuredImports = this.workerPoolConfig?.modules?.imports ??
|
|
123
|
-
WorkThreadEnvironment_1.DefaultWorkThreadEnvironment.modules.imports;
|
|
124
|
-
const imports = [];
|
|
125
|
-
const append = (specifier) => {
|
|
126
|
-
if (!specifier)
|
|
127
|
-
return;
|
|
128
|
-
if (!imports.includes(specifier))
|
|
129
|
-
imports.push(specifier);
|
|
130
|
-
};
|
|
131
|
-
append(adapterDescriptor.adapterModule);
|
|
132
|
-
for (const specifier of configuredImports) {
|
|
133
|
-
if (specifier === adapterDescriptor.adapterModule)
|
|
134
|
-
continue;
|
|
135
|
-
append(specifier);
|
|
136
|
-
}
|
|
137
|
-
return { imports };
|
|
138
67
|
}
|
|
139
68
|
async push(task, track = false, ...args) {
|
|
140
69
|
const { ctx, log } = (await this.logCtx(args, db_decorators_1.OperationKeys.CREATE, true)).for(this.push);
|
|
@@ -216,13 +145,10 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
216
145
|
async start(...args) {
|
|
217
146
|
const { ctx } = (await this.logCtx(args, "run", true)).for(this.start);
|
|
218
147
|
await this.lock.acquire();
|
|
219
|
-
if (this.running)
|
|
220
|
-
this.lock.release();
|
|
148
|
+
if (this.running)
|
|
221
149
|
return;
|
|
222
|
-
}
|
|
223
150
|
this.running = true;
|
|
224
151
|
this.lock.release();
|
|
225
|
-
await this.spawnWorkers();
|
|
226
152
|
void this.loop(ctx);
|
|
227
153
|
}
|
|
228
154
|
async stop(...args) {
|
|
@@ -238,7 +164,7 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
238
164
|
.execute(ctx);
|
|
239
165
|
const timeout = ctx.getOrUndefined?.("gracefulShutdownMsTimeout") ??
|
|
240
166
|
this.config.gracefulShutdownMsTimeout;
|
|
241
|
-
|
|
167
|
+
return new Promise((resolve, reject) => {
|
|
242
168
|
const timer = setTimeout(() => {
|
|
243
169
|
log.error(`Graceful shutdown interrupted after ${timeout} ms...`);
|
|
244
170
|
resolve();
|
|
@@ -260,249 +186,6 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
260
186
|
reject(err);
|
|
261
187
|
});
|
|
262
188
|
});
|
|
263
|
-
await this.shutdownWorkers();
|
|
264
|
-
}
|
|
265
|
-
// -------------------------
|
|
266
|
-
// Worker pool orchestration
|
|
267
|
-
// -------------------------
|
|
268
|
-
async spawnWorkers() {
|
|
269
|
-
if (!this.hasWorkerPool())
|
|
270
|
-
return;
|
|
271
|
-
const target = this.getWorkerCount();
|
|
272
|
-
const creations = [];
|
|
273
|
-
while (this.workerThreads.length < target) {
|
|
274
|
-
const ready = this.createWorker();
|
|
275
|
-
if (ready)
|
|
276
|
-
creations.push(ready);
|
|
277
|
-
}
|
|
278
|
-
if (creations.length) {
|
|
279
|
-
await Promise.all(creations);
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
createWorker() {
|
|
283
|
-
if (!this.workerPoolConfig)
|
|
284
|
-
return undefined;
|
|
285
|
-
let resolveReady;
|
|
286
|
-
let rejectReady;
|
|
287
|
-
const readyPromise = new Promise((resolve, reject) => {
|
|
288
|
-
resolveReady = resolve;
|
|
289
|
-
rejectReady = reject;
|
|
290
|
-
});
|
|
291
|
-
const entry = this.workerPoolConfig.entry;
|
|
292
|
-
const workerId = `${this.config.workerId}-${this.workerCounter++}`;
|
|
293
|
-
const env = {
|
|
294
|
-
workerId,
|
|
295
|
-
mode: this.workerPoolConfig.mode ?? "node",
|
|
296
|
-
persistence: Object.assign({}, this.config.workerAdapter ?? WorkThreadEnvironment_1.DefaultWorkThreadEnvironment.persistence, {
|
|
297
|
-
alias: this.adapter.alias,
|
|
298
|
-
flavour: this.adapter.flavour,
|
|
299
|
-
}),
|
|
300
|
-
taskEngine: {
|
|
301
|
-
concurrency: this.workerThreadCapacity,
|
|
302
|
-
leaseMs: this.config.leaseMs,
|
|
303
|
-
pollMsBusy: this.config.pollMsBusy,
|
|
304
|
-
pollMsIdle: this.config.pollMsIdle,
|
|
305
|
-
logTailMax: this.config.logTailMax,
|
|
306
|
-
streamBufferSize: this.config.streamBufferSize,
|
|
307
|
-
maxLoggingBuffer: this.config.maxLoggingBuffer,
|
|
308
|
-
loggingBufferTruncation: this.config.loggingBufferTruncation,
|
|
309
|
-
gracefulShutdownMsTimeout: this.config.gracefulShutdownMsTimeout,
|
|
310
|
-
},
|
|
311
|
-
modules: this.computeWorkerModules(),
|
|
312
|
-
};
|
|
313
|
-
const worker = new worker_threads_1.Worker(entry, {
|
|
314
|
-
workerData: { environment: env },
|
|
315
|
-
});
|
|
316
|
-
const state = {
|
|
317
|
-
id: workerId,
|
|
318
|
-
worker,
|
|
319
|
-
ready: false,
|
|
320
|
-
activeJobs: 0,
|
|
321
|
-
capacity: this.workerThreadCapacity,
|
|
322
|
-
readyPromise,
|
|
323
|
-
resolveReady,
|
|
324
|
-
rejectReady,
|
|
325
|
-
};
|
|
326
|
-
this.workerThreads.push(state);
|
|
327
|
-
worker.on("message", (msg) => this.handleWorkerMessage(state, msg));
|
|
328
|
-
worker.on("error", (err) => this.handleWorkerError(state, err));
|
|
329
|
-
worker.on("exit", (code) => this.handleWorkerExit(state, code));
|
|
330
|
-
return readyPromise;
|
|
331
|
-
}
|
|
332
|
-
async shutdownWorkers() {
|
|
333
|
-
for (const state of this.workerThreads.splice(0)) {
|
|
334
|
-
const message = {
|
|
335
|
-
type: "control",
|
|
336
|
-
command: "shutdown",
|
|
337
|
-
};
|
|
338
|
-
try {
|
|
339
|
-
state.worker.postMessage(message);
|
|
340
|
-
}
|
|
341
|
-
catch {
|
|
342
|
-
// ignore
|
|
343
|
-
}
|
|
344
|
-
await state.worker.terminate();
|
|
345
|
-
}
|
|
346
|
-
for (const job of this.workerJobQueue.splice(0)) {
|
|
347
|
-
job.reject(new db_decorators_1.InternalError(`Worker pool shutting down before job ${job.id} could start`));
|
|
348
|
-
}
|
|
349
|
-
for (const job of this.workerJobs.values()) {
|
|
350
|
-
job.reject(new db_decorators_1.InternalError(`Worker terminated before finishing job ${job.id}`));
|
|
351
|
-
}
|
|
352
|
-
this.workerJobs.clear();
|
|
353
|
-
}
|
|
354
|
-
handleWorkerError(state, err) {
|
|
355
|
-
this.log.error(`worker ${state.id} error: ${err.message}`, err);
|
|
356
|
-
if (state.rejectReady) {
|
|
357
|
-
state.rejectReady(err);
|
|
358
|
-
state.resolveReady = undefined;
|
|
359
|
-
state.rejectReady = undefined;
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
handleWorkerExit(state, code) {
|
|
363
|
-
this.log.info(`worker ${state.id} exited with code ${code}`);
|
|
364
|
-
if (state.rejectReady) {
|
|
365
|
-
state.rejectReady(new Error(`worker ${state.id} exited before reporting ready`));
|
|
366
|
-
state.resolveReady = undefined;
|
|
367
|
-
state.rejectReady = undefined;
|
|
368
|
-
}
|
|
369
|
-
const idx = this.workerThreads.indexOf(state);
|
|
370
|
-
if (idx >= 0)
|
|
371
|
-
this.workerThreads.splice(idx, 1);
|
|
372
|
-
for (const [jobId, job] of this.workerJobs.entries()) {
|
|
373
|
-
if (job.worker === state) {
|
|
374
|
-
job.worker = undefined;
|
|
375
|
-
this.workerJobs.delete(jobId);
|
|
376
|
-
this.workerJobQueue.unshift(job);
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
if (this.running) {
|
|
380
|
-
void this.spawnWorkers().catch((err) => this.log.error(`failed to respawn worker`, err));
|
|
381
|
-
this.processWorkerQueue();
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
handleWorkerMessage(state, msg) {
|
|
385
|
-
if (msg.type === "ready") {
|
|
386
|
-
state.ready = true;
|
|
387
|
-
if (state.resolveReady) {
|
|
388
|
-
state.resolveReady();
|
|
389
|
-
state.resolveReady = undefined;
|
|
390
|
-
state.rejectReady = undefined;
|
|
391
|
-
}
|
|
392
|
-
this.log.info(`worker ${state.id} ready`);
|
|
393
|
-
this.processWorkerQueue();
|
|
394
|
-
return;
|
|
395
|
-
}
|
|
396
|
-
if (msg.type === "error") {
|
|
397
|
-
this.log.error(`worker ${state.id} reported error: ${msg.error}`);
|
|
398
|
-
return;
|
|
399
|
-
}
|
|
400
|
-
if (msg.type === "log") {
|
|
401
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
402
|
-
if (!job)
|
|
403
|
-
return;
|
|
404
|
-
void this.appendLog(job.ctx, job.task, msg.entries)
|
|
405
|
-
.then(([updated, entries]) => {
|
|
406
|
-
job.task = updated;
|
|
407
|
-
return this.emitLog(job.ctx, job.task.id, entries);
|
|
408
|
-
})
|
|
409
|
-
.catch((err) => this.log.error(`Failed to append worker log`, err));
|
|
410
|
-
return;
|
|
411
|
-
}
|
|
412
|
-
if (msg.type === "progress") {
|
|
413
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
414
|
-
if (!job)
|
|
415
|
-
return;
|
|
416
|
-
void this.emitProgress(job.ctx, job.task.id, msg.payload);
|
|
417
|
-
return;
|
|
418
|
-
}
|
|
419
|
-
if (msg.type === "heartbeat") {
|
|
420
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
421
|
-
if (!job)
|
|
422
|
-
return;
|
|
423
|
-
job.task.leaseExpiry = new Date(Date.now() + this.config.leaseMs);
|
|
424
|
-
void this.tasks.update(job.task).catch(() => null);
|
|
425
|
-
return;
|
|
426
|
-
}
|
|
427
|
-
if (msg.type === "result") {
|
|
428
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
429
|
-
if (!job)
|
|
430
|
-
return;
|
|
431
|
-
this.workerJobs.delete(job.id);
|
|
432
|
-
state.activeJobs = Math.max(0, state.activeJobs - 1);
|
|
433
|
-
this.applyWorkerCache(job.ctx, msg.cache);
|
|
434
|
-
switch (msg.status) {
|
|
435
|
-
case "success":
|
|
436
|
-
job.resolve(msg.output);
|
|
437
|
-
break;
|
|
438
|
-
case "error": {
|
|
439
|
-
const err = new Error(msg.error.message);
|
|
440
|
-
if (msg.error.name)
|
|
441
|
-
err.name = msg.error.name;
|
|
442
|
-
if (msg.error.stack)
|
|
443
|
-
err.stack = msg.error.stack;
|
|
444
|
-
job.reject(err);
|
|
445
|
-
break;
|
|
446
|
-
}
|
|
447
|
-
case "state-change":
|
|
448
|
-
job.reject(new TaskStateChangeError_1.TaskStateChangeError(msg.request));
|
|
449
|
-
break;
|
|
450
|
-
}
|
|
451
|
-
this.processWorkerQueue();
|
|
452
|
-
return;
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
applyWorkerCache(ctx, cache) {
|
|
456
|
-
if (!cache)
|
|
457
|
-
return;
|
|
458
|
-
Object.entries(cache).forEach(([key, value]) => {
|
|
459
|
-
ctx.cacheResult(key, value);
|
|
460
|
-
});
|
|
461
|
-
}
|
|
462
|
-
processWorkerQueue() {
|
|
463
|
-
if (!this.hasWorkerPool())
|
|
464
|
-
return;
|
|
465
|
-
const available = this.workerThreads
|
|
466
|
-
.filter((state) => state.ready && state.activeJobs < state.capacity)
|
|
467
|
-
.sort((a, b) => a.activeJobs - b.activeJobs);
|
|
468
|
-
for (const state of available) {
|
|
469
|
-
while (state.activeJobs < state.capacity &&
|
|
470
|
-
this.workerJobQueue.length > 0) {
|
|
471
|
-
const job = this.workerJobQueue.shift();
|
|
472
|
-
if (!job)
|
|
473
|
-
break;
|
|
474
|
-
this.assignWorker(state, job);
|
|
475
|
-
}
|
|
476
|
-
if (!this.workerJobQueue.length)
|
|
477
|
-
break;
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
assignWorker(state, job) {
|
|
481
|
-
if (!this.workerPoolConfig)
|
|
482
|
-
return;
|
|
483
|
-
const payload = {
|
|
484
|
-
jobId: job.id,
|
|
485
|
-
taskId: job.task.id,
|
|
486
|
-
classification: job.classification,
|
|
487
|
-
input: job.input,
|
|
488
|
-
attempt: job.task.attempt ?? 0,
|
|
489
|
-
resultCache: job.ctx.resultCache ?? {},
|
|
490
|
-
streamBufferSize: this.config.streamBufferSize,
|
|
491
|
-
maxLoggingBuffer: this.config.maxLoggingBuffer,
|
|
492
|
-
loggingBufferTruncation: this.config.loggingBufferTruncation,
|
|
493
|
-
};
|
|
494
|
-
job.worker = state;
|
|
495
|
-
this.workerJobs.set(job.id, job);
|
|
496
|
-
state.activeJobs += 1;
|
|
497
|
-
const message = {
|
|
498
|
-
type: "execute",
|
|
499
|
-
job: payload,
|
|
500
|
-
};
|
|
501
|
-
state.worker.postMessage(message);
|
|
502
|
-
}
|
|
503
|
-
enqueueWorkerJob(job) {
|
|
504
|
-
this.workerJobQueue.push(job);
|
|
505
|
-
this.processWorkerQueue();
|
|
506
189
|
}
|
|
507
190
|
// -------------------------
|
|
508
191
|
// Worker loop
|
|
@@ -537,11 +220,10 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
537
220
|
.or(condLeaseExpired)
|
|
538
221
|
.or(condScheduled);
|
|
539
222
|
// Fetch more than concurrency because some will fail to claim due to conflicts
|
|
540
|
-
const concurrency = this.getExecutionConcurrency();
|
|
541
223
|
const candidates = await this.tasks
|
|
542
224
|
.select()
|
|
543
225
|
.where(runnable)
|
|
544
|
-
.limit(Math.max(concurrency * 4, 20))
|
|
226
|
+
.limit(Math.max(this.config.concurrency * 4, 20))
|
|
545
227
|
.execute();
|
|
546
228
|
log.verbose(`claimBatch candidates:${candidates.length}`);
|
|
547
229
|
const out = [];
|
|
@@ -549,7 +231,7 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
549
231
|
const claimed = await this.tryClaim(c, ctx);
|
|
550
232
|
if (claimed)
|
|
551
233
|
out.push(claimed);
|
|
552
|
-
if (out.length >= concurrency)
|
|
234
|
+
if (out.length >= this.config.concurrency)
|
|
553
235
|
break;
|
|
554
236
|
}
|
|
555
237
|
log.verbose(`claimBatch claimed:${out.length}`);
|
|
@@ -585,39 +267,9 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
585
267
|
// -------------------------
|
|
586
268
|
// Execution
|
|
587
269
|
// -------------------------
|
|
588
|
-
async runHandlerInline(classification, input, ctx) {
|
|
589
|
-
const handler = this.registry.get(classification);
|
|
590
|
-
if (!handler)
|
|
591
|
-
throw new db_decorators_1.InternalError(`No task handler registered for type: ${classification}`);
|
|
592
|
-
return handler.run(input, ctx);
|
|
593
|
-
}
|
|
594
|
-
async dispatchToWorker(classification, input, task, ctx) {
|
|
595
|
-
if (!this.canDispatchToWorkers()) {
|
|
596
|
-
return this.runHandlerInline(classification, input, ctx);
|
|
597
|
-
}
|
|
598
|
-
const uuid = await index_1.UUID.instance.generate();
|
|
599
|
-
return new Promise((resolve, reject) => {
|
|
600
|
-
const job = {
|
|
601
|
-
id: uuid,
|
|
602
|
-
classification,
|
|
603
|
-
input,
|
|
604
|
-
task,
|
|
605
|
-
ctx,
|
|
606
|
-
resolve,
|
|
607
|
-
reject,
|
|
608
|
-
};
|
|
609
|
-
this.enqueueWorkerJob(job);
|
|
610
|
-
});
|
|
611
|
-
}
|
|
612
|
-
async invokeHandler(classification, input, task, ctx) {
|
|
613
|
-
if (!this.hasWorkerPool()) {
|
|
614
|
-
return this.runHandlerInline(classification, input, ctx);
|
|
615
|
-
}
|
|
616
|
-
return this.dispatchToWorker(classification, input, task, ctx);
|
|
617
|
-
}
|
|
618
270
|
async executeClaimed(task) {
|
|
619
271
|
const { ctx, log } = (await this.logCtx([], task.classification, true)).for(this.executeClaimed);
|
|
620
|
-
const taskCtx =
|
|
272
|
+
const taskCtx = new TaskContext_1.TaskContext(ctx).accumulate({
|
|
621
273
|
taskId: task.id,
|
|
622
274
|
logger: new logging_2.TaskLogger(log, this.config.streamBufferSize, this.config.maxLoggingBuffer),
|
|
623
275
|
attempt: task.attempt,
|
|
@@ -627,7 +279,7 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
627
279
|
await this.emitLog(taskCtx, task.id, logs);
|
|
628
280
|
},
|
|
629
281
|
flush: async () => {
|
|
630
|
-
|
|
282
|
+
return taskCtx.logger.flush(taskCtx.pipe);
|
|
631
283
|
},
|
|
632
284
|
progress: async (data) => {
|
|
633
285
|
await this.emitProgress(taskCtx, task.id, data);
|
|
@@ -662,8 +314,11 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
662
314
|
}
|
|
663
315
|
}
|
|
664
316
|
else {
|
|
665
|
-
|
|
666
|
-
|
|
317
|
+
const handler = this.registry.get(task.classification);
|
|
318
|
+
log.debug(`handler type for ${task.id} is ${handler?.constructor?.name ?? "none"}`);
|
|
319
|
+
if (!handler)
|
|
320
|
+
throw new db_decorators_1.InternalError(`No task handler registered for type: ${task.classification}`);
|
|
321
|
+
output = await handler.run(task.input, taskCtx);
|
|
667
322
|
log.verbose(`handler finished for ${task.id}`);
|
|
668
323
|
}
|
|
669
324
|
task.status = constants_1.TaskStatus.SUCCEEDED;
|
|
@@ -821,12 +476,15 @@ class TaskEngine extends ContextualLoggedClass_1.AbsContextual {
|
|
|
821
476
|
}
|
|
822
477
|
while (idx < steps.length) {
|
|
823
478
|
const step = steps[idx];
|
|
479
|
+
const handler = this.registry.get(step.classification);
|
|
480
|
+
if (!handler)
|
|
481
|
+
throw new Error(`No task handler registered for composite step: ${step.classification}`);
|
|
824
482
|
await context.pipe([
|
|
825
483
|
logging_1.LogLevel.info,
|
|
826
484
|
`Composite step ${idx + 1}/${steps.length}: ${step.classification}`,
|
|
827
485
|
]);
|
|
828
486
|
try {
|
|
829
|
-
const out = await
|
|
487
|
+
const out = await handler.run(step.input, context);
|
|
830
488
|
const stepIndex = idx;
|
|
831
489
|
const now = new Date();
|
|
832
490
|
results[stepIndex] = new TaskStepResultModel_1.TaskStepResultModel({
|
|
@@ -3,27 +3,29 @@ import { Repo } from "../repository/Repository";
|
|
|
3
3
|
import { TaskEventModel } from "./models/TaskEventModel";
|
|
4
4
|
import { TaskHandlerRegistry } from "./TaskHandlerRegistry";
|
|
5
5
|
import { TaskEventBus } from "./TaskEventBus";
|
|
6
|
+
import { TaskStepResultModel } from "./models/TaskStepResultModel";
|
|
7
|
+
import { TaskLogEntryModel } from "./models/TaskLogEntryModel";
|
|
8
|
+
import { TaskBackoffModel } from "./models/TaskBackoffModel";
|
|
9
|
+
import { TaskStepSpecModel } from "./models/TaskStepSpecModel";
|
|
10
|
+
import { TaskEventType, TaskStatus } from "./constants";
|
|
6
11
|
import { Adapter } from "../persistence/Adapter";
|
|
7
12
|
import { Context } from "../persistence/Context";
|
|
8
13
|
import { ContextOf, FlagsOf } from "../persistence/types";
|
|
9
|
-
import {
|
|
14
|
+
import { LogLevel } from "@decaf-ts/logging";
|
|
15
|
+
import { AbsContextual, ContextualArgs, MaybeContextualArg } from "../utils/ContextualLoggedClass";
|
|
10
16
|
import { OperationKeys } from "@decaf-ts/db-decorators";
|
|
11
17
|
import { TaskContext } from "./TaskContext";
|
|
18
|
+
import { TaskStateChangeRequest } from "./TaskStateChangeError";
|
|
12
19
|
import { DateTarget } from "@decaf-ts/decorator-validation";
|
|
13
20
|
import { Constructor } from "@decaf-ts/decoration";
|
|
14
21
|
import { TaskEngineConfig, TaskFlags } from "./types";
|
|
15
22
|
import { TaskTracker } from "./TaskTracker";
|
|
16
|
-
|
|
17
|
-
|
|
23
|
+
import { Lock } from "@decaf-ts/transactional-decorators";
|
|
24
|
+
export declare class TaskEngine<A extends Adapter<any, any, any, any>, C extends TaskEngineConfig<A> = TaskEngineConfig<A>> extends AbsContextual<ContextOf<A>> {
|
|
25
|
+
protected config: C;
|
|
18
26
|
private _tasks?;
|
|
19
27
|
private _events?;
|
|
20
|
-
|
|
21
|
-
private workerThreads;
|
|
22
|
-
private workerJobQueue;
|
|
23
|
-
private workerJobs;
|
|
24
|
-
private workerCounter;
|
|
25
|
-
private workerThreadCapacity;
|
|
26
|
-
private lock;
|
|
28
|
+
protected lock: Lock;
|
|
27
29
|
protected get Context(): Constructor<ContextOf<A>>;
|
|
28
30
|
protected get adapter(): A;
|
|
29
31
|
protected get registry(): TaskHandlerRegistry;
|
|
@@ -32,14 +34,7 @@ export declare class TaskEngine<A extends Adapter<any, any, any, any>> extends A
|
|
|
32
34
|
protected get events(): Repo<TaskEventModel>;
|
|
33
35
|
protected running: boolean;
|
|
34
36
|
static createTaskContext(base?: Context<any>, overrides?: Partial<TaskFlags>): TaskContext;
|
|
35
|
-
constructor(config:
|
|
36
|
-
private normalizeWorkerPoolConfig;
|
|
37
|
-
private hasWorkerPool;
|
|
38
|
-
private getWorkerCount;
|
|
39
|
-
private getWorkerExecutionSlots;
|
|
40
|
-
private canDispatchToWorkers;
|
|
41
|
-
private getExecutionConcurrency;
|
|
42
|
-
private computeWorkerModules;
|
|
37
|
+
constructor(config: C);
|
|
43
38
|
push<I, O>(task: TaskModel<I, O>, ...args: MaybeContextualArg<any>): Promise<TaskModel<I, O>>;
|
|
44
39
|
push<I, O>(task: TaskModel<I, O>, track: false, ...args: MaybeContextualArg<any>): Promise<TaskModel<I, O>>;
|
|
45
40
|
push<I, O>(task: TaskModel<I, O>, track: true, ...args: MaybeContextualArg<any>): Promise<{
|
|
@@ -62,38 +57,25 @@ export declare class TaskEngine<A extends Adapter<any, any, any, any>> extends A
|
|
|
62
57
|
task: TaskModel<any, any>;
|
|
63
58
|
tracker: TaskTracker<any>;
|
|
64
59
|
}>;
|
|
65
|
-
|
|
60
|
+
protected ensureTaskError(task: TaskModel, ctx: Context): Promise<TaskModel>;
|
|
66
61
|
cancel(id: string, ...args: MaybeContextualArg<any>): Promise<TaskModel>;
|
|
67
62
|
isRunning(): Promise<boolean>;
|
|
68
63
|
start(...args: MaybeContextualArg<any>): Promise<void>;
|
|
69
64
|
stop(...args: MaybeContextualArg<any>): Promise<void>;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
private dispatchToWorker;
|
|
85
|
-
private invokeHandler;
|
|
86
|
-
private executeClaimed;
|
|
87
|
-
private handleTaskStateChange;
|
|
88
|
-
private runComposite;
|
|
89
|
-
private normalizeBackoff;
|
|
90
|
-
private normalizeSteps;
|
|
91
|
-
private normalizeStepResults;
|
|
92
|
-
private appendLog;
|
|
93
|
-
private emitStatus;
|
|
94
|
-
private emitLog;
|
|
95
|
-
private emitProgress;
|
|
96
|
-
private persistEvent;
|
|
65
|
+
protected loop(...args: ContextualArgs<any>): Promise<void>;
|
|
66
|
+
protected claimBatch(ctx: Context<any>): Promise<TaskModel[]>;
|
|
67
|
+
protected tryClaim(task: TaskModel, ctx: Context): Promise<TaskModel | null>;
|
|
68
|
+
protected executeClaimed(task: TaskModel): Promise<void>;
|
|
69
|
+
protected handleTaskStateChange(request: TaskStateChangeRequest, task: TaskModel, ctx: TaskContext): Promise<void>;
|
|
70
|
+
protected runComposite(task: TaskModel, context: TaskContext): Promise<any>;
|
|
71
|
+
protected normalizeBackoff(backoff: TaskBackoffModel | string | object | any): TaskBackoffModel;
|
|
72
|
+
protected normalizeSteps(steps: TaskStepSpecModel[] | string | undefined): TaskStepSpecModel[];
|
|
73
|
+
protected normalizeStepResults(results: TaskStepResultModel[] | string | undefined): TaskStepResultModel[];
|
|
74
|
+
protected appendLog(ctx: TaskContext | Context, task: TaskModel, logEntries: [LogLevel, string] | [LogLevel, string, any] | ([LogLevel, string] | [LogLevel, string, any])[]): Promise<[TaskModel, TaskLogEntryModel[]]>;
|
|
75
|
+
protected emitStatus(ctx: TaskContext | Context, task: TaskModel, status: TaskStatus, outputOrError?: any | Error, originalError?: Error): Promise<void>;
|
|
76
|
+
protected emitLog(ctx: TaskContext | Context, taskId: string, entries: TaskLogEntryModel[]): Promise<void>;
|
|
77
|
+
protected emitProgress(ctx: TaskContext | Context, taskId: string, data: any): Promise<void>;
|
|
78
|
+
protected persistEvent(ctx: TaskContext | Context, taskId: string, type: TaskEventType, payload: any): Promise<TaskEventModel>;
|
|
97
79
|
toString(): string;
|
|
98
80
|
context(operation: ((...args: any[]) => any) | OperationKeys.CREATE | OperationKeys.READ | OperationKeys.UPDATE | OperationKeys.DELETE | string, overrides: Partial<FlagsOf<ContextOf<A>>>, ...args: any[]): Promise<ContextOf<A>>;
|
|
99
81
|
}
|