@decaf-ts/core 0.8.52 → 0.8.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/core.cjs +1 -1
- package/dist/core.cjs.map +1 -1
- package/dist/core.js +1 -1
- package/dist/core.js.map +1 -1
- package/lib/esm/index.d.ts +1 -1
- package/lib/esm/index.js +1 -1
- package/lib/esm/tasks/TaskEngine.d.ts +28 -46
- package/lib/esm/tasks/TaskEngine.js +16 -358
- package/lib/esm/tasks/TaskEngine.js.map +1 -1
- package/lib/esm/tasks/TaskService.d.ts +4 -1
- package/lib/esm/tasks/TaskService.js +1 -4
- package/lib/esm/tasks/TaskService.js.map +1 -1
- package/lib/esm/tasks/TaskTracker.d.ts +1 -1
- package/lib/esm/tasks/TaskTracker.js.map +1 -1
- package/lib/esm/tasks/constants.js +0 -1
- package/lib/esm/tasks/constants.js.map +1 -1
- package/lib/esm/tasks/types.d.ts +0 -12
- package/lib/esm/workers/TaskEngine.d.ts +40 -0
- package/lib/esm/workers/TaskEngine.js +496 -0
- package/lib/esm/workers/TaskEngine.js.map +1 -0
- package/lib/esm/workers/WorkThreadEnvironment.js.map +1 -0
- package/lib/esm/workers/index.d.ts +4 -0
- package/lib/esm/workers/index.js +5 -0
- package/lib/esm/workers/index.js.map +1 -0
- package/lib/{tasks → esm}/workers/messages.d.ts +1 -1
- package/lib/{tasks → esm}/workers/messages.js.map +1 -1
- package/lib/esm/workers/types.d.ts +41 -0
- package/lib/esm/workers/types.js +2 -0
- package/lib/esm/workers/types.js.map +1 -0
- package/lib/esm/workers/workerThread.d.ts +1 -0
- package/lib/esm/{tasks/workers → workers}/workerThread.js +5 -8
- package/lib/esm/workers/workerThread.js.map +1 -0
- package/lib/index.cjs +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/tasks/TaskEngine.cjs +15 -357
- package/lib/tasks/TaskEngine.d.ts +28 -46
- package/lib/tasks/TaskEngine.js.map +1 -1
- package/lib/tasks/TaskService.cjs +1 -4
- package/lib/tasks/TaskService.d.ts +4 -1
- package/lib/tasks/TaskService.js.map +1 -1
- package/lib/tasks/TaskTracker.d.ts +1 -1
- package/lib/tasks/TaskTracker.js.map +1 -1
- package/lib/tasks/constants.cjs +0 -1
- package/lib/tasks/constants.js.map +1 -1
- package/lib/tasks/types.d.ts +0 -12
- package/lib/workers/TaskEngine.cjs +500 -0
- package/lib/workers/TaskEngine.d.ts +40 -0
- package/lib/workers/TaskEngine.js.map +1 -0
- package/lib/workers/WorkThreadEnvironment.js.map +1 -0
- package/lib/workers/index.cjs +21 -0
- package/lib/workers/index.d.ts +4 -0
- package/lib/workers/index.js.map +1 -0
- package/lib/{esm/tasks/workers → workers}/messages.d.ts +1 -1
- package/lib/workers/messages.js.map +1 -0
- package/lib/workers/types.cjs +3 -0
- package/lib/workers/types.d.ts +41 -0
- package/lib/workers/types.js.map +1 -0
- package/lib/{tasks/workers → workers}/workerThread.cjs +14 -17
- package/lib/workers/workerThread.d.ts +1 -0
- package/lib/workers/workerThread.js.map +1 -0
- package/package.json +6 -1
- package/lib/esm/tasks/workers/WorkThreadEnvironment.js.map +0 -1
- package/lib/esm/tasks/workers/messages.js.map +0 -1
- package/lib/esm/tasks/workers/workerThread.d.ts +0 -1
- package/lib/esm/tasks/workers/workerThread.js.map +0 -1
- package/lib/tasks/workers/WorkThreadEnvironment.js.map +0 -1
- package/lib/tasks/workers/workerThread.d.ts +0 -1
- package/lib/tasks/workers/workerThread.js.map +0 -1
- /package/lib/esm/{tasks/workers → workers}/WorkThreadEnvironment.d.ts +0 -0
- /package/lib/esm/{tasks/workers → workers}/WorkThreadEnvironment.js +0 -0
- /package/lib/esm/{tasks/workers → workers}/messages.js +0 -0
- /package/lib/{tasks/workers → workers}/WorkThreadEnvironment.cjs +0 -0
- /package/lib/{tasks/workers → workers}/WorkThreadEnvironment.d.ts +0 -0
- /package/lib/{tasks/workers → workers}/messages.cjs +0 -0
package/lib/esm/index.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ export * from "./persistence";
|
|
|
22
22
|
* @const VERSION
|
|
23
23
|
* @memberOf module:core
|
|
24
24
|
*/
|
|
25
|
-
export declare const VERSION = "0.8.
|
|
25
|
+
export declare const VERSION = "0.8.53";
|
|
26
26
|
/**
|
|
27
27
|
* @description Stores the current package version
|
|
28
28
|
* @summary A constant representing the version of the core package
|
package/lib/esm/index.js
CHANGED
|
@@ -32,7 +32,7 @@ export * from "./persistence/index.js";
|
|
|
32
32
|
* @const VERSION
|
|
33
33
|
* @memberOf module:core
|
|
34
34
|
*/
|
|
35
|
-
export const VERSION = "0.8.
|
|
35
|
+
export const VERSION = "0.8.53";
|
|
36
36
|
/**
|
|
37
37
|
* @description Stores the current package version
|
|
38
38
|
* @summary A constant representing the version of the core package
|
|
@@ -3,27 +3,29 @@ import { Repo } from "../repository/Repository";
|
|
|
3
3
|
import { TaskEventModel } from "./models/TaskEventModel";
|
|
4
4
|
import { TaskHandlerRegistry } from "./TaskHandlerRegistry";
|
|
5
5
|
import { TaskEventBus } from "./TaskEventBus";
|
|
6
|
+
import { TaskStepResultModel } from "./models/TaskStepResultModel";
|
|
7
|
+
import { TaskLogEntryModel } from "./models/TaskLogEntryModel";
|
|
8
|
+
import { TaskBackoffModel } from "./models/TaskBackoffModel";
|
|
9
|
+
import { TaskStepSpecModel } from "./models/TaskStepSpecModel";
|
|
10
|
+
import { TaskEventType, TaskStatus } from "./constants";
|
|
6
11
|
import { Adapter } from "../persistence/Adapter";
|
|
7
12
|
import { Context } from "../persistence/Context";
|
|
8
13
|
import { ContextOf, FlagsOf } from "../persistence/types";
|
|
9
|
-
import {
|
|
14
|
+
import { LogLevel } from "@decaf-ts/logging";
|
|
15
|
+
import { AbsContextual, ContextualArgs, MaybeContextualArg } from "../utils/ContextualLoggedClass";
|
|
10
16
|
import { OperationKeys } from "@decaf-ts/db-decorators";
|
|
11
17
|
import { TaskContext } from "./TaskContext";
|
|
18
|
+
import { TaskStateChangeRequest } from "./TaskStateChangeError";
|
|
12
19
|
import { DateTarget } from "@decaf-ts/decorator-validation";
|
|
13
20
|
import { Constructor } from "@decaf-ts/decoration";
|
|
14
21
|
import { TaskEngineConfig, TaskFlags } from "./types";
|
|
15
22
|
import { TaskTracker } from "./TaskTracker";
|
|
16
|
-
|
|
17
|
-
|
|
23
|
+
import { Lock } from "@decaf-ts/transactional-decorators";
|
|
24
|
+
export declare class TaskEngine<A extends Adapter<any, any, any, any>, C extends TaskEngineConfig<A> = TaskEngineConfig<A>> extends AbsContextual<ContextOf<A>> {
|
|
25
|
+
protected config: C;
|
|
18
26
|
private _tasks?;
|
|
19
27
|
private _events?;
|
|
20
|
-
|
|
21
|
-
private workerThreads;
|
|
22
|
-
private workerJobQueue;
|
|
23
|
-
private workerJobs;
|
|
24
|
-
private workerCounter;
|
|
25
|
-
private workerThreadCapacity;
|
|
26
|
-
private lock;
|
|
28
|
+
protected lock: Lock;
|
|
27
29
|
protected get Context(): Constructor<ContextOf<A>>;
|
|
28
30
|
protected get adapter(): A;
|
|
29
31
|
protected get registry(): TaskHandlerRegistry;
|
|
@@ -32,14 +34,7 @@ export declare class TaskEngine<A extends Adapter<any, any, any, any>> extends A
|
|
|
32
34
|
protected get events(): Repo<TaskEventModel>;
|
|
33
35
|
protected running: boolean;
|
|
34
36
|
static createTaskContext(base?: Context<any>, overrides?: Partial<TaskFlags>): TaskContext;
|
|
35
|
-
constructor(config:
|
|
36
|
-
private normalizeWorkerPoolConfig;
|
|
37
|
-
private hasWorkerPool;
|
|
38
|
-
private getWorkerCount;
|
|
39
|
-
private getWorkerExecutionSlots;
|
|
40
|
-
private canDispatchToWorkers;
|
|
41
|
-
private getExecutionConcurrency;
|
|
42
|
-
private computeWorkerModules;
|
|
37
|
+
constructor(config: C);
|
|
43
38
|
push<I, O>(task: TaskModel<I, O>, ...args: MaybeContextualArg<any>): Promise<TaskModel<I, O>>;
|
|
44
39
|
push<I, O>(task: TaskModel<I, O>, track: false, ...args: MaybeContextualArg<any>): Promise<TaskModel<I, O>>;
|
|
45
40
|
push<I, O>(task: TaskModel<I, O>, track: true, ...args: MaybeContextualArg<any>): Promise<{
|
|
@@ -62,38 +57,25 @@ export declare class TaskEngine<A extends Adapter<any, any, any, any>> extends A
|
|
|
62
57
|
task: TaskModel<any, any>;
|
|
63
58
|
tracker: TaskTracker<any>;
|
|
64
59
|
}>;
|
|
65
|
-
|
|
60
|
+
protected ensureTaskError(task: TaskModel, ctx: Context): Promise<TaskModel>;
|
|
66
61
|
cancel(id: string, ...args: MaybeContextualArg<any>): Promise<TaskModel>;
|
|
67
62
|
isRunning(): Promise<boolean>;
|
|
68
63
|
start(...args: MaybeContextualArg<any>): Promise<void>;
|
|
69
64
|
stop(...args: MaybeContextualArg<any>): Promise<void>;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
private dispatchToWorker;
|
|
85
|
-
private invokeHandler;
|
|
86
|
-
private executeClaimed;
|
|
87
|
-
private handleTaskStateChange;
|
|
88
|
-
private runComposite;
|
|
89
|
-
private normalizeBackoff;
|
|
90
|
-
private normalizeSteps;
|
|
91
|
-
private normalizeStepResults;
|
|
92
|
-
private appendLog;
|
|
93
|
-
private emitStatus;
|
|
94
|
-
private emitLog;
|
|
95
|
-
private emitProgress;
|
|
96
|
-
private persistEvent;
|
|
65
|
+
protected loop(...args: ContextualArgs<any>): Promise<void>;
|
|
66
|
+
protected claimBatch(ctx: Context<any>): Promise<TaskModel[]>;
|
|
67
|
+
protected tryClaim(task: TaskModel, ctx: Context): Promise<TaskModel | null>;
|
|
68
|
+
protected executeClaimed(task: TaskModel): Promise<void>;
|
|
69
|
+
protected handleTaskStateChange(request: TaskStateChangeRequest, task: TaskModel, ctx: TaskContext): Promise<void>;
|
|
70
|
+
protected runComposite(task: TaskModel, context: TaskContext): Promise<any>;
|
|
71
|
+
protected normalizeBackoff(backoff: TaskBackoffModel | string | object | any): TaskBackoffModel;
|
|
72
|
+
protected normalizeSteps(steps: TaskStepSpecModel[] | string | undefined): TaskStepSpecModel[];
|
|
73
|
+
protected normalizeStepResults(results: TaskStepResultModel[] | string | undefined): TaskStepResultModel[];
|
|
74
|
+
protected appendLog(ctx: TaskContext | Context, task: TaskModel, logEntries: [LogLevel, string] | [LogLevel, string, any] | ([LogLevel, string] | [LogLevel, string, any])[]): Promise<[TaskModel, TaskLogEntryModel[]]>;
|
|
75
|
+
protected emitStatus(ctx: TaskContext | Context, task: TaskModel, status: TaskStatus, outputOrError?: any | Error, originalError?: Error): Promise<void>;
|
|
76
|
+
protected emitLog(ctx: TaskContext | Context, taskId: string, entries: TaskLogEntryModel[]): Promise<void>;
|
|
77
|
+
protected emitProgress(ctx: TaskContext | Context, taskId: string, data: any): Promise<void>;
|
|
78
|
+
protected persistEvent(ctx: TaskContext | Context, taskId: string, type: TaskEventType, payload: any): Promise<TaskEventModel>;
|
|
97
79
|
toString(): string;
|
|
98
80
|
context(operation: ((...args: any[]) => any) | OperationKeys.CREATE | OperationKeys.READ | OperationKeys.UPDATE | OperationKeys.DELETE | string, overrides: Partial<FlagsOf<ContextOf<A>>>, ...args: any[]): Promise<ContextOf<A>>;
|
|
99
81
|
}
|
|
@@ -8,7 +8,6 @@ import { TaskStepResultModel } from "./models/TaskStepResultModel.js";
|
|
|
8
8
|
import { TaskLogEntryModel } from "./models/TaskLogEntryModel.js";
|
|
9
9
|
import { TaskBackoffModel } from "./models/TaskBackoffModel.js";
|
|
10
10
|
import { TaskStepSpecModel } from "./models/TaskStepSpecModel.js";
|
|
11
|
-
import { Worker } from "node:worker_threads";
|
|
12
11
|
import { DefaultTaskEngineConfig, TaskEventType, TaskStatus, TaskType, } from "./constants.js";
|
|
13
12
|
import { LogLevel } from "@decaf-ts/logging";
|
|
14
13
|
import { AbsContextual, } from "./../utils/ContextualLoggedClass.js";
|
|
@@ -20,8 +19,7 @@ import { TaskLogger } from "./logging.js";
|
|
|
20
19
|
import { TaskErrorModel } from "./models/TaskErrorModel.js";
|
|
21
20
|
import { TaskTracker } from "./TaskTracker.js";
|
|
22
21
|
import { Lock } from "@decaf-ts/transactional-decorators";
|
|
23
|
-
import { PersistenceKeys
|
|
24
|
-
import { DefaultWorkThreadEnvironment, } from "./workers/WorkThreadEnvironment.js";
|
|
22
|
+
import { PersistenceKeys } from "./../persistence/index.js";
|
|
25
23
|
export class TaskEngine extends AbsContextual {
|
|
26
24
|
get Context() {
|
|
27
25
|
return TaskContext;
|
|
@@ -39,16 +37,12 @@ export class TaskEngine extends AbsContextual {
|
|
|
39
37
|
if (this._tasks)
|
|
40
38
|
return this._tasks;
|
|
41
39
|
this._tasks = Repository.forModel(TaskModel, this.adapter.alias);
|
|
42
|
-
if (this.config.overrides)
|
|
43
|
-
this._tasks = this._tasks.override(this.config.overrides);
|
|
44
40
|
return this._tasks;
|
|
45
41
|
}
|
|
46
42
|
get events() {
|
|
47
43
|
if (this._events)
|
|
48
44
|
return this._events;
|
|
49
45
|
this._events = Repository.forModel(TaskEventModel, this.config.adapter.alias);
|
|
50
|
-
if (this.config.overrides)
|
|
51
|
-
this._events = this._events.override(this.config.overrides);
|
|
52
46
|
return this._events;
|
|
53
47
|
}
|
|
54
48
|
static createTaskContext(base, overrides) {
|
|
@@ -61,77 +55,12 @@ export class TaskEngine extends AbsContextual {
|
|
|
61
55
|
constructor(config) {
|
|
62
56
|
super();
|
|
63
57
|
this.config = config;
|
|
64
|
-
this.workerThreads = [];
|
|
65
|
-
this.workerJobQueue = [];
|
|
66
|
-
this.workerJobs = new Map();
|
|
67
|
-
this.workerCounter = 0;
|
|
68
|
-
this.workerThreadCapacity = 1;
|
|
69
58
|
this.lock = new Lock();
|
|
70
59
|
this.running = false;
|
|
71
|
-
if (config.workerPool && !config.workerAdapter) {
|
|
72
|
-
throw new InternalError("Worker pool requires workerAdapter descriptor in TaskEngineConfig");
|
|
73
|
-
}
|
|
74
60
|
this.config = Object.assign({}, DefaultTaskEngineConfig, config, {
|
|
75
61
|
bus: config.bus || new TaskEventBus(),
|
|
76
62
|
registry: config.registry || new TaskHandlerRegistry(),
|
|
77
63
|
});
|
|
78
|
-
this.workerThreadCapacity = Math.max(1, this.config.workerConcurrency ?? 1);
|
|
79
|
-
this.workerPoolConfig = this.normalizeWorkerPoolConfig(this.config.workerPool);
|
|
80
|
-
}
|
|
81
|
-
normalizeWorkerPoolConfig(pool) {
|
|
82
|
-
if (!pool)
|
|
83
|
-
return undefined;
|
|
84
|
-
if (!pool.entry) {
|
|
85
|
-
throw new InternalError("Worker pool configuration requires an explicit entry file path");
|
|
86
|
-
}
|
|
87
|
-
if (pool.size != null && pool.size !== this.config.concurrency) {
|
|
88
|
-
throw new InternalError("TaskEngine concurrency must match workerPool.size when worker pool is enabled");
|
|
89
|
-
}
|
|
90
|
-
return Object.assign({}, pool, {
|
|
91
|
-
size: this.config.concurrency,
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
hasWorkerPool() {
|
|
95
|
-
return !!this.workerPoolConfig && (this.workerPoolConfig.size ?? 0) > 0;
|
|
96
|
-
}
|
|
97
|
-
getWorkerCount() {
|
|
98
|
-
if (!this.workerPoolConfig)
|
|
99
|
-
return 0;
|
|
100
|
-
return this.workerPoolConfig.size ?? 0;
|
|
101
|
-
}
|
|
102
|
-
getWorkerExecutionSlots() {
|
|
103
|
-
return this.getWorkerCount() * this.workerThreadCapacity;
|
|
104
|
-
}
|
|
105
|
-
canDispatchToWorkers() {
|
|
106
|
-
return this.hasWorkerPool();
|
|
107
|
-
}
|
|
108
|
-
getExecutionConcurrency() {
|
|
109
|
-
if (this.hasWorkerPool()) {
|
|
110
|
-
return Math.max(1, this.getWorkerExecutionSlots());
|
|
111
|
-
}
|
|
112
|
-
return this.config.concurrency;
|
|
113
|
-
}
|
|
114
|
-
computeWorkerModules() {
|
|
115
|
-
const adapterDescriptor = this.config.workerAdapter ?? DefaultWorkThreadEnvironment.persistence;
|
|
116
|
-
if (!adapterDescriptor?.adapterModule) {
|
|
117
|
-
throw new InternalError("Worker adapter descriptor must include adapterModule");
|
|
118
|
-
}
|
|
119
|
-
const configuredImports = this.workerPoolConfig?.modules?.imports ??
|
|
120
|
-
DefaultWorkThreadEnvironment.modules.imports;
|
|
121
|
-
const imports = [];
|
|
122
|
-
const append = (specifier) => {
|
|
123
|
-
if (!specifier)
|
|
124
|
-
return;
|
|
125
|
-
if (!imports.includes(specifier))
|
|
126
|
-
imports.push(specifier);
|
|
127
|
-
};
|
|
128
|
-
append(adapterDescriptor.adapterModule);
|
|
129
|
-
for (const specifier of configuredImports) {
|
|
130
|
-
if (specifier === adapterDescriptor.adapterModule)
|
|
131
|
-
continue;
|
|
132
|
-
append(specifier);
|
|
133
|
-
}
|
|
134
|
-
return { imports };
|
|
135
64
|
}
|
|
136
65
|
async push(task, track = false, ...args) {
|
|
137
66
|
const { ctx, log } = (await this.logCtx(args, OperationKeys.CREATE, true)).for(this.push);
|
|
@@ -213,13 +142,10 @@ export class TaskEngine extends AbsContextual {
|
|
|
213
142
|
async start(...args) {
|
|
214
143
|
const { ctx } = (await this.logCtx(args, "run", true)).for(this.start);
|
|
215
144
|
await this.lock.acquire();
|
|
216
|
-
if (this.running)
|
|
217
|
-
this.lock.release();
|
|
145
|
+
if (this.running)
|
|
218
146
|
return;
|
|
219
|
-
}
|
|
220
147
|
this.running = true;
|
|
221
148
|
this.lock.release();
|
|
222
|
-
await this.spawnWorkers();
|
|
223
149
|
void this.loop(ctx);
|
|
224
150
|
}
|
|
225
151
|
async stop(...args) {
|
|
@@ -235,7 +161,7 @@ export class TaskEngine extends AbsContextual {
|
|
|
235
161
|
.execute(ctx);
|
|
236
162
|
const timeout = ctx.getOrUndefined?.("gracefulShutdownMsTimeout") ??
|
|
237
163
|
this.config.gracefulShutdownMsTimeout;
|
|
238
|
-
|
|
164
|
+
return new Promise((resolve, reject) => {
|
|
239
165
|
const timer = setTimeout(() => {
|
|
240
166
|
log.error(`Graceful shutdown interrupted after ${timeout} ms...`);
|
|
241
167
|
resolve();
|
|
@@ -257,249 +183,6 @@ export class TaskEngine extends AbsContextual {
|
|
|
257
183
|
reject(err);
|
|
258
184
|
});
|
|
259
185
|
});
|
|
260
|
-
await this.shutdownWorkers();
|
|
261
|
-
}
|
|
262
|
-
// -------------------------
|
|
263
|
-
// Worker pool orchestration
|
|
264
|
-
// -------------------------
|
|
265
|
-
async spawnWorkers() {
|
|
266
|
-
if (!this.hasWorkerPool())
|
|
267
|
-
return;
|
|
268
|
-
const target = this.getWorkerCount();
|
|
269
|
-
const creations = [];
|
|
270
|
-
while (this.workerThreads.length < target) {
|
|
271
|
-
const ready = this.createWorker();
|
|
272
|
-
if (ready)
|
|
273
|
-
creations.push(ready);
|
|
274
|
-
}
|
|
275
|
-
if (creations.length) {
|
|
276
|
-
await Promise.all(creations);
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
createWorker() {
|
|
280
|
-
if (!this.workerPoolConfig)
|
|
281
|
-
return undefined;
|
|
282
|
-
let resolveReady;
|
|
283
|
-
let rejectReady;
|
|
284
|
-
const readyPromise = new Promise((resolve, reject) => {
|
|
285
|
-
resolveReady = resolve;
|
|
286
|
-
rejectReady = reject;
|
|
287
|
-
});
|
|
288
|
-
const entry = this.workerPoolConfig.entry;
|
|
289
|
-
const workerId = `${this.config.workerId}-${this.workerCounter++}`;
|
|
290
|
-
const env = {
|
|
291
|
-
workerId,
|
|
292
|
-
mode: this.workerPoolConfig.mode ?? "node",
|
|
293
|
-
persistence: Object.assign({}, this.config.workerAdapter ?? DefaultWorkThreadEnvironment.persistence, {
|
|
294
|
-
alias: this.adapter.alias,
|
|
295
|
-
flavour: this.adapter.flavour,
|
|
296
|
-
}),
|
|
297
|
-
taskEngine: {
|
|
298
|
-
concurrency: this.workerThreadCapacity,
|
|
299
|
-
leaseMs: this.config.leaseMs,
|
|
300
|
-
pollMsBusy: this.config.pollMsBusy,
|
|
301
|
-
pollMsIdle: this.config.pollMsIdle,
|
|
302
|
-
logTailMax: this.config.logTailMax,
|
|
303
|
-
streamBufferSize: this.config.streamBufferSize,
|
|
304
|
-
maxLoggingBuffer: this.config.maxLoggingBuffer,
|
|
305
|
-
loggingBufferTruncation: this.config.loggingBufferTruncation,
|
|
306
|
-
gracefulShutdownMsTimeout: this.config.gracefulShutdownMsTimeout,
|
|
307
|
-
},
|
|
308
|
-
modules: this.computeWorkerModules(),
|
|
309
|
-
};
|
|
310
|
-
const worker = new Worker(entry, {
|
|
311
|
-
workerData: { environment: env },
|
|
312
|
-
});
|
|
313
|
-
const state = {
|
|
314
|
-
id: workerId,
|
|
315
|
-
worker,
|
|
316
|
-
ready: false,
|
|
317
|
-
activeJobs: 0,
|
|
318
|
-
capacity: this.workerThreadCapacity,
|
|
319
|
-
readyPromise,
|
|
320
|
-
resolveReady,
|
|
321
|
-
rejectReady,
|
|
322
|
-
};
|
|
323
|
-
this.workerThreads.push(state);
|
|
324
|
-
worker.on("message", (msg) => this.handleWorkerMessage(state, msg));
|
|
325
|
-
worker.on("error", (err) => this.handleWorkerError(state, err));
|
|
326
|
-
worker.on("exit", (code) => this.handleWorkerExit(state, code));
|
|
327
|
-
return readyPromise;
|
|
328
|
-
}
|
|
329
|
-
async shutdownWorkers() {
|
|
330
|
-
for (const state of this.workerThreads.splice(0)) {
|
|
331
|
-
const message = {
|
|
332
|
-
type: "control",
|
|
333
|
-
command: "shutdown",
|
|
334
|
-
};
|
|
335
|
-
try {
|
|
336
|
-
state.worker.postMessage(message);
|
|
337
|
-
}
|
|
338
|
-
catch {
|
|
339
|
-
// ignore
|
|
340
|
-
}
|
|
341
|
-
await state.worker.terminate();
|
|
342
|
-
}
|
|
343
|
-
for (const job of this.workerJobQueue.splice(0)) {
|
|
344
|
-
job.reject(new InternalError(`Worker pool shutting down before job ${job.id} could start`));
|
|
345
|
-
}
|
|
346
|
-
for (const job of this.workerJobs.values()) {
|
|
347
|
-
job.reject(new InternalError(`Worker terminated before finishing job ${job.id}`));
|
|
348
|
-
}
|
|
349
|
-
this.workerJobs.clear();
|
|
350
|
-
}
|
|
351
|
-
handleWorkerError(state, err) {
|
|
352
|
-
this.log.error(`worker ${state.id} error: ${err.message}`, err);
|
|
353
|
-
if (state.rejectReady) {
|
|
354
|
-
state.rejectReady(err);
|
|
355
|
-
state.resolveReady = undefined;
|
|
356
|
-
state.rejectReady = undefined;
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
handleWorkerExit(state, code) {
|
|
360
|
-
this.log.info(`worker ${state.id} exited with code ${code}`);
|
|
361
|
-
if (state.rejectReady) {
|
|
362
|
-
state.rejectReady(new Error(`worker ${state.id} exited before reporting ready`));
|
|
363
|
-
state.resolveReady = undefined;
|
|
364
|
-
state.rejectReady = undefined;
|
|
365
|
-
}
|
|
366
|
-
const idx = this.workerThreads.indexOf(state);
|
|
367
|
-
if (idx >= 0)
|
|
368
|
-
this.workerThreads.splice(idx, 1);
|
|
369
|
-
for (const [jobId, job] of this.workerJobs.entries()) {
|
|
370
|
-
if (job.worker === state) {
|
|
371
|
-
job.worker = undefined;
|
|
372
|
-
this.workerJobs.delete(jobId);
|
|
373
|
-
this.workerJobQueue.unshift(job);
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
if (this.running) {
|
|
377
|
-
void this.spawnWorkers().catch((err) => this.log.error(`failed to respawn worker`, err));
|
|
378
|
-
this.processWorkerQueue();
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
handleWorkerMessage(state, msg) {
|
|
382
|
-
if (msg.type === "ready") {
|
|
383
|
-
state.ready = true;
|
|
384
|
-
if (state.resolveReady) {
|
|
385
|
-
state.resolveReady();
|
|
386
|
-
state.resolveReady = undefined;
|
|
387
|
-
state.rejectReady = undefined;
|
|
388
|
-
}
|
|
389
|
-
this.log.info(`worker ${state.id} ready`);
|
|
390
|
-
this.processWorkerQueue();
|
|
391
|
-
return;
|
|
392
|
-
}
|
|
393
|
-
if (msg.type === "error") {
|
|
394
|
-
this.log.error(`worker ${state.id} reported error: ${msg.error}`);
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
if (msg.type === "log") {
|
|
398
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
399
|
-
if (!job)
|
|
400
|
-
return;
|
|
401
|
-
void this.appendLog(job.ctx, job.task, msg.entries)
|
|
402
|
-
.then(([updated, entries]) => {
|
|
403
|
-
job.task = updated;
|
|
404
|
-
return this.emitLog(job.ctx, job.task.id, entries);
|
|
405
|
-
})
|
|
406
|
-
.catch((err) => this.log.error(`Failed to append worker log`, err));
|
|
407
|
-
return;
|
|
408
|
-
}
|
|
409
|
-
if (msg.type === "progress") {
|
|
410
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
411
|
-
if (!job)
|
|
412
|
-
return;
|
|
413
|
-
void this.emitProgress(job.ctx, job.task.id, msg.payload);
|
|
414
|
-
return;
|
|
415
|
-
}
|
|
416
|
-
if (msg.type === "heartbeat") {
|
|
417
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
418
|
-
if (!job)
|
|
419
|
-
return;
|
|
420
|
-
job.task.leaseExpiry = new Date(Date.now() + this.config.leaseMs);
|
|
421
|
-
void this.tasks.update(job.task).catch(() => null);
|
|
422
|
-
return;
|
|
423
|
-
}
|
|
424
|
-
if (msg.type === "result") {
|
|
425
|
-
const job = this.workerJobs.get(msg.jobId);
|
|
426
|
-
if (!job)
|
|
427
|
-
return;
|
|
428
|
-
this.workerJobs.delete(job.id);
|
|
429
|
-
state.activeJobs = Math.max(0, state.activeJobs - 1);
|
|
430
|
-
this.applyWorkerCache(job.ctx, msg.cache);
|
|
431
|
-
switch (msg.status) {
|
|
432
|
-
case "success":
|
|
433
|
-
job.resolve(msg.output);
|
|
434
|
-
break;
|
|
435
|
-
case "error": {
|
|
436
|
-
const err = new Error(msg.error.message);
|
|
437
|
-
if (msg.error.name)
|
|
438
|
-
err.name = msg.error.name;
|
|
439
|
-
if (msg.error.stack)
|
|
440
|
-
err.stack = msg.error.stack;
|
|
441
|
-
job.reject(err);
|
|
442
|
-
break;
|
|
443
|
-
}
|
|
444
|
-
case "state-change":
|
|
445
|
-
job.reject(new TaskStateChangeError(msg.request));
|
|
446
|
-
break;
|
|
447
|
-
}
|
|
448
|
-
this.processWorkerQueue();
|
|
449
|
-
return;
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
applyWorkerCache(ctx, cache) {
|
|
453
|
-
if (!cache)
|
|
454
|
-
return;
|
|
455
|
-
Object.entries(cache).forEach(([key, value]) => {
|
|
456
|
-
ctx.cacheResult(key, value);
|
|
457
|
-
});
|
|
458
|
-
}
|
|
459
|
-
processWorkerQueue() {
|
|
460
|
-
if (!this.hasWorkerPool())
|
|
461
|
-
return;
|
|
462
|
-
const available = this.workerThreads
|
|
463
|
-
.filter((state) => state.ready && state.activeJobs < state.capacity)
|
|
464
|
-
.sort((a, b) => a.activeJobs - b.activeJobs);
|
|
465
|
-
for (const state of available) {
|
|
466
|
-
while (state.activeJobs < state.capacity &&
|
|
467
|
-
this.workerJobQueue.length > 0) {
|
|
468
|
-
const job = this.workerJobQueue.shift();
|
|
469
|
-
if (!job)
|
|
470
|
-
break;
|
|
471
|
-
this.assignWorker(state, job);
|
|
472
|
-
}
|
|
473
|
-
if (!this.workerJobQueue.length)
|
|
474
|
-
break;
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
assignWorker(state, job) {
|
|
478
|
-
if (!this.workerPoolConfig)
|
|
479
|
-
return;
|
|
480
|
-
const payload = {
|
|
481
|
-
jobId: job.id,
|
|
482
|
-
taskId: job.task.id,
|
|
483
|
-
classification: job.classification,
|
|
484
|
-
input: job.input,
|
|
485
|
-
attempt: job.task.attempt ?? 0,
|
|
486
|
-
resultCache: job.ctx.resultCache ?? {},
|
|
487
|
-
streamBufferSize: this.config.streamBufferSize,
|
|
488
|
-
maxLoggingBuffer: this.config.maxLoggingBuffer,
|
|
489
|
-
loggingBufferTruncation: this.config.loggingBufferTruncation,
|
|
490
|
-
};
|
|
491
|
-
job.worker = state;
|
|
492
|
-
this.workerJobs.set(job.id, job);
|
|
493
|
-
state.activeJobs += 1;
|
|
494
|
-
const message = {
|
|
495
|
-
type: "execute",
|
|
496
|
-
job: payload,
|
|
497
|
-
};
|
|
498
|
-
state.worker.postMessage(message);
|
|
499
|
-
}
|
|
500
|
-
enqueueWorkerJob(job) {
|
|
501
|
-
this.workerJobQueue.push(job);
|
|
502
|
-
this.processWorkerQueue();
|
|
503
186
|
}
|
|
504
187
|
// -------------------------
|
|
505
188
|
// Worker loop
|
|
@@ -534,11 +217,10 @@ export class TaskEngine extends AbsContextual {
|
|
|
534
217
|
.or(condLeaseExpired)
|
|
535
218
|
.or(condScheduled);
|
|
536
219
|
// Fetch more than concurrency because some will fail to claim due to conflicts
|
|
537
|
-
const concurrency = this.getExecutionConcurrency();
|
|
538
220
|
const candidates = await this.tasks
|
|
539
221
|
.select()
|
|
540
222
|
.where(runnable)
|
|
541
|
-
.limit(Math.max(concurrency * 4, 20))
|
|
223
|
+
.limit(Math.max(this.config.concurrency * 4, 20))
|
|
542
224
|
.execute();
|
|
543
225
|
log.verbose(`claimBatch candidates:${candidates.length}`);
|
|
544
226
|
const out = [];
|
|
@@ -546,7 +228,7 @@ export class TaskEngine extends AbsContextual {
|
|
|
546
228
|
const claimed = await this.tryClaim(c, ctx);
|
|
547
229
|
if (claimed)
|
|
548
230
|
out.push(claimed);
|
|
549
|
-
if (out.length >= concurrency)
|
|
231
|
+
if (out.length >= this.config.concurrency)
|
|
550
232
|
break;
|
|
551
233
|
}
|
|
552
234
|
log.verbose(`claimBatch claimed:${out.length}`);
|
|
@@ -582,39 +264,9 @@ export class TaskEngine extends AbsContextual {
|
|
|
582
264
|
// -------------------------
|
|
583
265
|
// Execution
|
|
584
266
|
// -------------------------
|
|
585
|
-
async runHandlerInline(classification, input, ctx) {
|
|
586
|
-
const handler = this.registry.get(classification);
|
|
587
|
-
if (!handler)
|
|
588
|
-
throw new InternalError(`No task handler registered for type: ${classification}`);
|
|
589
|
-
return handler.run(input, ctx);
|
|
590
|
-
}
|
|
591
|
-
async dispatchToWorker(classification, input, task, ctx) {
|
|
592
|
-
if (!this.canDispatchToWorkers()) {
|
|
593
|
-
return this.runHandlerInline(classification, input, ctx);
|
|
594
|
-
}
|
|
595
|
-
const uuid = await UUID.instance.generate();
|
|
596
|
-
return new Promise((resolve, reject) => {
|
|
597
|
-
const job = {
|
|
598
|
-
id: uuid,
|
|
599
|
-
classification,
|
|
600
|
-
input,
|
|
601
|
-
task,
|
|
602
|
-
ctx,
|
|
603
|
-
resolve,
|
|
604
|
-
reject,
|
|
605
|
-
};
|
|
606
|
-
this.enqueueWorkerJob(job);
|
|
607
|
-
});
|
|
608
|
-
}
|
|
609
|
-
async invokeHandler(classification, input, task, ctx) {
|
|
610
|
-
if (!this.hasWorkerPool()) {
|
|
611
|
-
return this.runHandlerInline(classification, input, ctx);
|
|
612
|
-
}
|
|
613
|
-
return this.dispatchToWorker(classification, input, task, ctx);
|
|
614
|
-
}
|
|
615
267
|
async executeClaimed(task) {
|
|
616
268
|
const { ctx, log } = (await this.logCtx([], task.classification, true)).for(this.executeClaimed);
|
|
617
|
-
const taskCtx =
|
|
269
|
+
const taskCtx = new TaskContext(ctx).accumulate({
|
|
618
270
|
taskId: task.id,
|
|
619
271
|
logger: new TaskLogger(log, this.config.streamBufferSize, this.config.maxLoggingBuffer),
|
|
620
272
|
attempt: task.attempt,
|
|
@@ -624,7 +276,7 @@ export class TaskEngine extends AbsContextual {
|
|
|
624
276
|
await this.emitLog(taskCtx, task.id, logs);
|
|
625
277
|
},
|
|
626
278
|
flush: async () => {
|
|
627
|
-
|
|
279
|
+
return taskCtx.logger.flush(taskCtx.pipe);
|
|
628
280
|
},
|
|
629
281
|
progress: async (data) => {
|
|
630
282
|
await this.emitProgress(taskCtx, task.id, data);
|
|
@@ -659,8 +311,11 @@ export class TaskEngine extends AbsContextual {
|
|
|
659
311
|
}
|
|
660
312
|
}
|
|
661
313
|
else {
|
|
662
|
-
|
|
663
|
-
|
|
314
|
+
const handler = this.registry.get(task.classification);
|
|
315
|
+
log.debug(`handler type for ${task.id} is ${handler?.constructor?.name ?? "none"}`);
|
|
316
|
+
if (!handler)
|
|
317
|
+
throw new InternalError(`No task handler registered for type: ${task.classification}`);
|
|
318
|
+
output = await handler.run(task.input, taskCtx);
|
|
664
319
|
log.verbose(`handler finished for ${task.id}`);
|
|
665
320
|
}
|
|
666
321
|
task.status = TaskStatus.SUCCEEDED;
|
|
@@ -818,12 +473,15 @@ export class TaskEngine extends AbsContextual {
|
|
|
818
473
|
}
|
|
819
474
|
while (idx < steps.length) {
|
|
820
475
|
const step = steps[idx];
|
|
476
|
+
const handler = this.registry.get(step.classification);
|
|
477
|
+
if (!handler)
|
|
478
|
+
throw new Error(`No task handler registered for composite step: ${step.classification}`);
|
|
821
479
|
await context.pipe([
|
|
822
480
|
LogLevel.info,
|
|
823
481
|
`Composite step ${idx + 1}/${steps.length}: ${step.classification}`,
|
|
824
482
|
]);
|
|
825
483
|
try {
|
|
826
|
-
const out = await
|
|
484
|
+
const out = await handler.run(step.input, context);
|
|
827
485
|
const stepIndex = idx;
|
|
828
486
|
const now = new Date();
|
|
829
487
|
results[stepIndex] = new TaskStepResultModel({
|