@sidequest/engine 1.15.1 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine.cjs +22 -1
- package/dist/engine.cjs.map +1 -1
- package/dist/engine.d.ts +44 -0
- package/dist/engine.js +22 -1
- package/dist/engine.js.map +1 -1
- package/dist/execution/dispatcher.cjs +5 -2
- package/dist/execution/dispatcher.cjs.map +1 -1
- package/dist/execution/dispatcher.js +5 -2
- package/dist/execution/dispatcher.js.map +1 -1
- package/dist/execution/executor-manager.cjs +62 -26
- package/dist/execution/executor-manager.cjs.map +1 -1
- package/dist/execution/executor-manager.d.ts +13 -1
- package/dist/execution/executor-manager.js +63 -27
- package/dist/execution/executor-manager.js.map +1 -1
- package/dist/index.cjs +4 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/job/job-builder.cjs +1 -0
- package/dist/job/job-builder.cjs.map +1 -1
- package/dist/job/job-builder.js +1 -0
- package/dist/job/job-builder.js.map +1 -1
- package/dist/routines/release-stale-jobs.cjs +1 -0
- package/dist/routines/release-stale-jobs.cjs.map +1 -1
- package/dist/routines/release-stale-jobs.js +1 -0
- package/dist/routines/release-stale-jobs.js.map +1 -1
- package/dist/shared-runner/inline-runner.cjs +44 -0
- package/dist/shared-runner/inline-runner.cjs.map +1 -0
- package/dist/shared-runner/inline-runner.d.ts +35 -0
- package/dist/shared-runner/inline-runner.js +42 -0
- package/dist/shared-runner/inline-runner.js.map +1 -0
- package/dist/shared-runner/job-runner.d.ts +24 -0
- package/dist/shared-runner/runner-pool.cjs +34 -2
- package/dist/shared-runner/runner-pool.cjs.map +1 -1
- package/dist/shared-runner/runner-pool.d.ts +10 -4
- package/dist/shared-runner/runner-pool.js +35 -3
- package/dist/shared-runner/runner-pool.js.map +1 -1
- package/dist/shared-runner/runner.cjs +45 -4
- package/dist/shared-runner/runner.cjs.map +1 -1
- package/dist/shared-runner/runner.d.ts +12 -2
- package/dist/shared-runner/runner.js +46 -5
- package/dist/shared-runner/runner.js.map +1 -1
- package/dist/workers/main.cjs +6 -76
- package/dist/workers/main.cjs.map +1 -1
- package/dist/workers/main.d.ts +1 -22
- package/dist/workers/main.js +6 -76
- package/dist/workers/main.js.map +1 -1
- package/dist/workers/worker-runtime.cjs +88 -0
- package/dist/workers/worker-runtime.cjs.map +1 -0
- package/dist/workers/worker-runtime.d.ts +44 -0
- package/dist/workers/worker-runtime.js +86 -0
- package/dist/workers/worker-runtime.js.map +1 -0
- package/package.json +4 -4
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { logger } from '@sidequest/core';
|
|
1
|
+
import { logger, serializeAbortReason } from '@sidequest/core';
|
|
2
|
+
import { MessageChannel } from 'node:worker_threads';
|
|
2
3
|
import Piscina from 'piscina';
|
|
3
4
|
import { DEFAULT_RUNNER_PATH } from '../constants.js';
|
|
4
5
|
|
|
@@ -25,13 +26,44 @@ class RunnerPool {
|
|
|
25
26
|
}
|
|
26
27
|
/**
|
|
27
28
|
* Runs a job in the worker pool.
|
|
29
|
+
*
|
|
30
|
+
* With `abortGracePeriodMs === 0` (default), an abort terminates the worker immediately. With a
|
|
31
|
+
* positive grace period, the abort is delivered to the job cooperatively over a transferred port
|
|
32
|
+
* (so it can stop via `this.abortSignal`), and the worker is only forcibly terminated if it has
|
|
33
|
+
* not finished within the grace period.
|
|
34
|
+
*
|
|
28
35
|
* @param job The job data to run.
|
|
29
|
-
* @param signal Optional
|
|
36
|
+
* @param signal Optional abort signal for cancellation/timeout.
|
|
30
37
|
* @returns A promise resolving to the job result.
|
|
31
38
|
*/
|
|
32
39
|
run(job, signal) {
|
|
33
40
|
logger("RunnerPool").debug(`Running job ${job.id} in pool`);
|
|
34
|
-
|
|
41
|
+
// Already aborted before we could start (e.g. canceled between claim and dispatch): don't run it.
|
|
42
|
+
if (signal?.aborted) {
|
|
43
|
+
return Promise.reject(signal.reason instanceof Error ? signal.reason : new Error("Job aborted before execution"));
|
|
44
|
+
}
|
|
45
|
+
const grace = this.nonNullConfig.abortGracePeriodMs;
|
|
46
|
+
if (!signal || grace <= 0) {
|
|
47
|
+
// Abort terminates the worker immediately.
|
|
48
|
+
return this.pool.run({ jobData: job, config: this.nonNullConfig }, { signal });
|
|
49
|
+
}
|
|
50
|
+
// Deliver the abort cooperatively first, then hard-terminate after the grace period.
|
|
51
|
+
const channel = new MessageChannel();
|
|
52
|
+
const hardKill = new AbortController();
|
|
53
|
+
let graceTimer;
|
|
54
|
+
const onAbort = () => {
|
|
55
|
+
channel.port1.postMessage(serializeAbortReason(signal.reason));
|
|
56
|
+
graceTimer = setTimeout(() => hardKill.abort(), grace);
|
|
57
|
+
};
|
|
58
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
59
|
+
return this.pool
|
|
60
|
+
.run({ jobData: job, config: this.nonNullConfig, abortPort: channel.port2 }, { transferList: [channel.port2], signal: hardKill.signal })
|
|
61
|
+
.finally(() => {
|
|
62
|
+
if (graceTimer)
|
|
63
|
+
clearTimeout(graceTimer);
|
|
64
|
+
signal.removeEventListener("abort", onAbort);
|
|
65
|
+
channel.port1.close();
|
|
66
|
+
});
|
|
35
67
|
}
|
|
36
68
|
/**
|
|
37
69
|
* Destroys the worker pool and releases resources.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner-pool.js","sources":["../../src/shared-runner/runner-pool.ts"],"sourcesContent":[null],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner-pool.js","sources":["../../src/shared-runner/runner-pool.ts"],"sourcesContent":[null],"names":[],"mappings":";;;;;AAOA;;AAEG;MACU,UAAU,CAAA;AAQD,IAAA,aAAA;;AANH,IAAA,IAAI;AAErB;;;AAGG;AACH,IAAA,WAAA,CAAoB,aAAsC,EAAA;QAAtC,IAAA,CAAA,aAAa,GAAb,aAAa;AAC/B,QAAA,IAAI,CAAC,IAAI,GAAG,IAAI,OAAO,CAAC;AACtB,YAAA,QAAQ,EAAE,mBAAmB;AAC7B,YAAA,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,UAAU;AACzC,YAAA,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,UAAU;AACzC,YAAA,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC,iBAAiB;AAClD,SAAA,CAAC;QACF,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CACxB,CAAA,6BAAA,EAAgC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAA,iBAAA,EAAoB,IAAI,CAAC,aAAa,CAAC,UAAU,CAAA,QAAA,CAAU,CACzH;IACH;AAEA;;;;;;;;;;;AAWG;IACH,GAAG,CAAC,GAAY,EAAE,MAAoB,EAAA;AACpC,QAAA,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAA,YAAA,EAAe,GAAG,CAAC,EAAE,CAAA,QAAA,CAAU,CAAC;;AAG3D,QAAA,IAAI,MAAM,EAAE,OAAO,EAAE;YACnB,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACnH;AAEA,QAAA,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,kBAAkB;AAEnD,QAAA,IAAI,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,EAAE;;YAEzB,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;QAChF;;AAGA,QAAA,MAAM,OAAO,GAAG,IAAI,cAAc,EAAE;AACpC,QAAA,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE;AACtC,QAAA,IAAI,UAAqD;QAEzD,MAAM,OAAO,GAAG,MAAK;AACnB,YAAA,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAC9D,YAAA,UAAU,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC;AACxD,QAAA,CAAC;AAED,QAAA,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEzD,OAAO,IAAI,CAAC;AACT,aAAA,GAAG,CACF,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,EACtE,EAAE,YAAY,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;aAE3D,OAAO,CAAC,MAAK;AACZ,YAAA,IAAI,UAAU;gBAAE,YAAY,CAAC,UAAU,CAAC;AACxC,YAAA,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC;AAC5C,YAAA,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE;AACvB,QAAA,CAAC,CAAC;IACN;AAEA;;AAEG;IACH,OAAO,GAAA;QACL,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC;AACpD,QAAA,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACxB,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;IACrD;AACD;;;;"}
|
|
@@ -8,14 +8,38 @@ var node_url = require('node:url');
|
|
|
8
8
|
var _import = require('../utils/import.cjs');
|
|
9
9
|
var manualLoader = require('./manual-loader.cjs');
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Builds an {@link AbortSignal} for a worker-thread job from an abort port.
|
|
13
|
+
*
|
|
14
|
+
* The thread runner cannot receive a live `AbortSignal` across the worker boundary, so the engine
|
|
15
|
+
* transfers a {@link MessagePort} and posts an abort message on it; this turns that message into a
|
|
16
|
+
* local signal carrying the proper `JobTimeout`/`JobCanceled` reason.
|
|
17
|
+
*/
|
|
18
|
+
function signalFromAbortPort(port) {
|
|
19
|
+
const controller = new AbortController();
|
|
20
|
+
port.on("message", (message) => {
|
|
21
|
+
controller.abort(core.deserializeAbortReason(message));
|
|
22
|
+
});
|
|
23
|
+
return controller.signal;
|
|
24
|
+
}
|
|
11
25
|
/**
|
|
12
26
|
* Runs a job by dynamically importing its script and executing the specified class.
|
|
13
|
-
* @param jobData The job data containing script and class information
|
|
27
|
+
* @param jobData The job data containing script and class information.
|
|
14
28
|
* @param config The non-nullable engine configuration.
|
|
29
|
+
* @param inline Whether the job runs inline in the host process. When true, the Sidequest config is
|
|
30
|
+
* not re-injected (the host process is already configured).
|
|
31
|
+
* @param signal Abort signal handed to the job as `this.abortSignal` (used by the inline runner,
|
|
32
|
+
* which executes in the same process).
|
|
33
|
+
* @param abortPort Port the thread runner uses to receive the abort cooperatively across the worker
|
|
34
|
+
* boundary; it is turned into the job's `this.abortSignal`. Mutually exclusive with `signal`.
|
|
15
35
|
* @returns A promise resolving to the job result.
|
|
16
36
|
*/
|
|
17
|
-
async function run({ jobData, config }) {
|
|
18
|
-
|
|
37
|
+
async function run({ jobData, config, inline, signal, abortPort, }) {
|
|
38
|
+
// In inline mode the job runs in the host process, where Sidequest is already configured, so
|
|
39
|
+
// re-injecting the config is redundant. In a worker thread the module is fresh and needs it.
|
|
40
|
+
if (!inline) {
|
|
41
|
+
await injectSidequestConfig(config);
|
|
42
|
+
}
|
|
19
43
|
let script = {};
|
|
20
44
|
try {
|
|
21
45
|
core.logger("Runner").debug(`Importing job script "${jobData.script}"`);
|
|
@@ -67,8 +91,25 @@ async function run({ jobData, config }) {
|
|
|
67
91
|
}
|
|
68
92
|
const job = new JobClass(...jobData.constructor_args);
|
|
69
93
|
job.injectJobData(jobData);
|
|
94
|
+
// Exactly one of these is provided: inline passes a live signal; the thread runner passes an abort
|
|
95
|
+
// port it turns into one.
|
|
96
|
+
let abortSignal;
|
|
97
|
+
if (signal) {
|
|
98
|
+
abortSignal = signal;
|
|
99
|
+
}
|
|
100
|
+
else if (abortPort) {
|
|
101
|
+
abortSignal = signalFromAbortPort(abortPort);
|
|
102
|
+
}
|
|
103
|
+
if (abortSignal) {
|
|
104
|
+
job.injectAbortSignal(abortSignal);
|
|
105
|
+
}
|
|
70
106
|
core.logger("Runner").debug(`Executing job class "${jobData.class}" with args:`, jobData.args);
|
|
71
|
-
|
|
107
|
+
try {
|
|
108
|
+
return await job.perform(...jobData.args);
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
abortPort?.close();
|
|
112
|
+
}
|
|
72
113
|
}
|
|
73
114
|
/**
|
|
74
115
|
* Injects the provided Sidequest engine configuration into the job script.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.cjs","sources":["../../src/shared-runner/runner.ts"],"sourcesContent":[null],"names":["logger","MANUAL_SCRIPT_TAG","findSidequestJobsScriptInParentDirs","resolveScriptPath","existsSync","fileURLToPath","toErrorData","resolveScriptPathForJob","importSidequest"],"mappings":";;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"runner.cjs","sources":["../../src/shared-runner/runner.ts"],"sourcesContent":[null],"names":["deserializeAbortReason","logger","MANUAL_SCRIPT_TAG","findSidequestJobsScriptInParentDirs","resolveScriptPath","existsSync","fileURLToPath","toErrorData","resolveScriptPathForJob","importSidequest"],"mappings":";;;;;;;;;;AAkBA;;;;;;AAMG;AACH,SAAS,mBAAmB,CAAC,IAAiB,EAAA;AAC5C,IAAA,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE;IACxC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAA2B,KAAI;QACjD,UAAU,CAAC,KAAK,CAACA,2BAAsB,CAAC,OAAO,CAAC,CAAC;AACnD,IAAA,CAAC,CAAC;IACF,OAAO,UAAU,CAAC,MAAM;AAC1B;AAEA;;;;;;;;;;;AAWG;AACY,eAAe,GAAG,CAAC,EAChC,OAAO,EACP,MAAM,EACN,MAAM,EACN,MAAM,EACN,SAAS,GAOV,EAAA;;;IAGC,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,MAAM,qBAAqB,CAAC,MAAM,CAAC;IACrC;IAEA,IAAI,MAAM,GAAiC,EAAE;AAC7C,IAAA,IAAI;AACF,QAAAC,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAA,sBAAA,EAAyB,OAAO,CAAC,MAAM,CAAA,CAAA,CAAG,CAAC;AAElE,QAAA,IAAI,SAAiB;AACrB,QAAA,IAAI,OAAO,CAAC,MAAM,KAAKC,8BAAiB,EAAE;YACxCD,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6EAA6E,CAAC;AACrG,YAAA,IAAI;;AAEF,gBAAA,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;;;oBAGxB,SAAS,GAAGE,gDAAmC,EAAE;gBACnD;qBAAO;;AAEL,oBAAA,SAAS,GAAGC,8BAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;oBAClD,IAAI,CAACC,kBAAU,CAACC,sBAAa,CAAC,SAAS,CAAC,CAAC,EAAE;AACzC,wBAAA,MAAM,IAAI,KAAK,CAAC,2DAA2D,SAAS,CAAA,CAAE,CAAC;oBACzF;gBACF;YACF;YAAE,OAAO,KAAK,EAAE;gBACd,MAAM,YAAY,GAAG,CAAA,gEAAA,EAAmE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE;gBAChJL,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;AACpC,gBAAA,MAAM,SAAS,GAAGM,gBAAW,CAAC,KAAc,CAAC;AAC7C,gBAAA,OAAO,EAAE,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE;YAC1E;QACF;aAAO;YACLN,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,oEAAoE,CAAC;;AAE5F,YAAA,SAAS,GAAGO,4BAAuB,CAAC,OAAO,CAAC,MAAM,CAAC;QACrD;QAEA,MAAM,IAAI,MAAM,OAAO,SAAS,CAAC,CAAiC;AAClE,QAAAP,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAA,kCAAA,EAAqC,OAAO,CAAC,MAAM,CAAA,CAAA,CAAG,CAAC;IAChF;IAAE,OAAO,KAAK,EAAE;QACd,MAAM,YAAY,GAAG,CAAA,6BAAA,EAAgC,OAAO,CAAC,MAAM,CAAA,GAAA,EAAM,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA,CAAE;QACjIA,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;AACpC,QAAA,MAAM,SAAS,GAAGM,gBAAW,CAAC,KAAc,CAAC;AAC7C,QAAA,OAAO,EAAE,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE;IAC1E;AAEA,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO;IACxD,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;AAC/C,QAAA,MAAM,KAAK,GAAG,CAAA,mBAAA,EAAsB,OAAO,CAAC,KAAK,EAAE;QACnDN,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,SAAS,GAAGM,gBAAW,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;AAC/C,QAAA,OAAO,EAAE,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE;IAC1E;IAEA,MAAM,GAAG,GAAQ,IAAI,QAAQ,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC;AAC1D,IAAA,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC;;;AAG1B,IAAA,IAAI,WAAoC;IACxC,IAAI,MAAM,EAAE;QACV,WAAW,GAAG,MAAM;IACtB;SAAO,IAAI,SAAS,EAAE;AACpB,QAAA,WAAW,GAAG,mBAAmB,CAAC,SAAS,CAAC;IAC9C;IACA,IAAI,WAAW,EAAE;AACf,QAAA,GAAG,CAAC,iBAAiB,CAAC,WAAW,CAAC;IACpC;AAEA,IAAAN,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAA,qBAAA,EAAwB,OAAO,CAAC,KAAK,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC;AACzF,IAAA,IAAI;QACF,OAAO,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC3C;YAAU;QACR,SAAS,EAAE,KAAK,EAAE;IACpB;AACF;AAEA;;;;;;;;;;AAUG;AACI,eAAe,qBAAqB,CAAC,MAAoB,EAAA;AAC9D,IAAA,IAAI;QACFA,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,4CAA4C,CAAC;AACpE,QAAA,MAAM,EAAE,SAAS,EAAE,GAAG,MAAMQ,uBAAe,EAAE;AAC7C,QAAA,MAAM,SAAS,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;QAC7DR,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,wCAAwC,CAAC;AAChE,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,KAAK,EAAE;QACdA,WAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CACnB,CAAA,mCAAA,EAAsC,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA,oBAAA,CAAsB,CACnH;AACD,QAAA,OAAO,KAAK;IACd;AACF;;;;;"}
|
|
@@ -1,15 +1,25 @@
|
|
|
1
1
|
import { JobData, JobResult } from '@sidequest/core';
|
|
2
|
+
import { MessagePort } from 'node:worker_threads';
|
|
2
3
|
import { EngineConfig } from '../engine.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* Runs a job by dynamically importing its script and executing the specified class.
|
|
6
|
-
* @param jobData The job data containing script and class information
|
|
7
|
+
* @param jobData The job data containing script and class information.
|
|
7
8
|
* @param config The non-nullable engine configuration.
|
|
9
|
+
* @param inline Whether the job runs inline in the host process. When true, the Sidequest config is
|
|
10
|
+
* not re-injected (the host process is already configured).
|
|
11
|
+
* @param signal Abort signal handed to the job as `this.abortSignal` (used by the inline runner,
|
|
12
|
+
* which executes in the same process).
|
|
13
|
+
* @param abortPort Port the thread runner uses to receive the abort cooperatively across the worker
|
|
14
|
+
* boundary; it is turned into the job's `this.abortSignal`. Mutually exclusive with `signal`.
|
|
8
15
|
* @returns A promise resolving to the job result.
|
|
9
16
|
*/
|
|
10
|
-
declare function run({ jobData, config }: {
|
|
17
|
+
declare function run({ jobData, config, inline, signal, abortPort, }: {
|
|
11
18
|
jobData: JobData;
|
|
12
19
|
config: EngineConfig;
|
|
20
|
+
inline?: boolean;
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
abortPort?: MessagePort;
|
|
13
23
|
}): Promise<JobResult>;
|
|
14
24
|
/**
|
|
15
25
|
* Injects the provided Sidequest engine configuration into the job script.
|
|
@@ -1,17 +1,41 @@
|
|
|
1
|
-
import { logger, toErrorData, resolveScriptPathForJob } from '@sidequest/core';
|
|
1
|
+
import { logger, toErrorData, resolveScriptPathForJob, deserializeAbortReason } from '@sidequest/core';
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { importSidequest } from '../utils/import.js';
|
|
5
5
|
import { MANUAL_SCRIPT_TAG, findSidequestJobsScriptInParentDirs, resolveScriptPath } from './manual-loader.js';
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Builds an {@link AbortSignal} for a worker-thread job from an abort port.
|
|
9
|
+
*
|
|
10
|
+
* The thread runner cannot receive a live `AbortSignal` across the worker boundary, so the engine
|
|
11
|
+
* transfers a {@link MessagePort} and posts an abort message on it; this turns that message into a
|
|
12
|
+
* local signal carrying the proper `JobTimeout`/`JobCanceled` reason.
|
|
13
|
+
*/
|
|
14
|
+
function signalFromAbortPort(port) {
|
|
15
|
+
const controller = new AbortController();
|
|
16
|
+
port.on("message", (message) => {
|
|
17
|
+
controller.abort(deserializeAbortReason(message));
|
|
18
|
+
});
|
|
19
|
+
return controller.signal;
|
|
20
|
+
}
|
|
7
21
|
/**
|
|
8
22
|
* Runs a job by dynamically importing its script and executing the specified class.
|
|
9
|
-
* @param jobData The job data containing script and class information
|
|
23
|
+
* @param jobData The job data containing script and class information.
|
|
10
24
|
* @param config The non-nullable engine configuration.
|
|
25
|
+
* @param inline Whether the job runs inline in the host process. When true, the Sidequest config is
|
|
26
|
+
* not re-injected (the host process is already configured).
|
|
27
|
+
* @param signal Abort signal handed to the job as `this.abortSignal` (used by the inline runner,
|
|
28
|
+
* which executes in the same process).
|
|
29
|
+
* @param abortPort Port the thread runner uses to receive the abort cooperatively across the worker
|
|
30
|
+
* boundary; it is turned into the job's `this.abortSignal`. Mutually exclusive with `signal`.
|
|
11
31
|
* @returns A promise resolving to the job result.
|
|
12
32
|
*/
|
|
13
|
-
async function run({ jobData, config }) {
|
|
14
|
-
|
|
33
|
+
async function run({ jobData, config, inline, signal, abortPort, }) {
|
|
34
|
+
// In inline mode the job runs in the host process, where Sidequest is already configured, so
|
|
35
|
+
// re-injecting the config is redundant. In a worker thread the module is fresh and needs it.
|
|
36
|
+
if (!inline) {
|
|
37
|
+
await injectSidequestConfig(config);
|
|
38
|
+
}
|
|
15
39
|
let script = {};
|
|
16
40
|
try {
|
|
17
41
|
logger("Runner").debug(`Importing job script "${jobData.script}"`);
|
|
@@ -63,8 +87,25 @@ async function run({ jobData, config }) {
|
|
|
63
87
|
}
|
|
64
88
|
const job = new JobClass(...jobData.constructor_args);
|
|
65
89
|
job.injectJobData(jobData);
|
|
90
|
+
// Exactly one of these is provided: inline passes a live signal; the thread runner passes an abort
|
|
91
|
+
// port it turns into one.
|
|
92
|
+
let abortSignal;
|
|
93
|
+
if (signal) {
|
|
94
|
+
abortSignal = signal;
|
|
95
|
+
}
|
|
96
|
+
else if (abortPort) {
|
|
97
|
+
abortSignal = signalFromAbortPort(abortPort);
|
|
98
|
+
}
|
|
99
|
+
if (abortSignal) {
|
|
100
|
+
job.injectAbortSignal(abortSignal);
|
|
101
|
+
}
|
|
66
102
|
logger("Runner").debug(`Executing job class "${jobData.class}" with args:`, jobData.args);
|
|
67
|
-
|
|
103
|
+
try {
|
|
104
|
+
return await job.perform(...jobData.args);
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
abortPort?.close();
|
|
108
|
+
}
|
|
68
109
|
}
|
|
69
110
|
/**
|
|
70
111
|
* Injects the provided Sidequest engine configuration into the job script.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sources":["../../src/shared-runner/runner.ts"],"sourcesContent":[null],"names":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"runner.js","sources":["../../src/shared-runner/runner.ts"],"sourcesContent":[null],"names":[],"mappings":";;;;;;AAkBA;;;;;;AAMG;AACH,SAAS,mBAAmB,CAAC,IAAiB,EAAA;AAC5C,IAAA,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE;IACxC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAA2B,KAAI;QACjD,UAAU,CAAC,KAAK,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;AACnD,IAAA,CAAC,CAAC;IACF,OAAO,UAAU,CAAC,MAAM;AAC1B;AAEA;;;;;;;;;;;AAWG;AACY,eAAe,GAAG,CAAC,EAChC,OAAO,EACP,MAAM,EACN,MAAM,EACN,MAAM,EACN,SAAS,GAOV,EAAA;;;IAGC,IAAI,CAAC,MAAM,EAAE;AACX,QAAA,MAAM,qBAAqB,CAAC,MAAM,CAAC;IACrC;IAEA,IAAI,MAAM,GAAiC,EAAE;AAC7C,IAAA,IAAI;AACF,QAAA,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAA,sBAAA,EAAyB,OAAO,CAAC,MAAM,CAAA,CAAA,CAAG,CAAC;AAElE,QAAA,IAAI,SAAiB;AACrB,QAAA,IAAI,OAAO,CAAC,MAAM,KAAK,iBAAiB,EAAE;YACxC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6EAA6E,CAAC;AACrG,YAAA,IAAI;;AAEF,gBAAA,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;;;oBAGxB,SAAS,GAAG,mCAAmC,EAAE;gBACnD;qBAAO;;AAEL,oBAAA,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;oBAClD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE;AACzC,wBAAA,MAAM,IAAI,KAAK,CAAC,2DAA2D,SAAS,CAAA,CAAE,CAAC;oBACzF;gBACF;YACF;YAAE,OAAO,KAAK,EAAE;gBACd,MAAM,YAAY,GAAG,CAAA,gEAAA,EAAmE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE;gBAChJ,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;AACpC,gBAAA,MAAM,SAAS,GAAG,WAAW,CAAC,KAAc,CAAC;AAC7C,gBAAA,OAAO,EAAE,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE;YAC1E;QACF;aAAO;YACL,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,oEAAoE,CAAC;;AAE5F,YAAA,SAAS,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC;QACrD;QAEA,MAAM,IAAI,MAAM,OAAO,SAAS,CAAC,CAAiC;AAClE,QAAA,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAA,kCAAA,EAAqC,OAAO,CAAC,MAAM,CAAA,CAAA,CAAG,CAAC;IAChF;IAAE,OAAO,KAAK,EAAE;QACd,MAAM,YAAY,GAAG,CAAA,6BAAA,EAAgC,OAAO,CAAC,MAAM,CAAA,GAAA,EAAM,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA,CAAE;QACjI,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;AACpC,QAAA,MAAM,SAAS,GAAG,WAAW,CAAC,KAAc,CAAC;AAC7C,QAAA,OAAO,EAAE,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE;IAC1E;AAEA,IAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO;IACxD,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;AAC/C,QAAA,MAAM,KAAK,GAAG,CAAA,mBAAA,EAAsB,OAAO,CAAC,KAAK,EAAE;QACnD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;AAC/C,QAAA,OAAO,EAAE,qBAAqB,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE;IAC1E;IAEA,MAAM,GAAG,GAAQ,IAAI,QAAQ,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC;AAC1D,IAAA,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC;;;AAG1B,IAAA,IAAI,WAAoC;IACxC,IAAI,MAAM,EAAE;QACV,WAAW,GAAG,MAAM;IACtB;SAAO,IAAI,SAAS,EAAE;AACpB,QAAA,WAAW,GAAG,mBAAmB,CAAC,SAAS,CAAC;IAC9C;IACA,IAAI,WAAW,EAAE;AACf,QAAA,GAAG,CAAC,iBAAiB,CAAC,WAAW,CAAC;IACpC;AAEA,IAAA,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAA,qBAAA,EAAwB,OAAO,CAAC,KAAK,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC;AACzF,IAAA,IAAI;QACF,OAAO,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAC3C;YAAU;QACR,SAAS,EAAE,KAAK,EAAE;IACpB;AACF;AAEA;;;;;;;;;;AAUG;AACI,eAAe,qBAAqB,CAAC,MAAoB,EAAA;AAC9D,IAAA,IAAI;QACF,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,4CAA4C,CAAC;AACpE,QAAA,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,eAAe,EAAE;AAC7C,QAAA,MAAM,SAAS,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;QAC7D,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,wCAAwC,CAAC;AAChE,QAAA,OAAO,IAAI;IACb;IAAE,OAAO,KAAK,EAAE;QACd,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CACnB,CAAA,mCAAA,EAAsC,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA,oBAAA,CAAsB,CACnH;AACD,QAAA,OAAO,KAAK;IACd;AACF;;;;"}
|
package/dist/workers/main.cjs
CHANGED
|
@@ -1,21 +1,15 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var core = require('@sidequest/core');
|
|
4
|
-
var nodeCron = require('node-cron');
|
|
5
4
|
var constants = require('../constants.cjs');
|
|
6
5
|
var engine = require('../engine.cjs');
|
|
7
|
-
var dispatcher = require('../execution/dispatcher.cjs');
|
|
8
|
-
var executorManager = require('../execution/executor-manager.cjs');
|
|
9
|
-
var queueManager = require('../execution/queue-manager.cjs');
|
|
10
|
-
var cleanupFinishedJob = require('../routines/cleanup-finished-job.cjs');
|
|
11
|
-
var releaseStaleJobs = require('../routines/release-stale-jobs.cjs');
|
|
12
6
|
var shutdown = require('../utils/shutdown.cjs');
|
|
7
|
+
var workerRuntime = require('./worker-runtime.cjs');
|
|
13
8
|
|
|
14
9
|
class MainWorker {
|
|
15
10
|
shuttingDown = false;
|
|
16
|
-
|
|
11
|
+
runtime;
|
|
17
12
|
engine = new engine.Engine();
|
|
18
|
-
backend;
|
|
19
13
|
/**
|
|
20
14
|
* Starts a Sidequest worker process with the given configuration.
|
|
21
15
|
* @param sidequestConfig The Sidequest configuration for the worker.
|
|
@@ -24,10 +18,8 @@ class MainWorker {
|
|
|
24
18
|
if (!this.shuttingDown) {
|
|
25
19
|
try {
|
|
26
20
|
const nonNullConfig = await this.engine.configure({ ...sidequestConfig, skipMigration: true });
|
|
27
|
-
this.
|
|
28
|
-
|
|
29
|
-
this.dispatcher.start();
|
|
30
|
-
await this.startCron(nonNullConfig.releaseStaleJobsIntervalMin, nonNullConfig.releaseStaleJobsMaxStaleMs, nonNullConfig.releaseStaleJobsMaxClaimedMs, nonNullConfig.cleanupFinishedJobsIntervalMin, nonNullConfig.cleanupFinishedJobsOlderThan);
|
|
21
|
+
this.runtime = new workerRuntime.WorkerRuntime(this.engine.getBackend(), nonNullConfig);
|
|
22
|
+
await this.runtime.start();
|
|
31
23
|
}
|
|
32
24
|
catch (error) {
|
|
33
25
|
core.logger("Worker").error(error);
|
|
@@ -44,75 +36,13 @@ class MainWorker {
|
|
|
44
36
|
async shutdown() {
|
|
45
37
|
if (!this.shuttingDown) {
|
|
46
38
|
this.shuttingDown = true;
|
|
47
|
-
core.logger("Worker").debug("Shutting down
|
|
48
|
-
await this.
|
|
39
|
+
core.logger("Worker").debug("Shutting down worker runtime");
|
|
40
|
+
await this.runtime?.shutdown();
|
|
49
41
|
core.logger("Worker").debug("Shutting down engine");
|
|
50
42
|
await this.engine.close();
|
|
51
43
|
core.logger("Worker").debug("Main worker completely shut down");
|
|
52
44
|
}
|
|
53
45
|
}
|
|
54
|
-
/**
|
|
55
|
-
* Starts cron job for releasing stale jobs.
|
|
56
|
-
* Also executes the task immediately.
|
|
57
|
-
*/
|
|
58
|
-
async startAndExecuteStaleJobsReleaseCron(intervalMin, maxStaleMs, maxClaimedMs) {
|
|
59
|
-
if (!this.backend) {
|
|
60
|
-
throw new Error("Backend is not initialized. Cannot start stale jobs release cron.");
|
|
61
|
-
}
|
|
62
|
-
core.logger("Worker").debug(`Starting stale jobs release cron with interval: ${intervalMin} minutes`);
|
|
63
|
-
const releaseTask = nodeCron.schedule(`*/${intervalMin} * * * *`, async () => {
|
|
64
|
-
try {
|
|
65
|
-
core.logger("Worker").debug("Running stale jobs release task");
|
|
66
|
-
await releaseStaleJobs.releaseStaleJobs(this.backend, maxStaleMs, maxClaimedMs);
|
|
67
|
-
}
|
|
68
|
-
catch (error) {
|
|
69
|
-
core.logger("Worker").error("Error on running ReleaseStaleJob!", error);
|
|
70
|
-
}
|
|
71
|
-
});
|
|
72
|
-
return releaseTask.execute();
|
|
73
|
-
}
|
|
74
|
-
/**
|
|
75
|
-
* Starts cron job for cleaning up finished jobs.
|
|
76
|
-
* Also executes the task immediately.
|
|
77
|
-
*/
|
|
78
|
-
async startAndExecuteFinishedJobsCleanupCron(intervalMin, cutoffMs) {
|
|
79
|
-
if (!this.backend) {
|
|
80
|
-
throw new Error("Backend is not initialized. Cannot start finished jobs cleanup cron.");
|
|
81
|
-
}
|
|
82
|
-
core.logger("Worker").debug(`Starting finished jobs cleanup cron with interval: ${intervalMin} minutes`);
|
|
83
|
-
const cleanupTask = nodeCron.schedule(`*/${intervalMin} * * * *`, async () => {
|
|
84
|
-
try {
|
|
85
|
-
core.logger("Worker").debug("Running finished jobs cleanup task");
|
|
86
|
-
await cleanupFinishedJob.cleanupFinishedJobs(this.backend, cutoffMs);
|
|
87
|
-
}
|
|
88
|
-
catch (error) {
|
|
89
|
-
core.logger("Worker").error("Error on running CleanupJob!", error);
|
|
90
|
-
}
|
|
91
|
-
});
|
|
92
|
-
return cleanupTask.execute();
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* Starts cron jobs for releasing stale jobs and cleaning up finished jobs.
|
|
96
|
-
*
|
|
97
|
-
* @param staleIntervalMin Interval in minutes for releasing stale jobs, or false to disable.
|
|
98
|
-
* @param maxStaleMs Maximum age in milliseconds for stale jobs.
|
|
99
|
-
* @param maxClaimedMs Maximum age in milliseconds for claimed jobs.
|
|
100
|
-
* @param cleanupIntervalMin Interval in minutes for cleaning up finished jobs, or false to disable
|
|
101
|
-
* @param cleanupCutoffMs Maximum age in milliseconds for finished jobs to be cleaned up.
|
|
102
|
-
*/
|
|
103
|
-
async startCron(staleIntervalMin, maxStaleMs, maxClaimedMs, cleanupIntervalMin, cleanupCutoffMs) {
|
|
104
|
-
core.logger("Worker").debug("Starting cron jobs");
|
|
105
|
-
const promises = [];
|
|
106
|
-
if (staleIntervalMin !== false) {
|
|
107
|
-
promises.push(this.startAndExecuteStaleJobsReleaseCron(staleIntervalMin, maxStaleMs, maxClaimedMs));
|
|
108
|
-
}
|
|
109
|
-
if (cleanupIntervalMin !== false) {
|
|
110
|
-
promises.push(this.startAndExecuteFinishedJobsCleanupCron(cleanupIntervalMin, cleanupCutoffMs));
|
|
111
|
-
}
|
|
112
|
-
await Promise.all(promises).catch((error) => {
|
|
113
|
-
core.logger("Worker").error(error);
|
|
114
|
-
});
|
|
115
|
-
}
|
|
116
46
|
}
|
|
117
47
|
// Gate the bootstrap on the explicit flag the engine passes when forking, not on `!!process.send`.
|
|
118
48
|
// Any process forked over IPC (including a Vitest `pool: 'forks'` test worker that transitively
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.cjs","sources":["../../src/workers/main.ts"],"sourcesContent":[null],"names":["Engine","
|
|
1
|
+
{"version":3,"file":"main.cjs","sources":["../../src/workers/main.ts"],"sourcesContent":[null],"names":["Engine","WorkerRuntime","logger","WORKER_PROCESS_FLAG","gracefulShutdown"],"mappings":";;;;;;;;MAMa,UAAU,CAAA;IACrB,YAAY,GAAG,KAAK;AACZ,IAAA,OAAO;AACP,IAAA,MAAM,GAAG,IAAIA,aAAM,EAAE;AAE7B;;;AAGG;IACH,MAAM,SAAS,CAAC,eAA6B,EAAA;AAC3C,QAAA,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;AACtB,YAAA,IAAI;AACF,gBAAA,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,GAAG,eAAe,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC9F,gBAAA,IAAI,CAAC,OAAO,GAAG,IAAIC,2BAAa,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAG,EAAE,aAAa,CAAC;AAC1E,gBAAA,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE;YAC5B;YAAE,OAAO,KAAK,EAAE;gBACdC,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;AAC7B,gBAAA,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACjB;QACF;aAAO;YACLA,WAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC;QAChF;IACF;AAEA;;AAEG;AACH,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;AACtB,YAAA,IAAI,CAAC,YAAY,GAAG,IAAI;YACxBA,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,8BAA8B,CAAC;AACtD,YAAA,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE;YAC9BA,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC;AAC9C,YAAA,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACzBA,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC;QAC5D;IACF;AACD;AAED;AACA;AACA;AACA;AACA,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAACC,6BAAmB,CAAC;AAEjE,IAAI,cAAc,EAAE;AAClB,IAAA,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE;IAE/B,OAAO,CAAC,EAAE,CACR,SAAS;;AAET,IAAA,OAAO,EAAE,IAAI,EAAE,eAAe,EAA+D,KAAI;AAC/F,QAAA,IAAI,IAAI,KAAK,OAAO,EAAE;YACpB,IAAI,CAAC,eAAe,EAAE;AACpB,gBAAA,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC;YACnE;AACA,YAAA,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;AACxB,gBAAAC,yBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,eAAe,CAAC,gBAAgB,CAAC;gBAC1FF,WAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,gDAAgD,CAAC;AACvE,gBAAA,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC;YAChD;iBAAO;gBACLA,WAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,yDAAyD,CAAC;YAClF;QACF;AAAO,aAAA,IAAI,IAAI,KAAK,UAAU,EAAE;AAC9B,YAAA,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;gBACxBA,WAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,oDAAoD,CAAC;AAC3E,gBAAA,MAAM,MAAM,CAAC,QAAQ,EAAE;gBACvBA,WAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC;AAClD,gBAAA,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACjB;iBAAO;gBACLA,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,4DAA4D,CAAC;YACtF;QACF;AACF,IAAA,CAAC,CACF;AAED,IAAA,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,MAAK;QAC5BA,WAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,yCAAyC,CAAC;QACjE,OAAO,CAAC,IAAI,EAAE;AAChB,IAAA,CAAC,CAAC;IAEF,IAAI,OAAO,CAAC,IAAI;AAAE,QAAA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;AACzC;;;;"}
|
package/dist/workers/main.d.ts
CHANGED
|
@@ -2,9 +2,8 @@ import { EngineConfig } from '../engine.js';
|
|
|
2
2
|
|
|
3
3
|
declare class MainWorker {
|
|
4
4
|
shuttingDown: boolean;
|
|
5
|
-
private
|
|
5
|
+
private runtime?;
|
|
6
6
|
private engine;
|
|
7
|
-
private backend?;
|
|
8
7
|
/**
|
|
9
8
|
* Starts a Sidequest worker process with the given configuration.
|
|
10
9
|
* @param sidequestConfig The Sidequest configuration for the worker.
|
|
@@ -14,26 +13,6 @@ declare class MainWorker {
|
|
|
14
13
|
* Gracefully shuts down the worker and releases resources.
|
|
15
14
|
*/
|
|
16
15
|
shutdown(): Promise<void>;
|
|
17
|
-
/**
|
|
18
|
-
* Starts cron job for releasing stale jobs.
|
|
19
|
-
* Also executes the task immediately.
|
|
20
|
-
*/
|
|
21
|
-
startAndExecuteStaleJobsReleaseCron(intervalMin: number, maxStaleMs: number, maxClaimedMs: number): Promise<unknown>;
|
|
22
|
-
/**
|
|
23
|
-
* Starts cron job for cleaning up finished jobs.
|
|
24
|
-
* Also executes the task immediately.
|
|
25
|
-
*/
|
|
26
|
-
startAndExecuteFinishedJobsCleanupCron(intervalMin: number, cutoffMs: number): Promise<unknown>;
|
|
27
|
-
/**
|
|
28
|
-
* Starts cron jobs for releasing stale jobs and cleaning up finished jobs.
|
|
29
|
-
*
|
|
30
|
-
* @param staleIntervalMin Interval in minutes for releasing stale jobs, or false to disable.
|
|
31
|
-
* @param maxStaleMs Maximum age in milliseconds for stale jobs.
|
|
32
|
-
* @param maxClaimedMs Maximum age in milliseconds for claimed jobs.
|
|
33
|
-
* @param cleanupIntervalMin Interval in minutes for cleaning up finished jobs, or false to disable
|
|
34
|
-
* @param cleanupCutoffMs Maximum age in milliseconds for finished jobs to be cleaned up.
|
|
35
|
-
*/
|
|
36
|
-
startCron(staleIntervalMin: number | false, maxStaleMs: number, maxClaimedMs: number, cleanupIntervalMin: number | false, cleanupCutoffMs: number): Promise<void>;
|
|
37
16
|
}
|
|
38
17
|
|
|
39
18
|
export { MainWorker };
|
package/dist/workers/main.js
CHANGED
|
@@ -1,19 +1,13 @@
|
|
|
1
1
|
import { logger } from '@sidequest/core';
|
|
2
|
-
import nodeCron from 'node-cron';
|
|
3
2
|
import { WORKER_PROCESS_FLAG } from '../constants.js';
|
|
4
3
|
import { Engine } from '../engine.js';
|
|
5
|
-
import { Dispatcher } from '../execution/dispatcher.js';
|
|
6
|
-
import { ExecutorManager } from '../execution/executor-manager.js';
|
|
7
|
-
import { QueueManager } from '../execution/queue-manager.js';
|
|
8
|
-
import { cleanupFinishedJobs } from '../routines/cleanup-finished-job.js';
|
|
9
|
-
import { releaseStaleJobs } from '../routines/release-stale-jobs.js';
|
|
10
4
|
import { gracefulShutdown } from '../utils/shutdown.js';
|
|
5
|
+
import { WorkerRuntime } from './worker-runtime.js';
|
|
11
6
|
|
|
12
7
|
class MainWorker {
|
|
13
8
|
shuttingDown = false;
|
|
14
|
-
|
|
9
|
+
runtime;
|
|
15
10
|
engine = new Engine();
|
|
16
|
-
backend;
|
|
17
11
|
/**
|
|
18
12
|
* Starts a Sidequest worker process with the given configuration.
|
|
19
13
|
* @param sidequestConfig The Sidequest configuration for the worker.
|
|
@@ -22,10 +16,8 @@ class MainWorker {
|
|
|
22
16
|
if (!this.shuttingDown) {
|
|
23
17
|
try {
|
|
24
18
|
const nonNullConfig = await this.engine.configure({ ...sidequestConfig, skipMigration: true });
|
|
25
|
-
this.
|
|
26
|
-
|
|
27
|
-
this.dispatcher.start();
|
|
28
|
-
await this.startCron(nonNullConfig.releaseStaleJobsIntervalMin, nonNullConfig.releaseStaleJobsMaxStaleMs, nonNullConfig.releaseStaleJobsMaxClaimedMs, nonNullConfig.cleanupFinishedJobsIntervalMin, nonNullConfig.cleanupFinishedJobsOlderThan);
|
|
19
|
+
this.runtime = new WorkerRuntime(this.engine.getBackend(), nonNullConfig);
|
|
20
|
+
await this.runtime.start();
|
|
29
21
|
}
|
|
30
22
|
catch (error) {
|
|
31
23
|
logger("Worker").error(error);
|
|
@@ -42,75 +34,13 @@ class MainWorker {
|
|
|
42
34
|
async shutdown() {
|
|
43
35
|
if (!this.shuttingDown) {
|
|
44
36
|
this.shuttingDown = true;
|
|
45
|
-
logger("Worker").debug("Shutting down
|
|
46
|
-
await this.
|
|
37
|
+
logger("Worker").debug("Shutting down worker runtime");
|
|
38
|
+
await this.runtime?.shutdown();
|
|
47
39
|
logger("Worker").debug("Shutting down engine");
|
|
48
40
|
await this.engine.close();
|
|
49
41
|
logger("Worker").debug("Main worker completely shut down");
|
|
50
42
|
}
|
|
51
43
|
}
|
|
52
|
-
/**
|
|
53
|
-
* Starts cron job for releasing stale jobs.
|
|
54
|
-
* Also executes the task immediately.
|
|
55
|
-
*/
|
|
56
|
-
async startAndExecuteStaleJobsReleaseCron(intervalMin, maxStaleMs, maxClaimedMs) {
|
|
57
|
-
if (!this.backend) {
|
|
58
|
-
throw new Error("Backend is not initialized. Cannot start stale jobs release cron.");
|
|
59
|
-
}
|
|
60
|
-
logger("Worker").debug(`Starting stale jobs release cron with interval: ${intervalMin} minutes`);
|
|
61
|
-
const releaseTask = nodeCron.schedule(`*/${intervalMin} * * * *`, async () => {
|
|
62
|
-
try {
|
|
63
|
-
logger("Worker").debug("Running stale jobs release task");
|
|
64
|
-
await releaseStaleJobs(this.backend, maxStaleMs, maxClaimedMs);
|
|
65
|
-
}
|
|
66
|
-
catch (error) {
|
|
67
|
-
logger("Worker").error("Error on running ReleaseStaleJob!", error);
|
|
68
|
-
}
|
|
69
|
-
});
|
|
70
|
-
return releaseTask.execute();
|
|
71
|
-
}
|
|
72
|
-
/**
|
|
73
|
-
* Starts cron job for cleaning up finished jobs.
|
|
74
|
-
* Also executes the task immediately.
|
|
75
|
-
*/
|
|
76
|
-
async startAndExecuteFinishedJobsCleanupCron(intervalMin, cutoffMs) {
|
|
77
|
-
if (!this.backend) {
|
|
78
|
-
throw new Error("Backend is not initialized. Cannot start finished jobs cleanup cron.");
|
|
79
|
-
}
|
|
80
|
-
logger("Worker").debug(`Starting finished jobs cleanup cron with interval: ${intervalMin} minutes`);
|
|
81
|
-
const cleanupTask = nodeCron.schedule(`*/${intervalMin} * * * *`, async () => {
|
|
82
|
-
try {
|
|
83
|
-
logger("Worker").debug("Running finished jobs cleanup task");
|
|
84
|
-
await cleanupFinishedJobs(this.backend, cutoffMs);
|
|
85
|
-
}
|
|
86
|
-
catch (error) {
|
|
87
|
-
logger("Worker").error("Error on running CleanupJob!", error);
|
|
88
|
-
}
|
|
89
|
-
});
|
|
90
|
-
return cleanupTask.execute();
|
|
91
|
-
}
|
|
92
|
-
/**
|
|
93
|
-
* Starts cron jobs for releasing stale jobs and cleaning up finished jobs.
|
|
94
|
-
*
|
|
95
|
-
* @param staleIntervalMin Interval in minutes for releasing stale jobs, or false to disable.
|
|
96
|
-
* @param maxStaleMs Maximum age in milliseconds for stale jobs.
|
|
97
|
-
* @param maxClaimedMs Maximum age in milliseconds for claimed jobs.
|
|
98
|
-
* @param cleanupIntervalMin Interval in minutes for cleaning up finished jobs, or false to disable
|
|
99
|
-
* @param cleanupCutoffMs Maximum age in milliseconds for finished jobs to be cleaned up.
|
|
100
|
-
*/
|
|
101
|
-
async startCron(staleIntervalMin, maxStaleMs, maxClaimedMs, cleanupIntervalMin, cleanupCutoffMs) {
|
|
102
|
-
logger("Worker").debug("Starting cron jobs");
|
|
103
|
-
const promises = [];
|
|
104
|
-
if (staleIntervalMin !== false) {
|
|
105
|
-
promises.push(this.startAndExecuteStaleJobsReleaseCron(staleIntervalMin, maxStaleMs, maxClaimedMs));
|
|
106
|
-
}
|
|
107
|
-
if (cleanupIntervalMin !== false) {
|
|
108
|
-
promises.push(this.startAndExecuteFinishedJobsCleanupCron(cleanupIntervalMin, cleanupCutoffMs));
|
|
109
|
-
}
|
|
110
|
-
await Promise.all(promises).catch((error) => {
|
|
111
|
-
logger("Worker").error(error);
|
|
112
|
-
});
|
|
113
|
-
}
|
|
114
44
|
}
|
|
115
45
|
// Gate the bootstrap on the explicit flag the engine passes when forking, not on `!!process.send`.
|
|
116
46
|
// Any process forked over IPC (including a Vitest `pool: 'forks'` test worker that transitively
|
package/dist/workers/main.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sources":["../../src/workers/main.ts"],"sourcesContent":[null],"names":[
|
|
1
|
+
{"version":3,"file":"main.js","sources":["../../src/workers/main.ts"],"sourcesContent":[null],"names":[],"mappings":";;;;;;MAMa,UAAU,CAAA;IACrB,YAAY,GAAG,KAAK;AACZ,IAAA,OAAO;AACP,IAAA,MAAM,GAAG,IAAI,MAAM,EAAE;AAE7B;;;AAGG;IACH,MAAM,SAAS,CAAC,eAA6B,EAAA;AAC3C,QAAA,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;AACtB,YAAA,IAAI;AACF,gBAAA,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,GAAG,eAAe,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC9F,gBAAA,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAG,EAAE,aAAa,CAAC;AAC1E,gBAAA,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE;YAC5B;YAAE,OAAO,KAAK,EAAE;gBACd,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;AAC7B,gBAAA,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACjB;QACF;aAAO;YACL,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC;QAChF;IACF;AAEA;;AAEG;AACH,IAAA,MAAM,QAAQ,GAAA;AACZ,QAAA,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;AACtB,YAAA,IAAI,CAAC,YAAY,GAAG,IAAI;YACxB,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,8BAA8B,CAAC;AACtD,YAAA,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE;YAC9B,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC;AAC9C,YAAA,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACzB,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC;QAC5D;IACF;AACD;AAED;AACA;AACA;AACA;AACA,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC;AAEjE,IAAI,cAAc,EAAE;AAClB,IAAA,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE;IAE/B,OAAO,CAAC,EAAE,CACR,SAAS;;AAET,IAAA,OAAO,EAAE,IAAI,EAAE,eAAe,EAA+D,KAAI;AAC/F,QAAA,IAAI,IAAI,KAAK,OAAO,EAAE;YACpB,IAAI,CAAC,eAAe,EAAE;AACpB,gBAAA,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC;YACnE;AACA,YAAA,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;AACxB,gBAAA,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,eAAe,CAAC,gBAAgB,CAAC;gBAC1F,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,gDAAgD,CAAC;AACvE,gBAAA,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC;YAChD;iBAAO;gBACL,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,yDAAyD,CAAC;YAClF;QACF;AAAO,aAAA,IAAI,IAAI,KAAK,UAAU,EAAE;AAC9B,YAAA,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;gBACxB,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,oDAAoD,CAAC;AAC3E,gBAAA,MAAM,MAAM,CAAC,QAAQ,EAAE;gBACvB,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC;AAClD,gBAAA,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACjB;iBAAO;gBACL,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,4DAA4D,CAAC;YACtF;QACF;AACF,IAAA,CAAC,CACF;AAED,IAAA,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,MAAK;QAC5B,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,yCAAyC,CAAC;QACjE,OAAO,CAAC,IAAI,EAAE;AAChB,IAAA,CAAC,CAAC;IAEF,IAAI,OAAO,CAAC,IAAI;AAAE,QAAA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;AACzC;;;;"}
|