@xylabs/threads 3.6.7 → 3.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/master/implementation.browser.js +3 -3
- package/dist/esm/master/implementation.node.js +11 -9
- package/dist/esm/master/pool.js +9 -9
- package/dist/esm/master/spawn.js +4 -4
- package/dist/esm/observable-promise.js +2 -2
- package/dist/esm/observable.js +1 -1
- package/dist/esm/worker/index.js +2 -2
- package/dist/master/implementation.browser.js +3 -3
- package/dist/master/implementation.node.js +11 -9
- package/dist/master/pool.js +9 -9
- package/dist/master/spawn.js +4 -4
- package/dist/observable-promise.js +2 -2
- package/dist/observable.js +1 -1
- package/dist/worker/index.js +2 -2
- package/package.json +3 -3
- package/src/master/implementation.browser.ts +3 -3
- package/src/master/implementation.node.ts +19 -17
- package/src/master/invocation-proxy.ts +0 -1
- package/src/master/pool-types.ts +28 -28
- package/src/master/pool.ts +10 -10
- package/src/master/spawn.ts +9 -9
- package/src/observable-promise.ts +11 -11
- package/src/observable.ts +1 -2
- package/src/ponyfills.ts +6 -6
- package/src/transferable.ts +0 -1
- package/src/types/master.ts +4 -3
- package/src/worker/implementation.tiny-worker.ts +0 -1
- package/src/worker/index.ts +2 -2
- package/test/lib/serialization.ts +2 -2
- package/test/observable-promise.test.ts +8 -9
- package/test/observable.test.ts +6 -6
- package/test/pool.test.ts +6 -7
- package/test/spawn.test.ts +3 -2
- package/test/streaming.test.ts +1 -1
- package/test/transferables.test.ts +2 -2
- package/test-tooling/webpack/app.ts +4 -5
- package/test-tooling/webpack/webpack.chromium.mocha.ts +1 -1
- package/test-tooling/webpack/webpack.test.ts +13 -7
|
@@ -29,9 +29,9 @@ function selectWorkerImplementation() {
|
|
|
29
29
|
url = createSourceBlobURL(`importScripts(${JSON.stringify(url)});`);
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
|
-
if (typeof url === 'string'
|
|
33
|
-
isAbsoluteURL(url)
|
|
34
|
-
(options?.CORSWorkaround ?? true)) {
|
|
32
|
+
if (typeof url === 'string'
|
|
33
|
+
&& isAbsoluteURL(url)
|
|
34
|
+
&& (options?.CORSWorkaround ?? true)) {
|
|
35
35
|
url = createSourceBlobURL(`importScripts(${JSON.stringify(url)});`);
|
|
36
36
|
}
|
|
37
37
|
super(url, options);
|
|
@@ -56,8 +56,8 @@ function resolveScriptPath(scriptPath, baseURL) {
|
|
|
56
56
|
const makeRelative = (filePath) => {
|
|
57
57
|
return node_path_1.default.isAbsolute(filePath) ? filePath : node_path_1.default.join(baseURL || eval('__dirname'), filePath);
|
|
58
58
|
};
|
|
59
|
-
return typeof __non_webpack_require__ === 'function'
|
|
60
|
-
__non_webpack_require__.resolve(makeRelative(scriptPath))
|
|
59
|
+
return typeof __non_webpack_require__ === 'function'
|
|
60
|
+
? __non_webpack_require__.resolve(makeRelative(scriptPath))
|
|
61
61
|
: eval('require').resolve(makeRelative(rebaseScriptPath(scriptPath, /[/\\]worker_threads[/\\]/)));
|
|
62
62
|
}
|
|
63
63
|
function initWorkerThreadsWorker() {
|
|
@@ -96,7 +96,7 @@ function initWorkerThreadsWorker() {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
const terminateWorkersAndMaster = () => {
|
|
99
|
-
Promise.all(allWorkers.map(
|
|
99
|
+
Promise.all(allWorkers.map(worker => worker.terminate())).then(() => process.exit(0), () => process.exit(1));
|
|
100
100
|
allWorkers = [];
|
|
101
101
|
};
|
|
102
102
|
process.on('SIGINT', () => terminateWorkersAndMaster());
|
|
@@ -120,8 +120,10 @@ function initTinyWorker() {
|
|
|
120
120
|
class Worker extends TinyWorker {
|
|
121
121
|
emitter;
|
|
122
122
|
constructor(scriptPath, options) {
|
|
123
|
-
const resolvedScriptPath = options && options.fromSource
|
|
124
|
-
|
|
123
|
+
const resolvedScriptPath = options && options.fromSource
|
|
124
|
+
? null
|
|
125
|
+
: process.platform === 'win32'
|
|
126
|
+
? `file:///${resolveScriptPath(scriptPath).replaceAll('\\', '/')}`
|
|
125
127
|
: resolveScriptPath(scriptPath);
|
|
126
128
|
if (!resolvedScriptPath) {
|
|
127
129
|
const sourceCode = scriptPath;
|
|
@@ -148,12 +150,12 @@ function initTinyWorker() {
|
|
|
148
150
|
this.emitter.removeListener(eventName, listener);
|
|
149
151
|
}
|
|
150
152
|
terminate() {
|
|
151
|
-
allWorkers = allWorkers.filter(
|
|
153
|
+
allWorkers = allWorkers.filter(worker => worker !== this);
|
|
152
154
|
return super.terminate();
|
|
153
155
|
}
|
|
154
156
|
}
|
|
155
157
|
const terminateWorkersAndMaster = () => {
|
|
156
|
-
Promise.all(allWorkers.map(
|
|
158
|
+
Promise.all(allWorkers.map(worker => worker.terminate())).then(() => process.exit(0), () => process.exit(1));
|
|
157
159
|
allWorkers = [];
|
|
158
160
|
};
|
|
159
161
|
process.on('SIGINT', () => terminateWorkersAndMaster());
|
|
@@ -195,8 +197,8 @@ function isWorkerRuntime() {
|
|
|
195
197
|
return self !== undefined && self['postMessage'] ? true : false;
|
|
196
198
|
}
|
|
197
199
|
else {
|
|
198
|
-
const isMainThread = typeof __non_webpack_require__ === 'function'
|
|
199
|
-
__non_webpack_require__('worker_threads').isMainThread
|
|
200
|
+
const isMainThread = typeof __non_webpack_require__ === 'function'
|
|
201
|
+
? __non_webpack_require__('worker_threads').isMainThread
|
|
200
202
|
: eval('require')('worker_threads').isMainThread;
|
|
201
203
|
return !isMainThread;
|
|
202
204
|
}
|
package/dist/esm/master/pool.js
CHANGED
|
@@ -19,7 +19,7 @@ function createArray(size) {
|
|
|
19
19
|
return array;
|
|
20
20
|
}
|
|
21
21
|
function delay(ms) {
|
|
22
|
-
return new Promise(
|
|
22
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
23
23
|
}
|
|
24
24
|
function flatMap(array, mapper) {
|
|
25
25
|
return array.reduce((flattened, element) => [...flattened, ...mapper(element)], []);
|
|
@@ -51,7 +51,7 @@ class WorkerPool {
|
|
|
51
51
|
this.options = options;
|
|
52
52
|
this.workers = spawnWorkers(spawnWorker, size);
|
|
53
53
|
this.eventObservable = (0, observable_fns_1.multicast)(observable_fns_1.Observable.from(this.eventSubject));
|
|
54
|
-
Promise.all(this.workers.map(
|
|
54
|
+
Promise.all(this.workers.map(worker => worker.init)).then(() => this.eventSubject.next({
|
|
55
55
|
size: this.workers.length,
|
|
56
56
|
type: pool_types_1.PoolEventType.initialized,
|
|
57
57
|
}), (error) => {
|
|
@@ -62,7 +62,7 @@ class WorkerPool {
|
|
|
62
62
|
}
|
|
63
63
|
findIdlingWorker() {
|
|
64
64
|
const { concurrency = 1 } = this.options;
|
|
65
|
-
return this.workers.find(
|
|
65
|
+
return this.workers.find(worker => worker.runningTasks.length < concurrency);
|
|
66
66
|
}
|
|
67
67
|
async runPoolTask(worker, task) {
|
|
68
68
|
const workerID = this.workers.indexOf(worker) + 1;
|
|
@@ -95,7 +95,7 @@ class WorkerPool {
|
|
|
95
95
|
async run(worker, task) {
|
|
96
96
|
const runPromise = (async () => {
|
|
97
97
|
const removeTaskFromWorkersRunningTasks = () => {
|
|
98
|
-
worker.runningTasks = worker.runningTasks.filter(
|
|
98
|
+
worker.runningTasks = worker.runningTasks.filter(someRunPromise => someRunPromise !== runPromise);
|
|
99
99
|
};
|
|
100
100
|
await delay(0);
|
|
101
101
|
try {
|
|
@@ -142,7 +142,7 @@ class WorkerPool {
|
|
|
142
142
|
});
|
|
143
143
|
}
|
|
144
144
|
async settled(allowResolvingImmediately = false) {
|
|
145
|
-
const getCurrentlyRunningTasks = () => flatMap(this.workers,
|
|
145
|
+
const getCurrentlyRunningTasks = () => flatMap(this.workers, worker => worker.runningTasks);
|
|
146
146
|
const taskFailures = [];
|
|
147
147
|
const failureSubscription = this.eventObservable.subscribe((event) => {
|
|
148
148
|
if (event.type === pool_types_1.PoolEventType.taskFailed) {
|
|
@@ -213,7 +213,7 @@ class WorkerPool {
|
|
|
213
213
|
cancel: () => {
|
|
214
214
|
if (!this.taskQueue.includes(task))
|
|
215
215
|
return;
|
|
216
|
-
this.taskQueue = this.taskQueue.filter(
|
|
216
|
+
this.taskQueue = this.taskQueue.filter(someTask => someTask !== task);
|
|
217
217
|
this.eventSubject.next({
|
|
218
218
|
taskID: task.id,
|
|
219
219
|
type: pool_types_1.PoolEventType.taskCanceled,
|
|
@@ -224,9 +224,9 @@ class WorkerPool {
|
|
|
224
224
|
then: taskCompletion.then.bind(taskCompletion),
|
|
225
225
|
};
|
|
226
226
|
if (this.taskQueue.length >= maxQueuedJobs) {
|
|
227
|
-
throw new Error('Maximum number of pool tasks queued. Refusing to queue another one.\n'
|
|
228
|
-
'This usually happens for one of two reasons: We are either at peak '
|
|
229
|
-
"workload right now or some tasks just won't finish, thus blocking the pool.");
|
|
227
|
+
throw new Error('Maximum number of pool tasks queued. Refusing to queue another one.\n'
|
|
228
|
+
+ 'This usually happens for one of two reasons: We are either at peak '
|
|
229
|
+
+ "workload right now or some tasks just won't finish, thus blocking the pool.");
|
|
230
230
|
}
|
|
231
231
|
this.debug(`Queueing task #${task.id}...`);
|
|
232
232
|
this.taskQueue.push(task);
|
package/dist/esm/master/spawn.js
CHANGED
|
@@ -16,8 +16,8 @@ const debugSpawn = (0, debug_1.default)('threads:master:spawn');
|
|
|
16
16
|
const debugThreadUtils = (0, debug_1.default)('threads:master:thread-utils');
|
|
17
17
|
const isInitMessage = (data) => data && data.type === 'init';
|
|
18
18
|
const isUncaughtErrorMessage = (data) => data && data.type === 'uncaughtError';
|
|
19
|
-
const initMessageTimeout = typeof process !== 'undefined' && process.env !== undefined && process.env.THREADS_WORKER_INIT_TIMEOUT
|
|
20
|
-
Number.parseInt(process.env.THREADS_WORKER_INIT_TIMEOUT, 10)
|
|
19
|
+
const initMessageTimeout = typeof process !== 'undefined' && process.env !== undefined && process.env.THREADS_WORKER_INIT_TIMEOUT
|
|
20
|
+
? Number.parseInt(process.env.THREADS_WORKER_INIT_TIMEOUT, 10)
|
|
21
21
|
: 10_000;
|
|
22
22
|
async function withTimeout(promise, timeoutInMs, errorMessage) {
|
|
23
23
|
let timeoutHandle;
|
|
@@ -85,8 +85,8 @@ function createTerminator(worker) {
|
|
|
85
85
|
}
|
|
86
86
|
function setPrivateThreadProps(raw, worker, workerEvents, terminate) {
|
|
87
87
|
const workerErrors = workerEvents
|
|
88
|
-
.filter(
|
|
89
|
-
.map(
|
|
88
|
+
.filter(event => event.type === master_1.WorkerEventType.internalError)
|
|
89
|
+
.map(errorEvent => errorEvent.error);
|
|
90
90
|
return Object.assign(raw, {
|
|
91
91
|
[symbols_1.$errors]: workerErrors,
|
|
92
92
|
[symbols_1.$events]: workerEvents,
|
|
@@ -115,8 +115,8 @@ class ObservablePromise extends observable_fns_1.Observable {
|
|
|
115
115
|
}, () => handler());
|
|
116
116
|
}
|
|
117
117
|
static from(thing) {
|
|
118
|
-
return isThenable(thing)
|
|
119
|
-
new ObservablePromise((observer) => {
|
|
118
|
+
return isThenable(thing)
|
|
119
|
+
? new ObservablePromise((observer) => {
|
|
120
120
|
const onFulfilled = (value) => {
|
|
121
121
|
observer.next(value);
|
|
122
122
|
observer.complete();
|
package/dist/esm/observable.js
CHANGED
|
@@ -9,7 +9,7 @@ class Subject extends observable_fns_1.Observable {
|
|
|
9
9
|
super((observer) => {
|
|
10
10
|
this[$observers] = [...(this[$observers] || []), observer];
|
|
11
11
|
const unsubscribe = () => {
|
|
12
|
-
this[$observers] = this[$observers].filter(
|
|
12
|
+
this[$observers] = this[$observers].filter(someObserver => someObserver !== observer);
|
|
13
13
|
};
|
|
14
14
|
return unsubscribe;
|
|
15
15
|
});
|
package/dist/esm/worker/index.js
CHANGED
|
@@ -95,7 +95,7 @@ async function runFunction(jobUID, fn, args) {
|
|
|
95
95
|
const resultType = isObservable(syncResult) ? 'observable' : 'promise';
|
|
96
96
|
postJobStartMessage(jobUID, resultType);
|
|
97
97
|
if (isObservable(syncResult)) {
|
|
98
|
-
const subscription = syncResult.subscribe(
|
|
98
|
+
const subscription = syncResult.subscribe(value => postJobResultMessage(jobUID, false, (0, common_1.serialize)(value)), (error) => {
|
|
99
99
|
postJobErrorMessage(jobUID, (0, common_1.serialize)(error));
|
|
100
100
|
activeSubscriptions.delete(jobUID);
|
|
101
101
|
}, () => {
|
|
@@ -136,7 +136,7 @@ function expose(exposed) {
|
|
|
136
136
|
runFunction(messageData.uid, exposed[messageData.method], messageData.args.map(common_1.deserialize));
|
|
137
137
|
}
|
|
138
138
|
});
|
|
139
|
-
const methodNames = Object.keys(exposed).filter(
|
|
139
|
+
const methodNames = Object.keys(exposed).filter(key => typeof exposed[key] === 'function');
|
|
140
140
|
postModuleInitMessage(methodNames);
|
|
141
141
|
}
|
|
142
142
|
else {
|
|
@@ -29,9 +29,9 @@ function selectWorkerImplementation() {
|
|
|
29
29
|
url = createSourceBlobURL(`importScripts(${JSON.stringify(url)});`);
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
|
-
if (typeof url === 'string'
|
|
33
|
-
isAbsoluteURL(url)
|
|
34
|
-
(options?.CORSWorkaround ?? true)) {
|
|
32
|
+
if (typeof url === 'string'
|
|
33
|
+
&& isAbsoluteURL(url)
|
|
34
|
+
&& (options?.CORSWorkaround ?? true)) {
|
|
35
35
|
url = createSourceBlobURL(`importScripts(${JSON.stringify(url)});`);
|
|
36
36
|
}
|
|
37
37
|
super(url, options);
|
|
@@ -56,8 +56,8 @@ function resolveScriptPath(scriptPath, baseURL) {
|
|
|
56
56
|
const makeRelative = (filePath) => {
|
|
57
57
|
return node_path_1.default.isAbsolute(filePath) ? filePath : node_path_1.default.join(baseURL || eval('__dirname'), filePath);
|
|
58
58
|
};
|
|
59
|
-
return typeof __non_webpack_require__ === 'function'
|
|
60
|
-
__non_webpack_require__.resolve(makeRelative(scriptPath))
|
|
59
|
+
return typeof __non_webpack_require__ === 'function'
|
|
60
|
+
? __non_webpack_require__.resolve(makeRelative(scriptPath))
|
|
61
61
|
: eval('require').resolve(makeRelative(rebaseScriptPath(scriptPath, /[/\\]worker_threads[/\\]/)));
|
|
62
62
|
}
|
|
63
63
|
function initWorkerThreadsWorker() {
|
|
@@ -96,7 +96,7 @@ function initWorkerThreadsWorker() {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
const terminateWorkersAndMaster = () => {
|
|
99
|
-
Promise.all(allWorkers.map(
|
|
99
|
+
Promise.all(allWorkers.map(worker => worker.terminate())).then(() => process.exit(0), () => process.exit(1));
|
|
100
100
|
allWorkers = [];
|
|
101
101
|
};
|
|
102
102
|
process.on('SIGINT', () => terminateWorkersAndMaster());
|
|
@@ -120,8 +120,10 @@ function initTinyWorker() {
|
|
|
120
120
|
class Worker extends TinyWorker {
|
|
121
121
|
emitter;
|
|
122
122
|
constructor(scriptPath, options) {
|
|
123
|
-
const resolvedScriptPath = options && options.fromSource
|
|
124
|
-
|
|
123
|
+
const resolvedScriptPath = options && options.fromSource
|
|
124
|
+
? null
|
|
125
|
+
: process.platform === 'win32'
|
|
126
|
+
? `file:///${resolveScriptPath(scriptPath).replaceAll('\\', '/')}`
|
|
125
127
|
: resolveScriptPath(scriptPath);
|
|
126
128
|
if (!resolvedScriptPath) {
|
|
127
129
|
const sourceCode = scriptPath;
|
|
@@ -148,12 +150,12 @@ function initTinyWorker() {
|
|
|
148
150
|
this.emitter.removeListener(eventName, listener);
|
|
149
151
|
}
|
|
150
152
|
terminate() {
|
|
151
|
-
allWorkers = allWorkers.filter(
|
|
153
|
+
allWorkers = allWorkers.filter(worker => worker !== this);
|
|
152
154
|
return super.terminate();
|
|
153
155
|
}
|
|
154
156
|
}
|
|
155
157
|
const terminateWorkersAndMaster = () => {
|
|
156
|
-
Promise.all(allWorkers.map(
|
|
158
|
+
Promise.all(allWorkers.map(worker => worker.terminate())).then(() => process.exit(0), () => process.exit(1));
|
|
157
159
|
allWorkers = [];
|
|
158
160
|
};
|
|
159
161
|
process.on('SIGINT', () => terminateWorkersAndMaster());
|
|
@@ -195,8 +197,8 @@ function isWorkerRuntime() {
|
|
|
195
197
|
return self !== undefined && self['postMessage'] ? true : false;
|
|
196
198
|
}
|
|
197
199
|
else {
|
|
198
|
-
const isMainThread = typeof __non_webpack_require__ === 'function'
|
|
199
|
-
__non_webpack_require__('worker_threads').isMainThread
|
|
200
|
+
const isMainThread = typeof __non_webpack_require__ === 'function'
|
|
201
|
+
? __non_webpack_require__('worker_threads').isMainThread
|
|
200
202
|
: eval('require')('worker_threads').isMainThread;
|
|
201
203
|
return !isMainThread;
|
|
202
204
|
}
|
package/dist/master/pool.js
CHANGED
|
@@ -19,7 +19,7 @@ function createArray(size) {
|
|
|
19
19
|
return array;
|
|
20
20
|
}
|
|
21
21
|
function delay(ms) {
|
|
22
|
-
return new Promise(
|
|
22
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
23
23
|
}
|
|
24
24
|
function flatMap(array, mapper) {
|
|
25
25
|
return array.reduce((flattened, element) => [...flattened, ...mapper(element)], []);
|
|
@@ -51,7 +51,7 @@ class WorkerPool {
|
|
|
51
51
|
this.options = options;
|
|
52
52
|
this.workers = spawnWorkers(spawnWorker, size);
|
|
53
53
|
this.eventObservable = (0, observable_fns_1.multicast)(observable_fns_1.Observable.from(this.eventSubject));
|
|
54
|
-
Promise.all(this.workers.map(
|
|
54
|
+
Promise.all(this.workers.map(worker => worker.init)).then(() => this.eventSubject.next({
|
|
55
55
|
size: this.workers.length,
|
|
56
56
|
type: pool_types_1.PoolEventType.initialized,
|
|
57
57
|
}), (error) => {
|
|
@@ -62,7 +62,7 @@ class WorkerPool {
|
|
|
62
62
|
}
|
|
63
63
|
findIdlingWorker() {
|
|
64
64
|
const { concurrency = 1 } = this.options;
|
|
65
|
-
return this.workers.find(
|
|
65
|
+
return this.workers.find(worker => worker.runningTasks.length < concurrency);
|
|
66
66
|
}
|
|
67
67
|
async runPoolTask(worker, task) {
|
|
68
68
|
const workerID = this.workers.indexOf(worker) + 1;
|
|
@@ -95,7 +95,7 @@ class WorkerPool {
|
|
|
95
95
|
async run(worker, task) {
|
|
96
96
|
const runPromise = (async () => {
|
|
97
97
|
const removeTaskFromWorkersRunningTasks = () => {
|
|
98
|
-
worker.runningTasks = worker.runningTasks.filter(
|
|
98
|
+
worker.runningTasks = worker.runningTasks.filter(someRunPromise => someRunPromise !== runPromise);
|
|
99
99
|
};
|
|
100
100
|
await delay(0);
|
|
101
101
|
try {
|
|
@@ -142,7 +142,7 @@ class WorkerPool {
|
|
|
142
142
|
});
|
|
143
143
|
}
|
|
144
144
|
async settled(allowResolvingImmediately = false) {
|
|
145
|
-
const getCurrentlyRunningTasks = () => flatMap(this.workers,
|
|
145
|
+
const getCurrentlyRunningTasks = () => flatMap(this.workers, worker => worker.runningTasks);
|
|
146
146
|
const taskFailures = [];
|
|
147
147
|
const failureSubscription = this.eventObservable.subscribe((event) => {
|
|
148
148
|
if (event.type === pool_types_1.PoolEventType.taskFailed) {
|
|
@@ -213,7 +213,7 @@ class WorkerPool {
|
|
|
213
213
|
cancel: () => {
|
|
214
214
|
if (!this.taskQueue.includes(task))
|
|
215
215
|
return;
|
|
216
|
-
this.taskQueue = this.taskQueue.filter(
|
|
216
|
+
this.taskQueue = this.taskQueue.filter(someTask => someTask !== task);
|
|
217
217
|
this.eventSubject.next({
|
|
218
218
|
taskID: task.id,
|
|
219
219
|
type: pool_types_1.PoolEventType.taskCanceled,
|
|
@@ -224,9 +224,9 @@ class WorkerPool {
|
|
|
224
224
|
then: taskCompletion.then.bind(taskCompletion),
|
|
225
225
|
};
|
|
226
226
|
if (this.taskQueue.length >= maxQueuedJobs) {
|
|
227
|
-
throw new Error('Maximum number of pool tasks queued. Refusing to queue another one.\n'
|
|
228
|
-
'This usually happens for one of two reasons: We are either at peak '
|
|
229
|
-
"workload right now or some tasks just won't finish, thus blocking the pool.");
|
|
227
|
+
throw new Error('Maximum number of pool tasks queued. Refusing to queue another one.\n'
|
|
228
|
+
+ 'This usually happens for one of two reasons: We are either at peak '
|
|
229
|
+
+ "workload right now or some tasks just won't finish, thus blocking the pool.");
|
|
230
230
|
}
|
|
231
231
|
this.debug(`Queueing task #${task.id}...`);
|
|
232
232
|
this.taskQueue.push(task);
|
package/dist/master/spawn.js
CHANGED
|
@@ -16,8 +16,8 @@ const debugSpawn = (0, debug_1.default)('threads:master:spawn');
|
|
|
16
16
|
const debugThreadUtils = (0, debug_1.default)('threads:master:thread-utils');
|
|
17
17
|
const isInitMessage = (data) => data && data.type === 'init';
|
|
18
18
|
const isUncaughtErrorMessage = (data) => data && data.type === 'uncaughtError';
|
|
19
|
-
const initMessageTimeout = typeof process !== 'undefined' && process.env !== undefined && process.env.THREADS_WORKER_INIT_TIMEOUT
|
|
20
|
-
Number.parseInt(process.env.THREADS_WORKER_INIT_TIMEOUT, 10)
|
|
19
|
+
const initMessageTimeout = typeof process !== 'undefined' && process.env !== undefined && process.env.THREADS_WORKER_INIT_TIMEOUT
|
|
20
|
+
? Number.parseInt(process.env.THREADS_WORKER_INIT_TIMEOUT, 10)
|
|
21
21
|
: 10_000;
|
|
22
22
|
async function withTimeout(promise, timeoutInMs, errorMessage) {
|
|
23
23
|
let timeoutHandle;
|
|
@@ -85,8 +85,8 @@ function createTerminator(worker) {
|
|
|
85
85
|
}
|
|
86
86
|
function setPrivateThreadProps(raw, worker, workerEvents, terminate) {
|
|
87
87
|
const workerErrors = workerEvents
|
|
88
|
-
.filter(
|
|
89
|
-
.map(
|
|
88
|
+
.filter(event => event.type === master_1.WorkerEventType.internalError)
|
|
89
|
+
.map(errorEvent => errorEvent.error);
|
|
90
90
|
return Object.assign(raw, {
|
|
91
91
|
[symbols_1.$errors]: workerErrors,
|
|
92
92
|
[symbols_1.$events]: workerEvents,
|
|
@@ -115,8 +115,8 @@ class ObservablePromise extends observable_fns_1.Observable {
|
|
|
115
115
|
}, () => handler());
|
|
116
116
|
}
|
|
117
117
|
static from(thing) {
|
|
118
|
-
return isThenable(thing)
|
|
119
|
-
new ObservablePromise((observer) => {
|
|
118
|
+
return isThenable(thing)
|
|
119
|
+
? new ObservablePromise((observer) => {
|
|
120
120
|
const onFulfilled = (value) => {
|
|
121
121
|
observer.next(value);
|
|
122
122
|
observer.complete();
|
package/dist/observable.js
CHANGED
|
@@ -9,7 +9,7 @@ class Subject extends observable_fns_1.Observable {
|
|
|
9
9
|
super((observer) => {
|
|
10
10
|
this[$observers] = [...(this[$observers] || []), observer];
|
|
11
11
|
const unsubscribe = () => {
|
|
12
|
-
this[$observers] = this[$observers].filter(
|
|
12
|
+
this[$observers] = this[$observers].filter(someObserver => someObserver !== observer);
|
|
13
13
|
};
|
|
14
14
|
return unsubscribe;
|
|
15
15
|
});
|
package/dist/worker/index.js
CHANGED
|
@@ -95,7 +95,7 @@ async function runFunction(jobUID, fn, args) {
|
|
|
95
95
|
const resultType = isObservable(syncResult) ? 'observable' : 'promise';
|
|
96
96
|
postJobStartMessage(jobUID, resultType);
|
|
97
97
|
if (isObservable(syncResult)) {
|
|
98
|
-
const subscription = syncResult.subscribe(
|
|
98
|
+
const subscription = syncResult.subscribe(value => postJobResultMessage(jobUID, false, (0, common_1.serialize)(value)), (error) => {
|
|
99
99
|
postJobErrorMessage(jobUID, (0, common_1.serialize)(error));
|
|
100
100
|
activeSubscriptions.delete(jobUID);
|
|
101
101
|
}, () => {
|
|
@@ -136,7 +136,7 @@ function expose(exposed) {
|
|
|
136
136
|
runFunction(messageData.uid, exposed[messageData.method], messageData.args.map(common_1.deserialize));
|
|
137
137
|
}
|
|
138
138
|
});
|
|
139
|
-
const methodNames = Object.keys(exposed).filter(
|
|
139
|
+
const methodNames = Object.keys(exposed).filter(key => typeof exposed[key] === 'function');
|
|
140
140
|
postModuleInitMessage(methodNames);
|
|
141
141
|
}
|
|
142
142
|
else {
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xylabs/threads",
|
|
3
|
-
"version": "3.6.
|
|
3
|
+
"version": "3.6.8",
|
|
4
4
|
"description": "Web workers & worker threads as simple as a function call",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"module": "dist/esm/index.js",
|
|
8
8
|
"scripts": {
|
|
9
9
|
"package-compile": "yarn build",
|
|
10
|
-
"clean": "rimraf dist
|
|
10
|
+
"clean": "rimraf ./dist ./dist-esm",
|
|
11
11
|
"dev": "npm run clean && tsc -p tsconfig.json --watch",
|
|
12
12
|
"build": "npm run clean && npm run build:cjs && npm run build:es",
|
|
13
13
|
"build:cjs": "tsc -p tsconfig.json",
|
|
@@ -105,7 +105,7 @@
|
|
|
105
105
|
"puppet-run": "^0.11.4",
|
|
106
106
|
"puppet-run-plugin-mocha": "^0.1.1",
|
|
107
107
|
"raw-loader": "^4.0.2",
|
|
108
|
-
"rimraf": "^
|
|
108
|
+
"rimraf": "^5.0.10",
|
|
109
109
|
"rollup": "^4.19.2",
|
|
110
110
|
"threads-plugin": "^1.4.0",
|
|
111
111
|
"tiny-worker": "^2.3.0",
|
|
@@ -37,10 +37,10 @@ function selectWorkerImplementation(): ImplementationExport {
|
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
if (
|
|
40
|
-
typeof url === 'string'
|
|
41
|
-
isAbsoluteURL(url)
|
|
40
|
+
typeof url === 'string'
|
|
41
|
+
&& isAbsoluteURL(url) // Create source code blob loading JS file via `importScripts()`
|
|
42
42
|
// to circumvent worker CORS restrictions
|
|
43
|
-
(options?.CORSWorkaround ?? true)
|
|
43
|
+
&& (options?.CORSWorkaround ?? true)
|
|
44
44
|
) {
|
|
45
45
|
url = createSourceBlobURL(`importScripts(${JSON.stringify(url)});`)
|
|
46
46
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
1
2
|
/* eslint-disable import/no-internal-modules */
|
|
2
3
|
/* eslint-disable unicorn/no-process-exit */
|
|
3
|
-
/* eslint-disable sonarjs/no-identical-functions */
|
|
4
4
|
/* eslint-disable unicorn/prefer-logical-operator-over-ternary */
|
|
5
5
|
/* eslint-disable unicorn/prefer-regexp-test */
|
|
6
|
-
|
|
6
|
+
|
|
7
7
|
/* eslint-disable unicorn/prefer-add-event-listener */
|
|
8
8
|
/* eslint-disable unicorn/prefer-event-target */
|
|
9
9
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
@@ -86,15 +86,15 @@ function resolveScriptPath(scriptPath: string, baseURL?: string | undefined) {
|
|
|
86
86
|
return path.isAbsolute(filePath) ? filePath : path.join(baseURL || eval('__dirname'), filePath)
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
-
return typeof __non_webpack_require__ === 'function'
|
|
90
|
-
|
|
89
|
+
return typeof __non_webpack_require__ === 'function'
|
|
90
|
+
? __non_webpack_require__.resolve(makeRelative(scriptPath))
|
|
91
91
|
: eval('require').resolve(makeRelative(rebaseScriptPath(scriptPath, /[/\\]worker_threads[/\\]/)))
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
function initWorkerThreadsWorker(): ImplementationExport {
|
|
95
95
|
// Webpack hack
|
|
96
|
-
const NativeWorker
|
|
97
|
-
typeof __non_webpack_require__ === 'function' ? __non_webpack_require__('worker_threads').Worker : eval('require')('worker_threads').Worker
|
|
96
|
+
const NativeWorker
|
|
97
|
+
= typeof __non_webpack_require__ === 'function' ? __non_webpack_require__('worker_threads').Worker : eval('require')('worker_threads').Worker
|
|
98
98
|
|
|
99
99
|
let allWorkers: Array<typeof NativeWorker> = []
|
|
100
100
|
|
|
@@ -137,7 +137,7 @@ function initWorkerThreadsWorker(): ImplementationExport {
|
|
|
137
137
|
|
|
138
138
|
const terminateWorkersAndMaster = () => {
|
|
139
139
|
// we should terminate all workers and then gracefully shutdown self process
|
|
140
|
-
Promise.all(allWorkers.map(
|
|
140
|
+
Promise.all(allWorkers.map(worker => worker.terminate())).then(
|
|
141
141
|
() => process.exit(0),
|
|
142
142
|
() => process.exit(1),
|
|
143
143
|
)
|
|
@@ -175,10 +175,12 @@ function initTinyWorker(): ImplementationExport {
|
|
|
175
175
|
constructor(scriptPath: string, options?: ThreadsWorkerOptions & { fromSource?: boolean }) {
|
|
176
176
|
// Need to apply a work-around for Windows or it will choke upon the absolute path
|
|
177
177
|
// (`Error [ERR_INVALID_PROTOCOL]: Protocol 'c:' not supported`)
|
|
178
|
-
const resolvedScriptPath
|
|
179
|
-
options && options.fromSource
|
|
180
|
-
|
|
181
|
-
|
|
178
|
+
const resolvedScriptPath
|
|
179
|
+
= options && options.fromSource
|
|
180
|
+
? null
|
|
181
|
+
: process.platform === 'win32'
|
|
182
|
+
? `file:///${resolveScriptPath(scriptPath).replaceAll('\\', '/')}`
|
|
183
|
+
: resolveScriptPath(scriptPath)
|
|
182
184
|
|
|
183
185
|
if (!resolvedScriptPath) {
|
|
184
186
|
// `options.fromSource` is true
|
|
@@ -209,14 +211,14 @@ function initTinyWorker(): ImplementationExport {
|
|
|
209
211
|
}
|
|
210
212
|
|
|
211
213
|
terminate() {
|
|
212
|
-
allWorkers = allWorkers.filter(
|
|
214
|
+
allWorkers = allWorkers.filter(worker => worker !== this)
|
|
213
215
|
return super.terminate()
|
|
214
216
|
}
|
|
215
217
|
}
|
|
216
218
|
|
|
217
219
|
const terminateWorkersAndMaster = () => {
|
|
218
220
|
// we should terminate all workers and then gracefully shutdown self process
|
|
219
|
-
Promise.all(allWorkers.map(
|
|
221
|
+
Promise.all(allWorkers.map(worker => worker.terminate())).then(
|
|
220
222
|
() => process.exit(0),
|
|
221
223
|
() => process.exit(1),
|
|
222
224
|
)
|
|
@@ -271,10 +273,10 @@ export function isWorkerRuntime() {
|
|
|
271
273
|
return self !== undefined && self['postMessage'] ? true : false
|
|
272
274
|
} else {
|
|
273
275
|
// Webpack hack
|
|
274
|
-
const isMainThread
|
|
275
|
-
typeof __non_webpack_require__ === 'function'
|
|
276
|
-
__non_webpack_require__('worker_threads').isMainThread
|
|
277
|
-
|
|
276
|
+
const isMainThread
|
|
277
|
+
= typeof __non_webpack_require__ === 'function'
|
|
278
|
+
? __non_webpack_require__('worker_threads').isMainThread
|
|
279
|
+
: eval('require')('worker_threads').isMainThread
|
|
278
280
|
return !isMainThread
|
|
279
281
|
}
|
|
280
282
|
}
|
package/src/master/pool-types.ts
CHANGED
|
@@ -19,41 +19,41 @@ export type TaskRunFunction<ThreadType extends Thread, Return> = (worker: Thread
|
|
|
19
19
|
/** Pool event. Subscribe to those events using `pool.events()`. Useful for debugging. */
|
|
20
20
|
export type PoolEvent<ThreadType extends Thread> =
|
|
21
21
|
| {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
type: PoolEventType.initialized
|
|
23
|
+
size: number
|
|
24
|
+
}
|
|
25
25
|
| {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
type: PoolEventType.taskQueued
|
|
27
|
+
taskID: number
|
|
28
|
+
}
|
|
29
29
|
| {
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
type: PoolEventType.taskQueueDrained
|
|
31
|
+
}
|
|
32
32
|
| {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
type: PoolEventType.taskStart
|
|
34
|
+
taskID: number
|
|
35
|
+
workerID: number
|
|
36
|
+
}
|
|
37
37
|
| {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
type: PoolEventType.taskCompleted
|
|
39
|
+
returnValue: any
|
|
40
|
+
taskID: number
|
|
41
|
+
workerID: number
|
|
42
|
+
}
|
|
43
43
|
| {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
44
|
+
type: PoolEventType.taskFailed
|
|
45
|
+
error: Error
|
|
46
|
+
taskID: number
|
|
47
|
+
workerID: number
|
|
48
|
+
}
|
|
49
49
|
| {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
type: PoolEventType.taskCanceled
|
|
51
|
+
taskID: number
|
|
52
|
+
}
|
|
53
53
|
| {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
54
|
+
type: PoolEventType.terminated
|
|
55
|
+
remainingQueue: Array<QueuedTask<ThreadType, any>>
|
|
56
|
+
}
|
|
57
57
|
|
|
58
58
|
export interface WorkerDescriptor<ThreadType extends Thread> {
|
|
59
59
|
init: Promise<ThreadType>
|
package/src/master/pool.ts
CHANGED
|
@@ -31,7 +31,7 @@ function createArray(size: number): number[] {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
function delay(ms: number) {
|
|
34
|
-
return new Promise(
|
|
34
|
+
return new Promise(resolve => setTimeout(resolve, ms))
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
function flatMap<In, Out>(array: In[], mapper: (element: In) => Out[]): Out[] {
|
|
@@ -133,7 +133,7 @@ class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
|
|
|
133
133
|
|
|
134
134
|
this.eventObservable = multicast(Observable.from(this.eventSubject))
|
|
135
135
|
|
|
136
|
-
Promise.all(this.workers.map(
|
|
136
|
+
Promise.all(this.workers.map(worker => worker.init)).then(
|
|
137
137
|
() =>
|
|
138
138
|
this.eventSubject.next({
|
|
139
139
|
size: this.workers.length,
|
|
@@ -149,7 +149,7 @@ class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
|
|
|
149
149
|
|
|
150
150
|
private findIdlingWorker(): WorkerDescriptor<ThreadType> | undefined {
|
|
151
151
|
const { concurrency = 1 } = this.options
|
|
152
|
-
return this.workers.find(
|
|
152
|
+
return this.workers.find(worker => worker.runningTasks.length < concurrency)
|
|
153
153
|
}
|
|
154
154
|
|
|
155
155
|
private async runPoolTask(worker: WorkerDescriptor<ThreadType>, task: QueuedTask<ThreadType, any>) {
|
|
@@ -186,7 +186,7 @@ class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
|
|
|
186
186
|
private async run(worker: WorkerDescriptor<ThreadType>, task: QueuedTask<ThreadType, any>) {
|
|
187
187
|
const runPromise = (async () => {
|
|
188
188
|
const removeTaskFromWorkersRunningTasks = () => {
|
|
189
|
-
worker.runningTasks = worker.runningTasks.filter(
|
|
189
|
+
worker.runningTasks = worker.runningTasks.filter(someRunPromise => someRunPromise !== runPromise)
|
|
190
190
|
}
|
|
191
191
|
|
|
192
192
|
// Defer task execution by one tick to give handlers time to subscribe
|
|
@@ -240,7 +240,7 @@ class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
|
|
|
240
240
|
}
|
|
241
241
|
|
|
242
242
|
async settled(allowResolvingImmediately: boolean = false): Promise<Error[]> {
|
|
243
|
-
const getCurrentlyRunningTasks = () => flatMap(this.workers,
|
|
243
|
+
const getCurrentlyRunningTasks = () => flatMap(this.workers, worker => worker.runningTasks)
|
|
244
244
|
|
|
245
245
|
const taskFailures: Error[] = []
|
|
246
246
|
|
|
@@ -327,7 +327,7 @@ class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
|
|
|
327
327
|
const task: QueuedTask<ThreadType, any> = {
|
|
328
328
|
cancel: () => {
|
|
329
329
|
if (!this.taskQueue.includes(task)) return
|
|
330
|
-
this.taskQueue = this.taskQueue.filter(
|
|
330
|
+
this.taskQueue = this.taskQueue.filter(someTask => someTask !== task)
|
|
331
331
|
this.eventSubject.next({
|
|
332
332
|
taskID: task.id,
|
|
333
333
|
type: PoolEventType.taskCanceled,
|
|
@@ -340,9 +340,9 @@ class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
|
|
|
340
340
|
|
|
341
341
|
if (this.taskQueue.length >= maxQueuedJobs) {
|
|
342
342
|
throw new Error(
|
|
343
|
-
'Maximum number of pool tasks queued. Refusing to queue another one.\n'
|
|
344
|
-
|
|
345
|
-
|
|
343
|
+
'Maximum number of pool tasks queued. Refusing to queue another one.\n'
|
|
344
|
+
+ 'This usually happens for one of two reasons: We are either at peak '
|
|
345
|
+
+ "workload right now or some tasks just won't finish, thus blocking the pool.",
|
|
346
346
|
)
|
|
347
347
|
}
|
|
348
348
|
|
|
@@ -368,7 +368,7 @@ class WorkerPool<ThreadType extends Thread> implements Pool<ThreadType> {
|
|
|
368
368
|
type: PoolEventType.terminated,
|
|
369
369
|
})
|
|
370
370
|
this.eventSubject.complete()
|
|
371
|
-
await Promise.all(this.workers.map(async
|
|
371
|
+
await Promise.all(this.workers.map(async worker => Thread.terminate(await worker.init)))
|
|
372
372
|
}
|
|
373
373
|
}
|
|
374
374
|
|
package/src/master/spawn.ts
CHANGED
|
@@ -28,9 +28,9 @@ type ArbitraryThreadType = FunctionThread<any, any> & ModuleThread<any>
|
|
|
28
28
|
|
|
29
29
|
export type ExposedToThreadType<Exposed extends WorkerFunction | WorkerModule<any>> =
|
|
30
30
|
Exposed extends ArbitraryWorkerInterface ? ArbitraryThreadType
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
: Exposed extends WorkerFunction ? FunctionThread<Parameters<Exposed>, StripAsync<ReturnType<Exposed>>>
|
|
32
|
+
: Exposed extends WorkerModule<any> ? ModuleThread<Exposed>
|
|
33
|
+
: never
|
|
34
34
|
|
|
35
35
|
const debugMessages = DebugLogger('threads:master:messages')
|
|
36
36
|
const debugSpawn = DebugLogger('threads:master:spawn')
|
|
@@ -39,10 +39,10 @@ const debugThreadUtils = DebugLogger('threads:master:thread-utils')
|
|
|
39
39
|
const isInitMessage = (data: any): data is WorkerInitMessage => data && data.type === ('init' as const)
|
|
40
40
|
const isUncaughtErrorMessage = (data: any): data is WorkerUncaughtErrorMessage => data && data.type === ('uncaughtError' as const)
|
|
41
41
|
|
|
42
|
-
const initMessageTimeout
|
|
43
|
-
typeof process !== 'undefined' && process.env !== undefined && process.env.THREADS_WORKER_INIT_TIMEOUT
|
|
44
|
-
Number.parseInt(process.env.THREADS_WORKER_INIT_TIMEOUT, 10)
|
|
45
|
-
|
|
42
|
+
const initMessageTimeout
|
|
43
|
+
= typeof process !== 'undefined' && process.env !== undefined && process.env.THREADS_WORKER_INIT_TIMEOUT
|
|
44
|
+
? Number.parseInt(process.env.THREADS_WORKER_INIT_TIMEOUT, 10)
|
|
45
|
+
: 10_000
|
|
46
46
|
|
|
47
47
|
async function withTimeout<T>(promise: Promise<T>, timeoutInMs: number, errorMessage: string): Promise<T> {
|
|
48
48
|
let timeoutHandle: any
|
|
@@ -122,8 +122,8 @@ function setPrivateThreadProps<T>(
|
|
|
122
122
|
terminate: () => Promise<void>,
|
|
123
123
|
): T & PrivateThreadProps {
|
|
124
124
|
const workerErrors = workerEvents
|
|
125
|
-
.filter(
|
|
126
|
-
.map(
|
|
125
|
+
.filter(event => event.type === WorkerEventType.internalError)
|
|
126
|
+
.map(errorEvent => (errorEvent as WorkerInternalErrorEvent).error)
|
|
127
127
|
|
|
128
128
|
return Object.assign(raw as any, {
|
|
129
129
|
[$errors]: workerErrors,
|
|
@@ -166,17 +166,17 @@ export class ObservablePromise<T> extends Observable<T> implements Promise<T> {
|
|
|
166
166
|
}
|
|
167
167
|
|
|
168
168
|
static from<T>(thing: Observable<T> | ObservableLike<T> | ArrayLike<T> | Thenable<T>): ObservablePromise<T> {
|
|
169
|
-
return isThenable(thing)
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
169
|
+
return isThenable(thing)
|
|
170
|
+
? new ObservablePromise((observer) => {
|
|
171
|
+
const onFulfilled = (value: T) => {
|
|
172
|
+
observer.next(value)
|
|
173
|
+
observer.complete()
|
|
174
|
+
}
|
|
175
|
+
const onRejected = (error: any) => {
|
|
176
|
+
observer.error(error)
|
|
177
|
+
}
|
|
178
|
+
thing.then(onFulfilled, onRejected)
|
|
179
|
+
})
|
|
180
180
|
: (super.from(thing) as ObservablePromise<T>)
|
|
181
181
|
}
|
|
182
182
|
}
|
package/src/observable.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/* eslint-disable sonarjs/prefer-immediate-return */
|
|
2
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
3
2
|
import { Observable, ObservableLike, SubscriptionObserver } from 'observable-fns'
|
|
4
3
|
|
|
@@ -20,7 +19,7 @@ export class Subject<T> extends Observable<T> implements ObservableLike<T> {
|
|
|
20
19
|
super((observer) => {
|
|
21
20
|
this[$observers] = [...(this[$observers] || []), observer]
|
|
22
21
|
const unsubscribe = () => {
|
|
23
|
-
this[$observers] = this[$observers].filter(
|
|
22
|
+
this[$observers] = this[$observers].filter(someObserver => someObserver !== observer)
|
|
24
23
|
}
|
|
25
24
|
return unsubscribe
|
|
26
25
|
})
|
package/src/ponyfills.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
2
|
export type SettlementResult<T> =
|
|
3
3
|
| {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
status: 'fulfilled'
|
|
5
|
+
value: T
|
|
6
|
+
}
|
|
7
7
|
| {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
reason: any
|
|
9
|
+
status: 'rejected'
|
|
10
|
+
}
|
|
11
11
|
|
|
12
12
|
// Based on <https://github.com/es-shims/Promise.allSettled/blob/master/implementation.js>
|
|
13
13
|
export function allSettled<T>(values: T[]): Promise<Array<SettlementResult<T>>> {
|
package/src/transferable.ts
CHANGED
package/src/types/master.ts
CHANGED
|
@@ -20,8 +20,8 @@ interface ObservableLike<T> {
|
|
|
20
20
|
|
|
21
21
|
export type StripAsync<Type> =
|
|
22
22
|
Type extends Promise<infer PromiseBaseType> ? PromiseBaseType
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
: Type extends ObservableLike<infer ObservableBaseType> ? ObservableBaseType
|
|
24
|
+
: Type
|
|
25
25
|
|
|
26
26
|
export type StripTransfer<Type> = Type extends TransferDescriptor<infer BaseType> ? BaseType : Type
|
|
27
27
|
|
|
@@ -32,7 +32,7 @@ export type ProxyableArgs<Args extends any[]> =
|
|
|
32
32
|
|
|
33
33
|
export type ProxyableFunction<Args extends any[], ReturnType> =
|
|
34
34
|
Args extends [] ? () => ObservablePromise<StripTransfer<StripAsync<ReturnType>>>
|
|
35
|
-
|
|
35
|
+
: (...args: ProxyableArgs<Args>) => ObservablePromise<StripTransfer<StripAsync<ReturnType>>>
|
|
36
36
|
|
|
37
37
|
export type ModuleProxy<Methods extends ModuleMethods> = {
|
|
38
38
|
[method in keyof Methods]: ProxyableFunction<Parameters<Methods[method]>, ReturnType<Methods[method]>>
|
|
@@ -55,6 +55,7 @@ interface AnyFunctionThread extends PrivateThreadProps {
|
|
|
55
55
|
}
|
|
56
56
|
|
|
57
57
|
// tslint:disable-next-line no-empty-interface
|
|
58
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-object-type
|
|
58
59
|
interface AnyModuleThread extends PrivateThreadProps {
|
|
59
60
|
// Not specifying an index signature here as that would make `ModuleThread` incompatible
|
|
60
61
|
}
|
package/src/worker/index.ts
CHANGED
|
@@ -131,7 +131,7 @@ async function runFunction(jobUID: number, fn: WorkerFunction, args: any[]) {
|
|
|
131
131
|
|
|
132
132
|
if (isObservable(syncResult)) {
|
|
133
133
|
const subscription = syncResult.subscribe(
|
|
134
|
-
|
|
134
|
+
value => postJobResultMessage(jobUID, false, serialize(value)),
|
|
135
135
|
(error) => {
|
|
136
136
|
postJobErrorMessage(jobUID, serialize(error) as any)
|
|
137
137
|
activeSubscriptions.delete(jobUID)
|
|
@@ -182,7 +182,7 @@ export function expose(exposed: WorkerFunction | WorkerModule<any>) {
|
|
|
182
182
|
}
|
|
183
183
|
})
|
|
184
184
|
|
|
185
|
-
const methodNames = Object.keys(exposed).filter(
|
|
185
|
+
const methodNames = Object.keys(exposed).filter(key => typeof exposed[key] === 'function')
|
|
186
186
|
postModuleInitMessage(methodNames)
|
|
187
187
|
} else {
|
|
188
188
|
throw new Error(`Invalid argument passed to expose(). Expected a function or an object, got: ${exposed}`)
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
/* eslint-disable import/no-internal-modules */
|
|
2
|
-
/* eslint-disable sonarjs/no-duplicate-string */
|
|
3
2
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
4
3
|
import test from 'ava'
|
|
5
4
|
import { Observable } from 'observable-fns'
|
|
6
5
|
|
|
7
6
|
import { ObservablePromise } from '../src/observable-promise'
|
|
8
7
|
|
|
9
|
-
const delay = (ms: number) => new Promise(
|
|
8
|
+
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
|
|
10
9
|
|
|
11
10
|
test('can create an observable promise', async (t) => {
|
|
12
11
|
t.plan(1)
|
|
@@ -43,9 +42,9 @@ test('can proxy a promise fulfillment', async (t) => {
|
|
|
43
42
|
}, 1)
|
|
44
43
|
})
|
|
45
44
|
|
|
46
|
-
const promise1 = async.then(
|
|
45
|
+
const promise1 = async.then(value => t.is(value, 123), t.fail)
|
|
47
46
|
await delay(10)
|
|
48
|
-
const promise2 = async.then(
|
|
47
|
+
const promise2 = async.then(value => t.is(value, 123), t.fail)
|
|
49
48
|
|
|
50
49
|
await Promise.all([promise1, promise2])
|
|
51
50
|
})
|
|
@@ -104,7 +103,7 @@ test('can subscribe to values and completion', async (t) => {
|
|
|
104
103
|
|
|
105
104
|
for (let index = 0; index < 2; index++) {
|
|
106
105
|
async.subscribe(
|
|
107
|
-
|
|
106
|
+
value => capturedValues.push(value),
|
|
108
107
|
() => {},
|
|
109
108
|
() => capturedCompletions++,
|
|
110
109
|
)
|
|
@@ -131,8 +130,8 @@ test('can subscribe to errors', async (t) => {
|
|
|
131
130
|
|
|
132
131
|
for (let index = 0; index < 2; index++) {
|
|
133
132
|
async.subscribe(
|
|
134
|
-
|
|
135
|
-
|
|
133
|
+
value => capturedValues.push(value),
|
|
134
|
+
error => capturedErrorMessages.push(error.message),
|
|
136
135
|
() => capturedCompletions++,
|
|
137
136
|
)
|
|
138
137
|
}
|
|
@@ -161,8 +160,8 @@ test('from(Observable) works', async (t) => {
|
|
|
161
160
|
|
|
162
161
|
for (let index = 0; index < 2; index++) {
|
|
163
162
|
async.subscribe(
|
|
164
|
-
|
|
165
|
-
|
|
163
|
+
value => capturedValues.push(value),
|
|
164
|
+
error => capturedErrorMessages.push(error.message),
|
|
166
165
|
() => capturedCompletions++,
|
|
167
166
|
)
|
|
168
167
|
}
|
package/test/observable.test.ts
CHANGED
|
@@ -17,17 +17,17 @@ test('Observable subject emits values and completion event', async (t) => {
|
|
|
17
17
|
const observable = Observable.from(subject)
|
|
18
18
|
|
|
19
19
|
const subscription1 = subject.subscribe(
|
|
20
|
-
|
|
20
|
+
value => values1.push(value),
|
|
21
21
|
undefined,
|
|
22
22
|
() => (completed1 = true),
|
|
23
23
|
)
|
|
24
24
|
subject.subscribe(
|
|
25
|
-
|
|
25
|
+
value => values2.push(value),
|
|
26
26
|
undefined,
|
|
27
27
|
() => (completed2 = true),
|
|
28
28
|
)
|
|
29
29
|
observable.subscribe(
|
|
30
|
-
|
|
30
|
+
value => values3.push(value),
|
|
31
31
|
undefined,
|
|
32
32
|
() => (completed3 = true),
|
|
33
33
|
)
|
|
@@ -59,17 +59,17 @@ test('Observable subject propagates errors', async (t) => {
|
|
|
59
59
|
|
|
60
60
|
const subscription1 = subject.subscribe(
|
|
61
61
|
() => {},
|
|
62
|
-
|
|
62
|
+
error => (error1 = error),
|
|
63
63
|
() => (completed1 = true),
|
|
64
64
|
)
|
|
65
65
|
subject.subscribe(
|
|
66
66
|
() => {},
|
|
67
|
-
|
|
67
|
+
error => (error2 = error),
|
|
68
68
|
() => (completed2 = true),
|
|
69
69
|
)
|
|
70
70
|
observable.subscribe(
|
|
71
71
|
() => {},
|
|
72
|
-
|
|
72
|
+
error => (error3 = error),
|
|
73
73
|
() => (completed3 = true),
|
|
74
74
|
)
|
|
75
75
|
|
package/test/pool.test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/* eslint-disable import/no-internal-modules */
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
/* eslint-disable require-await */
|
|
4
4
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
5
5
|
|
|
@@ -22,14 +22,14 @@ test.serial('thread pool basics work and events are emitted', async (t) => {
|
|
|
22
22
|
return spawn<() => string>(new Worker(workerPath))
|
|
23
23
|
}
|
|
24
24
|
const pool = Pool(spawnHelloWorld, 3)
|
|
25
|
-
pool.events().subscribe(
|
|
25
|
+
pool.events().subscribe(event => events.push(event))
|
|
26
26
|
|
|
27
27
|
// Just to make sure all worker threads are initialized before starting to queue
|
|
28
28
|
// This is only necessary for testing to make sure that this is the first event recorded
|
|
29
29
|
await new Promise((resolve, reject) => {
|
|
30
30
|
pool
|
|
31
31
|
.events()
|
|
32
|
-
.filter(
|
|
32
|
+
.filter(event => event.type === PoolEventType.initialized)
|
|
33
33
|
.subscribe(resolve, reject)
|
|
34
34
|
})
|
|
35
35
|
|
|
@@ -112,7 +112,6 @@ test.serial('pool.completed(true) works', async (t) => {
|
|
|
112
112
|
})
|
|
113
113
|
|
|
114
114
|
test.serial('pool.settled() does not reject on task failure', async (t) => {
|
|
115
|
-
// eslint-disable-next-line sonarjs/no-unused-collection
|
|
116
115
|
const returned: any[] = []
|
|
117
116
|
|
|
118
117
|
const spawnHelloWorld = () => spawn(new Worker(workerPath))
|
|
@@ -130,7 +129,7 @@ test.serial('pool.settled() does not reject on task failure', async (t) => {
|
|
|
130
129
|
|
|
131
130
|
const errors = await pool.settled()
|
|
132
131
|
t.is(errors.length, 2)
|
|
133
|
-
t.deepEqual(errors.map(
|
|
132
|
+
t.deepEqual(errors.map(error => error.message).sort(), ['Test error one', 'Test error two'])
|
|
134
133
|
})
|
|
135
134
|
|
|
136
135
|
test.serial('pool.settled(true) works', async (t) => {
|
|
@@ -146,7 +145,7 @@ test.serial('task.cancel() works', async (t) => {
|
|
|
146
145
|
const spawnHelloWorld = () => spawn(new Worker(workerPath))
|
|
147
146
|
const pool = Pool(spawnHelloWorld, 1)
|
|
148
147
|
|
|
149
|
-
pool.events().subscribe(
|
|
148
|
+
pool.events().subscribe(event => events.push(event))
|
|
150
149
|
|
|
151
150
|
let executionCount = 0
|
|
152
151
|
const tasks: QueuedTask<any, any>[] = []
|
|
@@ -165,7 +164,7 @@ test.serial('task.cancel() works', async (t) => {
|
|
|
165
164
|
await pool.completed()
|
|
166
165
|
t.is(executionCount, 2)
|
|
167
166
|
|
|
168
|
-
const cancellationEvents = events.filter(
|
|
167
|
+
const cancellationEvents = events.filter(event => event.type === 'taskCanceled')
|
|
169
168
|
t.deepEqual(cancellationEvents, [
|
|
170
169
|
{
|
|
171
170
|
taskID: 3,
|
package/test/spawn.test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/* eslint-disable import/no-internal-modules */
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
4
4
|
import test from 'ava'
|
|
5
5
|
import { Observable } from 'observable-fns'
|
|
@@ -28,7 +28,7 @@ test('can subscribe to an observable returned by a thread call', async (t) => {
|
|
|
28
28
|
const encounteredValues: any[] = []
|
|
29
29
|
|
|
30
30
|
const observable = countToFive()
|
|
31
|
-
observable.subscribe(
|
|
31
|
+
observable.subscribe(value => encounteredValues.push(value))
|
|
32
32
|
await observable
|
|
33
33
|
|
|
34
34
|
t.deepEqual(encounteredValues, [1, 2, 3, 4, 5])
|
|
@@ -52,6 +52,7 @@ test('thread job errors are handled', async (t) => {
|
|
|
52
52
|
})
|
|
53
53
|
|
|
54
54
|
test('thread transfer errors are handled', async (t) => {
|
|
55
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
55
56
|
const builtin = require('node:module').builtinModules
|
|
56
57
|
if (builtin.includes('worker_threads')) {
|
|
57
58
|
// test is actual for native worker_threads only
|
package/test/streaming.test.ts
CHANGED
|
@@ -6,7 +6,7 @@ test('can use worker returning an observable subject', async (t) => {
|
|
|
6
6
|
const captured: Array<{ max: number; min: number }> = []
|
|
7
7
|
|
|
8
8
|
const minmax = await spawn(new Worker('./workers/minmax'))
|
|
9
|
-
minmax.values().subscribe(
|
|
9
|
+
minmax.values().subscribe(values => captured.push(values))
|
|
10
10
|
|
|
11
11
|
await minmax.push(2)
|
|
12
12
|
await minmax.push(3)
|
|
@@ -25,7 +25,7 @@ function replaceArrayBufferWithPlaceholder<In extends any>(
|
|
|
25
25
|
if ((obj as any) === arrayBuffer) {
|
|
26
26
|
return arrayBufferPlaceholder as any
|
|
27
27
|
} else if (Array.isArray(obj)) {
|
|
28
|
-
return (obj as any[]).map(
|
|
28
|
+
return (obj as any[]).map(element => replaceArrayBufferWithPlaceholder(element, arrayBuffer)) as any
|
|
29
29
|
} else if (obj && typeof obj === 'object') {
|
|
30
30
|
const result: In = Object.create(Object.getPrototypeOf(obj))
|
|
31
31
|
|
|
@@ -44,7 +44,7 @@ test('can pass transferable objects on thread call', async (t) => {
|
|
|
44
44
|
const worker = new Worker('./workers/arraybuffer-xor')
|
|
45
45
|
const postMessageCalls: Array<any[]> = []
|
|
46
46
|
|
|
47
|
-
worker.postMessage = spyOn(worker.postMessage.bind(worker),
|
|
47
|
+
worker.postMessage = spyOn(worker.postMessage.bind(worker), postMessage => (...args) => {
|
|
48
48
|
postMessageCalls.push(replaceArrayBufferWithPlaceholder(args, testData))
|
|
49
49
|
return postMessage(...args)
|
|
50
50
|
})
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/* eslint-disable sonarjs/no-use-of-empty-return-value */
|
|
2
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
3
2
|
|
|
4
3
|
import { isWorkerRuntime, Pool, spawn, Worker } from '../../src/index'
|
|
@@ -9,10 +8,10 @@ type HelloWorker = (text: string) => string
|
|
|
9
8
|
async function test() {
|
|
10
9
|
const pool = Pool(() => spawn<HelloWorker>(new Worker('./pool-worker')))
|
|
11
10
|
const results = await Promise.all([
|
|
12
|
-
pool.queue(
|
|
13
|
-
pool.queue(
|
|
14
|
-
pool.queue(
|
|
15
|
-
pool.queue(
|
|
11
|
+
pool.queue(hello => hello('World')),
|
|
12
|
+
pool.queue(hello => hello('World')),
|
|
13
|
+
pool.queue(hello => hello('World')),
|
|
14
|
+
pool.queue(hello => hello('World')),
|
|
16
15
|
])
|
|
17
16
|
await pool.terminate()
|
|
18
17
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-unused-expressions */
|
|
2
|
+
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
1
3
|
/* eslint-disable require-await */
|
|
2
4
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
3
|
-
|
|
5
|
+
|
|
4
6
|
import path from 'node:path'
|
|
5
7
|
|
|
6
8
|
import test from 'ava'
|
|
@@ -10,16 +12,20 @@ const browserConfig = require('./webpack.web.config')
|
|
|
10
12
|
const serverConfig = require('./webpack.node.config')
|
|
11
13
|
|
|
12
14
|
const stringifyWebpackError = (error: any) =>
|
|
13
|
-
error
|
|
14
|
-
typeof error.stack === 'string'
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
error
|
|
16
|
+
? typeof error.stack === 'string'
|
|
17
|
+
? error.stack
|
|
18
|
+
: typeof error.message === 'string'
|
|
19
|
+
? error.message
|
|
20
|
+
: error
|
|
21
|
+
: ''
|
|
18
22
|
|
|
19
23
|
async function runWebpack(config: any) {
|
|
20
24
|
return new Promise<Webpack.Stats>((resolve, reject) => {
|
|
21
25
|
Webpack(config).run((error, stats) => {
|
|
22
|
-
|
|
26
|
+
if (stats) {
|
|
27
|
+
error ? reject(error) : resolve(stats)
|
|
28
|
+
}
|
|
23
29
|
})
|
|
24
30
|
})
|
|
25
31
|
}
|