@parcel/workers 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/index.js +5 -0
- package/package.json +33 -0
- package/src/.babelrc +8 -0
- package/src/.eslintrc.json +3 -0
- package/src/Worker.js +176 -0
- package/src/WorkerFarm.js +300 -0
- package/src/child.js +144 -0
- package/src/cpuCount.js +11 -0
- package/test/.babelrc +9 -0
- package/test/.eslintrc.json +6 -0
- package/test/integration/workerfarm/echo.js +10 -0
- package/test/integration/workerfarm/init.js +12 -0
- package/test/integration/workerfarm/ipc-pid.js +25 -0
- package/test/integration/workerfarm/ipc.js +17 -0
- package/test/integration/workerfarm/master-process-id.js +3 -0
- package/test/integration/workerfarm/master-sum.js +3 -0
- package/test/integration/workerfarm/ping.js +10 -0
- package/test/mocha.opts +3 -0
- package/test/workerfarm.js +178 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2017-present Devon Govett
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/index.js
ADDED
package/package.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@parcel/workers",
|
|
3
|
+
"version": "1.11.0",
|
|
4
|
+
"description": "Blazing fast, zero configuration web application bundler",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "https://github.com/parcel-bundler/parcel.git"
|
|
10
|
+
},
|
|
11
|
+
"engines": {
|
|
12
|
+
"node": ">= 6.0.0"
|
|
13
|
+
},
|
|
14
|
+
"publishConfig": {
|
|
15
|
+
"access": "public"
|
|
16
|
+
},
|
|
17
|
+
"scripts": {
|
|
18
|
+
"test": "cross-env NODE_ENV=test mocha",
|
|
19
|
+
"test-ci": "yarn build && yarn test",
|
|
20
|
+
"format": "prettier --write \"./{src,bin,test}/**/*.{js,json,md}\"",
|
|
21
|
+
"lint": "eslint . && prettier \"./{src,bin,test}/**/*.{js,json,md}\" --list-different",
|
|
22
|
+
"build": "babel src -d lib",
|
|
23
|
+
"prepublish": "yarn build"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"mocha": "^5.2.0"
|
|
27
|
+
},
|
|
28
|
+
"dependencies": {
|
|
29
|
+
"@parcel/utils": "^1.11.0",
|
|
30
|
+
"physical-cpu-count": "^2.0.0"
|
|
31
|
+
},
|
|
32
|
+
"gitHead": "34eb91e8e6991073e594bff731c333d09b0403b5"
|
|
33
|
+
}
|
package/src/.babelrc
ADDED
package/src/Worker.js
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
const childProcess = require('child_process');
|
|
2
|
+
const {EventEmitter} = require('events');
|
|
3
|
+
const {errorUtils} = require('@parcel/utils');
|
|
4
|
+
|
|
5
|
+
const childModule = require.resolve('./child');
|
|
6
|
+
|
|
7
|
+
let WORKER_ID = 0;
|
|
8
|
+
class Worker extends EventEmitter {
|
|
9
|
+
constructor(options) {
|
|
10
|
+
super();
|
|
11
|
+
|
|
12
|
+
this.options = options;
|
|
13
|
+
this.id = WORKER_ID++;
|
|
14
|
+
|
|
15
|
+
this.sendQueue = [];
|
|
16
|
+
this.processQueue = true;
|
|
17
|
+
|
|
18
|
+
this.calls = new Map();
|
|
19
|
+
this.exitCode = null;
|
|
20
|
+
this.callId = 0;
|
|
21
|
+
|
|
22
|
+
this.ready = false;
|
|
23
|
+
this.stopped = false;
|
|
24
|
+
this.isStopping = false;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async fork(forkModule, bundlerOptions) {
|
|
28
|
+
let filteredArgs = process.execArgv.filter(
|
|
29
|
+
v => !/^--(debug|inspect)/.test(v)
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
let options = {
|
|
33
|
+
execArgv: filteredArgs,
|
|
34
|
+
env: process.env,
|
|
35
|
+
cwd: process.cwd()
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
this.child = childProcess.fork(childModule, process.argv, options);
|
|
39
|
+
|
|
40
|
+
this.child.on('message', data => this.receive(data));
|
|
41
|
+
|
|
42
|
+
this.child.once('exit', code => {
|
|
43
|
+
this.exitCode = code;
|
|
44
|
+
this.emit('exit', code);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
this.child.on('error', err => {
|
|
48
|
+
this.emit('error', err);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
await new Promise((resolve, reject) => {
|
|
52
|
+
this.call({
|
|
53
|
+
method: 'childInit',
|
|
54
|
+
args: [forkModule],
|
|
55
|
+
retries: 0,
|
|
56
|
+
resolve,
|
|
57
|
+
reject
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
await this.init(bundlerOptions);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async init(bundlerOptions) {
|
|
65
|
+
this.ready = false;
|
|
66
|
+
|
|
67
|
+
return new Promise((resolve, reject) => {
|
|
68
|
+
this.call({
|
|
69
|
+
method: 'init',
|
|
70
|
+
args: [bundlerOptions],
|
|
71
|
+
retries: 0,
|
|
72
|
+
resolve: (...args) => {
|
|
73
|
+
this.ready = true;
|
|
74
|
+
this.emit('ready');
|
|
75
|
+
resolve(...args);
|
|
76
|
+
},
|
|
77
|
+
reject
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
send(data) {
|
|
83
|
+
if (!this.processQueue) {
|
|
84
|
+
return this.sendQueue.push(data);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
let result = this.child.send(data, error => {
|
|
88
|
+
if (error && error instanceof Error) {
|
|
89
|
+
// Ignore this, the workerfarm handles child errors
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
this.processQueue = true;
|
|
94
|
+
|
|
95
|
+
if (this.sendQueue.length > 0) {
|
|
96
|
+
let queueCopy = this.sendQueue.slice(0);
|
|
97
|
+
this.sendQueue = [];
|
|
98
|
+
queueCopy.forEach(entry => this.send(entry));
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
if (!result || /^win/.test(process.platform)) {
|
|
103
|
+
// Queue is handling too much messages throttle it
|
|
104
|
+
this.processQueue = false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
call(call) {
|
|
109
|
+
if (this.stopped || this.isStopping) {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
let idx = this.callId++;
|
|
114
|
+
this.calls.set(idx, call);
|
|
115
|
+
|
|
116
|
+
this.send({
|
|
117
|
+
type: 'request',
|
|
118
|
+
idx: idx,
|
|
119
|
+
child: this.id,
|
|
120
|
+
method: call.method,
|
|
121
|
+
args: call.args
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
receive(data) {
|
|
126
|
+
if (this.stopped || this.isStopping) {
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
let idx = data.idx;
|
|
131
|
+
let type = data.type;
|
|
132
|
+
let content = data.content;
|
|
133
|
+
let contentType = data.contentType;
|
|
134
|
+
|
|
135
|
+
if (type === 'request') {
|
|
136
|
+
this.emit('request', data);
|
|
137
|
+
} else if (type === 'response') {
|
|
138
|
+
let call = this.calls.get(idx);
|
|
139
|
+
if (!call) {
|
|
140
|
+
// Return for unknown calls, these might accur if a third party process uses workers
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (contentType === 'error') {
|
|
145
|
+
call.reject(errorUtils.jsonToError(content));
|
|
146
|
+
} else {
|
|
147
|
+
call.resolve(content);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
this.calls.delete(idx);
|
|
151
|
+
this.emit('response', data);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async stop() {
|
|
156
|
+
if (!this.stopped) {
|
|
157
|
+
this.stopped = true;
|
|
158
|
+
|
|
159
|
+
if (this.child) {
|
|
160
|
+
this.child.send('die');
|
|
161
|
+
|
|
162
|
+
let forceKill = setTimeout(
|
|
163
|
+
() => this.child.kill('SIGINT'),
|
|
164
|
+
this.options.forcedKillTime
|
|
165
|
+
);
|
|
166
|
+
await new Promise(resolve => {
|
|
167
|
+
this.child.once('exit', resolve);
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
clearTimeout(forceKill);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
module.exports = Worker;
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
const {EventEmitter} = require('events');
|
|
2
|
+
const {errorUtils} = require('@parcel/utils');
|
|
3
|
+
const Worker = require('./Worker');
|
|
4
|
+
const cpuCount = require('./cpuCount');
|
|
5
|
+
|
|
6
|
+
let shared = null;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* workerPath should always be defined inside farmOptions
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
class WorkerFarm extends EventEmitter {
|
|
13
|
+
constructor(options, farmOptions = {}) {
|
|
14
|
+
super();
|
|
15
|
+
this.options = {
|
|
16
|
+
maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
|
|
17
|
+
maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(),
|
|
18
|
+
forcedKillTime: 500,
|
|
19
|
+
warmWorkers: true,
|
|
20
|
+
useLocalWorker: true
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
if (farmOptions) {
|
|
24
|
+
this.options = Object.assign(this.options, farmOptions);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
this.warmWorkers = 0;
|
|
28
|
+
this.workers = new Map();
|
|
29
|
+
this.callQueue = [];
|
|
30
|
+
|
|
31
|
+
if (!this.options.workerPath) {
|
|
32
|
+
throw new Error('Please provide a worker path!');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
this.localWorker = require(this.options.workerPath);
|
|
36
|
+
this.run = this.mkhandle('run');
|
|
37
|
+
|
|
38
|
+
this.init(options);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
warmupWorker(method, args) {
|
|
42
|
+
// Workers are already stopping
|
|
43
|
+
if (this.ending) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Workers are not warmed up yet.
|
|
48
|
+
// Send the job to a remote worker in the background,
|
|
49
|
+
// but use the result from the local worker - it will be faster.
|
|
50
|
+
let promise = this.addCall(method, [...args, true]);
|
|
51
|
+
if (promise) {
|
|
52
|
+
promise
|
|
53
|
+
.then(() => {
|
|
54
|
+
this.warmWorkers++;
|
|
55
|
+
if (this.warmWorkers >= this.workers.size) {
|
|
56
|
+
this.emit('warmedup');
|
|
57
|
+
}
|
|
58
|
+
})
|
|
59
|
+
.catch(() => {});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
shouldStartRemoteWorkers() {
|
|
64
|
+
return (
|
|
65
|
+
this.options.maxConcurrentWorkers > 1 ||
|
|
66
|
+
process.env.NODE_ENV === 'test' ||
|
|
67
|
+
!this.options.useLocalWorker
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
mkhandle(method) {
|
|
72
|
+
return (...args) => {
|
|
73
|
+
// Child process workers are slow to start (~600ms).
|
|
74
|
+
// While we're waiting, just run on the main thread.
|
|
75
|
+
// This significantly speeds up startup time.
|
|
76
|
+
if (this.shouldUseRemoteWorkers()) {
|
|
77
|
+
return this.addCall(method, [...args, false]);
|
|
78
|
+
} else {
|
|
79
|
+
if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
|
|
80
|
+
this.warmupWorker(method, args);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return this.localWorker[method](...args, false);
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
onError(error, worker) {
|
|
89
|
+
// Handle ipc errors
|
|
90
|
+
if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
|
|
91
|
+
return this.stopWorker(worker);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
startChild() {
|
|
96
|
+
let worker = new Worker(this.options);
|
|
97
|
+
|
|
98
|
+
worker.fork(this.options.workerPath, this.bundlerOptions);
|
|
99
|
+
|
|
100
|
+
worker.on('request', data => this.processRequest(data, worker));
|
|
101
|
+
|
|
102
|
+
worker.on('ready', () => this.processQueue());
|
|
103
|
+
worker.on('response', () => this.processQueue());
|
|
104
|
+
|
|
105
|
+
worker.on('error', err => this.onError(err, worker));
|
|
106
|
+
worker.once('exit', () => this.stopWorker(worker));
|
|
107
|
+
|
|
108
|
+
this.workers.set(worker.id, worker);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async stopWorker(worker) {
|
|
112
|
+
if (!worker.stopped) {
|
|
113
|
+
this.workers.delete(worker.id);
|
|
114
|
+
|
|
115
|
+
worker.isStopping = true;
|
|
116
|
+
|
|
117
|
+
if (worker.calls.size) {
|
|
118
|
+
for (let call of worker.calls.values()) {
|
|
119
|
+
call.retries++;
|
|
120
|
+
this.callQueue.unshift(call);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
worker.calls = null;
|
|
125
|
+
|
|
126
|
+
await worker.stop();
|
|
127
|
+
|
|
128
|
+
// Process any requests that failed and start a new worker
|
|
129
|
+
this.processQueue();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async processQueue() {
|
|
134
|
+
if (this.ending || !this.callQueue.length) return;
|
|
135
|
+
|
|
136
|
+
if (this.workers.size < this.options.maxConcurrentWorkers) {
|
|
137
|
+
this.startChild();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
for (let worker of this.workers.values()) {
|
|
141
|
+
if (!this.callQueue.length) {
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
if (!worker.ready || worker.stopped || worker.isStopping) {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
|
|
150
|
+
worker.call(this.callQueue.shift());
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async processRequest(data, worker = false) {
|
|
156
|
+
let result = {
|
|
157
|
+
idx: data.idx,
|
|
158
|
+
type: 'response'
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
let method = data.method;
|
|
162
|
+
let args = data.args;
|
|
163
|
+
let location = data.location;
|
|
164
|
+
let awaitResponse = data.awaitResponse;
|
|
165
|
+
|
|
166
|
+
if (!location) {
|
|
167
|
+
throw new Error('Unknown request');
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const mod = require(location);
|
|
171
|
+
try {
|
|
172
|
+
result.contentType = 'data';
|
|
173
|
+
if (method) {
|
|
174
|
+
result.content = await mod[method](...args);
|
|
175
|
+
} else {
|
|
176
|
+
result.content = await mod(...args);
|
|
177
|
+
}
|
|
178
|
+
} catch (e) {
|
|
179
|
+
result.contentType = 'error';
|
|
180
|
+
result.content = errorUtils.errorToJson(e);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (awaitResponse) {
|
|
184
|
+
if (worker) {
|
|
185
|
+
worker.send(result);
|
|
186
|
+
} else {
|
|
187
|
+
return result;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
addCall(method, args) {
|
|
193
|
+
if (this.ending) {
|
|
194
|
+
throw new Error('Cannot add a worker call if workerfarm is ending.');
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
return new Promise((resolve, reject) => {
|
|
198
|
+
this.callQueue.push({
|
|
199
|
+
method,
|
|
200
|
+
args: args,
|
|
201
|
+
retries: 0,
|
|
202
|
+
resolve,
|
|
203
|
+
reject
|
|
204
|
+
});
|
|
205
|
+
this.processQueue();
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async end() {
|
|
210
|
+
this.ending = true;
|
|
211
|
+
await Promise.all(
|
|
212
|
+
Array.from(this.workers.values()).map(worker => this.stopWorker(worker))
|
|
213
|
+
);
|
|
214
|
+
this.ending = false;
|
|
215
|
+
shared = null;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
init(bundlerOptions) {
|
|
219
|
+
this.bundlerOptions = bundlerOptions;
|
|
220
|
+
|
|
221
|
+
if (this.shouldStartRemoteWorkers()) {
|
|
222
|
+
this.persistBundlerOptions();
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
this.localWorker.init(bundlerOptions);
|
|
226
|
+
this.startMaxWorkers();
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
persistBundlerOptions() {
|
|
230
|
+
for (let worker of this.workers.values()) {
|
|
231
|
+
worker.init(this.bundlerOptions);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
startMaxWorkers() {
|
|
236
|
+
// Starts workers untill the maximum is reached
|
|
237
|
+
if (this.workers.size < this.options.maxConcurrentWorkers) {
|
|
238
|
+
for (
|
|
239
|
+
let i = 0;
|
|
240
|
+
i < this.options.maxConcurrentWorkers - this.workers.size;
|
|
241
|
+
i++
|
|
242
|
+
) {
|
|
243
|
+
this.startChild();
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
shouldUseRemoteWorkers() {
|
|
249
|
+
return (
|
|
250
|
+
!this.options.useLocalWorker ||
|
|
251
|
+
(this.warmWorkers >= this.workers.size || !this.options.warmWorkers)
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
static async getShared(options, farmOptions) {
|
|
256
|
+
// Farm options shouldn't be considered safe to overwrite
|
|
257
|
+
// and require an entire new instance to be created
|
|
258
|
+
if (shared && farmOptions) {
|
|
259
|
+
await shared.end();
|
|
260
|
+
shared = null;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
if (!shared) {
|
|
264
|
+
shared = new WorkerFarm(options, farmOptions);
|
|
265
|
+
} else if (options) {
|
|
266
|
+
shared.init(options);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (!shared && !options) {
|
|
270
|
+
throw new Error('Workerfarm should be initialised using options');
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return shared;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
static getNumWorkers() {
|
|
277
|
+
return process.env.PARCEL_WORKERS
|
|
278
|
+
? parseInt(process.env.PARCEL_WORKERS, 10)
|
|
279
|
+
: cpuCount();
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
static async callMaster(request, awaitResponse = true) {
|
|
283
|
+
if (WorkerFarm.isWorker()) {
|
|
284
|
+
const child = require('./child');
|
|
285
|
+
return child.addCall(request, awaitResponse);
|
|
286
|
+
} else {
|
|
287
|
+
return (await WorkerFarm.getShared()).processRequest(request);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
static isWorker() {
|
|
292
|
+
return process.send && require.main.filename === require.resolve('./child');
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
static getConcurrentCallsPerWorker() {
|
|
296
|
+
return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 5;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
module.exports = WorkerFarm;
|
package/src/child.js
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
const {errorUtils} = require('@parcel/utils');
|
|
2
|
+
|
|
3
|
+
class Child {
|
|
4
|
+
constructor() {
|
|
5
|
+
if (!process.send) {
|
|
6
|
+
throw new Error('Only create Child instances in a worker!');
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
this.module = undefined;
|
|
10
|
+
this.childId = undefined;
|
|
11
|
+
|
|
12
|
+
this.callQueue = [];
|
|
13
|
+
this.responseQueue = new Map();
|
|
14
|
+
this.responseId = 0;
|
|
15
|
+
this.maxConcurrentCalls = 10;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
messageListener(data) {
|
|
19
|
+
if (data === 'die') {
|
|
20
|
+
return this.end();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
let type = data.type;
|
|
24
|
+
if (type === 'response') {
|
|
25
|
+
return this.handleResponse(data);
|
|
26
|
+
} else if (type === 'request') {
|
|
27
|
+
return this.handleRequest(data);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async send(data) {
|
|
32
|
+
process.send(data, err => {
|
|
33
|
+
if (err && err instanceof Error) {
|
|
34
|
+
if (err.code === 'ERR_IPC_CHANNEL_CLOSED') {
|
|
35
|
+
// IPC connection closed
|
|
36
|
+
// no need to keep the worker running if it can't send or receive data
|
|
37
|
+
return this.end();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
childInit(module, childId) {
|
|
44
|
+
this.module = require(module);
|
|
45
|
+
this.childId = childId;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async handleRequest(data) {
|
|
49
|
+
let idx = data.idx;
|
|
50
|
+
let child = data.child;
|
|
51
|
+
let method = data.method;
|
|
52
|
+
let args = data.args;
|
|
53
|
+
|
|
54
|
+
let result = {idx, child, type: 'response'};
|
|
55
|
+
try {
|
|
56
|
+
result.contentType = 'data';
|
|
57
|
+
if (method === 'childInit') {
|
|
58
|
+
result.content = this.childInit(...args, child);
|
|
59
|
+
} else {
|
|
60
|
+
result.content = await this.module[method](...args);
|
|
61
|
+
}
|
|
62
|
+
} catch (e) {
|
|
63
|
+
result.contentType = 'error';
|
|
64
|
+
result.content = errorUtils.errorToJson(e);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
this.send(result);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async handleResponse(data) {
|
|
71
|
+
let idx = data.idx;
|
|
72
|
+
let contentType = data.contentType;
|
|
73
|
+
let content = data.content;
|
|
74
|
+
let call = this.responseQueue.get(idx);
|
|
75
|
+
|
|
76
|
+
if (contentType === 'error') {
|
|
77
|
+
call.reject(errorUtils.jsonToError(content));
|
|
78
|
+
} else {
|
|
79
|
+
call.resolve(content);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
this.responseQueue.delete(idx);
|
|
83
|
+
|
|
84
|
+
// Process the next call
|
|
85
|
+
this.processQueue();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Keep in mind to make sure responses to these calls are JSON.Stringify safe
|
|
89
|
+
async addCall(request, awaitResponse = true) {
|
|
90
|
+
let call = request;
|
|
91
|
+
call.type = 'request';
|
|
92
|
+
call.child = this.childId;
|
|
93
|
+
call.awaitResponse = awaitResponse;
|
|
94
|
+
|
|
95
|
+
let promise;
|
|
96
|
+
if (awaitResponse) {
|
|
97
|
+
promise = new Promise((resolve, reject) => {
|
|
98
|
+
call.resolve = resolve;
|
|
99
|
+
call.reject = reject;
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
this.callQueue.push(call);
|
|
104
|
+
this.processQueue();
|
|
105
|
+
|
|
106
|
+
return promise;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async sendRequest(call) {
|
|
110
|
+
let idx;
|
|
111
|
+
if (call.awaitResponse) {
|
|
112
|
+
idx = this.responseId++;
|
|
113
|
+
this.responseQueue.set(idx, call);
|
|
114
|
+
}
|
|
115
|
+
this.send({
|
|
116
|
+
idx: idx,
|
|
117
|
+
child: call.child,
|
|
118
|
+
type: call.type,
|
|
119
|
+
location: call.location,
|
|
120
|
+
method: call.method,
|
|
121
|
+
args: call.args,
|
|
122
|
+
awaitResponse: call.awaitResponse
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async processQueue() {
|
|
127
|
+
if (!this.callQueue.length) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (this.responseQueue.size < this.maxConcurrentCalls) {
|
|
132
|
+
this.sendRequest(this.callQueue.shift());
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
end() {
|
|
137
|
+
process.exit();
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
let child = new Child();
|
|
142
|
+
process.on('message', child.messageListener.bind(child));
|
|
143
|
+
|
|
144
|
+
module.exports = child;
|
package/src/cpuCount.js
ADDED
package/test/.babelrc
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
const WorkerFarm = require(`../../../${
|
|
2
|
+
parseInt(process.versions.node, 10) < 8 ? 'lib' : 'src'
|
|
3
|
+
}/WorkerFarm`);
|
|
4
|
+
|
|
5
|
+
function run() {
|
|
6
|
+
let result = [process.pid];
|
|
7
|
+
return new Promise((resolve, reject) => {
|
|
8
|
+
WorkerFarm.callMaster({
|
|
9
|
+
location: require.resolve('./master-process-id.js'),
|
|
10
|
+
args: []
|
|
11
|
+
})
|
|
12
|
+
.then(pid => {
|
|
13
|
+
result.push(pid);
|
|
14
|
+
resolve(result);
|
|
15
|
+
})
|
|
16
|
+
.catch(reject);
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function init() {
|
|
21
|
+
// Do nothing
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
exports.run = run;
|
|
25
|
+
exports.init = init;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
const WorkerFarm = require(`../../../${
|
|
2
|
+
parseInt(process.versions.node, 10) < 8 ? 'lib' : 'src'
|
|
3
|
+
}/WorkerFarm`);
|
|
4
|
+
|
|
5
|
+
function run(a, b) {
|
|
6
|
+
return WorkerFarm.callMaster({
|
|
7
|
+
location: require.resolve('./master-sum.js'),
|
|
8
|
+
args: [a, b]
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function init() {
|
|
13
|
+
// Do nothing
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
exports.run = run;
|
|
17
|
+
exports.init = init;
|
package/test/mocha.opts
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
const assert = require('assert');
|
|
2
|
+
const WorkerFarm = require('../index');
|
|
3
|
+
|
|
4
|
+
describe('WorkerFarm', () => {
|
|
5
|
+
it('Should start up workers', async () => {
|
|
6
|
+
let workerfarm = new WorkerFarm(
|
|
7
|
+
{},
|
|
8
|
+
{
|
|
9
|
+
warmWorkers: false,
|
|
10
|
+
useLocalWorker: false,
|
|
11
|
+
workerPath: require.resolve('./integration/workerfarm/ping.js')
|
|
12
|
+
}
|
|
13
|
+
);
|
|
14
|
+
|
|
15
|
+
assert.equal(await workerfarm.run(), 'pong');
|
|
16
|
+
|
|
17
|
+
await workerfarm.end();
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it('Should handle 1000 requests without any issue', async () => {
|
|
21
|
+
let workerfarm = new WorkerFarm(
|
|
22
|
+
{},
|
|
23
|
+
{
|
|
24
|
+
warmWorkers: false,
|
|
25
|
+
useLocalWorker: false,
|
|
26
|
+
workerPath: require.resolve('./integration/workerfarm/echo.js')
|
|
27
|
+
}
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
let promises = [];
|
|
31
|
+
for (let i = 0; i < 1000; i++) {
|
|
32
|
+
promises.push(workerfarm.run(i));
|
|
33
|
+
}
|
|
34
|
+
await Promise.all(promises);
|
|
35
|
+
|
|
36
|
+
await workerfarm.end();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('Should consistently initialise workers, even after 100 re-inits', async () => {
|
|
40
|
+
let options = {
|
|
41
|
+
key: 0
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
let workerfarm = new WorkerFarm(options, {
|
|
45
|
+
warmWorkers: false,
|
|
46
|
+
useLocalWorker: false,
|
|
47
|
+
workerPath: require.resolve('./integration/workerfarm/init.js')
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
for (let i = 0; i < 100; i++) {
|
|
51
|
+
options.key = i;
|
|
52
|
+
workerfarm.init(options);
|
|
53
|
+
|
|
54
|
+
for (let i = 0; i < workerfarm.workers.size; i++) {
|
|
55
|
+
assert.equal((await workerfarm.run()).key, options.key);
|
|
56
|
+
}
|
|
57
|
+
assert.equal(workerfarm.shouldUseRemoteWorkers(), true);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
await workerfarm.end();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it('Should warm up workers', async () => {
|
|
64
|
+
let workerfarm = new WorkerFarm(
|
|
65
|
+
{},
|
|
66
|
+
{
|
|
67
|
+
warmWorkers: true,
|
|
68
|
+
useLocalWorker: true,
|
|
69
|
+
workerPath: require.resolve('./integration/workerfarm/echo.js')
|
|
70
|
+
}
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
for (let i = 0; i < 100; i++) {
|
|
74
|
+
assert.equal(await workerfarm.run(i), i);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
await new Promise(resolve => workerfarm.once('warmedup', resolve));
|
|
78
|
+
|
|
79
|
+
assert(workerfarm.workers.size > 0, 'Should have spawned workers.');
|
|
80
|
+
assert(
|
|
81
|
+
workerfarm.warmWorkers >= workerfarm.workers.size,
|
|
82
|
+
'Should have warmed up workers.'
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
await workerfarm.end();
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('Should use the local worker', async () => {
|
|
89
|
+
let workerfarm = new WorkerFarm(
|
|
90
|
+
{},
|
|
91
|
+
{
|
|
92
|
+
warmWorkers: true,
|
|
93
|
+
useLocalWorker: true,
|
|
94
|
+
workerPath: require.resolve('./integration/workerfarm/echo.js')
|
|
95
|
+
}
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
assert.equal(await workerfarm.run('hello world'), 'hello world');
|
|
99
|
+
assert.equal(workerfarm.shouldUseRemoteWorkers(), false);
|
|
100
|
+
|
|
101
|
+
await workerfarm.end();
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it('Should be able to use bi-directional communication', async () => {
|
|
105
|
+
let workerfarm = new WorkerFarm(
|
|
106
|
+
{},
|
|
107
|
+
{
|
|
108
|
+
warmWorkers: false,
|
|
109
|
+
useLocalWorker: false,
|
|
110
|
+
workerPath: require.resolve('./integration/workerfarm/ipc.js')
|
|
111
|
+
}
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
assert.equal(await workerfarm.run(1, 2), 3);
|
|
115
|
+
|
|
116
|
+
await workerfarm.end();
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('Should be able to handle 1000 bi-directional calls', async () => {
|
|
120
|
+
let workerfarm = new WorkerFarm(
|
|
121
|
+
{},
|
|
122
|
+
{
|
|
123
|
+
warmWorkers: false,
|
|
124
|
+
useLocalWorker: false,
|
|
125
|
+
workerPath: require.resolve('./integration/workerfarm/ipc.js')
|
|
126
|
+
}
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
for (let i = 0; i < 1000; i++) {
|
|
130
|
+
assert.equal(await workerfarm.run(1 + i, 2), 3 + i);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
await workerfarm.end();
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('Bi-directional call should return masters pid', async () => {
|
|
137
|
+
let workerfarm = new WorkerFarm(
|
|
138
|
+
{},
|
|
139
|
+
{
|
|
140
|
+
warmWorkers: false,
|
|
141
|
+
useLocalWorker: false,
|
|
142
|
+
workerPath: require.resolve('./integration/workerfarm/ipc-pid.js')
|
|
143
|
+
}
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
let result = await workerfarm.run();
|
|
147
|
+
assert.equal(result.length, 2);
|
|
148
|
+
assert.equal(result[1], process.pid);
|
|
149
|
+
assert.notEqual(result[0], process.pid);
|
|
150
|
+
|
|
151
|
+
await workerfarm.end();
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it('Should handle 10 big concurrent requests without any issue', async () => {
|
|
155
|
+
// This emulates the node.js ipc bug for win32
|
|
156
|
+
let workerfarm = new WorkerFarm(
|
|
157
|
+
{},
|
|
158
|
+
{
|
|
159
|
+
warmWorkers: false,
|
|
160
|
+
useLocalWorker: false,
|
|
161
|
+
workerPath: require.resolve('./integration/workerfarm/echo.js')
|
|
162
|
+
}
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
let bigData = [];
|
|
166
|
+
for (let i = 0; i < 10000; i++) {
|
|
167
|
+
bigData.push('This is some big data');
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
let promises = [];
|
|
171
|
+
for (let i = 0; i < 10; i++) {
|
|
172
|
+
promises.push(workerfarm.run(bigData));
|
|
173
|
+
}
|
|
174
|
+
await Promise.all(promises);
|
|
175
|
+
|
|
176
|
+
await workerfarm.end();
|
|
177
|
+
});
|
|
178
|
+
});
|