@parcel/workers 2.0.0-nightly.149 → 2.0.0-nightly.1490
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +23 -0
- package/lib/Handle.js +16 -58
- package/lib/Worker.js +103 -62
- package/lib/WorkerFarm.js +272 -192
- package/lib/backend.js +4 -6
- package/lib/bus.js +11 -13
- package/lib/child.js +140 -116
- package/lib/childState.js +2 -4
- package/lib/core-worker.browser.js +4 -0
- package/lib/core-worker.js +4 -0
- package/lib/cpuCount.js +36 -25
- package/lib/index.js +35 -32
- package/lib/process/ProcessChild.js +18 -24
- package/lib/process/ProcessWorker.js +27 -38
- package/lib/threads/ThreadsChild.js +26 -28
- package/lib/threads/ThreadsWorker.js +25 -31
- package/lib/web/WebChild.js +44 -0
- package/lib/web/WebWorker.js +85 -0
- package/package.json +19 -8
- package/src/Handle.js +10 -39
- package/src/Worker.js +95 -22
- package/src/WorkerFarm.js +267 -62
- package/src/backend.js +5 -0
- package/src/bus.js +3 -2
- package/src/child.js +95 -26
- package/src/core-worker.browser.js +3 -0
- package/src/core-worker.js +2 -0
- package/src/cpuCount.js +23 -10
- package/src/index.js +8 -2
- package/src/process/ProcessChild.js +2 -1
- package/src/process/ProcessWorker.js +1 -1
- package/src/threads/ThreadsWorker.js +2 -2
- package/src/types.js +1 -1
- package/src/web/WebChild.js +50 -0
- package/src/web/WebWorker.js +85 -0
- package/test/cpuCount.test.js +1 -1
- package/test/integration/workerfarm/console.js +1 -1
- package/test/integration/workerfarm/logging.js +1 -1
- package/test/integration/workerfarm/reverse-handle.js +2 -2
- package/test/workerfarm.js +5 -5
- package/lib/Profiler.js +0 -70
- package/lib/Trace.js +0 -126
- package/src/Profiler.js +0 -93
- package/src/Trace.js +0 -121
package/lib/WorkerFarm.js
CHANGED
@@ -10,226 +10,279 @@ Object.defineProperty(exports, "Handle", {
|
|
10
10
|
}
|
11
11
|
});
|
12
12
|
exports.default = void 0;
|
13
|
-
|
14
|
-
var
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
13
|
+
var coreWorker = _interopRequireWildcard(require("./core-worker"));
|
14
|
+
var bus = _interopRequireWildcard(require("./bus"));
|
15
|
+
function _assert() {
|
16
|
+
const data = _interopRequireDefault(require("assert"));
|
17
|
+
_assert = function () {
|
18
|
+
return data;
|
19
|
+
};
|
20
|
+
return data;
|
21
|
+
}
|
22
|
+
function _nullthrows() {
|
23
|
+
const data = _interopRequireDefault(require("nullthrows"));
|
24
|
+
_nullthrows = function () {
|
25
|
+
return data;
|
26
|
+
};
|
27
|
+
return data;
|
28
|
+
}
|
29
|
+
function _events() {
|
30
|
+
const data = _interopRequireDefault(require("events"));
|
31
|
+
_events = function () {
|
32
|
+
return data;
|
33
|
+
};
|
34
|
+
return data;
|
35
|
+
}
|
36
|
+
function _core() {
|
37
|
+
const data = require("@parcel/core");
|
38
|
+
_core = function () {
|
39
|
+
return data;
|
40
|
+
};
|
41
|
+
return data;
|
42
|
+
}
|
43
|
+
function _diagnostic() {
|
44
|
+
const data = _interopRequireWildcard(require("@parcel/diagnostic"));
|
45
|
+
_diagnostic = function () {
|
46
|
+
return data;
|
47
|
+
};
|
48
|
+
return data;
|
49
|
+
}
|
24
50
|
var _Worker = _interopRequireDefault(require("./Worker"));
|
25
|
-
|
26
51
|
var _cpuCount = _interopRequireDefault(require("./cpuCount"));
|
27
|
-
|
28
52
|
var _Handle = _interopRequireDefault(require("./Handle"));
|
29
|
-
|
30
53
|
var _childState = require("./childState");
|
31
|
-
|
32
54
|
var _backend = require("./backend");
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
55
|
+
function _profiler() {
|
56
|
+
const data = require("@parcel/profiler");
|
57
|
+
_profiler = function () {
|
58
|
+
return data;
|
59
|
+
};
|
60
|
+
return data;
|
61
|
+
}
|
62
|
+
function _fs() {
|
63
|
+
const data = _interopRequireDefault(require("fs"));
|
64
|
+
_fs = function () {
|
65
|
+
return data;
|
66
|
+
};
|
67
|
+
return data;
|
68
|
+
}
|
69
|
+
function _logger() {
|
70
|
+
const data = _interopRequireDefault(require("@parcel/logger"));
|
71
|
+
_logger = function () {
|
72
|
+
return data;
|
73
|
+
};
|
74
|
+
return data;
|
75
|
+
}
|
46
76
|
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
47
|
-
|
48
|
-
function
|
49
|
-
|
50
|
-
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
51
|
-
|
52
|
-
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
53
|
-
|
54
|
-
let profileId = 1;
|
77
|
+
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
|
78
|
+
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
|
55
79
|
let referenceId = 1;
|
80
|
+
const DEFAULT_MAX_CONCURRENT_CALLS = 30;
|
56
81
|
|
57
82
|
/**
|
58
83
|
* workerPath should always be defined inside farmOptions
|
59
84
|
*/
|
60
|
-
|
85
|
+
|
86
|
+
class WorkerFarm extends _events().default {
|
87
|
+
callQueue = [];
|
88
|
+
ending = false;
|
89
|
+
warmWorkers = 0;
|
90
|
+
readyWorkers = 0;
|
91
|
+
workers = new Map();
|
92
|
+
handles = new Map();
|
93
|
+
sharedReferences = new Map();
|
94
|
+
sharedReferencesByValue = new Map();
|
95
|
+
serializedSharedReferences = new Map();
|
61
96
|
constructor(farmOptions = {}) {
|
97
|
+
var _process$stdout;
|
62
98
|
super();
|
63
|
-
|
64
|
-
_defineProperty(this, "callQueue", []);
|
65
|
-
|
66
|
-
_defineProperty(this, "ending", false);
|
67
|
-
|
68
|
-
_defineProperty(this, "localWorker", void 0);
|
69
|
-
|
70
|
-
_defineProperty(this, "options", void 0);
|
71
|
-
|
72
|
-
_defineProperty(this, "run", void 0);
|
73
|
-
|
74
|
-
_defineProperty(this, "warmWorkers", 0);
|
75
|
-
|
76
|
-
_defineProperty(this, "workers", new Map());
|
77
|
-
|
78
|
-
_defineProperty(this, "handles", new Map());
|
79
|
-
|
80
|
-
_defineProperty(this, "sharedReferences", new Map());
|
81
|
-
|
82
|
-
_defineProperty(this, "sharedReferencesByValue", new Map());
|
83
|
-
|
84
|
-
_defineProperty(this, "profiler", void 0);
|
85
|
-
|
86
|
-
_defineProperty(this, "workerApi", {
|
87
|
-
callMaster: async (request, awaitResponse = true) => {
|
88
|
-
// $FlowFixMe
|
89
|
-
let result = await this.processRequest(_objectSpread({}, request, {
|
90
|
-
awaitResponse
|
91
|
-
}));
|
92
|
-
return (0, _core.deserialize)((0, _core.serialize)(result));
|
93
|
-
},
|
94
|
-
createReverseHandle: fn => this.createReverseHandle(fn),
|
95
|
-
callChild: (childId, request) => new Promise((resolve, reject) => {
|
96
|
-
(0, _nullthrows.default)(this.workers.get(childId)).call(_objectSpread({}, request, {
|
97
|
-
resolve,
|
98
|
-
reject,
|
99
|
-
retries: 0
|
100
|
-
}));
|
101
|
-
}),
|
102
|
-
getSharedReference: ref => this.sharedReferences.get(ref),
|
103
|
-
resolveSharedReference: value => this.sharedReferencesByValue.get(value)
|
104
|
-
});
|
105
|
-
|
106
|
-
this.options = _objectSpread({
|
99
|
+
this.options = {
|
107
100
|
maxConcurrentWorkers: WorkerFarm.getNumWorkers(),
|
108
|
-
maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(),
|
101
|
+
maxConcurrentCallsPerWorker: WorkerFarm.getConcurrentCallsPerWorker(farmOptions.shouldTrace ? 1 : DEFAULT_MAX_CONCURRENT_CALLS),
|
109
102
|
forcedKillTime: 500,
|
110
103
|
warmWorkers: false,
|
111
104
|
useLocalWorker: true,
|
112
105
|
// TODO: setting this to false makes some tests fail, figure out why
|
113
|
-
backend: (0, _backend.detectBackend)()
|
114
|
-
|
115
|
-
|
106
|
+
backend: (0, _backend.detectBackend)(),
|
107
|
+
...farmOptions
|
108
|
+
};
|
116
109
|
if (!this.options.workerPath) {
|
117
110
|
throw new Error('Please provide a worker path!');
|
118
|
-
}
|
119
|
-
|
111
|
+
}
|
120
112
|
|
121
|
-
|
113
|
+
// $FlowFixMe
|
114
|
+
if (process.browser) {
|
115
|
+
if (this.options.workerPath === '@parcel/core/src/worker.js') {
|
116
|
+
this.localWorker = coreWorker;
|
117
|
+
} else {
|
118
|
+
throw new Error('No dynamic require possible: ' + this.options.workerPath);
|
119
|
+
}
|
120
|
+
} else {
|
121
|
+
// $FlowFixMe this must be dynamic
|
122
|
+
this.localWorker = require(this.options.workerPath);
|
123
|
+
}
|
124
|
+
this.localWorkerInit = this.localWorker.childInit != null ? this.localWorker.childInit() : null;
|
122
125
|
this.run = this.createHandle('run');
|
126
|
+
|
127
|
+
// Worker thread stdout is by default piped into the process stdout, if there are enough worker
|
128
|
+
// threads to exceed the default listener limit, then anything else piping into stdout will trigger
|
129
|
+
// the `MaxListenersExceededWarning`, so we should ensure the max listeners is at least equal to the
|
130
|
+
// number of workers + 1 for the main thread.
|
131
|
+
//
|
132
|
+
// Note this can't be fixed easily where other things pipe into stdout - even after starting > 10 worker
|
133
|
+
// threads `process.stdout.getMaxListeners()` will still return 10, however adding another pipe into `stdout`
|
134
|
+
// will give the warning with `<worker count + 1>` as the number of listeners.
|
135
|
+
(_process$stdout = process.stdout) === null || _process$stdout === void 0 || _process$stdout.setMaxListeners(Math.max(process.stdout.getMaxListeners(), WorkerFarm.getNumWorkers() + 1));
|
123
136
|
this.startMaxWorkers();
|
124
137
|
}
|
125
|
-
|
138
|
+
workerApi = {
|
139
|
+
callMaster: async (request, awaitResponse = true) => {
|
140
|
+
// $FlowFixMe
|
141
|
+
let result = await this.processRequest({
|
142
|
+
...request,
|
143
|
+
awaitResponse
|
144
|
+
});
|
145
|
+
return (0, _core().deserialize)((0, _core().serialize)(result));
|
146
|
+
},
|
147
|
+
createReverseHandle: fn => this.createReverseHandle(fn),
|
148
|
+
callChild: (childId, request) => new Promise((resolve, reject) => {
|
149
|
+
(0, _nullthrows().default)(this.workers.get(childId)).call({
|
150
|
+
...request,
|
151
|
+
resolve,
|
152
|
+
reject,
|
153
|
+
retries: 0
|
154
|
+
});
|
155
|
+
}),
|
156
|
+
runHandle: (handle, args) => this.workerApi.callChild((0, _nullthrows().default)(handle.childId), {
|
157
|
+
handle: handle.id,
|
158
|
+
args
|
159
|
+
}),
|
160
|
+
getSharedReference: ref => this.sharedReferences.get(ref),
|
161
|
+
resolveSharedReference: value => this.sharedReferencesByValue.get(value)
|
162
|
+
};
|
126
163
|
warmupWorker(method, args) {
|
127
164
|
// Workers are already stopping
|
128
165
|
if (this.ending) {
|
129
166
|
return;
|
130
|
-
}
|
167
|
+
}
|
168
|
+
|
169
|
+
// Workers are not warmed up yet.
|
131
170
|
// Send the job to a remote worker in the background,
|
132
171
|
// but use the result from the local worker - it will be faster.
|
133
|
-
|
134
|
-
|
135
172
|
let promise = this.addCall(method, [...args, true]);
|
136
|
-
|
137
173
|
if (promise) {
|
138
174
|
promise.then(() => {
|
139
175
|
this.warmWorkers++;
|
140
|
-
|
141
176
|
if (this.warmWorkers >= this.workers.size) {
|
142
177
|
this.emit('warmedup');
|
143
178
|
}
|
144
179
|
}).catch(() => {});
|
145
180
|
}
|
146
181
|
}
|
147
|
-
|
148
182
|
shouldStartRemoteWorkers() {
|
149
183
|
return this.options.maxConcurrentWorkers > 0 || !this.options.useLocalWorker;
|
150
184
|
}
|
151
|
-
|
152
|
-
|
153
|
-
|
185
|
+
createHandle(method, useMainThread = false) {
|
186
|
+
if (!this.options.useLocalWorker) {
|
187
|
+
useMainThread = false;
|
188
|
+
}
|
189
|
+
return async (...args) => {
|
154
190
|
// Child process workers are slow to start (~600ms).
|
155
191
|
// While we're waiting, just run on the main thread.
|
156
192
|
// This significantly speeds up startup time.
|
157
|
-
if (this.shouldUseRemoteWorkers()) {
|
193
|
+
if (this.shouldUseRemoteWorkers() && !useMainThread) {
|
158
194
|
return this.addCall(method, [...args, false]);
|
159
195
|
} else {
|
160
196
|
if (this.options.warmWorkers && this.shouldStartRemoteWorkers()) {
|
161
197
|
this.warmupWorker(method, args);
|
162
198
|
}
|
163
|
-
|
164
|
-
|
199
|
+
let processedArgs;
|
200
|
+
if (!useMainThread) {
|
201
|
+
processedArgs = (0, _core().restoreDeserializedObject)((0, _core().prepareForSerialization)([...args, false]));
|
202
|
+
} else {
|
203
|
+
processedArgs = args;
|
204
|
+
}
|
205
|
+
if (this.localWorkerInit != null) {
|
206
|
+
await this.localWorkerInit;
|
207
|
+
this.localWorkerInit = null;
|
208
|
+
}
|
165
209
|
return this.localWorker[method](this.workerApi, ...processedArgs);
|
166
210
|
}
|
167
211
|
};
|
168
212
|
}
|
169
|
-
|
170
213
|
onError(error, worker) {
|
171
214
|
// Handle ipc errors
|
172
215
|
if (error.code === 'ERR_IPC_CHANNEL_CLOSED') {
|
173
216
|
return this.stopWorker(worker);
|
217
|
+
} else {
|
218
|
+
_logger().default.error(error, '@parcel/workers');
|
174
219
|
}
|
175
220
|
}
|
176
|
-
|
177
221
|
startChild() {
|
178
222
|
let worker = new _Worker.default({
|
179
223
|
forcedKillTime: this.options.forcedKillTime,
|
180
224
|
backend: this.options.backend,
|
181
|
-
|
225
|
+
shouldPatchConsole: this.options.shouldPatchConsole,
|
226
|
+
shouldTrace: this.options.shouldTrace,
|
227
|
+
sharedReferences: this.sharedReferences
|
182
228
|
});
|
183
|
-
worker.fork((0, _nullthrows.default)(this.options.workerPath));
|
229
|
+
worker.fork((0, _nullthrows().default)(this.options.workerPath));
|
184
230
|
worker.on('request', data => this.processRequest(data, worker));
|
185
|
-
worker.on('ready', () =>
|
231
|
+
worker.on('ready', () => {
|
232
|
+
this.readyWorkers++;
|
233
|
+
if (this.readyWorkers === this.options.maxConcurrentWorkers) {
|
234
|
+
this.emit('ready');
|
235
|
+
}
|
236
|
+
this.processQueue();
|
237
|
+
});
|
186
238
|
worker.on('response', () => this.processQueue());
|
187
239
|
worker.on('error', err => this.onError(err, worker));
|
188
240
|
worker.once('exit', () => this.stopWorker(worker));
|
189
241
|
this.workers.set(worker.id, worker);
|
190
242
|
}
|
191
|
-
|
192
243
|
async stopWorker(worker) {
|
193
244
|
if (!worker.stopped) {
|
194
245
|
this.workers.delete(worker.id);
|
195
246
|
worker.isStopping = true;
|
196
|
-
|
197
247
|
if (worker.calls.size) {
|
198
248
|
for (let call of worker.calls.values()) {
|
199
249
|
call.retries++;
|
200
250
|
this.callQueue.unshift(call);
|
201
251
|
}
|
202
252
|
}
|
203
|
-
|
204
253
|
worker.calls.clear();
|
205
|
-
await worker.stop();
|
254
|
+
await worker.stop();
|
206
255
|
|
256
|
+
// Process any requests that failed and start a new worker
|
207
257
|
this.processQueue();
|
208
258
|
}
|
209
259
|
}
|
210
|
-
|
211
260
|
processQueue() {
|
212
261
|
if (this.ending || !this.callQueue.length) return;
|
213
|
-
|
214
262
|
if (this.workers.size < this.options.maxConcurrentWorkers) {
|
215
263
|
this.startChild();
|
216
264
|
}
|
217
|
-
|
218
|
-
for (let worker of
|
265
|
+
let workers = [...this.workers.values()].sort((a, b) => a.calls.size - b.calls.size);
|
266
|
+
for (let worker of workers) {
|
219
267
|
if (!this.callQueue.length) {
|
220
268
|
break;
|
221
269
|
}
|
222
|
-
|
223
270
|
if (!worker.ready || worker.stopped || worker.isStopping) {
|
224
271
|
continue;
|
225
272
|
}
|
226
|
-
|
227
273
|
if (worker.calls.size < this.options.maxConcurrentCallsPerWorker) {
|
228
|
-
|
274
|
+
this.callWorker(worker, this.callQueue.shift());
|
229
275
|
}
|
230
276
|
}
|
231
277
|
}
|
232
|
-
|
278
|
+
async callWorker(worker, call) {
|
279
|
+
for (let ref of this.sharedReferences.keys()) {
|
280
|
+
if (!worker.sentSharedReferences.has(ref)) {
|
281
|
+
await worker.sendSharedReference(ref, this.getSerializedSharedReference(ref));
|
282
|
+
}
|
283
|
+
}
|
284
|
+
worker.call(call);
|
285
|
+
}
|
233
286
|
async processRequest(data, worker) {
|
234
287
|
let {
|
235
288
|
method,
|
@@ -240,73 +293,72 @@ class WorkerFarm extends _events.default {
|
|
240
293
|
handle: handleId
|
241
294
|
} = data;
|
242
295
|
let mod;
|
243
|
-
|
244
296
|
if (handleId != null) {
|
245
|
-
|
297
|
+
var _this$handles$get;
|
298
|
+
mod = (0, _nullthrows().default)((_this$handles$get = this.handles.get(handleId)) === null || _this$handles$get === void 0 ? void 0 : _this$handles$get.fn);
|
246
299
|
} else if (location) {
|
247
|
-
// $FlowFixMe
|
248
|
-
|
300
|
+
// $FlowFixMe
|
301
|
+
if (process.browser) {
|
302
|
+
if (location === '@parcel/workers/src/bus.js') {
|
303
|
+
mod = bus;
|
304
|
+
} else {
|
305
|
+
throw new Error('No dynamic require possible: ' + location);
|
306
|
+
}
|
307
|
+
} else {
|
308
|
+
// $FlowFixMe this must be dynamic
|
309
|
+
mod = require(location);
|
310
|
+
}
|
249
311
|
} else {
|
250
312
|
throw new Error('Unknown request');
|
251
313
|
}
|
252
|
-
|
253
314
|
const responseFromContent = content => ({
|
254
315
|
idx,
|
255
316
|
type: 'response',
|
256
317
|
contentType: 'data',
|
257
318
|
content
|
258
319
|
});
|
259
|
-
|
260
320
|
const errorResponseFromError = e => ({
|
261
321
|
idx,
|
262
322
|
type: 'response',
|
263
323
|
contentType: 'error',
|
264
|
-
content: (0, _diagnostic.anyToDiagnostic)(e)
|
324
|
+
content: (0, _diagnostic().anyToDiagnostic)(e)
|
265
325
|
});
|
266
|
-
|
267
326
|
let result;
|
268
|
-
|
269
327
|
if (method == null) {
|
270
328
|
try {
|
271
|
-
result = responseFromContent(
|
329
|
+
result = responseFromContent(await mod(...args));
|
272
330
|
} catch (e) {
|
273
331
|
result = errorResponseFromError(e);
|
274
332
|
}
|
275
333
|
} else {
|
276
334
|
// ESModule default interop
|
277
|
-
// $FlowFixMe
|
278
335
|
if (mod.__esModule && !mod[method] && mod.default) {
|
279
336
|
mod = mod.default;
|
280
337
|
}
|
281
|
-
|
282
338
|
try {
|
283
339
|
// $FlowFixMe
|
284
|
-
result = responseFromContent(
|
340
|
+
result = responseFromContent(await mod[method](...args));
|
285
341
|
} catch (e) {
|
286
342
|
result = errorResponseFromError(e);
|
287
343
|
}
|
288
344
|
}
|
289
|
-
|
290
345
|
if (awaitResponse) {
|
291
346
|
if (worker) {
|
292
347
|
worker.send(result);
|
293
348
|
} else {
|
294
349
|
if (result.contentType === 'error') {
|
295
|
-
throw new _diagnostic.default({
|
350
|
+
throw new (_diagnostic().default)({
|
296
351
|
diagnostic: result.content
|
297
352
|
});
|
298
353
|
}
|
299
|
-
|
300
354
|
return result.content;
|
301
355
|
}
|
302
356
|
}
|
303
357
|
}
|
304
|
-
|
305
358
|
addCall(method, args) {
|
306
359
|
if (this.ending) {
|
307
360
|
throw new Error('Cannot add a worker call if workerfarm is ending.');
|
308
361
|
}
|
309
|
-
|
310
362
|
return new Promise((resolve, reject) => {
|
311
363
|
this.callQueue.push({
|
312
364
|
method,
|
@@ -318,91 +370,87 @@ class WorkerFarm extends _events.default {
|
|
318
370
|
this.processQueue();
|
319
371
|
});
|
320
372
|
}
|
321
|
-
|
322
373
|
async end() {
|
323
374
|
this.ending = true;
|
324
|
-
|
375
|
+
await Promise.all(Array.from(this.workers.values()).map(worker => this.stopWorker(worker)));
|
325
376
|
for (let handle of this.handles.values()) {
|
326
377
|
handle.dispose();
|
327
378
|
}
|
328
|
-
|
329
379
|
this.handles = new Map();
|
330
380
|
this.sharedReferences = new Map();
|
331
381
|
this.sharedReferencesByValue = new Map();
|
332
|
-
await Promise.all(Array.from(this.workers.values()).map(worker => this.stopWorker(worker)));
|
333
382
|
this.ending = false;
|
334
383
|
}
|
335
|
-
|
336
384
|
startMaxWorkers() {
|
337
385
|
// Starts workers until the maximum is reached
|
338
386
|
if (this.workers.size < this.options.maxConcurrentWorkers) {
|
339
387
|
let toStart = this.options.maxConcurrentWorkers - this.workers.size;
|
340
|
-
|
341
388
|
while (toStart--) {
|
342
389
|
this.startChild();
|
343
390
|
}
|
344
391
|
}
|
345
392
|
}
|
346
|
-
|
347
393
|
shouldUseRemoteWorkers() {
|
348
394
|
return !this.options.useLocalWorker || (this.warmWorkers >= this.workers.size || !this.options.warmWorkers) && this.options.maxConcurrentWorkers > 0;
|
349
395
|
}
|
350
|
-
|
351
396
|
createReverseHandle(fn) {
|
352
397
|
let handle = new _Handle.default({
|
353
|
-
fn
|
354
|
-
workerApi: this.workerApi
|
398
|
+
fn
|
355
399
|
});
|
356
400
|
this.handles.set(handle.id, handle);
|
357
401
|
return handle;
|
358
402
|
}
|
359
|
-
|
360
|
-
async createSharedReference(value) {
|
403
|
+
createSharedReference(value, isCacheable = true) {
|
361
404
|
let ref = referenceId++;
|
362
405
|
this.sharedReferences.set(ref, value);
|
363
406
|
this.sharedReferencesByValue.set(value, ref);
|
364
|
-
|
365
|
-
|
366
|
-
for (let worker of this.workers.values()) {
|
367
|
-
promises.push(new Promise((resolve, reject) => {
|
368
|
-
worker.call({
|
369
|
-
method: 'createSharedReference',
|
370
|
-
args: [ref, value],
|
371
|
-
resolve,
|
372
|
-
reject,
|
373
|
-
retries: 0
|
374
|
-
});
|
375
|
-
}));
|
407
|
+
if (!isCacheable) {
|
408
|
+
this.serializedSharedReferences.set(ref, null);
|
376
409
|
}
|
377
|
-
|
378
|
-
await Promise.all(promises);
|
379
410
|
return {
|
380
411
|
ref,
|
381
412
|
dispose: () => {
|
382
413
|
this.sharedReferences.delete(ref);
|
383
414
|
this.sharedReferencesByValue.delete(value);
|
415
|
+
this.serializedSharedReferences.delete(ref);
|
384
416
|
let promises = [];
|
385
|
-
|
386
417
|
for (let worker of this.workers.values()) {
|
418
|
+
if (!worker.sentSharedReferences.has(ref)) {
|
419
|
+
continue;
|
420
|
+
}
|
421
|
+
worker.sentSharedReferences.delete(ref);
|
387
422
|
promises.push(new Promise((resolve, reject) => {
|
388
423
|
worker.call({
|
389
424
|
method: 'deleteSharedReference',
|
390
425
|
args: [ref],
|
391
426
|
resolve,
|
392
427
|
reject,
|
428
|
+
skipReadyCheck: true,
|
393
429
|
retries: 0
|
394
430
|
});
|
395
431
|
}));
|
396
432
|
}
|
397
|
-
|
398
433
|
return Promise.all(promises);
|
399
434
|
}
|
400
435
|
};
|
401
436
|
}
|
437
|
+
getSerializedSharedReference(ref) {
|
438
|
+
let cached = this.serializedSharedReferences.get(ref);
|
439
|
+
if (cached) {
|
440
|
+
return cached;
|
441
|
+
}
|
442
|
+
let value = this.sharedReferences.get(ref);
|
443
|
+
let buf = (0, _core().serialize)(value).buffer;
|
402
444
|
|
445
|
+
// If the reference was created with the isCacheable option set to false,
|
446
|
+
// serializedSharedReferences will contain `null` as the value.
|
447
|
+
if (cached !== null) {
|
448
|
+
this.serializedSharedReferences.set(ref, buf);
|
449
|
+
}
|
450
|
+
return buf;
|
451
|
+
}
|
403
452
|
async startProfile() {
|
404
453
|
let promises = [];
|
405
|
-
|
406
454
|
for (let worker of this.workers.values()) {
|
407
455
|
promises.push(new Promise((resolve, reject) => {
|
408
456
|
worker.call({
|
@@ -410,24 +458,21 @@ class WorkerFarm extends _events.default {
|
|
410
458
|
args: [],
|
411
459
|
resolve,
|
412
460
|
reject,
|
413
|
-
retries: 0
|
461
|
+
retries: 0,
|
462
|
+
skipReadyCheck: true
|
414
463
|
});
|
415
464
|
}));
|
416
465
|
}
|
417
|
-
|
418
|
-
this.profiler = new _Profiler.default();
|
466
|
+
this.profiler = new (_profiler().SamplingProfiler)();
|
419
467
|
promises.push(this.profiler.startProfiling());
|
420
468
|
await Promise.all(promises);
|
421
469
|
}
|
422
|
-
|
423
470
|
async endProfile() {
|
424
471
|
if (!this.profiler) {
|
425
472
|
return;
|
426
473
|
}
|
427
|
-
|
428
474
|
let promises = [this.profiler.stopProfiling()];
|
429
475
|
let names = ['Master'];
|
430
|
-
|
431
476
|
for (let worker of this.workers.values()) {
|
432
477
|
names.push('Worker ' + worker.id);
|
433
478
|
promises.push(new Promise((resolve, reject) => {
|
@@ -436,48 +481,83 @@ class WorkerFarm extends _events.default {
|
|
436
481
|
args: [],
|
437
482
|
resolve,
|
438
483
|
reject,
|
439
|
-
retries: 0
|
484
|
+
retries: 0,
|
485
|
+
skipReadyCheck: true
|
440
486
|
});
|
441
487
|
}));
|
442
488
|
}
|
443
|
-
|
444
489
|
var profiles = await Promise.all(promises);
|
445
|
-
let trace = new
|
446
|
-
let filename = `profile-${
|
447
|
-
let stream = trace.pipe(_fs.default.createWriteStream(filename));
|
448
|
-
|
490
|
+
let trace = new (_profiler().Trace)();
|
491
|
+
let filename = `profile-${getTimeId()}.trace`;
|
492
|
+
let stream = trace.pipe(_fs().default.createWriteStream(filename));
|
449
493
|
for (let profile of profiles) {
|
450
494
|
trace.addCPUProfile(names.shift(), profile);
|
451
495
|
}
|
452
|
-
|
453
496
|
trace.flush();
|
454
497
|
await new Promise(resolve => {
|
455
498
|
stream.once('finish', resolve);
|
456
499
|
});
|
457
|
-
|
458
|
-
_logger.default.info({
|
500
|
+
_logger().default.info({
|
459
501
|
origin: '@parcel/workers',
|
460
|
-
message: `Wrote profile to ${filename}`
|
502
|
+
message: (0, _diagnostic().md)`Wrote profile to ${filename}`
|
461
503
|
});
|
462
504
|
}
|
463
|
-
|
505
|
+
async callAllWorkers(method, args) {
|
506
|
+
let promises = [];
|
507
|
+
for (let worker of this.workers.values()) {
|
508
|
+
promises.push(new Promise((resolve, reject) => {
|
509
|
+
worker.call({
|
510
|
+
method,
|
511
|
+
args,
|
512
|
+
resolve,
|
513
|
+
reject,
|
514
|
+
retries: 0
|
515
|
+
});
|
516
|
+
}));
|
517
|
+
}
|
518
|
+
promises.push(this.localWorker[method](this.workerApi, ...args));
|
519
|
+
await Promise.all(promises);
|
520
|
+
}
|
521
|
+
async takeHeapSnapshot() {
|
522
|
+
let snapshotId = getTimeId();
|
523
|
+
try {
|
524
|
+
let snapshotPaths = await Promise.all([...this.workers.values()].map(worker => new Promise((resolve, reject) => {
|
525
|
+
worker.call({
|
526
|
+
method: 'takeHeapSnapshot',
|
527
|
+
args: [snapshotId],
|
528
|
+
resolve,
|
529
|
+
reject,
|
530
|
+
retries: 0,
|
531
|
+
skipReadyCheck: true
|
532
|
+
});
|
533
|
+
})));
|
534
|
+
_logger().default.info({
|
535
|
+
origin: '@parcel/workers',
|
536
|
+
message: (0, _diagnostic().md)`Wrote heap snapshots to the following paths:\n${snapshotPaths.join('\n')}`
|
537
|
+
});
|
538
|
+
} catch {
|
539
|
+
_logger().default.error({
|
540
|
+
origin: '@parcel/workers',
|
541
|
+
message: 'Unable to take heap snapshots. Note: requires Node 11.13.0+'
|
542
|
+
});
|
543
|
+
}
|
544
|
+
}
|
464
545
|
static getNumWorkers() {
|
465
|
-
return process.env.PARCEL_WORKERS ? parseInt(process.env.PARCEL_WORKERS, 10) : (0, _cpuCount.default)();
|
546
|
+
return process.env.PARCEL_WORKERS ? parseInt(process.env.PARCEL_WORKERS, 10) : Math.min(4, Math.ceil((0, _cpuCount.default)() / 2));
|
466
547
|
}
|
467
|
-
|
468
548
|
static isWorker() {
|
469
549
|
return !!_childState.child;
|
470
550
|
}
|
471
|
-
|
472
551
|
static getWorkerApi() {
|
473
|
-
(0, _assert.default)(_childState.child != null, 'WorkerFarm.getWorkerApi can only be called within workers');
|
552
|
+
(0, _assert().default)(_childState.child != null, 'WorkerFarm.getWorkerApi can only be called within workers');
|
474
553
|
return _childState.child.workerApi;
|
475
554
|
}
|
476
|
-
|
477
|
-
|
478
|
-
return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || 5;
|
555
|
+
static getConcurrentCallsPerWorker(defaultValue = DEFAULT_MAX_CONCURRENT_CALLS) {
|
556
|
+
return parseInt(process.env.PARCEL_MAX_CONCURRENT_CALLS, 10) || defaultValue;
|
479
557
|
}
|
480
|
-
|
481
558
|
}
|
482
|
-
|
483
|
-
|
559
|
+
exports.default = WorkerFarm;
|
560
|
+
function getTimeId() {
|
561
|
+
let now = new Date();
|
562
|
+
return String(now.getFullYear()) + String(now.getMonth() + 1).padStart(2, '0') + String(now.getDate()).padStart(2, '0') + '-' + String(now.getHours()).padStart(2, '0') + String(now.getMinutes()).padStart(2, '0') + String(now.getSeconds()).padStart(2, '0');
|
563
|
+
}
|