@loaders.gl/worker-utils 4.0.0-alpha.5 → 4.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es5/index.js +148 -0
- package/dist/es5/index.js.map +1 -0
- package/dist/es5/lib/async-queue/async-queue.js +100 -0
- package/dist/es5/lib/async-queue/async-queue.js.map +1 -0
- package/dist/es5/lib/env-utils/assert.js +12 -0
- package/dist/es5/lib/env-utils/assert.js.map +1 -0
- package/dist/es5/lib/env-utils/globals.js +32 -0
- package/dist/es5/lib/env-utils/globals.js.map +1 -0
- package/dist/es5/lib/env-utils/version.js +13 -0
- package/dist/es5/lib/env-utils/version.js.map +1 -0
- package/dist/es5/lib/library-utils/library-utils.js +154 -0
- package/dist/es5/lib/library-utils/library-utils.js.map +1 -0
- package/dist/es5/lib/node/require-utils.node.js +71 -0
- package/dist/es5/lib/node/require-utils.node.js.map +1 -0
- package/dist/es5/lib/node/worker_threads-browser.js +23 -0
- package/dist/es5/lib/node/worker_threads-browser.js.map +1 -0
- package/dist/es5/lib/node/worker_threads.js +27 -0
- package/dist/es5/lib/node/worker_threads.js.map +1 -0
- package/dist/es5/lib/process-utils/child-process-proxy.js +193 -0
- package/dist/es5/lib/process-utils/child-process-proxy.js.map +1 -0
- package/dist/es5/lib/process-utils/process-utils.js +33 -0
- package/dist/es5/lib/process-utils/process-utils.js.map +1 -0
- package/dist/es5/lib/worker-api/create-worker.js +166 -0
- package/dist/es5/lib/worker-api/create-worker.js.map +1 -0
- package/dist/es5/lib/worker-api/get-worker-url.js +38 -0
- package/dist/es5/lib/worker-api/get-worker-url.js.map +1 -0
- package/dist/es5/lib/worker-api/process-on-worker.js +137 -0
- package/dist/es5/lib/worker-api/process-on-worker.js.map +1 -0
- package/dist/es5/lib/worker-api/validate-worker-version.js +25 -0
- package/dist/es5/lib/worker-api/validate-worker-version.js.map +1 -0
- package/dist/es5/lib/worker-farm/worker-body.js +109 -0
- package/dist/es5/lib/worker-farm/worker-body.js.map +1 -0
- package/dist/es5/lib/worker-farm/worker-farm.js +113 -0
- package/dist/es5/lib/worker-farm/worker-farm.js.map +1 -0
- package/dist/es5/lib/worker-farm/worker-job.js +56 -0
- package/dist/es5/lib/worker-farm/worker-job.js.map +1 -0
- package/dist/es5/lib/worker-farm/worker-pool.js +211 -0
- package/dist/es5/lib/worker-farm/worker-pool.js.map +1 -0
- package/dist/es5/lib/worker-farm/worker-thread.js +136 -0
- package/dist/es5/lib/worker-farm/worker-thread.js.map +1 -0
- package/dist/es5/lib/worker-utils/get-loadable-worker-url.js +41 -0
- package/dist/es5/lib/worker-utils/get-loadable-worker-url.js.map +1 -0
- package/dist/es5/lib/worker-utils/get-transfer-list.js +59 -0
- package/dist/es5/lib/worker-utils/get-transfer-list.js.map +1 -0
- package/dist/es5/lib/worker-utils/remove-nontransferable-options.js +25 -0
- package/dist/es5/lib/worker-utils/remove-nontransferable-options.js.map +1 -0
- package/dist/es5/types.js +2 -0
- package/dist/es5/types.js.map +1 -0
- package/dist/es5/workers/null-worker.js +23 -0
- package/dist/es5/workers/null-worker.js.map +1 -0
- package/dist/esm/index.js +26 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/lib/async-queue/async-queue.js +76 -0
- package/dist/esm/lib/async-queue/async-queue.js.map +1 -0
- package/dist/esm/lib/env-utils/assert.js +6 -0
- package/dist/esm/lib/env-utils/assert.js.map +1 -0
- package/dist/esm/lib/env-utils/globals.js +17 -0
- package/dist/esm/lib/env-utils/globals.js.map +1 -0
- package/dist/esm/lib/env-utils/version.js +6 -0
- package/dist/esm/lib/env-utils/version.js.map +1 -0
- package/dist/esm/lib/library-utils/library-utils.js +74 -0
- package/dist/esm/lib/library-utils/library-utils.js.map +1 -0
- package/dist/esm/lib/node/require-utils.node.js +36 -0
- package/dist/esm/lib/node/require-utils.node.js.map +1 -0
- package/dist/esm/lib/node/worker_threads-browser.js +7 -0
- package/dist/esm/lib/node/worker_threads-browser.js.map +1 -0
- package/dist/esm/lib/node/worker_threads.js +4 -0
- package/dist/esm/lib/node/worker_threads.js.map +1 -0
- package/dist/esm/lib/process-utils/child-process-proxy.js +104 -0
- package/dist/esm/lib/process-utils/child-process-proxy.js.map +1 -0
- package/dist/esm/lib/process-utils/process-utils.js +26 -0
- package/dist/esm/lib/process-utils/process-utils.js.map +1 -0
- package/dist/esm/lib/worker-api/create-worker.js +84 -0
- package/dist/esm/lib/worker-api/create-worker.js.map +1 -0
- package/dist/esm/lib/worker-api/get-worker-url.js +31 -0
- package/dist/esm/lib/worker-api/get-worker-url.js.map +1 -0
- package/dist/esm/lib/worker-api/process-on-worker.js +75 -0
- package/dist/esm/lib/worker-api/process-on-worker.js.map +1 -0
- package/dist/esm/lib/worker-api/validate-worker-version.js +19 -0
- package/dist/esm/lib/worker-api/validate-worker-version.js.map +1 -0
- package/dist/esm/lib/worker-farm/worker-body.js +86 -0
- package/dist/esm/lib/worker-farm/worker-body.js.map +1 -0
- package/dist/esm/lib/worker-farm/worker-farm.js +72 -0
- package/dist/esm/lib/worker-farm/worker-farm.js.map +1 -0
- package/dist/esm/lib/worker-farm/worker-job.js +36 -0
- package/dist/esm/lib/worker-farm/worker-job.js.map +1 -0
- package/dist/esm/lib/worker-farm/worker-pool.js +124 -0
- package/dist/esm/lib/worker-farm/worker-pool.js.map +1 -0
- package/dist/esm/lib/worker-farm/worker-thread.js +105 -0
- package/dist/esm/lib/worker-farm/worker-thread.js.map +1 -0
- package/dist/esm/lib/worker-utils/get-loadable-worker-url.js +35 -0
- package/dist/esm/lib/worker-utils/get-loadable-worker-url.js.map +1 -0
- package/dist/esm/lib/worker-utils/get-transfer-list.js +50 -0
- package/dist/esm/lib/worker-utils/get-transfer-list.js.map +1 -0
- package/dist/esm/lib/worker-utils/remove-nontransferable-options.js +17 -0
- package/dist/esm/lib/worker-utils/remove-nontransferable-options.js.map +1 -0
- package/dist/esm/types.js +2 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/esm/workers/null-worker.js +5 -0
- package/dist/esm/workers/null-worker.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +58 -25
- package/dist/lib/async-queue/async-queue.js +81 -91
- package/dist/lib/env-utils/assert.js +12 -5
- package/dist/lib/env-utils/globals.js +25 -10
- package/dist/lib/env-utils/version.js +10 -5
- package/dist/lib/library-utils/library-utils.d.ts.map +1 -1
- package/dist/lib/library-utils/library-utils.js +161 -79
- package/dist/lib/node/require-utils.node.js +58 -41
- package/dist/lib/node/worker_threads-browser.d.ts +7 -0
- package/dist/lib/node/worker_threads-browser.d.ts.map +1 -0
- package/dist/lib/node/worker_threads-browser.js +15 -0
- package/dist/lib/node/worker_threads.d.ts +6 -0
- package/dist/lib/node/worker_threads.d.ts.map +1 -0
- package/dist/lib/node/worker_threads.js +33 -0
- package/dist/lib/process-utils/child-process-proxy.d.ts +1 -1
- package/dist/lib/process-utils/child-process-proxy.d.ts.map +1 -1
- package/dist/lib/process-utils/child-process-proxy.js +126 -113
- package/dist/lib/process-utils/process-utils.js +33 -27
- package/dist/lib/worker-api/create-worker.d.ts +1 -1
- package/dist/lib/worker-api/create-worker.d.ts.map +1 -1
- package/dist/lib/worker-api/create-worker.js +84 -92
- package/dist/lib/worker-api/get-worker-url.d.ts.map +1 -1
- package/dist/lib/worker-api/get-worker-url.js +55 -33
- package/dist/lib/worker-api/process-on-worker.d.ts +1 -1
- package/dist/lib/worker-api/process-on-worker.d.ts.map +1 -1
- package/dist/lib/worker-api/process-on-worker.js +79 -71
- package/dist/lib/worker-api/validate-worker-version.js +32 -18
- package/dist/lib/worker-farm/worker-body.d.ts +2 -0
- package/dist/lib/worker-farm/worker-body.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-body.js +104 -58
- package/dist/lib/worker-farm/worker-farm.d.ts +7 -3
- package/dist/lib/worker-farm/worker-farm.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-farm.js +83 -77
- package/dist/lib/worker-farm/worker-job.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-job.js +46 -53
- package/dist/lib/worker-farm/worker-pool.d.ts +6 -4
- package/dist/lib/worker-farm/worker-pool.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-pool.js +145 -147
- package/dist/lib/worker-farm/worker-thread.d.ts +10 -3
- package/dist/lib/worker-farm/worker-thread.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-thread.js +128 -102
- package/dist/lib/worker-utils/get-loadable-worker-url.js +63 -33
- package/dist/lib/worker-utils/get-transfer-list.d.ts +5 -0
- package/dist/lib/worker-utils/get-transfer-list.d.ts.map +1 -1
- package/dist/lib/worker-utils/get-transfer-list.js +83 -36
- package/dist/lib/worker-utils/remove-nontransferable-options.d.ts +3 -3
- package/dist/lib/worker-utils/remove-nontransferable-options.d.ts.map +1 -1
- package/dist/lib/worker-utils/remove-nontransferable-options.js +25 -21
- package/dist/null-worker.js +45 -14
- package/dist/null-worker.js.map +2 -2
- package/dist/types.d.ts +16 -11
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -2
- package/dist/workers/null-worker.js +6 -4
- package/package.json +8 -4
- package/src/index.ts +1 -1
- package/src/lib/library-utils/library-utils.ts +4 -1
- package/src/lib/node/worker_threads-browser.ts +13 -0
- package/src/lib/node/worker_threads.ts +6 -0
- package/src/lib/worker-api/create-worker.ts +1 -2
- package/src/lib/worker-api/get-worker-url.ts +3 -1
- package/src/lib/worker-api/process-on-worker.ts +8 -4
- package/src/lib/worker-farm/worker-body.ts +61 -19
- package/src/lib/worker-farm/worker-farm.ts +10 -5
- package/src/lib/worker-farm/worker-job.ts +3 -6
- package/src/lib/worker-farm/worker-pool.ts +5 -0
- package/src/lib/worker-farm/worker-thread.ts +45 -7
- package/src/lib/worker-utils/get-loadable-worker-url.ts +0 -2
- package/src/lib/worker-utils/get-transfer-list.ts +28 -0
- package/src/lib/worker-utils/remove-nontransferable-options.ts +17 -25
- package/src/types.ts +3 -2
- package/dist/index.js.map +0 -1
- package/dist/lib/async-queue/async-queue.js.map +0 -1
- package/dist/lib/env-utils/assert.js.map +0 -1
- package/dist/lib/env-utils/globals.js.map +0 -1
- package/dist/lib/env-utils/version.js.map +0 -1
- package/dist/lib/library-utils/library-utils.js.map +0 -1
- package/dist/lib/node/require-utils.node.js.map +0 -1
- package/dist/lib/process-utils/child-process-proxy.js.map +0 -1
- package/dist/lib/process-utils/process-utils.js.map +0 -1
- package/dist/lib/worker-api/create-worker.js.map +0 -1
- package/dist/lib/worker-api/get-worker-url.js.map +0 -1
- package/dist/lib/worker-api/process-on-worker.js.map +0 -1
- package/dist/lib/worker-api/validate-worker-version.js.map +0 -1
- package/dist/lib/worker-farm/worker-body.js.map +0 -1
- package/dist/lib/worker-farm/worker-farm.js.map +0 -1
- package/dist/lib/worker-farm/worker-job.js.map +0 -1
- package/dist/lib/worker-farm/worker-pool.js.map +0 -1
- package/dist/lib/worker-farm/worker-thread.js.map +0 -1
- package/dist/lib/worker-utils/get-loadable-worker-url.js.map +0 -1
- package/dist/lib/worker-utils/get-transfer-list.js.map +0 -1
- package/dist/lib/worker-utils/remove-nontransferable-options.js.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/workers/null-worker.js.map +0 -1
|
@@ -1,75 +1,83 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.processOnWorker = exports.canProcessOnWorker = void 0;
|
|
7
|
+
const worker_farm_1 = __importDefault(require("../worker-farm/worker-farm"));
|
|
8
|
+
const get_worker_url_1 = require("./get-worker-url");
|
|
9
|
+
const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
|
|
10
|
+
/**
|
|
11
|
+
* Determines if we can parse with worker
|
|
12
|
+
* @param loader
|
|
13
|
+
* @param data
|
|
14
|
+
* @param options
|
|
15
|
+
*/
|
|
16
|
+
function canProcessOnWorker(worker, options) {
|
|
17
|
+
if (!worker_farm_1.default.isSupported()) {
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
return worker.worker && options?.worker;
|
|
10
21
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
22
|
+
exports.canProcessOnWorker = canProcessOnWorker;
|
|
23
|
+
/**
|
|
24
|
+
* This function expects that the worker thread sends certain messages,
|
|
25
|
+
* Creating such a worker can be automated if the worker is wrapper by a call to
|
|
26
|
+
* createWorker in @loaders.gl/worker-utils.
|
|
27
|
+
*/
|
|
28
|
+
async function processOnWorker(worker, data, options = {}, context = {}) {
|
|
29
|
+
const name = (0, get_worker_url_1.getWorkerName)(worker);
|
|
30
|
+
const workerFarm = worker_farm_1.default.getWorkerFarm(options);
|
|
31
|
+
const { source } = options;
|
|
32
|
+
const workerPoolProps = { name, source };
|
|
33
|
+
if (!source) {
|
|
34
|
+
workerPoolProps.url = (0, get_worker_url_1.getWorkerURL)(worker, options);
|
|
35
|
+
}
|
|
36
|
+
const workerPool = workerFarm.getWorkerPool(workerPoolProps);
|
|
37
|
+
const jobName = options.jobName || worker.name;
|
|
38
|
+
const job = await workerPool.startJob(jobName,
|
|
39
|
+
// eslint-disable-next-line
|
|
40
|
+
onMessage.bind(null, context));
|
|
41
|
+
// Kick off the processing in the worker
|
|
42
|
+
const transferableOptions = (0, get_transfer_list_1.getTransferListForWriter)(options);
|
|
43
|
+
job.postMessage('process', { input: data, options: transferableOptions });
|
|
44
|
+
const result = await job.result;
|
|
45
|
+
return result.result;
|
|
28
46
|
}
|
|
29
|
-
|
|
47
|
+
exports.processOnWorker = processOnWorker;
|
|
48
|
+
/**
|
|
49
|
+
* Job completes when we receive the result
|
|
50
|
+
* @param job
|
|
51
|
+
* @param message
|
|
52
|
+
*/
|
|
30
53
|
async function onMessage(context, job, type, payload) {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
error
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
});
|
|
61
|
-
} catch (error) {
|
|
62
|
-
const message = error instanceof Error ? error.message : 'unknown error';
|
|
63
|
-
job.postMessage('error', {
|
|
64
|
-
id,
|
|
65
|
-
error: message
|
|
66
|
-
});
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
break;
|
|
70
|
-
|
|
71
|
-
default:
|
|
72
|
-
console.warn("process-on-worker: unknown message ".concat(type));
|
|
73
|
-
}
|
|
54
|
+
switch (type) {
|
|
55
|
+
case 'done':
|
|
56
|
+
// Worker is done
|
|
57
|
+
job.done(payload);
|
|
58
|
+
break;
|
|
59
|
+
case 'error':
|
|
60
|
+
// Worker encountered an error
|
|
61
|
+
job.error(new Error(payload.error));
|
|
62
|
+
break;
|
|
63
|
+
case 'process':
|
|
64
|
+
// Worker is asking for us (main thread) to process something
|
|
65
|
+
const { id, input, options } = payload;
|
|
66
|
+
try {
|
|
67
|
+
if (!context.process) {
|
|
68
|
+
job.postMessage('error', { id, error: 'Worker not set up to process on main thread' });
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const result = await context.process(input, options);
|
|
72
|
+
job.postMessage('done', { id, result });
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
const message = error instanceof Error ? error.message : 'unknown error';
|
|
76
|
+
job.postMessage('error', { id, error: message });
|
|
77
|
+
}
|
|
78
|
+
break;
|
|
79
|
+
default:
|
|
80
|
+
// eslint-disable-next-line
|
|
81
|
+
console.warn(`process-on-worker: unknown message ${type}`);
|
|
82
|
+
}
|
|
74
83
|
}
|
|
75
|
-
//# sourceMappingURL=process-on-worker.js.map
|
|
@@ -1,21 +1,35 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.validateWorkerVersion = void 0;
|
|
4
|
+
const assert_1 = require("../env-utils/assert");
|
|
5
|
+
const version_1 = require("../env-utils/version");
|
|
6
|
+
/**
|
|
7
|
+
* Check if worker is compatible with this library version
|
|
8
|
+
* @param worker
|
|
9
|
+
* @param libVersion
|
|
10
|
+
* @returns `true` if the two versions are compatible
|
|
11
|
+
*/
|
|
12
|
+
function validateWorkerVersion(worker, coreVersion = version_1.VERSION) {
|
|
13
|
+
(0, assert_1.assert)(worker, 'no worker provided');
|
|
14
|
+
const workerVersion = worker.version;
|
|
15
|
+
if (!coreVersion || !workerVersion) {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
// TODO enable when fix the __version__ injection
|
|
19
|
+
// const coreVersions = parseVersion(coreVersion);
|
|
20
|
+
// const workerVersions = parseVersion(workerVersion);
|
|
21
|
+
// assert(
|
|
22
|
+
// coreVersion.major === workerVersion.major && coreVersion.minor <= workerVersion.minor,
|
|
23
|
+
// `worker: ${worker.name} is not compatible. ${coreVersion.major}.${
|
|
24
|
+
// coreVersion.minor
|
|
25
|
+
// }+ is required.`
|
|
26
|
+
// );
|
|
27
|
+
return true;
|
|
12
28
|
}
|
|
13
|
-
|
|
29
|
+
exports.validateWorkerVersion = validateWorkerVersion;
|
|
30
|
+
// @ts-ignore
|
|
31
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
14
32
|
function parseVersion(version) {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
major: parts[0],
|
|
18
|
-
minor: parts[1]
|
|
19
|
-
};
|
|
33
|
+
const parts = version.split('.').map(Number);
|
|
34
|
+
return { major: parts[0], minor: parts[1] };
|
|
20
35
|
}
|
|
21
|
-
//# sourceMappingURL=validate-worker-version.js.map
|
|
@@ -3,6 +3,8 @@ import type { WorkerMessageType, WorkerMessagePayload } from '../../types';
|
|
|
3
3
|
* Type safe wrapper for worker code
|
|
4
4
|
*/
|
|
5
5
|
export default class WorkerBody {
|
|
6
|
+
/** Check that we are actually in a worker thread */
|
|
7
|
+
static inWorkerThread(): boolean;
|
|
6
8
|
static set onmessage(onMessage: (type: WorkerMessageType, payload: WorkerMessagePayload) => any);
|
|
7
9
|
static addEventListener(onMessage: (type: WorkerMessageType, payload: WorkerMessagePayload) => any): void;
|
|
8
10
|
static removeEventListener(onMessage: (type: WorkerMessageType, payload: WorkerMessagePayload) => any): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-body.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-body.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAoB,iBAAiB,EAAE,oBAAoB,EAAC,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-body.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-body.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAoB,iBAAiB,EAAE,oBAAoB,EAAC,MAAM,aAAa,CAAC;AAkB5F;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU;IAC7B,oDAAoD;IACpD,MAAM,CAAC,cAAc,IAAI,OAAO;IAOhC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,oBAAoB,KAAK,GAAG,EAqB9F;IAED,MAAM,CAAC,gBAAgB,CACrB,SAAS,EAAE,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,oBAAoB,KAAK,GAAG;IAyB5E,MAAM,CAAC,mBAAmB,CACxB,SAAS,EAAE,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,oBAAoB,KAAK,GAAG;IAY5E;;;;OAIG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,oBAAoB,GAAG,IAAI;CAcjF"}
|
|
@@ -1,65 +1,111 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const get_transfer_list_1 = require("../worker-utils/get-transfer-list");
|
|
4
|
+
/** Vile hack to defeat over-zealous bundlers from stripping out the require */
|
|
5
|
+
function getParentPort() {
|
|
6
|
+
// const isNode = globalThis.process;
|
|
7
|
+
let parentPort;
|
|
8
|
+
try {
|
|
9
|
+
// prettier-ignore
|
|
10
|
+
eval('globalThis.parentPort = require(\'worker_threads\').parentPort'); // eslint-disable-line no-eval
|
|
11
|
+
parentPort = globalThis.parentPort;
|
|
12
|
+
// eslint-disable-next-line no-empty
|
|
13
|
+
}
|
|
14
|
+
catch { }
|
|
15
|
+
return parentPort;
|
|
16
|
+
}
|
|
2
17
|
const onMessageWrapperMap = new Map();
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
18
|
+
/**
|
|
19
|
+
* Type safe wrapper for worker code
|
|
20
|
+
*/
|
|
21
|
+
class WorkerBody {
|
|
22
|
+
/** Check that we are actually in a worker thread */
|
|
23
|
+
static inWorkerThread() {
|
|
24
|
+
return typeof self !== 'undefined' || Boolean(getParentPort());
|
|
25
|
+
}
|
|
26
|
+
/*
|
|
27
|
+
* (type: WorkerMessageType, payload: WorkerMessagePayload) => any
|
|
28
|
+
*/
|
|
29
|
+
static set onmessage(onMessage) {
|
|
30
|
+
function handleMessage(message) {
|
|
31
|
+
// Confusingly the message itself also has a 'type' field which is always set to 'message'
|
|
32
|
+
const parentPort = getParentPort();
|
|
33
|
+
const { type, payload } = parentPort ? message : message.data;
|
|
34
|
+
// if (!isKnownMessage(message)) {
|
|
35
|
+
// return;
|
|
36
|
+
// }
|
|
37
|
+
onMessage(type, payload);
|
|
38
|
+
}
|
|
39
|
+
const parentPort = getParentPort();
|
|
40
|
+
if (parentPort) {
|
|
41
|
+
parentPort.on('message', handleMessage);
|
|
42
|
+
// if (message == 'exit') { parentPort.unref(); }
|
|
43
|
+
// eslint-disable-next-line
|
|
44
|
+
parentPort.on('exit', () => console.debug('Node worker closing'));
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
// eslint-disable-next-line no-restricted-globals
|
|
48
|
+
globalThis.onmessage = handleMessage;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
static addEventListener(onMessage) {
|
|
52
|
+
let onMessageWrapper = onMessageWrapperMap.get(onMessage);
|
|
53
|
+
if (!onMessageWrapper) {
|
|
54
|
+
onMessageWrapper = (message) => {
|
|
55
|
+
if (!isKnownMessage(message)) {
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
// Confusingly in the browser, the message itself also has a 'type' field which is always set to 'message'
|
|
59
|
+
const parentPort = getParentPort();
|
|
60
|
+
const { type, payload } = parentPort ? message : message.data;
|
|
61
|
+
onMessage(type, payload);
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
const parentPort = getParentPort();
|
|
65
|
+
if (parentPort) {
|
|
66
|
+
console.error('not implemented'); // eslint-disable-line
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
globalThis.addEventListener('message', onMessageWrapper);
|
|
25
70
|
}
|
|
26
|
-
|
|
27
|
-
const {
|
|
28
|
-
type,
|
|
29
|
-
payload
|
|
30
|
-
} = message.data;
|
|
31
|
-
onMessage(type, payload);
|
|
32
|
-
};
|
|
33
71
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
72
|
+
static removeEventListener(onMessage) {
|
|
73
|
+
const onMessageWrapper = onMessageWrapperMap.get(onMessage);
|
|
74
|
+
onMessageWrapperMap.delete(onMessage);
|
|
75
|
+
const parentPort = getParentPort();
|
|
76
|
+
if (parentPort) {
|
|
77
|
+
console.error('not implemented'); // eslint-disable-line
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
globalThis.removeEventListener('message', onMessageWrapper);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Send a message from a worker to creating thread (main thread)
|
|
85
|
+
* @param type
|
|
86
|
+
* @param payload
|
|
87
|
+
*/
|
|
88
|
+
static postMessage(type, payload) {
|
|
89
|
+
const data = { source: 'loaders.gl', type, payload };
|
|
90
|
+
// console.log('posting message', data);
|
|
91
|
+
const transferList = (0, get_transfer_list_1.getTransferList)(payload);
|
|
92
|
+
const parentPort = getParentPort();
|
|
93
|
+
if (parentPort) {
|
|
94
|
+
parentPort.postMessage(data, transferList);
|
|
95
|
+
// console.log('posted message', data);
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
// @ts-ignore
|
|
99
|
+
globalThis.postMessage(data, transferList);
|
|
100
|
+
}
|
|
53
101
|
}
|
|
54
|
-
}
|
|
55
|
-
|
|
56
102
|
}
|
|
57
|
-
|
|
103
|
+
exports.default = WorkerBody;
|
|
104
|
+
// Filter out noise messages sent to workers
|
|
58
105
|
function isKnownMessage(message) {
|
|
59
|
-
|
|
60
|
-
type
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
106
|
+
const { type, data } = message;
|
|
107
|
+
return (type === 'message' &&
|
|
108
|
+
data &&
|
|
109
|
+
typeof data.source === 'string' &&
|
|
110
|
+
data.source.startsWith('loaders.gl'));
|
|
64
111
|
}
|
|
65
|
-
//# sourceMappingURL=worker-body.js.map
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import WorkerPool from './worker-pool';
|
|
2
2
|
/**
|
|
3
|
-
* @param maxConcurrency
|
|
3
|
+
* @param maxConcurrency - max count of workers
|
|
4
|
+
* @param maxMobileConcurrency - max count of workers on mobile
|
|
5
|
+
* @param maxConcurrency - max count of workers
|
|
6
|
+
* @param reuseWorkers - if false, destroys workers when task is completed
|
|
7
|
+
* @param onDebug - callback intended to allow application to log worker pool activity
|
|
4
8
|
*/
|
|
5
|
-
export
|
|
9
|
+
export type WorkerFarmProps = {
|
|
6
10
|
maxConcurrency?: number;
|
|
7
11
|
maxMobileConcurrency?: number;
|
|
8
12
|
reuseWorkers?: boolean;
|
|
@@ -15,7 +19,7 @@ export default class WorkerFarm {
|
|
|
15
19
|
private props;
|
|
16
20
|
private workerPools;
|
|
17
21
|
private static _workerFarm?;
|
|
18
|
-
/**
|
|
22
|
+
/** Checks if workers are supported on this platform */
|
|
19
23
|
static isSupported(): boolean;
|
|
20
24
|
/** Get the singleton instance of the global worker farm */
|
|
21
25
|
static getWorkerFarm(props?: WorkerFarmProps): WorkerFarm;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-farm.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-farm.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,eAAe,CAAC;AAGvC
|
|
1
|
+
{"version":3,"file":"worker-farm.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-farm.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,eAAe,CAAC;AAGvC;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;CACtB,CAAC;AASF;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU;IAC7B,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,WAAW,CAAiC;IAEpD,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAa;IAExC,uDAAuD;IACvD,MAAM,CAAC,WAAW,IAAI,OAAO;IAI7B,2DAA2D;IAC3D,MAAM,CAAC,aAAa,CAAC,KAAK,GAAE,eAAoB,GAAG,UAAU;IAM7D,0DAA0D;IAC1D,OAAO;IAOP;;;OAGG;IACH,OAAO,IAAI,IAAI;IAOf;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI;IAQtC;;;;;;;;OAQG;IACH,aAAa,CAAC,OAAO,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAC,GAAG,UAAU;IAejF,mBAAmB;;;;;;CAQpB"}
|
|
@@ -1,83 +1,89 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const worker_pool_1 = __importDefault(require("./worker-pool"));
|
|
7
|
+
const worker_thread_1 = __importDefault(require("./worker-thread"));
|
|
4
8
|
const DEFAULT_PROPS = {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
+
maxConcurrency: 3,
|
|
10
|
+
maxMobileConcurrency: 1,
|
|
11
|
+
reuseWorkers: true,
|
|
12
|
+
onDebug: () => { }
|
|
9
13
|
};
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Process multiple jobs with a "farm" of different workers in worker pools.
|
|
16
|
+
*/
|
|
17
|
+
class WorkerFarm {
|
|
18
|
+
/** Checks if workers are supported on this platform */
|
|
19
|
+
static isSupported() {
|
|
20
|
+
return worker_thread_1.default.isSupported();
|
|
21
|
+
}
|
|
22
|
+
/** Get the singleton instance of the global worker farm */
|
|
23
|
+
static getWorkerFarm(props = {}) {
|
|
24
|
+
WorkerFarm._workerFarm = WorkerFarm._workerFarm || new WorkerFarm({});
|
|
25
|
+
WorkerFarm._workerFarm.setProps(props);
|
|
26
|
+
return WorkerFarm._workerFarm;
|
|
27
|
+
}
|
|
28
|
+
/** get global instance with WorkerFarm.getWorkerFarm() */
|
|
29
|
+
constructor(props) {
|
|
30
|
+
this.workerPools = new Map();
|
|
31
|
+
this.props = { ...DEFAULT_PROPS };
|
|
32
|
+
this.setProps(props);
|
|
33
|
+
/** @type Map<string, WorkerPool>} */
|
|
34
|
+
this.workerPools = new Map();
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Terminate all workers in the farm
|
|
38
|
+
* @note Can free up significant memory
|
|
39
|
+
*/
|
|
40
|
+
destroy() {
|
|
41
|
+
for (const workerPool of this.workerPools.values()) {
|
|
42
|
+
workerPool.destroy();
|
|
43
|
+
}
|
|
44
|
+
this.workerPools = new Map();
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Set props used when initializing worker pools
|
|
48
|
+
* @param props
|
|
49
|
+
*/
|
|
50
|
+
setProps(props) {
|
|
51
|
+
this.props = { ...this.props, ...props };
|
|
52
|
+
// Update worker pool props
|
|
53
|
+
for (const workerPool of this.workerPools.values()) {
|
|
54
|
+
workerPool.setProps(this._getWorkerPoolProps());
|
|
55
|
+
}
|
|
37
56
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Returns a worker pool for the specified worker
|
|
59
|
+
* @param options - only used first time for a specific worker name
|
|
60
|
+
* @param options.name - the name of the worker - used to identify worker pool
|
|
61
|
+
* @param options.url -
|
|
62
|
+
* @param options.source -
|
|
63
|
+
* @example
|
|
64
|
+
* const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
|
|
65
|
+
*/
|
|
66
|
+
getWorkerPool(options) {
|
|
67
|
+
const { name, source, url } = options;
|
|
68
|
+
let workerPool = this.workerPools.get(name);
|
|
69
|
+
if (!workerPool) {
|
|
70
|
+
workerPool = new worker_pool_1.default({
|
|
71
|
+
name,
|
|
72
|
+
source,
|
|
73
|
+
url
|
|
74
|
+
});
|
|
75
|
+
workerPool.setProps(this._getWorkerPoolProps());
|
|
76
|
+
this.workerPools.set(name, workerPool);
|
|
77
|
+
}
|
|
78
|
+
return workerPool;
|
|
47
79
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
} = options;
|
|
56
|
-
let workerPool = this.workerPools.get(name);
|
|
57
|
-
|
|
58
|
-
if (!workerPool) {
|
|
59
|
-
workerPool = new WorkerPool({
|
|
60
|
-
name,
|
|
61
|
-
source,
|
|
62
|
-
url
|
|
63
|
-
});
|
|
64
|
-
workerPool.setProps(this._getWorkerPoolProps());
|
|
65
|
-
this.workerPools.set(name, workerPool);
|
|
80
|
+
_getWorkerPoolProps() {
|
|
81
|
+
return {
|
|
82
|
+
maxConcurrency: this.props.maxConcurrency,
|
|
83
|
+
maxMobileConcurrency: this.props.maxMobileConcurrency,
|
|
84
|
+
reuseWorkers: this.props.reuseWorkers,
|
|
85
|
+
onDebug: this.props.onDebug
|
|
86
|
+
};
|
|
66
87
|
}
|
|
67
|
-
|
|
68
|
-
return workerPool;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
_getWorkerPoolProps() {
|
|
72
|
-
return {
|
|
73
|
-
maxConcurrency: this.props.maxConcurrency,
|
|
74
|
-
maxMobileConcurrency: this.props.maxMobileConcurrency,
|
|
75
|
-
reuseWorkers: this.props.reuseWorkers,
|
|
76
|
-
onDebug: this.props.onDebug
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
|
|
80
88
|
}
|
|
81
|
-
|
|
82
|
-
_defineProperty(WorkerFarm, "_workerFarm", void 0);
|
|
83
|
-
//# sourceMappingURL=worker-farm.js.map
|
|
89
|
+
exports.default = WorkerFarm;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-job.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-job.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,iBAAiB,EAAE,oBAAoB,EAAC,MAAM,aAAa,CAAC;AACzE,OAAO,YAAY,MAAM,iBAAiB,CAAC;AAG3C;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,SAAS;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC,SAAS,EAAE,OAAO,
|
|
1
|
+
{"version":3,"file":"worker-job.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-job.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,iBAAiB,EAAE,oBAAoB,EAAC,MAAM,aAAa,CAAC;AACzE,OAAO,YAAY,MAAM,iBAAiB,CAAC;AAG3C;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,SAAS;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC,SAAS,EAAE,OAAO,CAAQ;IAC1B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAE9B,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,OAAO,CAAoC;gBAEvC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY;IASvD;;;OAGG;IACH,WAAW,CAAC,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAQzE;;OAEG;IACH,IAAI,CAAC,KAAK,EAAE,GAAG,GAAG,IAAI;IAMtB;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;CAK1B"}
|