@loaders.gl/worker-utils 4.2.0-alpha.4 → 4.2.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +225 -86
- package/dist/index.cjs.map +7 -0
- package/dist/index.d.ts +18 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +19 -8
- package/dist/lib/async-queue/async-queue.js +83 -68
- package/dist/lib/env-utils/assert.js +10 -4
- package/dist/lib/env-utils/globals.js +19 -7
- package/dist/lib/env-utils/version.js +20 -11
- package/dist/lib/library-utils/library-utils.js +148 -73
- package/dist/lib/node/require-utils.node.js +72 -43
- package/dist/lib/node/worker_threads-browser.js +9 -2
- package/dist/lib/node/worker_threads.js +4 -2
- package/dist/lib/process-utils/child-process-proxy.d.ts +1 -3
- package/dist/lib/process-utils/child-process-proxy.d.ts.map +1 -1
- package/dist/lib/process-utils/child-process-proxy.js +103 -93
- package/dist/lib/process-utils/process-utils.js +28 -23
- package/dist/lib/worker-api/create-worker.d.ts +1 -1
- package/dist/lib/worker-api/create-worker.d.ts.map +1 -1
- package/dist/lib/worker-api/create-worker.js +77 -74
- package/dist/lib/worker-api/get-worker-url.d.ts +1 -1
- package/dist/lib/worker-api/get-worker-url.d.ts.map +1 -1
- package/dist/lib/worker-api/get-worker-url.js +52 -26
- package/dist/lib/worker-api/process-on-worker.d.ts +1 -1
- package/dist/lib/worker-api/process-on-worker.d.ts.map +1 -1
- package/dist/lib/worker-api/process-on-worker.js +70 -67
- package/dist/lib/worker-api/validate-worker-version.d.ts +1 -1
- package/dist/lib/worker-api/validate-worker-version.d.ts.map +1 -1
- package/dist/lib/worker-api/validate-worker-version.js +29 -14
- package/dist/lib/worker-farm/worker-body.d.ts +1 -1
- package/dist/lib/worker-farm/worker-body.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-body.js +109 -68
- package/dist/lib/worker-farm/worker-farm.d.ts +1 -1
- package/dist/lib/worker-farm/worker-farm.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-farm.js +80 -62
- package/dist/lib/worker-farm/worker-job.d.ts +2 -2
- package/dist/lib/worker-farm/worker-job.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-job.js +48 -32
- package/dist/lib/worker-farm/worker-pool.d.ts +3 -3
- package/dist/lib/worker-farm/worker-pool.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-pool.js +153 -111
- package/dist/lib/worker-farm/worker-thread.d.ts +1 -1
- package/dist/lib/worker-farm/worker-thread.d.ts.map +1 -1
- package/dist/lib/worker-farm/worker-thread.js +126 -94
- package/dist/lib/worker-utils/get-loadable-worker-url.js +54 -24
- package/dist/lib/worker-utils/get-transfer-list.js +79 -44
- package/dist/lib/worker-utils/remove-nontransferable-options.js +23 -14
- package/dist/null-worker-node.js +6 -1
- package/dist/null-worker-node.js.map +2 -2
- package/dist/null-worker.js +3 -1
- package/dist/null-worker.js.map +2 -2
- package/dist/types.js +3 -1
- package/dist/workers/null-worker.js +6 -3
- package/package.json +11 -11
- package/src/lib/process-utils/child-process-proxy.ts +1 -1
- package/src/lib/worker-farm/worker-body.ts +7 -2
- package/dist/index.js.map +0 -1
- package/dist/lib/async-queue/async-queue.js.map +0 -1
- package/dist/lib/env-utils/assert.js.map +0 -1
- package/dist/lib/env-utils/globals.js.map +0 -1
- package/dist/lib/env-utils/version.js.map +0 -1
- package/dist/lib/library-utils/library-utils.js.map +0 -1
- package/dist/lib/node/require-utils.node.js.map +0 -1
- package/dist/lib/node/worker_threads-browser.js.map +0 -1
- package/dist/lib/node/worker_threads.js.map +0 -1
- package/dist/lib/process-utils/child-process-proxy.js.map +0 -1
- package/dist/lib/process-utils/process-utils.js.map +0 -1
- package/dist/lib/worker-api/create-worker.js.map +0 -1
- package/dist/lib/worker-api/get-worker-url.js.map +0 -1
- package/dist/lib/worker-api/process-on-worker.js.map +0 -1
- package/dist/lib/worker-api/validate-worker-version.js.map +0 -1
- package/dist/lib/worker-farm/worker-body.js.map +0 -1
- package/dist/lib/worker-farm/worker-farm.js.map +0 -1
- package/dist/lib/worker-farm/worker-job.js.map +0 -1
- package/dist/lib/worker-farm/worker-pool.js.map +0 -1
- package/dist/lib/worker-farm/worker-thread.js.map +0 -1
- package/dist/lib/worker-utils/get-loadable-worker-url.js.map +0 -1
- package/dist/lib/worker-utils/get-transfer-list.js.map +0 -1
- package/dist/lib/worker-utils/remove-nontransferable-options.js.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/workers/null-worker.js.map +0 -1
|
@@ -1,125 +1,167 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { isMobile, isBrowser } from "../env-utils/globals.js";
|
|
2
5
|
import WorkerThread from "./worker-thread.js";
|
|
3
6
|
import WorkerJob from "./worker-job.js";
|
|
7
|
+
/**
|
|
8
|
+
* Process multiple data messages with small pool of identical workers
|
|
9
|
+
*/
|
|
4
10
|
export default class WorkerPool {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
this
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
this.isDestroyed = false;
|
|
21
|
-
this.source = props.source;
|
|
22
|
-
this.url = props.url;
|
|
23
|
-
this.setProps(props);
|
|
24
|
-
}
|
|
25
|
-
destroy() {
|
|
26
|
-
this.idleQueue.forEach(worker => worker.destroy());
|
|
27
|
-
this.isDestroyed = true;
|
|
28
|
-
}
|
|
29
|
-
setProps(props) {
|
|
30
|
-
this.props = {
|
|
31
|
-
...this.props,
|
|
32
|
-
...props
|
|
33
|
-
};
|
|
34
|
-
if (props.name !== undefined) {
|
|
35
|
-
this.name = props.name;
|
|
11
|
+
name = 'unnamed';
|
|
12
|
+
source; // | Function;
|
|
13
|
+
url;
|
|
14
|
+
maxConcurrency = 1;
|
|
15
|
+
maxMobileConcurrency = 1;
|
|
16
|
+
onDebug = () => { };
|
|
17
|
+
reuseWorkers = true;
|
|
18
|
+
props = {};
|
|
19
|
+
jobQueue = [];
|
|
20
|
+
idleQueue = [];
|
|
21
|
+
count = 0;
|
|
22
|
+
isDestroyed = false;
|
|
23
|
+
/** Checks if workers are supported on this platform */
|
|
24
|
+
static isSupported() {
|
|
25
|
+
return WorkerThread.isSupported();
|
|
36
26
|
}
|
|
37
|
-
|
|
38
|
-
|
|
27
|
+
/**
|
|
28
|
+
* @param processor - worker function
|
|
29
|
+
* @param maxConcurrency - max count of workers
|
|
30
|
+
*/
|
|
31
|
+
constructor(props) {
|
|
32
|
+
this.source = props.source;
|
|
33
|
+
this.url = props.url;
|
|
34
|
+
this.setProps(props);
|
|
39
35
|
}
|
|
40
|
-
|
|
41
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Terminates all workers in the pool
|
|
38
|
+
* @note Can free up significant memory
|
|
39
|
+
*/
|
|
40
|
+
destroy() {
|
|
41
|
+
// Destroy idle workers, active Workers will be destroyed on completion
|
|
42
|
+
this.idleQueue.forEach((worker) => worker.destroy());
|
|
43
|
+
this.isDestroyed = true;
|
|
42
44
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
+
setProps(props) {
|
|
46
|
+
this.props = { ...this.props, ...props };
|
|
47
|
+
if (props.name !== undefined) {
|
|
48
|
+
this.name = props.name;
|
|
49
|
+
}
|
|
50
|
+
if (props.maxConcurrency !== undefined) {
|
|
51
|
+
this.maxConcurrency = props.maxConcurrency;
|
|
52
|
+
}
|
|
53
|
+
if (props.maxMobileConcurrency !== undefined) {
|
|
54
|
+
this.maxMobileConcurrency = props.maxMobileConcurrency;
|
|
55
|
+
}
|
|
56
|
+
if (props.reuseWorkers !== undefined) {
|
|
57
|
+
this.reuseWorkers = props.reuseWorkers;
|
|
58
|
+
}
|
|
59
|
+
if (props.onDebug !== undefined) {
|
|
60
|
+
this.onDebug = props.onDebug;
|
|
61
|
+
}
|
|
45
62
|
}
|
|
46
|
-
|
|
47
|
-
|
|
63
|
+
async startJob(name, onMessage = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
|
|
64
|
+
// Promise resolves when thread starts working on this job
|
|
65
|
+
const startPromise = new Promise((onStart) => {
|
|
66
|
+
// Promise resolves when thread completes or fails working on this job
|
|
67
|
+
this.jobQueue.push({ name, onMessage, onError, onStart });
|
|
68
|
+
return this;
|
|
69
|
+
});
|
|
70
|
+
this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
|
|
71
|
+
return await startPromise;
|
|
48
72
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
73
|
+
// PRIVATE
|
|
74
|
+
/**
|
|
75
|
+
* Starts first queued job if worker is available or can be created
|
|
76
|
+
* Called when job is started and whenever a worker returns to the idleQueue
|
|
77
|
+
*/
|
|
78
|
+
async _startQueuedJob() {
|
|
79
|
+
if (!this.jobQueue.length) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const workerThread = this._getAvailableWorker();
|
|
83
|
+
if (!workerThread) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
// We have a worker, dequeue and start the job
|
|
87
|
+
const queuedJob = this.jobQueue.shift();
|
|
88
|
+
if (queuedJob) {
|
|
89
|
+
// Emit a debug event
|
|
90
|
+
// @ts-ignore
|
|
91
|
+
this.onDebug({
|
|
92
|
+
message: 'Starting job',
|
|
93
|
+
name: queuedJob.name,
|
|
94
|
+
workerThread,
|
|
95
|
+
backlog: this.jobQueue.length
|
|
96
|
+
});
|
|
97
|
+
// Create a worker job to let the app access thread and manage job completion
|
|
98
|
+
const job = new WorkerJob(queuedJob.name, workerThread);
|
|
99
|
+
// Set the worker thread's message handlers
|
|
100
|
+
workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
|
|
101
|
+
workerThread.onError = (error) => queuedJob.onError(job, error);
|
|
102
|
+
// Resolve the start promise so that the app can start sending messages to worker
|
|
103
|
+
queuedJob.onStart(job);
|
|
104
|
+
// Wait for the app to signal that the job is complete, then return worker to queue
|
|
105
|
+
try {
|
|
106
|
+
await job.result;
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
// eslint-disable-next-line no-console
|
|
110
|
+
console.error(`Worker exception: ${error}`);
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
this.returnWorkerToQueue(workerThread);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
68
116
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
117
|
+
/**
|
|
118
|
+
* Returns a worker to the idle queue
|
|
119
|
+
* Destroys the worker if
|
|
120
|
+
* - pool is destroyed
|
|
121
|
+
* - if this pool doesn't reuse workers
|
|
122
|
+
* - if maxConcurrency has been lowered
|
|
123
|
+
* @param worker
|
|
124
|
+
*/
|
|
125
|
+
returnWorkerToQueue(worker) {
|
|
126
|
+
const shouldDestroyWorker =
|
|
127
|
+
// Workers on Node.js prevent the process from exiting.
|
|
128
|
+
// Until we figure out how to close them before exit, we always destroy them
|
|
129
|
+
!isBrowser ||
|
|
130
|
+
// If the pool is destroyed, there is no reason to keep the worker around
|
|
131
|
+
this.isDestroyed ||
|
|
132
|
+
// If the app has disabled worker reuse, any completed workers should be destroyed
|
|
133
|
+
!this.reuseWorkers ||
|
|
134
|
+
// If concurrency has been lowered, this worker might be surplus to requirements
|
|
135
|
+
this.count > this._getMaxConcurrency();
|
|
136
|
+
if (shouldDestroyWorker) {
|
|
137
|
+
worker.destroy();
|
|
138
|
+
this.count--;
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
this.idleQueue.push(worker);
|
|
142
|
+
}
|
|
143
|
+
if (!this.isDestroyed) {
|
|
144
|
+
this._startQueuedJob(); // eslint-disable-line @typescript-eslint/no-floating-promises
|
|
145
|
+
}
|
|
72
146
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
} finally {
|
|
90
|
-
this.returnWorkerToQueue(workerThread);
|
|
91
|
-
}
|
|
147
|
+
/**
|
|
148
|
+
* Returns idle worker or creates new worker if maxConcurrency has not been reached
|
|
149
|
+
*/
|
|
150
|
+
_getAvailableWorker() {
|
|
151
|
+
// If a worker has completed and returned to the queue, it can be used
|
|
152
|
+
if (this.idleQueue.length > 0) {
|
|
153
|
+
return this.idleQueue.shift() || null;
|
|
154
|
+
}
|
|
155
|
+
// Create fresh worker if we haven't yet created the max amount of worker threads for this worker source
|
|
156
|
+
if (this.count < this._getMaxConcurrency()) {
|
|
157
|
+
this.count++;
|
|
158
|
+
const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
|
|
159
|
+
return new WorkerThread({ name, source: this.source, url: this.url });
|
|
160
|
+
}
|
|
161
|
+
// No worker available, have to wait
|
|
162
|
+
return null;
|
|
92
163
|
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const shouldDestroyWorker = !isBrowser || this.isDestroyed || !this.reuseWorkers || this.count > this._getMaxConcurrency();
|
|
96
|
-
if (shouldDestroyWorker) {
|
|
97
|
-
worker.destroy();
|
|
98
|
-
this.count--;
|
|
99
|
-
} else {
|
|
100
|
-
this.idleQueue.push(worker);
|
|
164
|
+
_getMaxConcurrency() {
|
|
165
|
+
return isMobile ? this.maxMobileConcurrency : this.maxConcurrency;
|
|
101
166
|
}
|
|
102
|
-
if (!this.isDestroyed) {
|
|
103
|
-
this._startQueuedJob();
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
_getAvailableWorker() {
|
|
107
|
-
if (this.idleQueue.length > 0) {
|
|
108
|
-
return this.idleQueue.shift() || null;
|
|
109
|
-
}
|
|
110
|
-
if (this.count < this._getMaxConcurrency()) {
|
|
111
|
-
this.count++;
|
|
112
|
-
const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
|
|
113
|
-
return new WorkerThread({
|
|
114
|
-
name,
|
|
115
|
-
source: this.source,
|
|
116
|
-
url: this.url
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
|
-
return null;
|
|
120
|
-
}
|
|
121
|
-
_getMaxConcurrency() {
|
|
122
|
-
return isMobile ? this.maxMobileConcurrency : this.maxConcurrency;
|
|
123
|
-
}
|
|
124
167
|
}
|
|
125
|
-
//# sourceMappingURL=worker-pool.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-thread.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-thread.ts"],"names":[],"mappings":"AAIA,OAAO,EAAa,cAAc,EAAC,
|
|
1
|
+
{"version":3,"file":"worker-thread.d.ts","sourceRoot":"","sources":["../../../src/lib/worker-farm/worker-thread.ts"],"names":[],"mappings":"AAIA,OAAO,EAAa,cAAc,EAAC,kCAA+B;AAQlE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,YAAY;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,UAAU,EAAE,OAAO,CAAS;IAC5B,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC;IAChC,SAAS,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,CAAC;IAClC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAEhC,OAAO,CAAC,YAAY,CAAc;IAElC,uDAAuD;IACvD,MAAM,CAAC,WAAW,IAAI,OAAO;gBAOjB,KAAK,EAAE,iBAAiB;IAYpC;;;OAGG;IACH,OAAO,IAAI,IAAI;IAOf,IAAI,SAAS,YAEZ;IAED;;;;OAIG;IACH,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,YAAY,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI;IAQlD;;;OAGG;IACH,uBAAuB,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK;IAiBjD;;OAEG;IACH,oBAAoB,IAAI,MAAM;IAsB9B;;;OAGG;IACH,iBAAiB,IAAI,cAAc;CA0BpC"}
|
|
@@ -1,104 +1,136 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { NodeWorker } from "../node/worker_threads.js";
|
|
2
5
|
import { isBrowser } from "../env-utils/globals.js";
|
|
3
6
|
import { assert } from "../env-utils/assert.js";
|
|
4
7
|
import { getLoadableWorkerURL } from "../worker-utils/get-loadable-worker-url.js";
|
|
5
8
|
import { getTransferList } from "../worker-utils/get-transfer-list.js";
|
|
6
|
-
const NOOP = () => {};
|
|
9
|
+
const NOOP = () => { };
|
|
10
|
+
/**
|
|
11
|
+
* Represents one worker thread
|
|
12
|
+
*/
|
|
7
13
|
export default class WorkerThread {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
this
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const {
|
|
21
|
-
name,
|
|
22
|
-
source,
|
|
23
|
-
url
|
|
24
|
-
} = props;
|
|
25
|
-
assert(source || url);
|
|
26
|
-
this.name = name;
|
|
27
|
-
this.source = source;
|
|
28
|
-
this.url = url;
|
|
29
|
-
this.onMessage = NOOP;
|
|
30
|
-
this.onError = error => console.log(error);
|
|
31
|
-
this.worker = isBrowser ? this._createBrowserWorker() : this._createNodeWorker();
|
|
32
|
-
}
|
|
33
|
-
destroy() {
|
|
34
|
-
this.onMessage = NOOP;
|
|
35
|
-
this.onError = NOOP;
|
|
36
|
-
this.worker.terminate();
|
|
37
|
-
this.terminated = true;
|
|
38
|
-
}
|
|
39
|
-
get isRunning() {
|
|
40
|
-
return Boolean(this.onMessage);
|
|
41
|
-
}
|
|
42
|
-
postMessage(data, transferList) {
|
|
43
|
-
transferList = transferList || getTransferList(data);
|
|
44
|
-
this.worker.postMessage(data, transferList);
|
|
45
|
-
}
|
|
46
|
-
_getErrorFromErrorEvent(event) {
|
|
47
|
-
let message = 'Failed to load ';
|
|
48
|
-
message += `worker ${this.name} from ${this.url}. `;
|
|
49
|
-
if (event.message) {
|
|
50
|
-
message += `${event.message} in `;
|
|
14
|
+
name;
|
|
15
|
+
source;
|
|
16
|
+
url;
|
|
17
|
+
terminated = false;
|
|
18
|
+
worker;
|
|
19
|
+
onMessage;
|
|
20
|
+
onError;
|
|
21
|
+
_loadableURL = '';
|
|
22
|
+
/** Checks if workers are supported on this platform */
|
|
23
|
+
static isSupported() {
|
|
24
|
+
return ((typeof Worker !== 'undefined' && isBrowser) ||
|
|
25
|
+
(typeof NodeWorker !== 'undefined' && !isBrowser));
|
|
51
26
|
}
|
|
52
|
-
|
|
53
|
-
|
|
27
|
+
constructor(props) {
|
|
28
|
+
const { name, source, url } = props;
|
|
29
|
+
assert(source || url); // Either source or url must be defined
|
|
30
|
+
this.name = name;
|
|
31
|
+
this.source = source;
|
|
32
|
+
this.url = url;
|
|
33
|
+
this.onMessage = NOOP;
|
|
34
|
+
this.onError = (error) => console.log(error); // eslint-disable-line
|
|
35
|
+
this.worker = isBrowser ? this._createBrowserWorker() : this._createNodeWorker();
|
|
54
36
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
37
|
+
/**
|
|
38
|
+
* Terminate this worker thread
|
|
39
|
+
* @note Can free up significant memory
|
|
40
|
+
*/
|
|
41
|
+
destroy() {
|
|
42
|
+
this.onMessage = NOOP;
|
|
43
|
+
this.onError = NOOP;
|
|
44
|
+
this.worker.terminate(); // eslint-disable-line @typescript-eslint/no-floating-promises
|
|
45
|
+
this.terminated = true;
|
|
46
|
+
}
|
|
47
|
+
get isRunning() {
|
|
48
|
+
return Boolean(this.onMessage);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Send a message to this worker thread
|
|
52
|
+
* @param data any data structure, ideally consisting mostly of transferrable objects
|
|
53
|
+
* @param transferList If not supplied, calculated automatically by traversing data
|
|
54
|
+
*/
|
|
55
|
+
postMessage(data, transferList) {
|
|
56
|
+
transferList = transferList || getTransferList(data);
|
|
57
|
+
// @ts-ignore
|
|
58
|
+
this.worker.postMessage(data, transferList);
|
|
59
|
+
}
|
|
60
|
+
// PRIVATE
|
|
61
|
+
/**
|
|
62
|
+
* Generate a standard Error from an ErrorEvent
|
|
63
|
+
* @param event
|
|
64
|
+
*/
|
|
65
|
+
_getErrorFromErrorEvent(event) {
|
|
66
|
+
// Note Error object does not have the expected fields if loading failed completely
|
|
67
|
+
// https://developer.mozilla.org/en-US/docs/Web/API/Worker#Event_handlers
|
|
68
|
+
// https://developer.mozilla.org/en-US/docs/Web/API/ErrorEvent
|
|
69
|
+
let message = 'Failed to load ';
|
|
70
|
+
message += `worker ${this.name} from ${this.url}. `;
|
|
71
|
+
if (event.message) {
|
|
72
|
+
message += `${event.message} in `;
|
|
73
|
+
}
|
|
74
|
+
// const hasFilename = event.filename && !event.filename.startsWith('blob:');
|
|
75
|
+
// message += hasFilename ? event.filename : this.source.slice(0, 100);
|
|
76
|
+
if (event.lineno) {
|
|
77
|
+
message += `:${event.lineno}:${event.colno}`;
|
|
78
|
+
}
|
|
79
|
+
return new Error(message);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Creates a worker thread on the browser
|
|
83
|
+
*/
|
|
84
|
+
_createBrowserWorker() {
|
|
85
|
+
this._loadableURL = getLoadableWorkerURL({ source: this.source, url: this.url });
|
|
86
|
+
const worker = new Worker(this._loadableURL, { name: this.name });
|
|
87
|
+
worker.onmessage = (event) => {
|
|
88
|
+
if (!event.data) {
|
|
89
|
+
this.onError(new Error('No data received'));
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
this.onMessage(event.data);
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
// This callback represents an uncaught exception in the worker thread
|
|
96
|
+
worker.onerror = (error) => {
|
|
97
|
+
this.onError(this._getErrorFromErrorEvent(error));
|
|
98
|
+
this.terminated = true;
|
|
99
|
+
};
|
|
100
|
+
// TODO - not clear when this would be called, for now just log in case it happens
|
|
101
|
+
worker.onmessageerror = (event) => console.error(event); // eslint-disable-line
|
|
102
|
+
return worker;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Creates a worker thread in node.js
|
|
106
|
+
* @todo https://nodejs.org/api/async_hooks.html#async-resource-worker-pool
|
|
107
|
+
*/
|
|
108
|
+
_createNodeWorker() {
|
|
109
|
+
let worker;
|
|
110
|
+
if (this.url) {
|
|
111
|
+
// Make sure relative URLs start with './'
|
|
112
|
+
const absolute = this.url.includes(':/') || this.url.startsWith('/');
|
|
113
|
+
const url = absolute ? this.url : `./${this.url}`;
|
|
114
|
+
// console.log('Starting work from', url);
|
|
115
|
+
worker = new NodeWorker(url, { eval: false });
|
|
116
|
+
}
|
|
117
|
+
else if (this.source) {
|
|
118
|
+
worker = new NodeWorker(this.source, { eval: true });
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
throw new Error('no worker');
|
|
122
|
+
}
|
|
123
|
+
worker.on('message', (data) => {
|
|
124
|
+
// console.error('message', data);
|
|
125
|
+
this.onMessage(data);
|
|
126
|
+
});
|
|
127
|
+
worker.on('error', (error) => {
|
|
128
|
+
// console.error('error', error);
|
|
129
|
+
this.onError(error);
|
|
130
|
+
});
|
|
131
|
+
worker.on('exit', (code) => {
|
|
132
|
+
// console.error('exit', code);
|
|
133
|
+
});
|
|
134
|
+
return worker;
|
|
93
135
|
}
|
|
94
|
-
worker.on('message', data => {
|
|
95
|
-
this.onMessage(data);
|
|
96
|
-
});
|
|
97
|
-
worker.on('error', error => {
|
|
98
|
-
this.onError(error);
|
|
99
|
-
});
|
|
100
|
-
worker.on('exit', code => {});
|
|
101
|
-
return worker;
|
|
102
|
-
}
|
|
103
136
|
}
|
|
104
|
-
//# sourceMappingURL=worker-thread.js.map
|
|
@@ -1,36 +1,67 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { assert } from "../env-utils/assert.js";
|
|
2
5
|
const workerURLCache = new Map();
|
|
6
|
+
/**
|
|
7
|
+
* Creates a loadable URL from worker source or URL
|
|
8
|
+
* that can be used to create `Worker` instances.
|
|
9
|
+
* Due to CORS issues it may be necessary to wrap a URL in a small importScripts
|
|
10
|
+
* @param props
|
|
11
|
+
* @param props.source Worker source
|
|
12
|
+
* @param props.url Worker URL
|
|
13
|
+
* @returns loadable url
|
|
14
|
+
*/
|
|
3
15
|
export function getLoadableWorkerURL(props) {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
16
|
+
assert((props.source && !props.url) || (!props.source && props.url)); // Either source or url must be defined
|
|
17
|
+
let workerURL = workerURLCache.get(props.source || props.url);
|
|
18
|
+
if (!workerURL) {
|
|
19
|
+
// Differentiate worker urls from worker source code
|
|
20
|
+
if (props.url) {
|
|
21
|
+
workerURL = getLoadableWorkerURLFromURL(props.url);
|
|
22
|
+
workerURLCache.set(props.url, workerURL);
|
|
23
|
+
}
|
|
24
|
+
if (props.source) {
|
|
25
|
+
workerURL = getLoadableWorkerURLFromSource(props.source);
|
|
26
|
+
workerURLCache.set(props.source, workerURL);
|
|
27
|
+
}
|
|
10
28
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
workerURLCache.set(props.source, workerURL);
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
assert(workerURL);
|
|
17
|
-
return workerURL;
|
|
29
|
+
assert(workerURL);
|
|
30
|
+
return workerURL;
|
|
18
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Build a loadable worker URL from worker URL
|
|
34
|
+
* @param url
|
|
35
|
+
* @returns loadable URL
|
|
36
|
+
*/
|
|
19
37
|
function getLoadableWorkerURLFromURL(url) {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
38
|
+
// A local script url, we can use it to initialize a Worker directly
|
|
39
|
+
if (!url.startsWith('http')) {
|
|
40
|
+
return url;
|
|
41
|
+
}
|
|
42
|
+
// A remote script, we need to use `importScripts` to load from different origin
|
|
43
|
+
const workerSource = buildScriptSource(url);
|
|
44
|
+
return getLoadableWorkerURLFromSource(workerSource);
|
|
25
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Build a loadable worker URL from worker source
|
|
48
|
+
* @param workerSource
|
|
49
|
+
* @returns loadable url
|
|
50
|
+
*/
|
|
26
51
|
function getLoadableWorkerURLFromSource(workerSource) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
});
|
|
30
|
-
return URL.createObjectURL(blob);
|
|
52
|
+
const blob = new Blob([workerSource], { type: 'application/javascript' });
|
|
53
|
+
return URL.createObjectURL(blob);
|
|
31
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Per spec, worker cannot be initialized with a script from a different origin
|
|
57
|
+
* However a local worker script can still import scripts from other origins,
|
|
58
|
+
* so we simply build a wrapper script.
|
|
59
|
+
*
|
|
60
|
+
* @param workerUrl
|
|
61
|
+
* @returns source
|
|
62
|
+
*/
|
|
32
63
|
function buildScriptSource(workerUrl) {
|
|
33
|
-
|
|
64
|
+
return `\
|
|
34
65
|
try {
|
|
35
66
|
importScripts('${workerUrl}');
|
|
36
67
|
} catch (error) {
|
|
@@ -38,4 +69,3 @@ try {
|
|
|
38
69
|
throw error;
|
|
39
70
|
}`;
|
|
40
71
|
}
|
|
41
|
-
//# sourceMappingURL=get-loadable-worker-url.js.map
|