@math.gl/tasks 0.0.0-stage → 5.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +147 -0
- package/LOADERS-GL-LICENSE.txt +52 -0
- package/README.md +26 -2
- package/THIRD-PARTY-NOTICES.md +5 -0
- package/dist/index.cjs +3454 -0
- package/dist/index.cjs.map +6 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/tasks/index.d.ts +9 -0
- package/dist/tasks/index.d.ts.map +1 -0
- package/dist/tasks/index.js +9 -0
- package/dist/tasks/index.js.map +1 -0
- package/dist/tasks/task-channel.d.ts +48 -0
- package/dist/tasks/task-channel.d.ts.map +1 -0
- package/dist/tasks/task-channel.js +166 -0
- package/dist/tasks/task-channel.js.map +1 -0
- package/dist/tasks/task-operation-queue.d.ts +26 -0
- package/dist/tasks/task-operation-queue.d.ts.map +1 -0
- package/dist/tasks/task-operation-queue.js +106 -0
- package/dist/tasks/task-operation-queue.js.map +1 -0
- package/dist/tasks/task-preload.d.ts +13 -0
- package/dist/tasks/task-preload.d.ts.map +1 -0
- package/dist/tasks/task-preload.js +96 -0
- package/dist/tasks/task-preload.js.map +1 -0
- package/dist/tasks/task-runner.d.ts +74 -0
- package/dist/tasks/task-runner.d.ts.map +1 -0
- package/dist/tasks/task-runner.js +811 -0
- package/dist/tasks/task-runner.js.map +1 -0
- package/dist/tasks/task-runtime.d.ts +27 -0
- package/dist/tasks/task-runtime.d.ts.map +1 -0
- package/dist/tasks/task-runtime.js +216 -0
- package/dist/tasks/task-runtime.js.map +1 -0
- package/dist/tasks/task-sync.d.ts +4 -0
- package/dist/tasks/task-sync.d.ts.map +1 -0
- package/dist/tasks/task-sync.js +95 -0
- package/dist/tasks/task-sync.js.map +1 -0
- package/dist/tasks/task-worker-channel.d.ts +11 -0
- package/dist/tasks/task-worker-channel.d.ts.map +1 -0
- package/dist/tasks/task-worker-channel.js +34 -0
- package/dist/tasks/task-worker-channel.js.map +1 -0
- package/dist/tasks/task.d.ts +132 -0
- package/dist/tasks/task.d.ts.map +1 -0
- package/dist/tasks/task.js +5 -0
- package/dist/tasks/task.js.map +1 -0
- package/dist/tasks/typed-task-dispatch.d.ts +30 -0
- package/dist/tasks/typed-task-dispatch.d.ts.map +1 -0
- package/dist/tasks/typed-task-dispatch.js +12 -0
- package/dist/tasks/typed-task-dispatch.js.map +1 -0
- package/dist/worker-utils/index.d.ts +25 -0
- package/dist/worker-utils/index.d.ts.map +1 -0
- package/dist/worker-utils/index.js +46 -0
- package/dist/worker-utils/index.js.map +1 -0
- package/dist/worker-utils/lib/async-queue/async-queue.d.ts +32 -0
- package/dist/worker-utils/lib/async-queue/async-queue.d.ts.map +1 -0
- package/dist/worker-utils/lib/async-queue/async-queue.js +88 -0
- package/dist/worker-utils/lib/async-queue/async-queue.js.map +1 -0
- package/dist/worker-utils/lib/env-utils/assert.d.ts +3 -0
- package/dist/worker-utils/lib/env-utils/assert.d.ts.map +1 -0
- package/dist/worker-utils/lib/env-utils/assert.js +13 -0
- package/dist/worker-utils/lib/env-utils/assert.js.map +1 -0
- package/dist/worker-utils/lib/env-utils/globals.d.ts +22 -0
- package/dist/worker-utils/lib/env-utils/globals.d.ts.map +1 -0
- package/dist/worker-utils/lib/env-utils/globals.js +30 -0
- package/dist/worker-utils/lib/env-utils/globals.js.map +1 -0
- package/dist/worker-utils/lib/env-utils/version.d.ts +3 -0
- package/dist/worker-utils/lib/env-utils/version.d.ts.map +1 -0
- package/dist/worker-utils/lib/env-utils/version.js +11 -0
- package/dist/worker-utils/lib/env-utils/version.js.map +1 -0
- package/dist/worker-utils/lib/library-utils/library-utils.d.ts +42 -0
- package/dist/worker-utils/lib/library-utils/library-utils.d.ts.map +1 -0
- package/dist/worker-utils/lib/library-utils/library-utils.js +251 -0
- package/dist/worker-utils/lib/library-utils/library-utils.js.map +1 -0
- package/dist/worker-utils/lib/node/worker_threads-browser.d.ts +15 -0
- package/dist/worker-utils/lib/node/worker_threads-browser.d.ts.map +1 -0
- package/dist/worker-utils/lib/node/worker_threads-browser.js +23 -0
- package/dist/worker-utils/lib/node/worker_threads-browser.js.map +1 -0
- package/dist/worker-utils/lib/node/worker_threads.d.ts +16 -0
- package/dist/worker-utils/lib/node/worker_threads.d.ts.map +1 -0
- package/dist/worker-utils/lib/node/worker_threads.js +7 -0
- package/dist/worker-utils/lib/node/worker_threads.js.map +1 -0
- package/dist/worker-utils/lib/npm-tag.d.ts +2 -0
- package/dist/worker-utils/lib/npm-tag.d.ts.map +1 -0
- package/dist/worker-utils/lib/npm-tag.js +5 -0
- package/dist/worker-utils/lib/npm-tag.js.map +1 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.d.ts +8 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.d.ts.map +1 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.js +18 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.js.map +1 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.d.ts +45 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.d.ts.map +1 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.js +128 -0
- package/dist/worker-utils/lib/process-utils/child-process-proxy.js.map +1 -0
- package/dist/worker-utils/lib/process-utils/process-utils.d.ts +2 -0
- package/dist/worker-utils/lib/process-utils/process-utils.d.ts.map +1 -0
- package/dist/worker-utils/lib/process-utils/process-utils.js +32 -0
- package/dist/worker-utils/lib/process-utils/process-utils.js.map +1 -0
- package/dist/worker-utils/lib/worker-api/create-worker.d.ts +9 -0
- package/dist/worker-utils/lib/worker-api/create-worker.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-api/create-worker.js +229 -0
- package/dist/worker-utils/lib/worker-api/create-worker.js.map +1 -0
- package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts +29 -0
- package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-api/get-worker-url.js +101 -0
- package/dist/worker-utils/lib/worker-api/get-worker-url.js.map +1 -0
- package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts +67 -0
- package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-api/process-on-worker.js +429 -0
- package/dist/worker-utils/lib/worker-api/process-on-worker.js.map +1 -0
- package/dist/worker-utils/lib/worker-api/shared-state.d.ts +34 -0
- package/dist/worker-utils/lib/worker-api/shared-state.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-api/shared-state.js +105 -0
- package/dist/worker-utils/lib/worker-api/shared-state.js.map +1 -0
- package/dist/worker-utils/lib/worker-api/validate-output-ownership.d.ts +3 -0
- package/dist/worker-utils/lib/worker-api/validate-output-ownership.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-api/validate-output-ownership.js +10 -0
- package/dist/worker-utils/lib/worker-api/validate-output-ownership.js.map +1 -0
- package/dist/worker-utils/lib/worker-api/validate-worker-version.d.ts +9 -0
- package/dist/worker-utils/lib/worker-api/validate-worker-version.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-api/validate-worker-version.js +35 -0
- package/dist/worker-utils/lib/worker-api/validate-worker-version.js.map +1 -0
- package/dist/worker-utils/lib/worker-api/worker-resource.d.ts +33 -0
- package/dist/worker-utils/lib/worker-api/worker-resource.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-api/worker-resource.js +156 -0
- package/dist/worker-utils/lib/worker-api/worker-resource.js.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-body.d.ts +20 -0
- package/dist/worker-utils/lib/worker-farm/worker-body.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-body.js +134 -0
- package/dist/worker-utils/lib/worker-farm/worker-body.js.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-farm.d.ts +73 -0
- package/dist/worker-utils/lib/worker-farm/worker-farm.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-farm.js +135 -0
- package/dist/worker-utils/lib/worker-farm/worker-farm.js.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-job.d.ts +47 -0
- package/dist/worker-utils/lib/worker-farm/worker-job.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-job.js +124 -0
- package/dist/worker-utils/lib/worker-farm/worker-job.js.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts +152 -0
- package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-pool.js +584 -0
- package/dist/worker-utils/lib/worker-farm/worker-pool.js.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-thread.d.ts +77 -0
- package/dist/worker-utils/lib/worker-farm/worker-thread.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-farm/worker-thread.js +215 -0
- package/dist/worker-utils/lib/worker-farm/worker-thread.js.map +1 -0
- package/dist/worker-utils/lib/worker-utils/get-input-byte-length.d.ts +5 -0
- package/dist/worker-utils/lib/worker-utils/get-input-byte-length.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-utils/get-input-byte-length.js +39 -0
- package/dist/worker-utils/lib/worker-utils/get-input-byte-length.js.map +1 -0
- package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.d.ts +15 -0
- package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.js +84 -0
- package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.js.map +1 -0
- package/dist/worker-utils/lib/worker-utils/get-transfer-list.d.ts +16 -0
- package/dist/worker-utils/lib/worker-utils/get-transfer-list.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-utils/get-transfer-list.js +127 -0
- package/dist/worker-utils/lib/worker-utils/get-transfer-list.js.map +1 -0
- package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.d.ts +6 -0
- package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.d.ts.map +1 -0
- package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.js +27 -0
- package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.js.map +1 -0
- package/dist/worker-utils/types.d.ts +165 -0
- package/dist/worker-utils/types.d.ts.map +1 -0
- package/dist/worker-utils/types.js +5 -0
- package/dist/worker-utils/types.js.map +1 -0
- package/dist/worker-utils/workers/null-worker.d.ts +2 -0
- package/dist/worker-utils/workers/null-worker.d.ts.map +1 -0
- package/dist/worker-utils/workers/null-worker.js +17 -0
- package/dist/worker-utils/workers/null-worker.js.map +1 -0
- package/package.json +43 -4
- package/src/index.ts +5 -0
- package/src/tasks/index.ts +33 -0
- package/src/tasks/task-channel.ts +191 -0
- package/src/tasks/task-operation-queue.ts +109 -0
- package/src/tasks/task-preload.ts +107 -0
- package/src/tasks/task-runner.ts +933 -0
- package/src/tasks/task-runtime.ts +244 -0
- package/src/tasks/task-sync.ts +100 -0
- package/src/tasks/task-worker-channel.ts +44 -0
- package/src/tasks/task.ts +168 -0
- package/src/tasks/typed-task-dispatch.ts +44 -0
- package/src/worker-utils/index.ts +105 -0
- package/src/worker-utils/lib/async-queue/async-queue.ts +100 -0
- package/src/worker-utils/lib/env-utils/assert.ts +14 -0
- package/src/worker-utils/lib/env-utils/globals.ts +40 -0
- package/src/worker-utils/lib/env-utils/version.ts +15 -0
- package/src/worker-utils/lib/library-utils/library-utils.ts +305 -0
- package/src/worker-utils/lib/node/worker_threads-browser.ts +44 -0
- package/src/worker-utils/lib/node/worker_threads.ts +21 -0
- package/src/worker-utils/lib/npm-tag.ts +5 -0
- package/src/worker-utils/lib/process-utils/child-process-proxy.browser.ts +23 -0
- package/src/worker-utils/lib/process-utils/child-process-proxy.ts +172 -0
- package/src/worker-utils/lib/process-utils/process-utils.ts +34 -0
- package/src/worker-utils/lib/worker-api/create-worker.ts +271 -0
- package/src/worker-utils/lib/worker-api/get-worker-url.ts +130 -0
- package/src/worker-utils/lib/worker-api/process-on-worker.ts +596 -0
- package/src/worker-utils/lib/worker-api/shared-state.ts +135 -0
- package/src/worker-utils/lib/worker-api/validate-output-ownership.ts +10 -0
- package/src/worker-utils/lib/worker-api/validate-worker-version.ts +44 -0
- package/src/worker-utils/lib/worker-api/worker-resource.ts +203 -0
- package/src/worker-utils/lib/worker-farm/worker-body.ts +154 -0
- package/src/worker-utils/lib/worker-farm/worker-farm.ts +172 -0
- package/src/worker-utils/lib/worker-farm/worker-job.ts +148 -0
- package/src/worker-utils/lib/worker-farm/worker-pool.ts +719 -0
- package/src/worker-utils/lib/worker-farm/worker-thread.ts +260 -0
- package/src/worker-utils/lib/worker-utils/get-input-byte-length.ts +34 -0
- package/src/worker-utils/lib/worker-utils/get-loadable-worker-url.ts +96 -0
- package/src/worker-utils/lib/worker-utils/get-transfer-list.ts +125 -0
- package/src/worker-utils/lib/worker-utils/remove-nontransferable-options.ts +27 -0
- package/src/worker-utils/types.ts +224 -0
- package/src/worker-utils/workers/null-worker.ts +20 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
/* eslint-disable no-console */
|
|
6
|
+
import {getAvailablePort} from './process-utils';
|
|
7
|
+
|
|
8
|
+
type NodeChildProcess = import('child_process').ChildProcess;
|
|
9
|
+
type SpawnOptions = import('child_process').SpawnOptions;
|
|
10
|
+
|
|
11
|
+
async function getChildProcessModule(): Promise<typeof import('child_process')> {
|
|
12
|
+
if (typeof process === 'undefined' || !process.versions?.node) {
|
|
13
|
+
throw new Error('ChildProcessProxy is only available in Node.js environments');
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
return await import('child_process');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export type ChildProcessProxyProps = {
|
|
20
|
+
command: string;
|
|
21
|
+
arguments: string[];
|
|
22
|
+
/** Whether to add a port specified arg */
|
|
23
|
+
portArg?: string;
|
|
24
|
+
/** Base port number */
|
|
25
|
+
port?: number;
|
|
26
|
+
/** Whether to search for an available port if the base port is occupied */
|
|
27
|
+
autoPort?: boolean;
|
|
28
|
+
/** Number of milliseconds to wait until concluding success */
|
|
29
|
+
/** wait: 0 - infinity */
|
|
30
|
+
wait?: number;
|
|
31
|
+
/** Options passed on to Node'.js `spawn` */
|
|
32
|
+
spawn?: SpawnOptions;
|
|
33
|
+
/** Should proceed if stderr stream recieved data */
|
|
34
|
+
ignoreStderr?: boolean;
|
|
35
|
+
/** Callback when the */
|
|
36
|
+
onStart?: (proxy: ChildProcessProxy) => void;
|
|
37
|
+
onSuccess?: (proxy: ChildProcessProxy) => void;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const DEFAULT_PROPS: ChildProcessProxyProps = {
|
|
41
|
+
command: '',
|
|
42
|
+
arguments: [],
|
|
43
|
+
port: 5000,
|
|
44
|
+
autoPort: true,
|
|
45
|
+
wait: 2000,
|
|
46
|
+
onSuccess: processProxy => {
|
|
47
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
48
|
+
console.log(`Started ${processProxy.props.command}`);
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Manager for a Node.js child process
|
|
54
|
+
* Prepares arguments, starts, stops and tracks output
|
|
55
|
+
*/
|
|
56
|
+
export default class ChildProcessProxy {
|
|
57
|
+
id: string;
|
|
58
|
+
props: ChildProcessProxyProps = {...DEFAULT_PROPS};
|
|
59
|
+
private childProcess: NodeChildProcess | null = null;
|
|
60
|
+
private port: number = 0;
|
|
61
|
+
private successTimer?: any; // NodeJS.Timeout;
|
|
62
|
+
|
|
63
|
+
// constructor(props?: {id?: string});
|
|
64
|
+
constructor({id = 'browser-driver'} = {}) {
|
|
65
|
+
this.id = id;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Starts a child process with the provided props */
|
|
69
|
+
async start(props: ChildProcessProxyProps): Promise<object> {
|
|
70
|
+
props = {...DEFAULT_PROPS, ...props};
|
|
71
|
+
this.props = props;
|
|
72
|
+
|
|
73
|
+
const childProcessModule = await getChildProcessModule();
|
|
74
|
+
|
|
75
|
+
const args = [...props.arguments];
|
|
76
|
+
|
|
77
|
+
// If portArg is set, we can look up an available port
|
|
78
|
+
this.port = Number(props.port);
|
|
79
|
+
if (props.portArg) {
|
|
80
|
+
if (props.autoPort) {
|
|
81
|
+
this.port = await getAvailablePort(props.port);
|
|
82
|
+
}
|
|
83
|
+
args.push(props.portArg, String(this.port));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return await new Promise((resolve, reject) => {
|
|
87
|
+
try {
|
|
88
|
+
this._setTimeout(() => {
|
|
89
|
+
if (props.onSuccess) {
|
|
90
|
+
props.onSuccess(this);
|
|
91
|
+
}
|
|
92
|
+
resolve({});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
96
|
+
console.log(`Spawning ${props.command} ${props.arguments.join(' ')}`);
|
|
97
|
+
if (props.spawn?.shell) {
|
|
98
|
+
throw new Error('ChildProcessProxy does not support shell execution');
|
|
99
|
+
}
|
|
100
|
+
const spawnOptions: SpawnOptions = {...props.spawn, shell: false};
|
|
101
|
+
const childProcess: NodeChildProcess = childProcessModule.spawn(
|
|
102
|
+
props.command,
|
|
103
|
+
args,
|
|
104
|
+
spawnOptions
|
|
105
|
+
);
|
|
106
|
+
this.childProcess = childProcess;
|
|
107
|
+
|
|
108
|
+
childProcess.stdout?.on('data', data => {
|
|
109
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
110
|
+
console.log(data.toString());
|
|
111
|
+
});
|
|
112
|
+
childProcess.stderr?.on('data', data => {
|
|
113
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
114
|
+
console.log(`Child process wrote to stderr: "${data}".`);
|
|
115
|
+
if (!props.ignoreStderr) {
|
|
116
|
+
this._clearTimeout();
|
|
117
|
+
reject(new Error(data));
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
childProcess.on('error', error => {
|
|
121
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
122
|
+
console.log(`Child process errored with ${error}`);
|
|
123
|
+
this._clearTimeout();
|
|
124
|
+
reject(error);
|
|
125
|
+
});
|
|
126
|
+
childProcess.on('close', code => {
|
|
127
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
128
|
+
console.log(`Child process exited with ${code}`);
|
|
129
|
+
this.childProcess = null;
|
|
130
|
+
this._clearTimeout();
|
|
131
|
+
resolve({});
|
|
132
|
+
});
|
|
133
|
+
} catch (error) {
|
|
134
|
+
reject(error);
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Stops a running child process */
|
|
140
|
+
async stop(): Promise<void> {
|
|
141
|
+
if (this.childProcess) {
|
|
142
|
+
this.childProcess.kill();
|
|
143
|
+
this.childProcess = null;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Exits this process */
|
|
148
|
+
async exit(statusCode: number = 0): Promise<void> {
|
|
149
|
+
try {
|
|
150
|
+
await this.stop();
|
|
151
|
+
// eslint-disable-next-line no-process-exit
|
|
152
|
+
process.exit(statusCode);
|
|
153
|
+
} catch (error) {
|
|
154
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
155
|
+
console.error((error as Error).message || error);
|
|
156
|
+
// eslint-disable-next-line no-process-exit
|
|
157
|
+
process.exit(1);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
_setTimeout(callback: (...args: any[]) => void) {
|
|
162
|
+
if (Number(this.props.wait) > 0) {
|
|
163
|
+
this.successTimer = setTimeout(callback, this.props.wait);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
_clearTimeout() {
|
|
168
|
+
if (this.successTimer) {
|
|
169
|
+
clearTimeout(this.successTimer);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import ChildProcess from 'child_process';
|
|
6
|
+
|
|
7
|
+
// Get an available port
|
|
8
|
+
// Works on Unix systems
|
|
9
|
+
export function getAvailablePort(defaultPort: number = 3000): Promise<number> {
|
|
10
|
+
return new Promise(resolve => {
|
|
11
|
+
// Get a list of all ports in use
|
|
12
|
+
ChildProcess.exec('lsof -i -P -n | grep LISTEN', (error, stdout) => {
|
|
13
|
+
if (error) {
|
|
14
|
+
// likely no permission, e.g. CI
|
|
15
|
+
resolve(defaultPort);
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const portsInUse: number[] = [];
|
|
20
|
+
const regex = /:(\d+) \(LISTEN\)/;
|
|
21
|
+
stdout.split('\n').forEach(line => {
|
|
22
|
+
const match = regex.exec(line);
|
|
23
|
+
if (match) {
|
|
24
|
+
portsInUse.push(Number(match[1]));
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
let port = defaultPort;
|
|
28
|
+
while (portsInUse.includes(port)) {
|
|
29
|
+
port++;
|
|
30
|
+
}
|
|
31
|
+
resolve(port);
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
}
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
WorkerProgress,
|
|
7
|
+
WorkerMessageType,
|
|
8
|
+
WorkerMessagePayload,
|
|
9
|
+
WorkerContext,
|
|
10
|
+
WorkerJobContext,
|
|
11
|
+
Process,
|
|
12
|
+
ProcessInBatches
|
|
13
|
+
} from '../../types';
|
|
14
|
+
import {SharedWorkerState} from './shared-state';
|
|
15
|
+
import AsyncQueue from '../async-queue/async-queue';
|
|
16
|
+
import WorkerBody from '../worker-farm/worker-body';
|
|
17
|
+
import {validateOutputOwnership} from './validate-output-ownership';
|
|
18
|
+
// import {validateWorkerVersion} from './validate-worker-version';
|
|
19
|
+
|
|
20
|
+
/** Counter for jobs */
|
|
21
|
+
let requestId = 0;
|
|
22
|
+
let activeController: AbortController | null = null;
|
|
23
|
+
let activeInputBatches: AsyncQueue<any> | null = null;
|
|
24
|
+
let activeOutputAcknowledgements: AsyncQueue<void> | null = null;
|
|
25
|
+
|
|
26
|
+
export type ProcessOnMainThread = (
|
|
27
|
+
data: any,
|
|
28
|
+
options?: {[key: string]: any},
|
|
29
|
+
jobContext?: WorkerJobContext
|
|
30
|
+
) => any;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Set up a WebWorkerGlobalScope to talk with the main thread
|
|
34
|
+
*/
|
|
35
|
+
export async function createWorker(
|
|
36
|
+
process: Process,
|
|
37
|
+
processInBatches?: ProcessInBatches,
|
|
38
|
+
preload?: () => Promise<void>
|
|
39
|
+
): Promise<void> {
|
|
40
|
+
if (!(await WorkerBody.inWorkerThread())) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const context: WorkerContext = {
|
|
45
|
+
process: processOnMainThread
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// eslint-disable-next-line complexity
|
|
49
|
+
WorkerBody.onmessage = async (type: WorkerMessageType, payload: WorkerMessagePayload) => {
|
|
50
|
+
if (type === 'cancel') {
|
|
51
|
+
activeController?.abort();
|
|
52
|
+
activeInputBatches?.close();
|
|
53
|
+
activeOutputAcknowledgements?.close();
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const controller =
|
|
57
|
+
type === 'process' || type === 'process-in-batches' ? new AbortController() : undefined;
|
|
58
|
+
if (controller) activeController = controller;
|
|
59
|
+
const startedAt = performance.now();
|
|
60
|
+
const reporter =
|
|
61
|
+
type === 'process' || type === 'process-in-batches' ? createProgressReporter() : undefined;
|
|
62
|
+
let sharedState: SharedWorkerState | undefined;
|
|
63
|
+
const throwIfAborted = () => {
|
|
64
|
+
controller?.signal.throwIfAborted();
|
|
65
|
+
sharedState?.throwIfCancelled();
|
|
66
|
+
};
|
|
67
|
+
const jobContext = {
|
|
68
|
+
...context,
|
|
69
|
+
signal: controller?.signal,
|
|
70
|
+
sharedState,
|
|
71
|
+
throwIfAborted,
|
|
72
|
+
reportProgress: (progress: WorkerProgress) => {
|
|
73
|
+
if (!controller?.signal.aborted && !sharedState?.snapshot.cancelled)
|
|
74
|
+
reporter?.report(progress);
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
try {
|
|
78
|
+
if (controller && payload.options?.['sharedState']) {
|
|
79
|
+
sharedState = new SharedWorkerState(payload.options['sharedState']);
|
|
80
|
+
jobContext.sharedState = sharedState;
|
|
81
|
+
}
|
|
82
|
+
switch (type) {
|
|
83
|
+
case 'preload':
|
|
84
|
+
await preload?.();
|
|
85
|
+
WorkerBody.postMessage('done', {});
|
|
86
|
+
break;
|
|
87
|
+
|
|
88
|
+
case 'process':
|
|
89
|
+
throwIfAborted();
|
|
90
|
+
validateOutputOwnership(payload.options?.['outputOwnership']);
|
|
91
|
+
if (!process) {
|
|
92
|
+
throw new Error('Worker does not support atomic processing');
|
|
93
|
+
}
|
|
94
|
+
const result = await process(
|
|
95
|
+
payload.input,
|
|
96
|
+
payload.options || {},
|
|
97
|
+
jobContext,
|
|
98
|
+
payload.context || {}
|
|
99
|
+
);
|
|
100
|
+
throwIfAborted();
|
|
101
|
+
reporter?.finish();
|
|
102
|
+
await WorkerBody.postMessage(
|
|
103
|
+
'done',
|
|
104
|
+
{
|
|
105
|
+
result,
|
|
106
|
+
executionMs: performance.now() - startedAt
|
|
107
|
+
},
|
|
108
|
+
payload.options?.['outputOwnership'] === 'copy'
|
|
109
|
+
);
|
|
110
|
+
break;
|
|
111
|
+
|
|
112
|
+
case 'process-in-batches':
|
|
113
|
+
throwIfAborted();
|
|
114
|
+
validateOutputOwnership(payload.options?.['outputOwnership']);
|
|
115
|
+
if (!processInBatches) {
|
|
116
|
+
throw new Error('Worker does not support batched processing');
|
|
117
|
+
}
|
|
118
|
+
const inputBatches = new AsyncQueue<any>();
|
|
119
|
+
const outputAcknowledgements = new AsyncQueue<void>();
|
|
120
|
+
activeInputBatches = inputBatches;
|
|
121
|
+
activeOutputAcknowledgements = outputAcknowledgements;
|
|
122
|
+
try {
|
|
123
|
+
const resultIterator = processInBatches(
|
|
124
|
+
createDemandDrivenIterator(inputBatches),
|
|
125
|
+
payload.options || {},
|
|
126
|
+
jobContext,
|
|
127
|
+
payload.context || {}
|
|
128
|
+
);
|
|
129
|
+
for await (const batch of resultIterator) {
|
|
130
|
+
throwIfAborted();
|
|
131
|
+
await WorkerBody.postMessage(
|
|
132
|
+
'output-batch',
|
|
133
|
+
{result: batch},
|
|
134
|
+
payload.options?.['outputOwnership'] === 'copy'
|
|
135
|
+
);
|
|
136
|
+
await outputAcknowledgements.next();
|
|
137
|
+
}
|
|
138
|
+
throwIfAborted();
|
|
139
|
+
reporter?.finish();
|
|
140
|
+
await WorkerBody.postMessage('done', {executionMs: performance.now() - startedAt});
|
|
141
|
+
} finally {
|
|
142
|
+
activeInputBatches = null;
|
|
143
|
+
activeOutputAcknowledgements = null;
|
|
144
|
+
}
|
|
145
|
+
break;
|
|
146
|
+
|
|
147
|
+
case 'input-batch':
|
|
148
|
+
if (!activeInputBatches) {
|
|
149
|
+
throw new Error('Worker has no active batched processing session');
|
|
150
|
+
}
|
|
151
|
+
activeInputBatches.push(payload.input);
|
|
152
|
+
break;
|
|
153
|
+
|
|
154
|
+
case 'input-done':
|
|
155
|
+
if (!activeInputBatches) {
|
|
156
|
+
throw new Error('Worker has no active batched processing session');
|
|
157
|
+
}
|
|
158
|
+
activeInputBatches.close();
|
|
159
|
+
break;
|
|
160
|
+
|
|
161
|
+
case 'output-ack':
|
|
162
|
+
if (!activeOutputAcknowledgements) {
|
|
163
|
+
throw new Error('Worker has no active batched processing session');
|
|
164
|
+
}
|
|
165
|
+
activeOutputAcknowledgements.push(undefined);
|
|
166
|
+
break;
|
|
167
|
+
|
|
168
|
+
default:
|
|
169
|
+
}
|
|
170
|
+
} catch (error) {
|
|
171
|
+
reporter?.finish(controller?.signal.aborted || sharedState?.snapshot.cancelled);
|
|
172
|
+
const message = error instanceof Error ? error.message : '';
|
|
173
|
+
await WorkerBody.postMessage('error', {
|
|
174
|
+
error: message,
|
|
175
|
+
errorName: error instanceof Error ? error.name : 'Error'
|
|
176
|
+
});
|
|
177
|
+
} finally {
|
|
178
|
+
if (controller && activeController === controller) activeController = null;
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
await WorkerBody.postMessage('ready', {});
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Requests exactly one input batch whenever the worker-side processor advances its iterator. */
|
|
185
|
+
async function* createDemandDrivenIterator(inputBatches: AsyncQueue<any>): AsyncIterable<any> {
|
|
186
|
+
while (true) {
|
|
187
|
+
await WorkerBody.postMessage('input-request', {});
|
|
188
|
+
const nextBatch = await inputBatches.next();
|
|
189
|
+
if (nextBatch.done) {
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
yield nextBatch.value;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function processOnMainThread(arrayBuffer: ArrayBuffer, options = {}, jobContext = {}) {
|
|
197
|
+
const signal = activeController?.signal;
|
|
198
|
+
signal?.throwIfAborted();
|
|
199
|
+
return new Promise((resolve, reject) => {
|
|
200
|
+
const id = requestId++;
|
|
201
|
+
const cleanup = () => {
|
|
202
|
+
signal?.removeEventListener('abort', abort);
|
|
203
|
+
void WorkerBody.removeEventListener(onMessage);
|
|
204
|
+
};
|
|
205
|
+
const abort = () => {
|
|
206
|
+
cleanup();
|
|
207
|
+
reject(signal?.reason);
|
|
208
|
+
};
|
|
209
|
+
const onMessage = (type: string, payload: WorkerMessagePayload) => {
|
|
210
|
+
if (payload.id !== id) return;
|
|
211
|
+
if (type === 'done') {
|
|
212
|
+
cleanup();
|
|
213
|
+
resolve(payload.result);
|
|
214
|
+
} else if (type === 'error') {
|
|
215
|
+
cleanup();
|
|
216
|
+
reject(payload.error);
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
void WorkerBody.addEventListener(onMessage)
|
|
220
|
+
.then(async () => {
|
|
221
|
+
if (signal?.aborted) {
|
|
222
|
+
abort();
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
signal?.addEventListener('abort', abort, {once: true});
|
|
226
|
+
await WorkerBody.postMessage('process', {
|
|
227
|
+
id,
|
|
228
|
+
input: arrayBuffer,
|
|
229
|
+
options,
|
|
230
|
+
context: jobContext
|
|
231
|
+
});
|
|
232
|
+
})
|
|
233
|
+
.catch(error => {
|
|
234
|
+
cleanup();
|
|
235
|
+
reject(error);
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Coalesces same-phase reports per event-loop turn and flushes before settlement. */
|
|
241
|
+
function createProgressReporter() {
|
|
242
|
+
let pending: WorkerProgress | undefined;
|
|
243
|
+
let finished = false;
|
|
244
|
+
let scheduled = false;
|
|
245
|
+
/** Posts the latest pending measurement. */
|
|
246
|
+
const flush = (): void => {
|
|
247
|
+
scheduled = false;
|
|
248
|
+
if (pending) {
|
|
249
|
+
WorkerBody.postMessage('progress', {progress: pending});
|
|
250
|
+
pending = undefined;
|
|
251
|
+
}
|
|
252
|
+
};
|
|
253
|
+
return {
|
|
254
|
+
/** Records progress while retaining phase transitions. */
|
|
255
|
+
report(progress: WorkerProgress): void {
|
|
256
|
+
if (finished) return;
|
|
257
|
+
if (pending && pending.phase !== progress.phase) flush();
|
|
258
|
+
pending = {...progress};
|
|
259
|
+
if (!scheduled) {
|
|
260
|
+
scheduled = true;
|
|
261
|
+
setTimeout(flush, 0);
|
|
262
|
+
}
|
|
263
|
+
},
|
|
264
|
+
/** Flushes final progress and suppresses reports from stale callbacks. */
|
|
265
|
+
finish(discard = false): void {
|
|
266
|
+
if (discard) pending = undefined;
|
|
267
|
+
flush();
|
|
268
|
+
finished = true;
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
// loaders.gl
|
|
6
|
+
// SPDX-License-Identifier: MIT
|
|
7
|
+
// Copyright (c) vis.gl contributors
|
|
8
|
+
|
|
9
|
+
import type {WorkerObject, WorkerOptions} from '../../types';
|
|
10
|
+
import {assert} from '../env-utils/assert';
|
|
11
|
+
import {isBrowser} from '../env-utils/globals';
|
|
12
|
+
import {VERSION} from '../env-utils/version';
|
|
13
|
+
import {NPM_TAG} from '../npm-tag';
|
|
14
|
+
|
|
15
|
+
const warnedWorkerVersionFallbacks = new Set<string>();
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Gets worker object's name (for debugging in Chrome thread inspector window)
|
|
19
|
+
*/
|
|
20
|
+
export function getWorkerName(worker: WorkerObject): string {
|
|
21
|
+
const warning = worker.version !== VERSION ? ` (worker-utils@${VERSION})` : '';
|
|
22
|
+
return `${worker.name}@${worker.version}${warning}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Returns an explicitly configured or test worker URL.
|
|
27
|
+
* @param worker Worker descriptor to resolve.
|
|
28
|
+
* @param options Worker options that can override the descriptor.
|
|
29
|
+
* @returns A worker URL, or `null` when URL-based fallback resolution is still required.
|
|
30
|
+
*/
|
|
31
|
+
export function getCustomWorkerURL(
|
|
32
|
+
worker: WorkerObject,
|
|
33
|
+
options: WorkerOptions = {}
|
|
34
|
+
): string | null {
|
|
35
|
+
const workerFile = isBrowser
|
|
36
|
+
? worker.workerFile || `${worker.id}-worker.js`
|
|
37
|
+
: worker.workerNode || `${worker.id}-worker-node.js`;
|
|
38
|
+
let url = getConfiguredWorkerURL(worker, options);
|
|
39
|
+
|
|
40
|
+
// If URL is test, generate local loaders.gl url
|
|
41
|
+
// @ts-ignore _workerType
|
|
42
|
+
const workerType = (options as any)._workerType || (options as any)?.core?._workerType;
|
|
43
|
+
if (!url && workerType === 'test') {
|
|
44
|
+
if (isBrowser) {
|
|
45
|
+
url = `modules/${worker.module}/dist/${workerFile}`;
|
|
46
|
+
} else {
|
|
47
|
+
// In the test environment the ts-node loader requires TypeScript code
|
|
48
|
+
const workerSourceDirectory =
|
|
49
|
+
worker.module === 'tasks' && worker.workerNode?.startsWith('worker-utils/')
|
|
50
|
+
? 'worker-utils/workers'
|
|
51
|
+
: 'workers';
|
|
52
|
+
const workerSourceNode =
|
|
53
|
+
worker.workerSourceNode || `${workerSourceDirectory}/${worker.id}-worker-node.ts`;
|
|
54
|
+
url = `modules/${worker.module}/src/${workerSourceNode}`;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return url || null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Returns an explicit worker URL override, excluding generated local test URLs. */
|
|
62
|
+
export function getConfiguredWorkerURL(
|
|
63
|
+
worker: WorkerObject,
|
|
64
|
+
options: WorkerOptions = {}
|
|
65
|
+
): string | null {
|
|
66
|
+
const workerOptions = options[worker.id] || {};
|
|
67
|
+
// Compression also supports the historical non-nested URL override.
|
|
68
|
+
return (
|
|
69
|
+
workerOptions.workerUrl || (worker.id === 'compression' ? options.workerUrl : null) || null
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Generates a worker URL using overrides first and the published CDN artifact as a fallback.
|
|
75
|
+
* @param worker Worker descriptor to resolve.
|
|
76
|
+
* @param options Worker options that can override the descriptor.
|
|
77
|
+
* @returns A loadable worker URL.
|
|
78
|
+
*/
|
|
79
|
+
export function getWorkerURL(worker: WorkerObject, options: WorkerOptions = {}): string {
|
|
80
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
81
|
+
const descriptorWorkerUrl = typeof worker.worker === 'string' ? worker.worker : null;
|
|
82
|
+
const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
|
|
83
|
+
|
|
84
|
+
assert(url);
|
|
85
|
+
|
|
86
|
+
// Allow user to override location
|
|
87
|
+
return url;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Returns the generated URL for a published pre-built worker.
|
|
92
|
+
* @param worker Worker descriptor to resolve.
|
|
93
|
+
* @param warn Whether to warn when an uninjected development version uses the npm tag.
|
|
94
|
+
* @returns The CDN worker URL.
|
|
95
|
+
*/
|
|
96
|
+
export function getDefaultWorkerURL(worker: WorkerObject, warn: boolean = false): string {
|
|
97
|
+
const workerFile = isBrowser
|
|
98
|
+
? worker.workerFile || `${worker.id}-worker.js`
|
|
99
|
+
: worker.workerNode || `${worker.id}-worker-node.js`;
|
|
100
|
+
let version = worker.version;
|
|
101
|
+
if (version === 'latest') {
|
|
102
|
+
version = NPM_TAG;
|
|
103
|
+
}
|
|
104
|
+
const versionTag = version ? `@${version}` : '';
|
|
105
|
+
const packageName = worker.packageName || `@loaders.gl/${worker.module}`;
|
|
106
|
+
const url = `https://unpkg.com/${packageName}${versionTag}/dist/${workerFile}`;
|
|
107
|
+
if (warn) {
|
|
108
|
+
warnIfUsingNpmTagFallback(worker, url);
|
|
109
|
+
}
|
|
110
|
+
return url;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Warn once when a worker falls back to the npm tag because __VERSION__ was not injected. */
|
|
114
|
+
function warnIfUsingNpmTagFallback(worker: WorkerObject, url: string): void {
|
|
115
|
+
if (worker.version !== 'latest') {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const workerId = `${worker.module}:${worker.id}`;
|
|
120
|
+
if (warnedWorkerVersionFallbacks.has(workerId)) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
warnedWorkerVersionFallbacks.add(workerId);
|
|
125
|
+
// eslint-disable-next-line no-console
|
|
126
|
+
// biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
|
|
127
|
+
console.warn(
|
|
128
|
+
`loaders.gl: ${worker.name} loader worker version is "latest" because __VERSION__ was not injected. Fetching ${url} from CDN.`
|
|
129
|
+
);
|
|
130
|
+
}
|