@math.gl/tasks 0.0.0-stage → 5.0.0-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/LICENSE +147 -0
  2. package/LOADERS-GL-LICENSE.txt +52 -0
  3. package/README.md +26 -2
  4. package/THIRD-PARTY-NOTICES.md +5 -0
  5. package/dist/index.cjs +3454 -0
  6. package/dist/index.cjs.map +6 -0
  7. package/dist/index.d.ts +2 -0
  8. package/dist/index.d.ts.map +1 -0
  9. package/dist/index.js +5 -0
  10. package/dist/index.js.map +1 -0
  11. package/dist/tasks/index.d.ts +9 -0
  12. package/dist/tasks/index.d.ts.map +1 -0
  13. package/dist/tasks/index.js +9 -0
  14. package/dist/tasks/index.js.map +1 -0
  15. package/dist/tasks/task-channel.d.ts +48 -0
  16. package/dist/tasks/task-channel.d.ts.map +1 -0
  17. package/dist/tasks/task-channel.js +166 -0
  18. package/dist/tasks/task-channel.js.map +1 -0
  19. package/dist/tasks/task-operation-queue.d.ts +26 -0
  20. package/dist/tasks/task-operation-queue.d.ts.map +1 -0
  21. package/dist/tasks/task-operation-queue.js +106 -0
  22. package/dist/tasks/task-operation-queue.js.map +1 -0
  23. package/dist/tasks/task-preload.d.ts +13 -0
  24. package/dist/tasks/task-preload.d.ts.map +1 -0
  25. package/dist/tasks/task-preload.js +96 -0
  26. package/dist/tasks/task-preload.js.map +1 -0
  27. package/dist/tasks/task-runner.d.ts +74 -0
  28. package/dist/tasks/task-runner.d.ts.map +1 -0
  29. package/dist/tasks/task-runner.js +811 -0
  30. package/dist/tasks/task-runner.js.map +1 -0
  31. package/dist/tasks/task-runtime.d.ts +27 -0
  32. package/dist/tasks/task-runtime.d.ts.map +1 -0
  33. package/dist/tasks/task-runtime.js +216 -0
  34. package/dist/tasks/task-runtime.js.map +1 -0
  35. package/dist/tasks/task-sync.d.ts +4 -0
  36. package/dist/tasks/task-sync.d.ts.map +1 -0
  37. package/dist/tasks/task-sync.js +95 -0
  38. package/dist/tasks/task-sync.js.map +1 -0
  39. package/dist/tasks/task-worker-channel.d.ts +11 -0
  40. package/dist/tasks/task-worker-channel.d.ts.map +1 -0
  41. package/dist/tasks/task-worker-channel.js +34 -0
  42. package/dist/tasks/task-worker-channel.js.map +1 -0
  43. package/dist/tasks/task.d.ts +132 -0
  44. package/dist/tasks/task.d.ts.map +1 -0
  45. package/dist/tasks/task.js +5 -0
  46. package/dist/tasks/task.js.map +1 -0
  47. package/dist/tasks/typed-task-dispatch.d.ts +30 -0
  48. package/dist/tasks/typed-task-dispatch.d.ts.map +1 -0
  49. package/dist/tasks/typed-task-dispatch.js +12 -0
  50. package/dist/tasks/typed-task-dispatch.js.map +1 -0
  51. package/dist/worker-utils/index.d.ts +25 -0
  52. package/dist/worker-utils/index.d.ts.map +1 -0
  53. package/dist/worker-utils/index.js +46 -0
  54. package/dist/worker-utils/index.js.map +1 -0
  55. package/dist/worker-utils/lib/async-queue/async-queue.d.ts +32 -0
  56. package/dist/worker-utils/lib/async-queue/async-queue.d.ts.map +1 -0
  57. package/dist/worker-utils/lib/async-queue/async-queue.js +88 -0
  58. package/dist/worker-utils/lib/async-queue/async-queue.js.map +1 -0
  59. package/dist/worker-utils/lib/env-utils/assert.d.ts +3 -0
  60. package/dist/worker-utils/lib/env-utils/assert.d.ts.map +1 -0
  61. package/dist/worker-utils/lib/env-utils/assert.js +13 -0
  62. package/dist/worker-utils/lib/env-utils/assert.js.map +1 -0
  63. package/dist/worker-utils/lib/env-utils/globals.d.ts +22 -0
  64. package/dist/worker-utils/lib/env-utils/globals.d.ts.map +1 -0
  65. package/dist/worker-utils/lib/env-utils/globals.js +30 -0
  66. package/dist/worker-utils/lib/env-utils/globals.js.map +1 -0
  67. package/dist/worker-utils/lib/env-utils/version.d.ts +3 -0
  68. package/dist/worker-utils/lib/env-utils/version.d.ts.map +1 -0
  69. package/dist/worker-utils/lib/env-utils/version.js +11 -0
  70. package/dist/worker-utils/lib/env-utils/version.js.map +1 -0
  71. package/dist/worker-utils/lib/library-utils/library-utils.d.ts +42 -0
  72. package/dist/worker-utils/lib/library-utils/library-utils.d.ts.map +1 -0
  73. package/dist/worker-utils/lib/library-utils/library-utils.js +251 -0
  74. package/dist/worker-utils/lib/library-utils/library-utils.js.map +1 -0
  75. package/dist/worker-utils/lib/node/worker_threads-browser.d.ts +15 -0
  76. package/dist/worker-utils/lib/node/worker_threads-browser.d.ts.map +1 -0
  77. package/dist/worker-utils/lib/node/worker_threads-browser.js +23 -0
  78. package/dist/worker-utils/lib/node/worker_threads-browser.js.map +1 -0
  79. package/dist/worker-utils/lib/node/worker_threads.d.ts +16 -0
  80. package/dist/worker-utils/lib/node/worker_threads.d.ts.map +1 -0
  81. package/dist/worker-utils/lib/node/worker_threads.js +7 -0
  82. package/dist/worker-utils/lib/node/worker_threads.js.map +1 -0
  83. package/dist/worker-utils/lib/npm-tag.d.ts +2 -0
  84. package/dist/worker-utils/lib/npm-tag.d.ts.map +1 -0
  85. package/dist/worker-utils/lib/npm-tag.js +5 -0
  86. package/dist/worker-utils/lib/npm-tag.js.map +1 -0
  87. package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.d.ts +8 -0
  88. package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.d.ts.map +1 -0
  89. package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.js +18 -0
  90. package/dist/worker-utils/lib/process-utils/child-process-proxy.browser.js.map +1 -0
  91. package/dist/worker-utils/lib/process-utils/child-process-proxy.d.ts +45 -0
  92. package/dist/worker-utils/lib/process-utils/child-process-proxy.d.ts.map +1 -0
  93. package/dist/worker-utils/lib/process-utils/child-process-proxy.js +128 -0
  94. package/dist/worker-utils/lib/process-utils/child-process-proxy.js.map +1 -0
  95. package/dist/worker-utils/lib/process-utils/process-utils.d.ts +2 -0
  96. package/dist/worker-utils/lib/process-utils/process-utils.d.ts.map +1 -0
  97. package/dist/worker-utils/lib/process-utils/process-utils.js +32 -0
  98. package/dist/worker-utils/lib/process-utils/process-utils.js.map +1 -0
  99. package/dist/worker-utils/lib/worker-api/create-worker.d.ts +9 -0
  100. package/dist/worker-utils/lib/worker-api/create-worker.d.ts.map +1 -0
  101. package/dist/worker-utils/lib/worker-api/create-worker.js +229 -0
  102. package/dist/worker-utils/lib/worker-api/create-worker.js.map +1 -0
  103. package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts +29 -0
  104. package/dist/worker-utils/lib/worker-api/get-worker-url.d.ts.map +1 -0
  105. package/dist/worker-utils/lib/worker-api/get-worker-url.js +101 -0
  106. package/dist/worker-utils/lib/worker-api/get-worker-url.js.map +1 -0
  107. package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts +67 -0
  108. package/dist/worker-utils/lib/worker-api/process-on-worker.d.ts.map +1 -0
  109. package/dist/worker-utils/lib/worker-api/process-on-worker.js +429 -0
  110. package/dist/worker-utils/lib/worker-api/process-on-worker.js.map +1 -0
  111. package/dist/worker-utils/lib/worker-api/shared-state.d.ts +34 -0
  112. package/dist/worker-utils/lib/worker-api/shared-state.d.ts.map +1 -0
  113. package/dist/worker-utils/lib/worker-api/shared-state.js +105 -0
  114. package/dist/worker-utils/lib/worker-api/shared-state.js.map +1 -0
  115. package/dist/worker-utils/lib/worker-api/validate-output-ownership.d.ts +3 -0
  116. package/dist/worker-utils/lib/worker-api/validate-output-ownership.d.ts.map +1 -0
  117. package/dist/worker-utils/lib/worker-api/validate-output-ownership.js +10 -0
  118. package/dist/worker-utils/lib/worker-api/validate-output-ownership.js.map +1 -0
  119. package/dist/worker-utils/lib/worker-api/validate-worker-version.d.ts +9 -0
  120. package/dist/worker-utils/lib/worker-api/validate-worker-version.d.ts.map +1 -0
  121. package/dist/worker-utils/lib/worker-api/validate-worker-version.js +35 -0
  122. package/dist/worker-utils/lib/worker-api/validate-worker-version.js.map +1 -0
  123. package/dist/worker-utils/lib/worker-api/worker-resource.d.ts +33 -0
  124. package/dist/worker-utils/lib/worker-api/worker-resource.d.ts.map +1 -0
  125. package/dist/worker-utils/lib/worker-api/worker-resource.js +156 -0
  126. package/dist/worker-utils/lib/worker-api/worker-resource.js.map +1 -0
  127. package/dist/worker-utils/lib/worker-farm/worker-body.d.ts +20 -0
  128. package/dist/worker-utils/lib/worker-farm/worker-body.d.ts.map +1 -0
  129. package/dist/worker-utils/lib/worker-farm/worker-body.js +134 -0
  130. package/dist/worker-utils/lib/worker-farm/worker-body.js.map +1 -0
  131. package/dist/worker-utils/lib/worker-farm/worker-farm.d.ts +73 -0
  132. package/dist/worker-utils/lib/worker-farm/worker-farm.d.ts.map +1 -0
  133. package/dist/worker-utils/lib/worker-farm/worker-farm.js +135 -0
  134. package/dist/worker-utils/lib/worker-farm/worker-farm.js.map +1 -0
  135. package/dist/worker-utils/lib/worker-farm/worker-job.d.ts +47 -0
  136. package/dist/worker-utils/lib/worker-farm/worker-job.d.ts.map +1 -0
  137. package/dist/worker-utils/lib/worker-farm/worker-job.js +124 -0
  138. package/dist/worker-utils/lib/worker-farm/worker-job.js.map +1 -0
  139. package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts +152 -0
  140. package/dist/worker-utils/lib/worker-farm/worker-pool.d.ts.map +1 -0
  141. package/dist/worker-utils/lib/worker-farm/worker-pool.js +584 -0
  142. package/dist/worker-utils/lib/worker-farm/worker-pool.js.map +1 -0
  143. package/dist/worker-utils/lib/worker-farm/worker-thread.d.ts +77 -0
  144. package/dist/worker-utils/lib/worker-farm/worker-thread.d.ts.map +1 -0
  145. package/dist/worker-utils/lib/worker-farm/worker-thread.js +215 -0
  146. package/dist/worker-utils/lib/worker-farm/worker-thread.js.map +1 -0
  147. package/dist/worker-utils/lib/worker-utils/get-input-byte-length.d.ts +5 -0
  148. package/dist/worker-utils/lib/worker-utils/get-input-byte-length.d.ts.map +1 -0
  149. package/dist/worker-utils/lib/worker-utils/get-input-byte-length.js +39 -0
  150. package/dist/worker-utils/lib/worker-utils/get-input-byte-length.js.map +1 -0
  151. package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.d.ts +15 -0
  152. package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.d.ts.map +1 -0
  153. package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.js +84 -0
  154. package/dist/worker-utils/lib/worker-utils/get-loadable-worker-url.js.map +1 -0
  155. package/dist/worker-utils/lib/worker-utils/get-transfer-list.d.ts +16 -0
  156. package/dist/worker-utils/lib/worker-utils/get-transfer-list.d.ts.map +1 -0
  157. package/dist/worker-utils/lib/worker-utils/get-transfer-list.js +127 -0
  158. package/dist/worker-utils/lib/worker-utils/get-transfer-list.js.map +1 -0
  159. package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.d.ts +6 -0
  160. package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.d.ts.map +1 -0
  161. package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.js +27 -0
  162. package/dist/worker-utils/lib/worker-utils/remove-nontransferable-options.js.map +1 -0
  163. package/dist/worker-utils/types.d.ts +165 -0
  164. package/dist/worker-utils/types.d.ts.map +1 -0
  165. package/dist/worker-utils/types.js +5 -0
  166. package/dist/worker-utils/types.js.map +1 -0
  167. package/dist/worker-utils/workers/null-worker.d.ts +2 -0
  168. package/dist/worker-utils/workers/null-worker.d.ts.map +1 -0
  169. package/dist/worker-utils/workers/null-worker.js +17 -0
  170. package/dist/worker-utils/workers/null-worker.js.map +1 -0
  171. package/package.json +43 -4
  172. package/src/index.ts +5 -0
  173. package/src/tasks/index.ts +33 -0
  174. package/src/tasks/task-channel.ts +191 -0
  175. package/src/tasks/task-operation-queue.ts +109 -0
  176. package/src/tasks/task-preload.ts +107 -0
  177. package/src/tasks/task-runner.ts +933 -0
  178. package/src/tasks/task-runtime.ts +244 -0
  179. package/src/tasks/task-sync.ts +100 -0
  180. package/src/tasks/task-worker-channel.ts +44 -0
  181. package/src/tasks/task.ts +168 -0
  182. package/src/tasks/typed-task-dispatch.ts +44 -0
  183. package/src/worker-utils/index.ts +105 -0
  184. package/src/worker-utils/lib/async-queue/async-queue.ts +100 -0
  185. package/src/worker-utils/lib/env-utils/assert.ts +14 -0
  186. package/src/worker-utils/lib/env-utils/globals.ts +40 -0
  187. package/src/worker-utils/lib/env-utils/version.ts +15 -0
  188. package/src/worker-utils/lib/library-utils/library-utils.ts +305 -0
  189. package/src/worker-utils/lib/node/worker_threads-browser.ts +44 -0
  190. package/src/worker-utils/lib/node/worker_threads.ts +21 -0
  191. package/src/worker-utils/lib/npm-tag.ts +5 -0
  192. package/src/worker-utils/lib/process-utils/child-process-proxy.browser.ts +23 -0
  193. package/src/worker-utils/lib/process-utils/child-process-proxy.ts +172 -0
  194. package/src/worker-utils/lib/process-utils/process-utils.ts +34 -0
  195. package/src/worker-utils/lib/worker-api/create-worker.ts +271 -0
  196. package/src/worker-utils/lib/worker-api/get-worker-url.ts +130 -0
  197. package/src/worker-utils/lib/worker-api/process-on-worker.ts +596 -0
  198. package/src/worker-utils/lib/worker-api/shared-state.ts +135 -0
  199. package/src/worker-utils/lib/worker-api/validate-output-ownership.ts +10 -0
  200. package/src/worker-utils/lib/worker-api/validate-worker-version.ts +44 -0
  201. package/src/worker-utils/lib/worker-api/worker-resource.ts +203 -0
  202. package/src/worker-utils/lib/worker-farm/worker-body.ts +154 -0
  203. package/src/worker-utils/lib/worker-farm/worker-farm.ts +172 -0
  204. package/src/worker-utils/lib/worker-farm/worker-job.ts +148 -0
  205. package/src/worker-utils/lib/worker-farm/worker-pool.ts +719 -0
  206. package/src/worker-utils/lib/worker-farm/worker-thread.ts +260 -0
  207. package/src/worker-utils/lib/worker-utils/get-input-byte-length.ts +34 -0
  208. package/src/worker-utils/lib/worker-utils/get-loadable-worker-url.ts +96 -0
  209. package/src/worker-utils/lib/worker-utils/get-transfer-list.ts +125 -0
  210. package/src/worker-utils/lib/worker-utils/remove-nontransferable-options.ts +27 -0
  211. package/src/worker-utils/types.ts +224 -0
  212. package/src/worker-utils/workers/null-worker.ts +20 -0
@@ -0,0 +1,596 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
+
5
+ import type {
6
+ WorkerObject,
7
+ WorkerOptions,
8
+ WorkerContext,
9
+ WorkerJobContext,
10
+ WorkerMessageType,
11
+ WorkerMessagePayload
12
+ } from '../../types';
13
+ import type WorkerJob from '../worker-farm/worker-job';
14
+ import AsyncQueue from '../async-queue/async-queue';
15
+ import WorkerFarm from '../worker-farm/worker-farm';
16
+ import type WorkerPool from '../worker-farm/worker-pool';
17
+ import type {WorkerPoolTarget, WorkerAdmission} from '../worker-farm/worker-pool';
18
+ import {
19
+ getCustomWorkerURL,
20
+ getConfiguredWorkerURL,
21
+ getDefaultWorkerURL,
22
+ getWorkerURL,
23
+ getWorkerName
24
+ } from './get-worker-url';
25
+ import {getTransferList, getTransferListForWriter} from '../worker-utils/get-transfer-list';
26
+ import {prepareSharedState, SharedWorkerState} from './shared-state';
27
+ import {isBrowser} from '../env-utils/globals';
28
+ import {getInputByteLength} from '../worker-utils/get-input-byte-length';
29
+ import {validateOutputOwnership} from './validate-output-ownership';
30
+
31
+ /** Options for worker processing */
32
+ export type ProcessOnWorkerOptions = WorkerOptions & {
33
+ /** Isolated pool, required when using an admission reservation. */
34
+ workerPool?: WorkerPool;
35
+ admission?: WorkerAdmission;
36
+ /** Cancels the job by terminating its worker. */
37
+ signal?: AbortSignal;
38
+ /** Diagnostic name shown for the worker job. */
39
+ jobName?: string;
40
+ [key: string]: any;
41
+ };
42
+
43
+ /** Options for preloading workers. */
44
+ export type PreloadWorkerOptions = {
45
+ /** Number of workers to warm in the worker pool. */
46
+ count?: number;
47
+ };
48
+
49
+ /**
50
+ * Determines if we can parse with worker
51
+ * @param loader
52
+ * @param data
53
+ * @param options
54
+ */
55
+ export function canProcessOnWorker(worker: WorkerObject, options?: WorkerOptions) {
56
+ if (!WorkerFarm.isSupported()) {
57
+ return false;
58
+ }
59
+
60
+ const workerOptions = options?.[worker.id];
61
+ return Boolean(
62
+ (worker.worker || worker.loadWorker || workerOptions?.workerUrl) && options?.worker
63
+ );
64
+ }
65
+
66
+ /**
67
+ * Checks whether a worker supports stateful batch processing.
68
+ * @param worker Worker descriptor declaring built-in batch support.
69
+ * @param options Worker options, including an explicit capability for custom URLs or source.
70
+ */
71
+ export function canProcessOnWorkerInBatches(
72
+ worker: WorkerObject,
73
+ options: WorkerOptions = {}
74
+ ): boolean {
75
+ if (!canProcessOnWorker(worker, options)) return false;
76
+ if (options.workerInBatches !== undefined) return options.workerInBatches;
77
+ return Boolean(
78
+ worker.workerInBatches && !options['source'] && !getConfiguredWorkerURL(worker, options)
79
+ );
80
+ }
81
+
82
+ /**
83
+ * This function expects that the worker thread sends certain messages,
84
+ * Creating such a worker can be automated if the worker is wrapper by a call to
85
+ * createWorker in @loaders.gl/core/worker-utils.
86
+ */
87
+ export async function processOnWorker<Input, Output, Options extends object>(
88
+ worker: WorkerObject<Input, Output, Options>,
89
+ data: Input,
90
+ options: ProcessOnWorkerOptions & Partial<Options> = {} as ProcessOnWorkerOptions &
91
+ Partial<Options>,
92
+ context: WorkerContext = {},
93
+ jobContext: WorkerJobContext = {}
94
+ ): Promise<Output> {
95
+ validateOutputOwnership(options.outputOwnership);
96
+ if (
97
+ options.inputOwnership !== undefined &&
98
+ !['copy', 'transfer', 'share'].includes(options.inputOwnership)
99
+ )
100
+ throw new Error('Invalid worker input ownership');
101
+ options = prepareSharedState(options);
102
+ throwIfAborted(options.signal);
103
+ validateSharedInput(options, [data, serializableOptions(options), jobContext]);
104
+ const name = getWorkerName(worker as WorkerObject);
105
+
106
+ const workerPool =
107
+ options.workerPool ??
108
+ WorkerFarm.getWorkerFarm(options).getWorkerPool(
109
+ getWorkerPoolTarget(worker as WorkerObject, options, name)
110
+ );
111
+
112
+ const jobName = options.jobName || worker.name;
113
+ const job = await workerPool.startJob(
114
+ jobName,
115
+ // eslint-disable-next-line
116
+ onMessage.bind(null, context, options),
117
+ undefined,
118
+ options.signal,
119
+ {
120
+ inputBytes: getInputByteLength([data, budgetOptions(options), jobContext]),
121
+ priority: options.priority,
122
+ admission: options.admission,
123
+ sharedState: options.sharedState ? new SharedWorkerState(options.sharedState) : undefined,
124
+ cancellationMode: options.cancellationMode,
125
+ cancellationTimeout: options.cancellationTimeout
126
+ }
127
+ );
128
+
129
+ try {
130
+ if (options.signal?.aborted && !job.isCancelled) job.abort(getAbortReason(options.signal));
131
+ // AbortSignal cannot be structured-cloned. It controls the job on this thread only.
132
+ const workerOptions = serializableOptions(options);
133
+ const transferableOptions = getTransferListForWriter(workerOptions);
134
+ const transferableContext = getTransferListForWriter(jobContext);
135
+ if (job.isRunning) {
136
+ job.postMessage(
137
+ 'process',
138
+ {
139
+ input: data,
140
+ options: transferableOptions,
141
+ context: transferableContext
142
+ },
143
+ options.inputOwnership === 'copy' || options.inputOwnership === 'share' ? [] : undefined
144
+ );
145
+ }
146
+
147
+ const result = await job.result;
148
+ return result.result;
149
+ } catch (error) {
150
+ if (!job.isCancelled) job.abort(error);
151
+ throw error;
152
+ }
153
+ }
154
+
155
+ /**
156
+ * Processes an input iterator on one leased worker and streams output batches back.
157
+ *
158
+ * The worker remains assigned to this job for the complete iterator lifetime, allowing
159
+ * `processInBatches` implementations to retain parser or encoder state between input batches.
160
+ * Input is demand-driven: the main thread advances the source only when that worker requests the
161
+ * next batch.
162
+ */
163
+ export function processOnWorkerInBatches<
164
+ InputBatch = any,
165
+ OutputBatch = any,
166
+ Options extends object = {[key: string]: any}
167
+ >(
168
+ worker: WorkerObject<InputBatch, OutputBatch, Options>,
169
+ input: AsyncIterable<InputBatch> | Iterable<InputBatch>,
170
+ options: ProcessOnWorkerOptions & Partial<Options> = {} as ProcessOnWorkerOptions &
171
+ Partial<Options>,
172
+ context: WorkerContext = {},
173
+ jobContext: WorkerJobContext = {}
174
+ ): AsyncIterable<OutputBatch> {
175
+ validateOutputOwnership(options.outputOwnership);
176
+ if (
177
+ options.inputOwnership !== undefined &&
178
+ !['copy', 'transfer', 'share'].includes(options.inputOwnership)
179
+ )
180
+ throw new Error('Invalid worker input ownership');
181
+ return processOnWorkerInBatchesIterator(
182
+ worker as WorkerObject,
183
+ input,
184
+ options,
185
+ context,
186
+ jobContext
187
+ );
188
+ }
189
+
190
+ /** Implements stateful worker batch processing as an async generator. */
191
+ async function* processOnWorkerInBatchesIterator<InputBatch, OutputBatch>(
192
+ worker: WorkerObject,
193
+ input: AsyncIterable<InputBatch> | Iterable<InputBatch>,
194
+ options: ProcessOnWorkerOptions,
195
+ context: WorkerContext,
196
+ jobContext: WorkerJobContext
197
+ ): AsyncIterable<OutputBatch> {
198
+ options = prepareSharedState(options);
199
+ throwIfAborted(options.signal);
200
+ const name = getWorkerName(worker);
201
+ const workerPool =
202
+ options.workerPool ??
203
+ WorkerFarm.getWorkerFarm(options).getWorkerPool(getWorkerPoolTarget(worker, options, name));
204
+ validateSharedInput(options, [serializableOptions(options), jobContext]);
205
+ const contextBytes = getInputByteLength([budgetOptions(options), jobContext]);
206
+ const inputBytes = options.inputBytes ?? contextBytes;
207
+ if (workerPool.requiresInputBytes && options.inputBytes === undefined) {
208
+ throw new Error('Byte-bounded batch sessions require inputBytes');
209
+ }
210
+ if (!Number.isSafeInteger(inputBytes) || inputBytes < contextBytes) {
211
+ throw new RangeError('Batch inputBytes must cover options and context buffers');
212
+ }
213
+ const outputBatches = new AsyncQueue<OutputBatch>();
214
+ const inputIterator = getAsyncIterator(input);
215
+ let outputFinished = false;
216
+ let inputFinished = false;
217
+ let inputCleanupRequested = false;
218
+ let inputRequest = Promise.resolve();
219
+ const batchJobFailed = new Error('Worker batch job failed');
220
+
221
+ /** Requests cleanup even when the output consumer is paused after yielding a batch. */
222
+ const closeJobInput = (): void => {
223
+ if (!inputFinished && !inputCleanupRequested) {
224
+ inputCleanupRequested = true;
225
+ closeInputIterator(inputIterator);
226
+ }
227
+ };
228
+
229
+ const finishOutput = (): void => {
230
+ if (!outputFinished) {
231
+ outputFinished = true;
232
+ outputBatches.close();
233
+ }
234
+ };
235
+ const failOutput = (_error: unknown): void => {
236
+ if (!outputFinished) {
237
+ outputFinished = true;
238
+ outputBatches.enqueue(batchJobFailed);
239
+ outputBatches.close();
240
+ }
241
+ };
242
+
243
+ const job = await workerPool
244
+ .startJob(
245
+ options.jobName || worker.name,
246
+ (activeJob, type, payload) => {
247
+ if (activeJob.isCancelled && type !== 'done' && type !== 'error') return;
248
+ switch (type) {
249
+ case 'progress':
250
+ reportProgress(activeJob, options, payload);
251
+ break;
252
+
253
+ case 'input-request':
254
+ inputRequest = inputRequest
255
+ .then(async () => {
256
+ if (!activeJob.isRunning) return;
257
+ const nextBatch = await inputIterator.next();
258
+ if (!activeJob.isRunning) {
259
+ return;
260
+ }
261
+ if (nextBatch.done) {
262
+ inputFinished = true;
263
+ activeJob.postMessage('input-done', {});
264
+ } else {
265
+ validateSharedInput(options, nextBatch.value);
266
+ if (
267
+ options.inputBytes !== undefined &&
268
+ getInputByteLength(nextBatch.value) + contextBytes > inputBytes
269
+ ) {
270
+ throw new RangeError('Worker batch exceeds reserved inputBytes');
271
+ }
272
+ activeJob.postMessage(
273
+ 'input-batch',
274
+ {input: nextBatch.value},
275
+ options.inputOwnership === 'copy' || options.inputOwnership === 'share'
276
+ ? []
277
+ : undefined
278
+ );
279
+ }
280
+ })
281
+ .catch(error => {
282
+ if (!activeJob.isCancelled) activeJob.abort(error);
283
+ });
284
+ break;
285
+
286
+ case 'output-batch':
287
+ outputBatches.push(payload.result as OutputBatch);
288
+ break;
289
+
290
+ case 'done':
291
+ activeJob.done(payload);
292
+ finishOutput();
293
+ break;
294
+
295
+ case 'error': {
296
+ const error = new Error(payload.error || 'Worker batch processing failed');
297
+ error.name = payload.errorName || 'Error';
298
+ activeJob.error(error);
299
+ failOutput(error);
300
+ break;
301
+ }
302
+
303
+ case 'process':
304
+ void onMessage(context, options, activeJob, type, payload);
305
+ break;
306
+
307
+ default: {
308
+ const error = new Error(`Unexpected worker batch message: ${type}`);
309
+ activeJob.error(error);
310
+ failOutput(error);
311
+ break;
312
+ }
313
+ }
314
+ },
315
+ undefined,
316
+ options.signal,
317
+ {
318
+ inputBytes,
319
+ priority: options.priority,
320
+ admission: options.admission,
321
+ sharedState: options.sharedState ? new SharedWorkerState(options.sharedState) : undefined,
322
+ cancellationMode: options.cancellationMode,
323
+ cancellationTimeout: options.cancellationTimeout
324
+ }
325
+ )
326
+ .catch(error => {
327
+ closeJobInput();
328
+ throw error;
329
+ });
330
+
331
+ void job.result.catch(error => {
332
+ failOutput(error);
333
+ closeJobInput();
334
+ });
335
+
336
+ try {
337
+ const workerOptions = serializableOptions(options);
338
+ if (job.isRunning) {
339
+ job.postMessage(
340
+ 'process-in-batches',
341
+ {
342
+ options: getTransferListForWriter(workerOptions),
343
+ context: getTransferListForWriter(jobContext)
344
+ },
345
+ options.inputOwnership === 'copy' || options.inputOwnership === 'share' ? [] : undefined
346
+ );
347
+ }
348
+
349
+ try {
350
+ for await (const outputBatch of outputBatches) {
351
+ yield outputBatch;
352
+ if (job.isRunning) {
353
+ job.postMessage('output-ack', {});
354
+ }
355
+ }
356
+ } catch (error) {
357
+ if (error === batchJobFailed) {
358
+ await job.result;
359
+ }
360
+ throw error;
361
+ }
362
+ await job.result;
363
+ } finally {
364
+ if (job.isRunning && !job.isCancelled) {
365
+ job.abort(createAbortError('Worker batch iterator was closed'));
366
+ }
367
+ closeJobInput();
368
+ }
369
+ }
370
+
371
+ /** Throws a cross-runtime abort error when a signal is already aborted. */
372
+ function throwIfAborted(signal?: AbortSignal): void {
373
+ if (!signal?.aborted) {
374
+ return;
375
+ }
376
+ throw getAbortReason(signal);
377
+ }
378
+
379
+ /** Returns a caller-provided abort reason or a cross-runtime `AbortError`. */
380
+ function getAbortReason(signal?: AbortSignal): unknown {
381
+ return signal ? signal.reason : createAbortError('Worker job was aborted');
382
+ }
383
+
384
+ /** Creates an abort error without requiring the DOMException global. */
385
+ function createAbortError(message: string): Error {
386
+ const error = new Error(message);
387
+ error.name = 'AbortError';
388
+ return error;
389
+ }
390
+
391
+ /** Returns an asynchronous iterator for either accepted source kind. */
392
+ function getAsyncIterator<T>(input: AsyncIterable<T> | Iterable<T>): AsyncIterator<T> {
393
+ if (Symbol.asyncIterator in input) {
394
+ return input[Symbol.asyncIterator]();
395
+ }
396
+ return (async function* iterateSynchronously() {
397
+ yield* input;
398
+ })();
399
+ }
400
+
401
+ /** Closes an input iterator without allowing a blocked return to delay cancellation. */
402
+ function closeInputIterator<InputBatch>(inputIterator: AsyncIterator<InputBatch>): void {
403
+ try {
404
+ void Promise.resolve(inputIterator.return?.()).catch(() => undefined);
405
+ } catch {
406
+ // The worker job has already been aborted; iterator cleanup is best effort.
407
+ }
408
+ }
409
+
410
+ /**
411
+ * Warm-start one or more workers in the same pool used by processOnWorker.
412
+ * @param worker Worker object to preload.
413
+ * @param options Worker options used to resolve the worker pool.
414
+ * @param preloadOptions Preload options.
415
+ */
416
+ export async function preloadWorker(
417
+ worker: WorkerObject,
418
+ options: ProcessOnWorkerOptions = {},
419
+ preloadOptions: PreloadWorkerOptions = {}
420
+ ): Promise<void> {
421
+ const name = getWorkerName(worker);
422
+ const workerPool =
423
+ options.workerPool ??
424
+ WorkerFarm.getWorkerFarm(options).getWorkerPool(getWorkerPoolTarget(worker, options, name));
425
+ const count =
426
+ preloadOptions.count ??
427
+ options.maxConcurrency ??
428
+ options.workerPool?.maxConcurrency ??
429
+ options['core']?.maxConcurrency ??
430
+ 1;
431
+ if (!Number.isSafeInteger(count) || count < 0) throw new RangeError('Invalid preload count');
432
+
433
+ const preloadJobs = Array.from({length: count}, async () => {
434
+ const job = await workerPool.startJob(
435
+ `${worker.name} preload`,
436
+ onPreloadMessage,
437
+ undefined,
438
+ options.signal
439
+ );
440
+ job.postMessage('preload', {});
441
+ return await job.result;
442
+ });
443
+
444
+ await Promise.all(preloadJobs);
445
+ }
446
+
447
+ /**
448
+ * Resolves the worker source while preserving URL overrides and classic-worker fallbacks.
449
+ * @param worker Worker descriptor to resolve.
450
+ * @param options Options for the active worker operation.
451
+ * @param name Versioned worker name.
452
+ * @returns A target suitable for worker-pool lookup.
453
+ */
454
+ export function getWorkerPoolTarget(
455
+ worker: WorkerObject,
456
+ options: ProcessOnWorkerOptions,
457
+ name: string
458
+ ): WorkerPoolTarget {
459
+ if (options['source']) {
460
+ return {name, source: options['source'], workerType: options.workerType};
461
+ }
462
+
463
+ const customWorkerUrl = getCustomWorkerURL(worker, options);
464
+ if (customWorkerUrl) {
465
+ return {name, url: customWorkerUrl, workerType: options.workerType};
466
+ }
467
+
468
+ if (isBrowser && worker.loadWorker) {
469
+ return {
470
+ name,
471
+ workerType: options.workerType,
472
+ loadWorker: worker.loadWorker,
473
+ getUrl: () => getWorkerURL(worker, options),
474
+ urlKey: getDefaultWorkerURL(worker)
475
+ };
476
+ }
477
+
478
+ return {name, url: getWorkerURL(worker, options), workerType: options.workerType};
479
+ }
480
+
481
+ /**
482
+ * Job completes when we receive the result
483
+ * @param job
484
+ * @param message
485
+ */
486
+ async function onMessage(
487
+ context: WorkerContext,
488
+ options: ProcessOnWorkerOptions,
489
+ job: WorkerJob,
490
+ type: WorkerMessageType,
491
+ payload: WorkerMessagePayload
492
+ ) {
493
+ if (job.isCancelled && type !== 'done' && type !== 'error') return;
494
+ switch (type) {
495
+ case 'progress':
496
+ reportProgress(job, options, payload);
497
+ break;
498
+
499
+ case 'done':
500
+ // Worker is done
501
+ job.done(payload);
502
+ break;
503
+
504
+ case 'error':
505
+ // Worker encountered an error
506
+ const error = new Error(payload.error);
507
+ error.name = payload.errorName || 'Error';
508
+ job.error(error);
509
+ break;
510
+
511
+ case 'process':
512
+ // Worker is asking for us (main thread) to process something
513
+ const {id, input, options: processOptions} = payload;
514
+ try {
515
+ if (!context.process) {
516
+ job.postMessage('error', {id, error: 'Worker not set up to process on main thread'});
517
+ return;
518
+ }
519
+ const result = await context.process(
520
+ input,
521
+ {...processOptions, signal: options.signal, onProgress: options.onProgress},
522
+ undefined,
523
+ payload.context || {}
524
+ );
525
+ job.postMessage('done', {id, result});
526
+ } catch (error) {
527
+ const message = error instanceof Error ? error.message : 'unknown error';
528
+ job.postMessage('error', {id, error: message});
529
+ }
530
+ break;
531
+
532
+ default:
533
+ // eslint-disable-next-line
534
+ // biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
535
+ console.warn(`process-on-worker: unknown message ${type}`);
536
+ }
537
+ }
538
+
539
+ /**
540
+ * Completes a preload job when the worker acknowledges the preload message.
541
+ * @param job Worker job.
542
+ * @param type Worker message type.
543
+ * @param payload Worker message payload.
544
+ */
545
+ function onPreloadMessage(job: WorkerJob, type: WorkerMessageType, payload: WorkerMessagePayload) {
546
+ switch (type) {
547
+ case 'done':
548
+ job.done(payload);
549
+ break;
550
+
551
+ case 'error':
552
+ job.error(new Error(payload.error));
553
+ break;
554
+
555
+ default:
556
+ // eslint-disable-next-line
557
+ // biome-ignore lint/suspicious/noConsole: Preserve upstream worker diagnostics.
558
+ console.warn(`process-on-worker: unknown preload message ${type}`);
559
+ }
560
+ }
561
+
562
+ /** Delivers progress; callback failures terminate the job and settle its caller. */
563
+ function reportProgress(
564
+ job: WorkerJob,
565
+ options: ProcessOnWorkerOptions,
566
+ payload: WorkerMessagePayload
567
+ ): void {
568
+ if (!job.isRunning || job.isCancelled || !payload.progress) return;
569
+ try {
570
+ options.onProgress?.(payload.progress);
571
+ } catch (error) {
572
+ job.abort(error);
573
+ }
574
+ }
575
+
576
+ /** Keep runtime objects, callbacks and cancellation controls on the calling thread. */
577
+ function serializableOptions(options: ProcessOnWorkerOptions): Record<string, any> {
578
+ const {signal, onProgress, onMetrics, workerPool, admission, workerType, ...serializable} =
579
+ options;
580
+ return serializable;
581
+ }
582
+
583
+ /** Share mode forbids accidentally cloning or detaching ordinary transferable storage. */
584
+ export function validateSharedInput(options: WorkerOptions, input: unknown): void {
585
+ if (options.inputOwnership === 'share' && getTransferList(input).length) {
586
+ throw new Error(
587
+ 'Share ownership requires SharedArrayBuffer storage; ordinary transferables are not allowed'
588
+ );
589
+ }
590
+ }
591
+
592
+ /** Runtime control storage is fixed overhead, not part of application input reservations. */
593
+ function budgetOptions(options: ProcessOnWorkerOptions): Record<string, any> {
594
+ const {sharedState, ...inputOptions} = serializableOptions(options);
595
+ return inputOptions;
596
+ }