@loaders.gl/core 5.0.0-alpha.2 → 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/dist.dev.js +187 -30
- package/dist/dist.min.js +3 -3
- package/dist/index.cjs +187 -32
- package/dist/index.cjs.map +3 -3
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/load.js +7 -0
- package/dist/lib/api/load.js.map +1 -1
- package/dist/lib/filesystems/browser-filesystem.d.ts.map +1 -1
- package/dist/lib/filesystems/browser-filesystem.js +25 -11
- package/dist/lib/filesystems/browser-filesystem.js.map +1 -1
- package/dist/lib/init.js +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +24 -10
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/null-loader.js +1 -1
- package/dist/null-worker-node.js +96 -56
- package/dist/null-worker.js +96 -56
- package/package.json +6 -6
- package/src/lib/api/load.ts +21 -1
- package/src/lib/filesystems/browser-filesystem.ts +25 -11
- package/src/lib/loader-utils/get-fetch-function.ts +23 -12
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get-fetch-function.js","sourceRoot":"","sources":["../../../src/lib/loader-utils/get-fetch-function.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,QAAQ,EAAC,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"get-fetch-function.js","sourceRoot":"","sources":["../../../src/lib/loader-utils/get-fetch-function.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,wBAAwB,EAAE,QAAQ,EAAC,MAAM,0BAA0B,CAAC;AAC5E,OAAO,EAAC,SAAS,EAAC,+BAA4B;AAC9C,OAAO,EAAC,sBAAsB,EAAC,0BAAuB;AAEtD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAuB,EACvB,OACmD;IAEnD,MAAM,aAAa,GAAG,sBAAsB,EAAE,CAAC;IAE/C,MAAM,aAAa,GAAG,OAAO,IAAI,aAAa,CAAC;IAC/C,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,IAAI,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC;IACrE,IAAI,aAAwB,CAAC;IAE7B,kCAAkC;IAClC,IAAI,OAAO,WAAW,KAAK,UAAU,EAAE,CAAC;QACtC,aAAa,GAAG,WAAW,CAAC;IAC9B,CAAC;SAAM,IAAI,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACjC,aAAa,GAAG,CAAC,GAAG,EAAE,cAAc,EAAE,EAAE,CACtC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,WAA0B,EAAE,cAAc,CAAC,CAAC,CAAC;IAClF,CAAC;SAAM,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QAC1B,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,aAAa,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,wBAAwB,CAAC;QAC9B,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,aAAa,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE;KACnD,CAAC,CAAC;AACL,CAAC;AAED,gGAAgG;AAChG,SAAS,iBAAiB,CAAC,cAA2B,EAAE,cAA4B;IAClF,MAAM,OAAO,GAAG,EAAC,GAAG,cAAc,EAAE,GAAG,cAAc,EAAC,CAAC;IACvD,IAAI,cAAc,CAAC,OAAO,IAAI,cAAc,EAAE,OAAO,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QACtF,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;IAC5B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/null-loader.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
5
5
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
6
|
+
const VERSION = typeof "5.0.0-alpha.3" !== 'undefined' ? "5.0.0-alpha.3" : 'latest';
|
|
7
7
|
/**
|
|
8
8
|
* Loads any data and returns null (or optionally passes through data unparsed)
|
|
9
9
|
*/
|
package/dist/null-worker-node.js
CHANGED
|
@@ -106,18 +106,18 @@ var WorkerBody = class {
|
|
|
106
106
|
static async addEventListener(onMessage) {
|
|
107
107
|
let onMessageWrapper = onMessageWrapperMap.get(onMessage);
|
|
108
108
|
if (!onMessageWrapper) {
|
|
109
|
-
onMessageWrapper =
|
|
110
|
-
|
|
109
|
+
onMessageWrapper = (message) => {
|
|
110
|
+
const messageData = getWorkerMessageData(message);
|
|
111
|
+
if (!messageData) {
|
|
111
112
|
return;
|
|
112
113
|
}
|
|
113
|
-
|
|
114
|
-
const { type, payload } = parentPort3 ? message : message.data;
|
|
115
|
-
onMessage(type, payload);
|
|
114
|
+
onMessage(messageData.type, messageData.payload);
|
|
116
115
|
};
|
|
116
|
+
onMessageWrapperMap.set(onMessage, onMessageWrapper);
|
|
117
117
|
}
|
|
118
118
|
const parentPort2 = await getParentPort();
|
|
119
119
|
if (parentPort2) {
|
|
120
|
-
|
|
120
|
+
parentPort2.on("message", onMessageWrapper);
|
|
121
121
|
} else {
|
|
122
122
|
globalThis.addEventListener("message", onMessageWrapper);
|
|
123
123
|
}
|
|
@@ -125,9 +125,12 @@ var WorkerBody = class {
|
|
|
125
125
|
static async removeEventListener(onMessage) {
|
|
126
126
|
const onMessageWrapper = onMessageWrapperMap.get(onMessage);
|
|
127
127
|
onMessageWrapperMap.delete(onMessage);
|
|
128
|
+
if (!onMessageWrapper) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
128
131
|
const parentPort2 = await getParentPort();
|
|
129
132
|
if (parentPort2) {
|
|
130
|
-
|
|
133
|
+
parentPort2.off("message", onMessageWrapper);
|
|
131
134
|
} else {
|
|
132
135
|
globalThis.removeEventListener("message", onMessageWrapper);
|
|
133
136
|
}
|
|
@@ -148,9 +151,12 @@ var WorkerBody = class {
|
|
|
148
151
|
}
|
|
149
152
|
}
|
|
150
153
|
};
|
|
151
|
-
function
|
|
152
|
-
|
|
153
|
-
|
|
154
|
+
function getWorkerMessageData(message) {
|
|
155
|
+
if (!message || typeof message !== "object") {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
const messageData = "data" in message && message.type === "message" ? message.data : message;
|
|
159
|
+
return messageData && typeof messageData.source === "string" && messageData.source.startsWith("loaders.gl") ? messageData : null;
|
|
154
160
|
}
|
|
155
161
|
|
|
156
162
|
// ../worker-utils/src/lib/async-queue/async-queue.ts
|
|
@@ -225,8 +231,8 @@ var AsyncQueue = class {
|
|
|
225
231
|
|
|
226
232
|
// ../worker-utils/src/lib/worker-api/create-worker.ts
|
|
227
233
|
var requestId = 0;
|
|
228
|
-
var
|
|
229
|
-
var
|
|
234
|
+
var activeInputBatches = null;
|
|
235
|
+
var activeOutputAcknowledgements = null;
|
|
230
236
|
async function createWorker(process, processInBatches) {
|
|
231
237
|
if (!await WorkerBody.inWorkerThread()) {
|
|
232
238
|
return;
|
|
@@ -256,24 +262,44 @@ async function createWorker(process, processInBatches) {
|
|
|
256
262
|
if (!processInBatches) {
|
|
257
263
|
throw new Error("Worker does not support batched processing");
|
|
258
264
|
}
|
|
259
|
-
inputBatches = new AsyncQueue();
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
265
|
+
const inputBatches = new AsyncQueue();
|
|
266
|
+
const outputAcknowledgements = new AsyncQueue();
|
|
267
|
+
activeInputBatches = inputBatches;
|
|
268
|
+
activeOutputAcknowledgements = outputAcknowledgements;
|
|
269
|
+
try {
|
|
270
|
+
const resultIterator = processInBatches(
|
|
271
|
+
createDemandDrivenIterator(inputBatches),
|
|
272
|
+
payload.options || {},
|
|
273
|
+
context,
|
|
274
|
+
payload.context || {}
|
|
275
|
+
);
|
|
276
|
+
for await (const batch of resultIterator) {
|
|
277
|
+
await WorkerBody.postMessage("output-batch", { result: batch });
|
|
278
|
+
await outputAcknowledgements.next();
|
|
279
|
+
}
|
|
280
|
+
await WorkerBody.postMessage("done", {});
|
|
281
|
+
} finally {
|
|
282
|
+
activeInputBatches = null;
|
|
283
|
+
activeOutputAcknowledgements = null;
|
|
269
284
|
}
|
|
270
|
-
WorkerBody.postMessage("done", {});
|
|
271
285
|
break;
|
|
272
286
|
case "input-batch":
|
|
273
|
-
|
|
287
|
+
if (!activeInputBatches) {
|
|
288
|
+
throw new Error("Worker has no active batched processing session");
|
|
289
|
+
}
|
|
290
|
+
activeInputBatches.push(payload.input);
|
|
274
291
|
break;
|
|
275
292
|
case "input-done":
|
|
276
|
-
|
|
293
|
+
if (!activeInputBatches) {
|
|
294
|
+
throw new Error("Worker has no active batched processing session");
|
|
295
|
+
}
|
|
296
|
+
activeInputBatches.close();
|
|
297
|
+
break;
|
|
298
|
+
case "output-ack":
|
|
299
|
+
if (!activeOutputAcknowledgements) {
|
|
300
|
+
throw new Error("Worker has no active batched processing session");
|
|
301
|
+
}
|
|
302
|
+
activeOutputAcknowledgements.push(void 0);
|
|
277
303
|
break;
|
|
278
304
|
default:
|
|
279
305
|
}
|
|
@@ -283,7 +309,17 @@ async function createWorker(process, processInBatches) {
|
|
|
283
309
|
}
|
|
284
310
|
};
|
|
285
311
|
}
|
|
286
|
-
function
|
|
312
|
+
async function* createDemandDrivenIterator(inputBatches) {
|
|
313
|
+
while (true) {
|
|
314
|
+
await WorkerBody.postMessage("input-request", {});
|
|
315
|
+
const nextBatch = await inputBatches.next();
|
|
316
|
+
if (nextBatch.done) {
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
yield nextBatch.value;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
function processOnMainThread(arrayBuffer, options = {}, jobContext = {}) {
|
|
287
323
|
return new Promise((resolve, reject) => {
|
|
288
324
|
const id = requestId++;
|
|
289
325
|
const onMessage = (type, payload2) => {
|
|
@@ -303,7 +339,7 @@ function processOnMainThread(arrayBuffer, options2 = {}, jobContext = {}) {
|
|
|
303
339
|
}
|
|
304
340
|
};
|
|
305
341
|
WorkerBody.addEventListener(onMessage);
|
|
306
|
-
const payload = { id, input: arrayBuffer, options
|
|
342
|
+
const payload = { id, input: arrayBuffer, options, context: jobContext };
|
|
307
343
|
WorkerBody.postMessage("process", payload);
|
|
308
344
|
});
|
|
309
345
|
}
|
|
@@ -311,21 +347,22 @@ function processOnMainThread(arrayBuffer, options2 = {}, jobContext = {}) {
|
|
|
311
347
|
// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
|
|
312
348
|
async function createLoaderWorker(loader) {
|
|
313
349
|
await createWorker(
|
|
314
|
-
|
|
315
|
-
const result = await parseData({
|
|
316
|
-
loader,
|
|
317
|
-
arrayBuffer: input,
|
|
318
|
-
options: options2,
|
|
319
|
-
context: {
|
|
320
|
-
...loaderContext,
|
|
321
|
-
coreApi: createWorkerCoreApi(),
|
|
322
|
-
_parse: createParseOnMainThread(workerContext == null ? void 0 : workerContext.process)
|
|
323
|
-
}
|
|
324
|
-
});
|
|
325
|
-
return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options2, loaderContext) : result;
|
|
326
|
-
}
|
|
350
|
+
(input, options, workerContext, loaderContext) => processLoaderWorkerData(loader, input, options, workerContext, loaderContext)
|
|
327
351
|
);
|
|
328
352
|
}
|
|
353
|
+
async function processLoaderWorkerData(loader, input, options = {}, workerContext, loaderContext = {}) {
|
|
354
|
+
const result = await parseData({
|
|
355
|
+
loader,
|
|
356
|
+
arrayBuffer: input,
|
|
357
|
+
options,
|
|
358
|
+
context: {
|
|
359
|
+
...loaderContext,
|
|
360
|
+
coreApi: createWorkerCoreApi(),
|
|
361
|
+
_parse: createParseOnMainThread(workerContext == null ? void 0 : workerContext.process)
|
|
362
|
+
}
|
|
363
|
+
});
|
|
364
|
+
return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options, loaderContext) : result;
|
|
365
|
+
}
|
|
329
366
|
function createWorkerCoreApi() {
|
|
330
367
|
const unavailable = (methodName) => () => {
|
|
331
368
|
throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
|
|
@@ -341,17 +378,17 @@ function createWorkerCoreApi() {
|
|
|
341
378
|
};
|
|
342
379
|
}
|
|
343
380
|
function createParseOnMainThread(processOnMainThread2) {
|
|
344
|
-
return (arrayBuffer, loaders,
|
|
381
|
+
return (arrayBuffer, loaders, options, context) => {
|
|
345
382
|
if (!processOnMainThread2) {
|
|
346
383
|
throw new Error("Worker not set up to parse on main thread");
|
|
347
384
|
}
|
|
348
|
-
const parseArguments = getMainThreadParseArguments(loaders,
|
|
385
|
+
const parseArguments = getMainThreadParseArguments(loaders, options, context);
|
|
349
386
|
return processOnMainThread2(arrayBuffer, parseArguments.options, parseArguments.context);
|
|
350
387
|
};
|
|
351
388
|
}
|
|
352
|
-
function getMainThreadParseArguments(loaders,
|
|
353
|
-
if (
|
|
354
|
-
return { options
|
|
389
|
+
function getMainThreadParseArguments(loaders, options, context) {
|
|
390
|
+
if (options) {
|
|
391
|
+
return { options, context: getSerializableLoaderContext(context) };
|
|
355
392
|
}
|
|
356
393
|
if (Array.isArray(loaders) || loaders && isLoaderObject(loaders)) {
|
|
357
394
|
return { options: void 0, context: getSerializableLoaderContext(context) };
|
|
@@ -374,7 +411,7 @@ function getSerializableLoaderContext(context) {
|
|
|
374
411
|
async function parseData({
|
|
375
412
|
loader,
|
|
376
413
|
arrayBuffer,
|
|
377
|
-
options
|
|
414
|
+
options,
|
|
378
415
|
context
|
|
379
416
|
}) {
|
|
380
417
|
let data;
|
|
@@ -389,19 +426,22 @@ async function parseData({
|
|
|
389
426
|
} else {
|
|
390
427
|
throw new Error(`Could not load data with ${loader.name} loader`);
|
|
391
428
|
}
|
|
392
|
-
|
|
393
|
-
...
|
|
394
|
-
modules:
|
|
429
|
+
options = {
|
|
430
|
+
...options,
|
|
431
|
+
modules: {
|
|
432
|
+
...loader && loader.options && loader.options.modules || {},
|
|
433
|
+
...options.modules || {}
|
|
434
|
+
},
|
|
395
435
|
core: {
|
|
396
|
-
...
|
|
436
|
+
...options.core,
|
|
397
437
|
worker: false
|
|
398
438
|
}
|
|
399
439
|
};
|
|
400
|
-
return await parser(data, { ...
|
|
440
|
+
return await parser(data, { ...options }, context, loader);
|
|
401
441
|
}
|
|
402
442
|
|
|
403
443
|
// src/null-loader.ts
|
|
404
|
-
var VERSION = true ? "5.0.0-alpha.
|
|
444
|
+
var VERSION = true ? "5.0.0-alpha.3" : "latest";
|
|
405
445
|
var NullLoader = {
|
|
406
446
|
dataType: null,
|
|
407
447
|
batchType: null,
|
|
@@ -411,11 +451,11 @@ var NullLoader = {
|
|
|
411
451
|
version: VERSION,
|
|
412
452
|
mimeTypes: ["application/x.empty"],
|
|
413
453
|
extensions: ["null"],
|
|
414
|
-
parse: async (arrayBuffer,
|
|
454
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
415
455
|
parseSync,
|
|
416
|
-
parseInBatches: async function* generator(asyncIterator,
|
|
456
|
+
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
417
457
|
for await (const batch of asyncIterator) {
|
|
418
|
-
yield parseSync(batch,
|
|
458
|
+
yield parseSync(batch, options, context);
|
|
419
459
|
}
|
|
420
460
|
},
|
|
421
461
|
tests: [() => false],
|
|
@@ -423,7 +463,7 @@ var NullLoader = {
|
|
|
423
463
|
null: {}
|
|
424
464
|
}
|
|
425
465
|
};
|
|
426
|
-
function parseSync(arrayBuffer,
|
|
466
|
+
function parseSync(arrayBuffer, options, context) {
|
|
427
467
|
return null;
|
|
428
468
|
}
|
|
429
469
|
|
package/dist/null-worker.js
CHANGED
|
@@ -71,18 +71,18 @@
|
|
|
71
71
|
static async addEventListener(onMessage) {
|
|
72
72
|
let onMessageWrapper = onMessageWrapperMap.get(onMessage);
|
|
73
73
|
if (!onMessageWrapper) {
|
|
74
|
-
onMessageWrapper =
|
|
75
|
-
|
|
74
|
+
onMessageWrapper = (message) => {
|
|
75
|
+
const messageData = getWorkerMessageData(message);
|
|
76
|
+
if (!messageData) {
|
|
76
77
|
return;
|
|
77
78
|
}
|
|
78
|
-
|
|
79
|
-
const { type, payload } = parentPort3 ? message : message.data;
|
|
80
|
-
onMessage(type, payload);
|
|
79
|
+
onMessage(messageData.type, messageData.payload);
|
|
81
80
|
};
|
|
81
|
+
onMessageWrapperMap.set(onMessage, onMessageWrapper);
|
|
82
82
|
}
|
|
83
83
|
const parentPort2 = await getParentPort();
|
|
84
84
|
if (parentPort2) {
|
|
85
|
-
|
|
85
|
+
parentPort2.on("message", onMessageWrapper);
|
|
86
86
|
} else {
|
|
87
87
|
globalThis.addEventListener("message", onMessageWrapper);
|
|
88
88
|
}
|
|
@@ -90,9 +90,12 @@
|
|
|
90
90
|
static async removeEventListener(onMessage) {
|
|
91
91
|
const onMessageWrapper = onMessageWrapperMap.get(onMessage);
|
|
92
92
|
onMessageWrapperMap.delete(onMessage);
|
|
93
|
+
if (!onMessageWrapper) {
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
93
96
|
const parentPort2 = await getParentPort();
|
|
94
97
|
if (parentPort2) {
|
|
95
|
-
|
|
98
|
+
parentPort2.off("message", onMessageWrapper);
|
|
96
99
|
} else {
|
|
97
100
|
globalThis.removeEventListener("message", onMessageWrapper);
|
|
98
101
|
}
|
|
@@ -113,9 +116,12 @@
|
|
|
113
116
|
}
|
|
114
117
|
}
|
|
115
118
|
};
|
|
116
|
-
function
|
|
117
|
-
|
|
118
|
-
|
|
119
|
+
function getWorkerMessageData(message) {
|
|
120
|
+
if (!message || typeof message !== "object") {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
const messageData = "data" in message && message.type === "message" ? message.data : message;
|
|
124
|
+
return messageData && typeof messageData.source === "string" && messageData.source.startsWith("loaders.gl") ? messageData : null;
|
|
119
125
|
}
|
|
120
126
|
|
|
121
127
|
// ../worker-utils/src/lib/async-queue/async-queue.ts
|
|
@@ -190,8 +196,8 @@
|
|
|
190
196
|
|
|
191
197
|
// ../worker-utils/src/lib/worker-api/create-worker.ts
|
|
192
198
|
var requestId = 0;
|
|
193
|
-
var
|
|
194
|
-
var
|
|
199
|
+
var activeInputBatches = null;
|
|
200
|
+
var activeOutputAcknowledgements = null;
|
|
195
201
|
async function createWorker(process, processInBatches) {
|
|
196
202
|
if (!await WorkerBody.inWorkerThread()) {
|
|
197
203
|
return;
|
|
@@ -221,24 +227,44 @@
|
|
|
221
227
|
if (!processInBatches) {
|
|
222
228
|
throw new Error("Worker does not support batched processing");
|
|
223
229
|
}
|
|
224
|
-
inputBatches = new AsyncQueue();
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
230
|
+
const inputBatches = new AsyncQueue();
|
|
231
|
+
const outputAcknowledgements = new AsyncQueue();
|
|
232
|
+
activeInputBatches = inputBatches;
|
|
233
|
+
activeOutputAcknowledgements = outputAcknowledgements;
|
|
234
|
+
try {
|
|
235
|
+
const resultIterator = processInBatches(
|
|
236
|
+
createDemandDrivenIterator(inputBatches),
|
|
237
|
+
payload.options || {},
|
|
238
|
+
context,
|
|
239
|
+
payload.context || {}
|
|
240
|
+
);
|
|
241
|
+
for await (const batch of resultIterator) {
|
|
242
|
+
await WorkerBody.postMessage("output-batch", { result: batch });
|
|
243
|
+
await outputAcknowledgements.next();
|
|
244
|
+
}
|
|
245
|
+
await WorkerBody.postMessage("done", {});
|
|
246
|
+
} finally {
|
|
247
|
+
activeInputBatches = null;
|
|
248
|
+
activeOutputAcknowledgements = null;
|
|
234
249
|
}
|
|
235
|
-
WorkerBody.postMessage("done", {});
|
|
236
250
|
break;
|
|
237
251
|
case "input-batch":
|
|
238
|
-
|
|
252
|
+
if (!activeInputBatches) {
|
|
253
|
+
throw new Error("Worker has no active batched processing session");
|
|
254
|
+
}
|
|
255
|
+
activeInputBatches.push(payload.input);
|
|
239
256
|
break;
|
|
240
257
|
case "input-done":
|
|
241
|
-
|
|
258
|
+
if (!activeInputBatches) {
|
|
259
|
+
throw new Error("Worker has no active batched processing session");
|
|
260
|
+
}
|
|
261
|
+
activeInputBatches.close();
|
|
262
|
+
break;
|
|
263
|
+
case "output-ack":
|
|
264
|
+
if (!activeOutputAcknowledgements) {
|
|
265
|
+
throw new Error("Worker has no active batched processing session");
|
|
266
|
+
}
|
|
267
|
+
activeOutputAcknowledgements.push(void 0);
|
|
242
268
|
break;
|
|
243
269
|
default:
|
|
244
270
|
}
|
|
@@ -248,7 +274,17 @@
|
|
|
248
274
|
}
|
|
249
275
|
};
|
|
250
276
|
}
|
|
251
|
-
function
|
|
277
|
+
async function* createDemandDrivenIterator(inputBatches) {
|
|
278
|
+
while (true) {
|
|
279
|
+
await WorkerBody.postMessage("input-request", {});
|
|
280
|
+
const nextBatch = await inputBatches.next();
|
|
281
|
+
if (nextBatch.done) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
yield nextBatch.value;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function processOnMainThread(arrayBuffer, options = {}, jobContext = {}) {
|
|
252
288
|
return new Promise((resolve, reject) => {
|
|
253
289
|
const id = requestId++;
|
|
254
290
|
const onMessage = (type, payload2) => {
|
|
@@ -268,7 +304,7 @@
|
|
|
268
304
|
}
|
|
269
305
|
};
|
|
270
306
|
WorkerBody.addEventListener(onMessage);
|
|
271
|
-
const payload = { id, input: arrayBuffer, options
|
|
307
|
+
const payload = { id, input: arrayBuffer, options, context: jobContext };
|
|
272
308
|
WorkerBody.postMessage("process", payload);
|
|
273
309
|
});
|
|
274
310
|
}
|
|
@@ -276,21 +312,22 @@
|
|
|
276
312
|
// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
|
|
277
313
|
async function createLoaderWorker(loader) {
|
|
278
314
|
await createWorker(
|
|
279
|
-
|
|
280
|
-
const result = await parseData({
|
|
281
|
-
loader,
|
|
282
|
-
arrayBuffer: input,
|
|
283
|
-
options: options2,
|
|
284
|
-
context: {
|
|
285
|
-
...loaderContext,
|
|
286
|
-
coreApi: createWorkerCoreApi(),
|
|
287
|
-
_parse: createParseOnMainThread(workerContext?.process)
|
|
288
|
-
}
|
|
289
|
-
});
|
|
290
|
-
return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options2, loaderContext) : result;
|
|
291
|
-
}
|
|
315
|
+
(input, options, workerContext, loaderContext) => processLoaderWorkerData(loader, input, options, workerContext, loaderContext)
|
|
292
316
|
);
|
|
293
317
|
}
|
|
318
|
+
async function processLoaderWorkerData(loader, input, options = {}, workerContext, loaderContext = {}) {
|
|
319
|
+
const result = await parseData({
|
|
320
|
+
loader,
|
|
321
|
+
arrayBuffer: input,
|
|
322
|
+
options,
|
|
323
|
+
context: {
|
|
324
|
+
...loaderContext,
|
|
325
|
+
coreApi: createWorkerCoreApi(),
|
|
326
|
+
_parse: createParseOnMainThread(workerContext?.process)
|
|
327
|
+
}
|
|
328
|
+
});
|
|
329
|
+
return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options, loaderContext) : result;
|
|
330
|
+
}
|
|
294
331
|
function createWorkerCoreApi() {
|
|
295
332
|
const unavailable = (methodName) => () => {
|
|
296
333
|
throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
|
|
@@ -306,17 +343,17 @@
|
|
|
306
343
|
};
|
|
307
344
|
}
|
|
308
345
|
function createParseOnMainThread(processOnMainThread2) {
|
|
309
|
-
return (arrayBuffer, loaders,
|
|
346
|
+
return (arrayBuffer, loaders, options, context) => {
|
|
310
347
|
if (!processOnMainThread2) {
|
|
311
348
|
throw new Error("Worker not set up to parse on main thread");
|
|
312
349
|
}
|
|
313
|
-
const parseArguments = getMainThreadParseArguments(loaders,
|
|
350
|
+
const parseArguments = getMainThreadParseArguments(loaders, options, context);
|
|
314
351
|
return processOnMainThread2(arrayBuffer, parseArguments.options, parseArguments.context);
|
|
315
352
|
};
|
|
316
353
|
}
|
|
317
|
-
function getMainThreadParseArguments(loaders,
|
|
318
|
-
if (
|
|
319
|
-
return { options
|
|
354
|
+
function getMainThreadParseArguments(loaders, options, context) {
|
|
355
|
+
if (options) {
|
|
356
|
+
return { options, context: getSerializableLoaderContext(context) };
|
|
320
357
|
}
|
|
321
358
|
if (Array.isArray(loaders) || loaders && isLoaderObject(loaders)) {
|
|
322
359
|
return { options: void 0, context: getSerializableLoaderContext(context) };
|
|
@@ -339,7 +376,7 @@
|
|
|
339
376
|
async function parseData({
|
|
340
377
|
loader,
|
|
341
378
|
arrayBuffer,
|
|
342
|
-
options
|
|
379
|
+
options,
|
|
343
380
|
context
|
|
344
381
|
}) {
|
|
345
382
|
let data;
|
|
@@ -354,19 +391,22 @@
|
|
|
354
391
|
} else {
|
|
355
392
|
throw new Error(`Could not load data with ${loader.name} loader`);
|
|
356
393
|
}
|
|
357
|
-
|
|
358
|
-
...
|
|
359
|
-
modules:
|
|
394
|
+
options = {
|
|
395
|
+
...options,
|
|
396
|
+
modules: {
|
|
397
|
+
...loader && loader.options && loader.options.modules || {},
|
|
398
|
+
...options.modules || {}
|
|
399
|
+
},
|
|
360
400
|
core: {
|
|
361
|
-
...
|
|
401
|
+
...options.core,
|
|
362
402
|
worker: false
|
|
363
403
|
}
|
|
364
404
|
};
|
|
365
|
-
return await parser(data, { ...
|
|
405
|
+
return await parser(data, { ...options }, context, loader);
|
|
366
406
|
}
|
|
367
407
|
|
|
368
408
|
// src/null-loader.ts
|
|
369
|
-
var VERSION = true ? "5.0.0-alpha.
|
|
409
|
+
var VERSION = true ? "5.0.0-alpha.3" : "latest";
|
|
370
410
|
var NullLoader = {
|
|
371
411
|
dataType: null,
|
|
372
412
|
batchType: null,
|
|
@@ -376,11 +416,11 @@
|
|
|
376
416
|
version: VERSION,
|
|
377
417
|
mimeTypes: ["application/x.empty"],
|
|
378
418
|
extensions: ["null"],
|
|
379
|
-
parse: async (arrayBuffer,
|
|
419
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
380
420
|
parseSync,
|
|
381
|
-
parseInBatches: async function* generator(asyncIterator,
|
|
421
|
+
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
382
422
|
for await (const batch of asyncIterator) {
|
|
383
|
-
yield parseSync(batch,
|
|
423
|
+
yield parseSync(batch, options, context);
|
|
384
424
|
}
|
|
385
425
|
},
|
|
386
426
|
tests: [() => false],
|
|
@@ -388,7 +428,7 @@
|
|
|
388
428
|
null: {}
|
|
389
429
|
}
|
|
390
430
|
};
|
|
391
|
-
function parseSync(arrayBuffer,
|
|
431
|
+
function parseSync(arrayBuffer, options, context) {
|
|
392
432
|
return null;
|
|
393
433
|
}
|
|
394
434
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/core",
|
|
3
|
-
"version": "5.0.0-alpha.
|
|
3
|
+
"version": "5.0.0-alpha.3",
|
|
4
4
|
"description": "The core API for working with loaders.gl loaders and writers",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -58,11 +58,11 @@
|
|
|
58
58
|
"build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
|
|
59
59
|
},
|
|
60
60
|
"dependencies": {
|
|
61
|
-
"@loaders.gl/loader-utils": "5.0.0-alpha.
|
|
62
|
-
"@loaders.gl/schema": "5.0.0-alpha.
|
|
63
|
-
"@loaders.gl/schema-utils": "5.0.0-alpha.
|
|
64
|
-
"@loaders.gl/worker-utils": "5.0.0-alpha.
|
|
61
|
+
"@loaders.gl/loader-utils": "5.0.0-alpha.3",
|
|
62
|
+
"@loaders.gl/schema": "5.0.0-alpha.3",
|
|
63
|
+
"@loaders.gl/schema-utils": "5.0.0-alpha.3",
|
|
64
|
+
"@loaders.gl/worker-utils": "5.0.0-alpha.3",
|
|
65
65
|
"@probe.gl/log": "^4.1.1"
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "6e6f532b04a2798191ee2d9a932ca73b7c7f3c8d"
|
|
68
68
|
}
|
package/src/lib/api/load.ts
CHANGED
|
@@ -13,7 +13,8 @@ import type {
|
|
|
13
13
|
LoaderReturnType,
|
|
14
14
|
LoaderArrayOptionsType,
|
|
15
15
|
LoaderArrayReturnType,
|
|
16
|
-
SourceLoader
|
|
16
|
+
SourceLoader,
|
|
17
|
+
LoaderWithParser
|
|
17
18
|
} from '@loaders.gl/loader-utils';
|
|
18
19
|
import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
|
|
19
20
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
@@ -27,6 +28,7 @@ import {parseSync} from './parse-sync';
|
|
|
27
28
|
import {parseInBatches} from './parse-in-batches';
|
|
28
29
|
import {loadInBatches} from './load-in-batches';
|
|
29
30
|
import {selectLoader} from './select-loader';
|
|
31
|
+
import {getLoaderImplementation} from './load-loader';
|
|
30
32
|
|
|
31
33
|
/**
|
|
32
34
|
* Parses `data` using a specified loader
|
|
@@ -155,6 +157,24 @@ export async function load(
|
|
|
155
157
|
}
|
|
156
158
|
);
|
|
157
159
|
}
|
|
160
|
+
|
|
161
|
+
if (selectedLoader && typeof url === 'string') {
|
|
162
|
+
const loaderImplementation = await getLoaderImplementation(
|
|
163
|
+
selectedLoader,
|
|
164
|
+
resolvedOptions,
|
|
165
|
+
url
|
|
166
|
+
);
|
|
167
|
+
const parseUrl = (
|
|
168
|
+
loaderImplementation as LoaderWithParser & {
|
|
169
|
+
parseUrl?: (
|
|
170
|
+
url: string,
|
|
171
|
+
options?: LoaderOptions,
|
|
172
|
+
context?: LoaderContext
|
|
173
|
+
) => Promise<unknown>;
|
|
174
|
+
}
|
|
175
|
+
).parseUrl;
|
|
176
|
+
if (parseUrl) return await parseUrl(url, resolvedOptions, context);
|
|
177
|
+
}
|
|
158
178
|
}
|
|
159
179
|
|
|
160
180
|
// Select fetch function
|