@loaders.gl/core 5.0.0-alpha.5 → 5.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.dev.js +379 -41
- package/dist/dist.min.js +3 -3
- package/dist/index.cjs +385 -42
- package/dist/index.cjs.map +3 -3
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +6 -2
- package/dist/lib/api/parse-in-batches.js.map +1 -1
- package/dist/lib/api/parse.js +2 -2
- package/dist/lib/api/parse.js.map +1 -1
- package/dist/lib/init.js +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +2 -1
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +6 -3
- package/dist/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +1 -0
- package/dist/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +12 -1
- package/dist/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/null-loader.js +1 -1
- package/dist/null-worker-node.js +29 -3
- package/dist/null-worker.js +29 -3
- package/package.json +6 -6
- package/src/lib/api/parse-in-batches.ts +11 -2
- package/src/lib/api/parse.ts +2 -2
- package/src/lib/loader-utils/get-fetch-function.ts +2 -1
- package/src/lib/loader-utils/loader-context.ts +6 -3
- package/src/lib/loader-utils/option-defaults.ts +1 -0
- package/src/lib/loader-utils/option-utils.ts +13 -1
package/dist/index.cjs
CHANGED
|
@@ -401,6 +401,10 @@ var WorkerThread = class {
|
|
|
401
401
|
name;
|
|
402
402
|
source;
|
|
403
403
|
url;
|
|
404
|
+
/** Lazily resolves the classic worker URL used as a fallback. */
|
|
405
|
+
getUrl;
|
|
406
|
+
/** Creates a browser Worker directly. */
|
|
407
|
+
loadWorker;
|
|
404
408
|
terminated = false;
|
|
405
409
|
worker;
|
|
406
410
|
onMessage;
|
|
@@ -411,11 +415,13 @@ var WorkerThread = class {
|
|
|
411
415
|
return typeof Worker !== "undefined" && isBrowser2 || typeof NodeWorker !== "undefined" && !isBrowser2;
|
|
412
416
|
}
|
|
413
417
|
constructor(props) {
|
|
414
|
-
const { name, source, url } = props;
|
|
415
|
-
assert2(source || url);
|
|
418
|
+
const { name, source, url, getUrl, loadWorker } = props;
|
|
419
|
+
assert2(source || url || getUrl || loadWorker);
|
|
416
420
|
this.name = name;
|
|
417
421
|
this.source = source;
|
|
418
422
|
this.url = url;
|
|
423
|
+
this.getUrl = getUrl;
|
|
424
|
+
this.loadWorker = loadWorker;
|
|
419
425
|
this.onMessage = NOOP;
|
|
420
426
|
this.onError = (error) => console.log(error);
|
|
421
427
|
this.worker = isBrowser2 ? this._createBrowserWorker() : this._createNodeWorker();
|
|
@@ -461,7 +467,7 @@ var WorkerThread = class {
|
|
|
461
467
|
*/
|
|
462
468
|
_getErrorFromErrorEvent(event) {
|
|
463
469
|
let message = "Failed to load ";
|
|
464
|
-
message += `worker ${this.name} from ${this.url}. `;
|
|
470
|
+
message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
|
|
465
471
|
if (event.message) {
|
|
466
472
|
message += `${event.message} in `;
|
|
467
473
|
}
|
|
@@ -474,8 +480,26 @@ var WorkerThread = class {
|
|
|
474
480
|
* Creates a worker thread on the browser
|
|
475
481
|
*/
|
|
476
482
|
_createBrowserWorker() {
|
|
477
|
-
|
|
483
|
+
var _a;
|
|
484
|
+
if (this.loadWorker) {
|
|
485
|
+
try {
|
|
486
|
+
const worker2 = this.loadWorker();
|
|
487
|
+
if (worker2) {
|
|
488
|
+
return this._initializeBrowserWorker(worker2);
|
|
489
|
+
}
|
|
490
|
+
} catch (error) {
|
|
491
|
+
if (!this.source && !this.url && !this.getUrl) {
|
|
492
|
+
throw error;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
const url = this.url || ((_a = this.getUrl) == null ? void 0 : _a.call(this));
|
|
497
|
+
this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
|
|
478
498
|
const worker = new Worker(this._loadableURL, { name: this.name });
|
|
499
|
+
return this._initializeBrowserWorker(worker);
|
|
500
|
+
}
|
|
501
|
+
/** Attaches protocol handlers to a browser worker. */
|
|
502
|
+
_initializeBrowserWorker(worker) {
|
|
479
503
|
worker.onmessage = (event) => {
|
|
480
504
|
if (!event.data) {
|
|
481
505
|
this.onError(new Error("No data received"));
|
|
@@ -525,6 +549,10 @@ var WorkerPool = class {
|
|
|
525
549
|
source;
|
|
526
550
|
// | Function;
|
|
527
551
|
url;
|
|
552
|
+
/** Lazily resolves a classic worker fallback URL. */
|
|
553
|
+
getUrl;
|
|
554
|
+
/** Creates a browser Worker directly. */
|
|
555
|
+
loadWorker;
|
|
528
556
|
maxConcurrency = 1;
|
|
529
557
|
maxMobileConcurrency = 1;
|
|
530
558
|
onDebug = () => {
|
|
@@ -546,6 +574,8 @@ var WorkerPool = class {
|
|
|
546
574
|
constructor(props) {
|
|
547
575
|
this.source = props.source;
|
|
548
576
|
this.url = props.url;
|
|
577
|
+
this.getUrl = props.getUrl;
|
|
578
|
+
this.loadWorker = props.loadWorker;
|
|
549
579
|
this.setProps(props);
|
|
550
580
|
}
|
|
551
581
|
/**
|
|
@@ -655,7 +685,13 @@ var WorkerPool = class {
|
|
|
655
685
|
if (this.count < this._getMaxConcurrency()) {
|
|
656
686
|
this.count++;
|
|
657
687
|
const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
|
|
658
|
-
return new WorkerThread({
|
|
688
|
+
return new WorkerThread({
|
|
689
|
+
name,
|
|
690
|
+
source: this.source,
|
|
691
|
+
url: this.url,
|
|
692
|
+
getUrl: this.getUrl,
|
|
693
|
+
loadWorker: this.loadWorker
|
|
694
|
+
});
|
|
659
695
|
}
|
|
660
696
|
return null;
|
|
661
697
|
}
|
|
@@ -672,6 +708,8 @@ var DEFAULT_PROPS = {
|
|
|
672
708
|
onDebug: () => {
|
|
673
709
|
}
|
|
674
710
|
};
|
|
711
|
+
var loadWorkerIds = /* @__PURE__ */ new WeakMap();
|
|
712
|
+
var nextLoadWorkerId = 1;
|
|
675
713
|
var WorkerFarm = class _WorkerFarm {
|
|
676
714
|
props;
|
|
677
715
|
workerPools = /* @__PURE__ */ new Map();
|
|
@@ -715,24 +753,28 @@ var WorkerFarm = class _WorkerFarm {
|
|
|
715
753
|
}
|
|
716
754
|
/**
|
|
717
755
|
* Returns a worker pool for the specified worker
|
|
718
|
-
* @param options
|
|
719
|
-
* @param options.name
|
|
720
|
-
* @param options.url
|
|
721
|
-
* @param options.source
|
|
756
|
+
* @param options Worker target used to identify and initialize the pool.
|
|
757
|
+
* @param options.name Worker name used as the base pool identity.
|
|
758
|
+
* @param options.url Worker script URL.
|
|
759
|
+
* @param options.source Inline worker source.
|
|
760
|
+
* @param options.loadWorker Built-in browser worker factory.
|
|
722
761
|
* @example
|
|
723
762
|
* const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
|
|
724
763
|
*/
|
|
725
764
|
getWorkerPool(options) {
|
|
726
|
-
const { name, source, url } = options;
|
|
727
|
-
|
|
765
|
+
const { name, source, url, getUrl, loadWorker } = options;
|
|
766
|
+
const workerPoolKey = getWorkerPoolKey(options);
|
|
767
|
+
let workerPool = this.workerPools.get(workerPoolKey);
|
|
728
768
|
if (!workerPool) {
|
|
729
769
|
workerPool = new WorkerPool({
|
|
730
770
|
name,
|
|
731
771
|
source,
|
|
732
|
-
url
|
|
772
|
+
url,
|
|
773
|
+
getUrl,
|
|
774
|
+
loadWorker
|
|
733
775
|
});
|
|
734
776
|
workerPool.setProps(this._getWorkerPoolProps());
|
|
735
|
-
this.workerPools.set(
|
|
777
|
+
this.workerPools.set(workerPoolKey, workerPool);
|
|
736
778
|
}
|
|
737
779
|
return workerPool;
|
|
738
780
|
}
|
|
@@ -745,6 +787,91 @@ var WorkerFarm = class _WorkerFarm {
|
|
|
745
787
|
};
|
|
746
788
|
}
|
|
747
789
|
};
|
|
790
|
+
function getWorkerPoolKey(options) {
|
|
791
|
+
const { name, source, url, urlKey, loadWorker } = options;
|
|
792
|
+
const sourceKey = source ? `source:${source}` : "";
|
|
793
|
+
const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
|
|
794
|
+
const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
|
|
795
|
+
return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
|
|
796
|
+
}
|
|
797
|
+
function getLoadWorkerId(loadWorker) {
|
|
798
|
+
let loadWorkerId = loadWorkerIds.get(loadWorker);
|
|
799
|
+
if (!loadWorkerId) {
|
|
800
|
+
loadWorkerId = nextLoadWorkerId++;
|
|
801
|
+
loadWorkerIds.set(loadWorker, loadWorkerId);
|
|
802
|
+
}
|
|
803
|
+
return loadWorkerId;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
// ../worker-utils/src/lib/async-queue/async-queue.ts
|
|
807
|
+
var AsyncQueue = class {
|
|
808
|
+
_values;
|
|
809
|
+
_settlers;
|
|
810
|
+
_closed;
|
|
811
|
+
constructor() {
|
|
812
|
+
this._values = [];
|
|
813
|
+
this._settlers = [];
|
|
814
|
+
this._closed = false;
|
|
815
|
+
}
|
|
816
|
+
/** Return an async iterator for this queue */
|
|
817
|
+
[Symbol.asyncIterator]() {
|
|
818
|
+
return this;
|
|
819
|
+
}
|
|
820
|
+
/** Push a new value - the async iterator will yield a promise resolved to this value */
|
|
821
|
+
push(value) {
|
|
822
|
+
return this.enqueue(value);
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* Push a new value - the async iterator will yield a promise resolved to this value
|
|
826
|
+
* Add an error - the async iterator will yield a promise rejected with this value
|
|
827
|
+
*/
|
|
828
|
+
enqueue(value) {
|
|
829
|
+
if (this._closed) {
|
|
830
|
+
throw new Error("Closed");
|
|
831
|
+
}
|
|
832
|
+
if (this._settlers.length > 0) {
|
|
833
|
+
if (this._values.length > 0) {
|
|
834
|
+
throw new Error("Illegal internal state");
|
|
835
|
+
}
|
|
836
|
+
const settler = this._settlers.shift();
|
|
837
|
+
if (value instanceof Error) {
|
|
838
|
+
settler.reject(value);
|
|
839
|
+
} else {
|
|
840
|
+
settler.resolve({ value });
|
|
841
|
+
}
|
|
842
|
+
} else {
|
|
843
|
+
this._values.push(value);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
/** Indicate that we not waiting for more values - The async iterator will be done */
|
|
847
|
+
close() {
|
|
848
|
+
while (this._settlers.length > 0) {
|
|
849
|
+
const settler = this._settlers.shift();
|
|
850
|
+
settler.resolve({ done: true });
|
|
851
|
+
}
|
|
852
|
+
this._closed = true;
|
|
853
|
+
}
|
|
854
|
+
// ITERATOR IMPLEMENTATION
|
|
855
|
+
/** @returns a Promise for an IteratorResult */
|
|
856
|
+
next() {
|
|
857
|
+
if (this._values.length > 0) {
|
|
858
|
+
const value = this._values.shift();
|
|
859
|
+
if (value instanceof Error) {
|
|
860
|
+
return Promise.reject(value);
|
|
861
|
+
}
|
|
862
|
+
return Promise.resolve({ done: false, value });
|
|
863
|
+
}
|
|
864
|
+
if (this._closed) {
|
|
865
|
+
if (this._settlers.length > 0) {
|
|
866
|
+
throw new Error("Illegal internal state");
|
|
867
|
+
}
|
|
868
|
+
return Promise.resolve({ done: true, value: void 0 });
|
|
869
|
+
}
|
|
870
|
+
return new Promise((resolve2, reject) => {
|
|
871
|
+
this._settlers.push({ resolve: resolve2, reject });
|
|
872
|
+
});
|
|
873
|
+
}
|
|
874
|
+
};
|
|
748
875
|
|
|
749
876
|
// ../worker-utils/src/lib/worker-api/get-worker-url.ts
|
|
750
877
|
var warnedWorkerVersionFallbacks = /* @__PURE__ */ new Set();
|
|
@@ -752,35 +879,42 @@ function getWorkerName(worker) {
|
|
|
752
879
|
const warning = worker.version !== VERSION2 ? ` (worker-utils@${VERSION2})` : "";
|
|
753
880
|
return `${worker.name}@${worker.version}${warning}`;
|
|
754
881
|
}
|
|
755
|
-
function
|
|
882
|
+
function getCustomWorkerURL(worker, options = {}) {
|
|
756
883
|
var _a;
|
|
757
884
|
const workerOptions = options[worker.id] || {};
|
|
758
|
-
const workerFile = isBrowser2 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
885
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
759
886
|
let url = workerOptions.workerUrl;
|
|
760
|
-
if (!url && typeof worker.worker === "string") {
|
|
761
|
-
url = worker.worker;
|
|
762
|
-
}
|
|
763
887
|
if (!url && worker.id === "compression") {
|
|
764
888
|
url = options.workerUrl;
|
|
765
889
|
}
|
|
766
890
|
const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
|
|
767
|
-
if (workerType === "test") {
|
|
891
|
+
if (!url && workerType === "test") {
|
|
768
892
|
if (isBrowser2) {
|
|
769
893
|
url = `modules/${worker.module}/dist/${workerFile}`;
|
|
770
894
|
} else {
|
|
771
895
|
url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
|
|
772
896
|
}
|
|
773
897
|
}
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
898
|
+
return url || null;
|
|
899
|
+
}
|
|
900
|
+
function getWorkerURL(worker, options = {}) {
|
|
901
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
902
|
+
const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
|
|
903
|
+
const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
|
|
904
|
+
assert2(url);
|
|
905
|
+
return url;
|
|
906
|
+
}
|
|
907
|
+
function getDefaultWorkerURL(worker, warn = false) {
|
|
908
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
909
|
+
let version2 = worker.version;
|
|
910
|
+
if (version2 === "latest") {
|
|
911
|
+
version2 = NPM_TAG;
|
|
912
|
+
}
|
|
913
|
+
const versionTag = version2 ? `@${version2}` : "";
|
|
914
|
+
const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
|
|
915
|
+
if (warn) {
|
|
781
916
|
warnIfUsingNpmTagFallback(worker, url);
|
|
782
917
|
}
|
|
783
|
-
assert2(url);
|
|
784
918
|
return url;
|
|
785
919
|
}
|
|
786
920
|
function warnIfUsingNpmTagFallback(worker, url) {
|
|
@@ -803,19 +937,16 @@ function canProcessOnWorker(worker, options) {
|
|
|
803
937
|
return false;
|
|
804
938
|
}
|
|
805
939
|
const workerOptions = options == null ? void 0 : options[worker.id];
|
|
806
|
-
return Boolean(
|
|
940
|
+
return Boolean(
|
|
941
|
+
(worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
|
|
942
|
+
);
|
|
807
943
|
}
|
|
808
944
|
async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
|
|
809
945
|
var _a, _b, _c;
|
|
810
946
|
throwIfAborted(options.signal);
|
|
811
947
|
const name = getWorkerName(worker);
|
|
812
948
|
const workerFarm = WorkerFarm.getWorkerFarm(options);
|
|
813
|
-
const
|
|
814
|
-
const workerPoolProps = { name, source };
|
|
815
|
-
if (!source) {
|
|
816
|
-
workerPoolProps.url = getWorkerURL(worker, options);
|
|
817
|
-
}
|
|
818
|
-
const workerPool = workerFarm.getWorkerPool(workerPoolProps);
|
|
949
|
+
const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
|
|
819
950
|
const jobName = options.jobName || worker.name;
|
|
820
951
|
const job = await workerPool.startJob(
|
|
821
952
|
jobName,
|
|
@@ -844,6 +975,115 @@ async function processOnWorker(worker, data, options = {}, context = {}, jobCont
|
|
|
844
975
|
(_c = options.signal) == null ? void 0 : _c.removeEventListener("abort", abortJob);
|
|
845
976
|
}
|
|
846
977
|
}
|
|
978
|
+
function processOnWorkerInBatches(worker, input, options = {}, context = {}, jobContext = {}) {
|
|
979
|
+
return processOnWorkerInBatchesIterator(worker, input, options, context, jobContext);
|
|
980
|
+
}
|
|
981
|
+
async function* processOnWorkerInBatchesIterator(worker, input, options, context, jobContext) {
|
|
982
|
+
var _a, _b, _c;
|
|
983
|
+
throwIfAborted(options.signal);
|
|
984
|
+
const name = getWorkerName(worker);
|
|
985
|
+
const workerFarm = WorkerFarm.getWorkerFarm(options);
|
|
986
|
+
const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
|
|
987
|
+
const outputBatches = new AsyncQueue();
|
|
988
|
+
const inputIterator = getAsyncIterator(input);
|
|
989
|
+
let outputFinished = false;
|
|
990
|
+
let inputFinished = false;
|
|
991
|
+
let inputRequest = Promise.resolve();
|
|
992
|
+
const batchJobFailed = new Error("Worker batch job failed");
|
|
993
|
+
const finishOutput = () => {
|
|
994
|
+
if (!outputFinished) {
|
|
995
|
+
outputFinished = true;
|
|
996
|
+
outputBatches.close();
|
|
997
|
+
}
|
|
998
|
+
};
|
|
999
|
+
const failOutput = (_error) => {
|
|
1000
|
+
if (!outputFinished) {
|
|
1001
|
+
outputFinished = true;
|
|
1002
|
+
outputBatches.enqueue(batchJobFailed);
|
|
1003
|
+
outputBatches.close();
|
|
1004
|
+
}
|
|
1005
|
+
};
|
|
1006
|
+
const job = await workerPool.startJob(
|
|
1007
|
+
options.jobName || worker.name,
|
|
1008
|
+
(activeJob, type, payload) => {
|
|
1009
|
+
switch (type) {
|
|
1010
|
+
case "input-request":
|
|
1011
|
+
inputRequest = inputRequest.then(async () => {
|
|
1012
|
+
const nextBatch = await inputIterator.next();
|
|
1013
|
+
if (!activeJob.isRunning) {
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
1016
|
+
if (nextBatch.done) {
|
|
1017
|
+
inputFinished = true;
|
|
1018
|
+
activeJob.postMessage("input-done", {});
|
|
1019
|
+
} else {
|
|
1020
|
+
activeJob.postMessage("input-batch", { input: nextBatch.value });
|
|
1021
|
+
}
|
|
1022
|
+
}).catch((error) => activeJob.abort(error));
|
|
1023
|
+
break;
|
|
1024
|
+
case "output-batch":
|
|
1025
|
+
outputBatches.push(payload.result);
|
|
1026
|
+
break;
|
|
1027
|
+
case "done":
|
|
1028
|
+
activeJob.done(payload);
|
|
1029
|
+
finishOutput();
|
|
1030
|
+
break;
|
|
1031
|
+
case "error": {
|
|
1032
|
+
const error = new Error(payload.error || "Worker batch processing failed");
|
|
1033
|
+
activeJob.error(error);
|
|
1034
|
+
failOutput(error);
|
|
1035
|
+
break;
|
|
1036
|
+
}
|
|
1037
|
+
case "process":
|
|
1038
|
+
void onMessage(context, activeJob, type, payload);
|
|
1039
|
+
break;
|
|
1040
|
+
default: {
|
|
1041
|
+
const error = new Error(`Unexpected worker batch message: ${type}`);
|
|
1042
|
+
activeJob.error(error);
|
|
1043
|
+
failOutput(error);
|
|
1044
|
+
break;
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
);
|
|
1049
|
+
const abortJob = () => job.abort(getAbortReason(options.signal));
|
|
1050
|
+
(_a = options.signal) == null ? void 0 : _a.addEventListener("abort", abortJob, { once: true });
|
|
1051
|
+
if ((_b = options.signal) == null ? void 0 : _b.aborted) {
|
|
1052
|
+
abortJob();
|
|
1053
|
+
}
|
|
1054
|
+
void job.result.catch(failOutput);
|
|
1055
|
+
try {
|
|
1056
|
+
const { signal: _signal, ...workerOptions } = options;
|
|
1057
|
+
if (job.isRunning) {
|
|
1058
|
+
job.postMessage("process-in-batches", {
|
|
1059
|
+
options: getTransferListForWriter(workerOptions),
|
|
1060
|
+
context: getTransferListForWriter(jobContext)
|
|
1061
|
+
});
|
|
1062
|
+
}
|
|
1063
|
+
try {
|
|
1064
|
+
for await (const outputBatch of outputBatches) {
|
|
1065
|
+
yield outputBatch;
|
|
1066
|
+
if (job.isRunning) {
|
|
1067
|
+
job.postMessage("output-ack", {});
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
} catch (error) {
|
|
1071
|
+
if (error === batchJobFailed) {
|
|
1072
|
+
await job.result;
|
|
1073
|
+
}
|
|
1074
|
+
throw error;
|
|
1075
|
+
}
|
|
1076
|
+
await job.result;
|
|
1077
|
+
} finally {
|
|
1078
|
+
(_c = options.signal) == null ? void 0 : _c.removeEventListener("abort", abortJob);
|
|
1079
|
+
if (job.isRunning) {
|
|
1080
|
+
job.abort(createAbortError2("Worker batch iterator was closed"));
|
|
1081
|
+
}
|
|
1082
|
+
if (!inputFinished) {
|
|
1083
|
+
closeInputIterator(inputIterator);
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
847
1087
|
function throwIfAborted(signal) {
|
|
848
1088
|
if (!(signal == null ? void 0 : signal.aborted)) {
|
|
849
1089
|
return;
|
|
@@ -858,6 +1098,39 @@ function createAbortError2(message) {
|
|
|
858
1098
|
error.name = "AbortError";
|
|
859
1099
|
return error;
|
|
860
1100
|
}
|
|
1101
|
+
function getAsyncIterator(input) {
|
|
1102
|
+
if (Symbol.asyncIterator in input) {
|
|
1103
|
+
return input[Symbol.asyncIterator]();
|
|
1104
|
+
}
|
|
1105
|
+
return (async function* iterateSynchronously() {
|
|
1106
|
+
yield* input;
|
|
1107
|
+
})();
|
|
1108
|
+
}
|
|
1109
|
+
function closeInputIterator(inputIterator) {
|
|
1110
|
+
var _a;
|
|
1111
|
+
try {
|
|
1112
|
+
void Promise.resolve((_a = inputIterator.return) == null ? void 0 : _a.call(inputIterator)).catch(() => void 0);
|
|
1113
|
+
} catch {
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
function getWorkerPoolTarget(worker, options, name) {
|
|
1117
|
+
if (options.source) {
|
|
1118
|
+
return { name, source: options.source };
|
|
1119
|
+
}
|
|
1120
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
1121
|
+
if (customWorkerUrl) {
|
|
1122
|
+
return { name, url: customWorkerUrl };
|
|
1123
|
+
}
|
|
1124
|
+
if (isBrowser2 && worker.loadWorker) {
|
|
1125
|
+
return {
|
|
1126
|
+
name,
|
|
1127
|
+
loadWorker: worker.loadWorker,
|
|
1128
|
+
getUrl: () => getWorkerURL(worker, options),
|
|
1129
|
+
urlKey: getDefaultWorkerURL(worker)
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
return { name, url: getWorkerURL(worker, options) };
|
|
1133
|
+
}
|
|
861
1134
|
async function onMessage(context, job, type, payload) {
|
|
862
1135
|
switch (type) {
|
|
863
1136
|
case "done":
|
|
@@ -911,12 +1184,30 @@ function canParseWithWorker(loader, options) {
|
|
|
911
1184
|
}
|
|
912
1185
|
return Boolean(canProcessOnWorker(loader, workerOptions));
|
|
913
1186
|
}
|
|
1187
|
+
function shouldParseWithWorker(loader, data, options, context, estimateLoader = loader) {
|
|
1188
|
+
var _a, _b;
|
|
1189
|
+
if (!canParseWithWorker(loader, options)) {
|
|
1190
|
+
return false;
|
|
1191
|
+
}
|
|
1192
|
+
if (((_a = options == null ? void 0 : options.core) == null ? void 0 : _a.worker) !== "auto" || !estimateLoader.getWorkerEstimate) {
|
|
1193
|
+
return true;
|
|
1194
|
+
}
|
|
1195
|
+
try {
|
|
1196
|
+
const estimate = estimateLoader.getWorkerEstimate(data, options, context);
|
|
1197
|
+
if (estimate === void 0 || !Number.isFinite(estimate) || estimate < 0 || estimate > 1) {
|
|
1198
|
+
return true;
|
|
1199
|
+
}
|
|
1200
|
+
return estimate >= (((_b = options.core) == null ? void 0 : _b.workerThreshold) ?? 0.1);
|
|
1201
|
+
} catch {
|
|
1202
|
+
return true;
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
914
1205
|
async function parseWithWorker(loader, data, options, context, parseOnMainThread, signal) {
|
|
915
1206
|
const workerSignal = signal || getWorkerAbortSignal(options);
|
|
916
1207
|
const result = await processOnWorker(
|
|
917
1208
|
loader,
|
|
918
1209
|
data,
|
|
919
|
-
{ ...getWorkerOptions(options), signal: workerSignal },
|
|
1210
|
+
{ ...getWorkerOptions(options, loader), signal: workerSignal },
|
|
920
1211
|
{
|
|
921
1212
|
process: async (input, processOptions, _workerContext, parseContext) => {
|
|
922
1213
|
if (!parseOnMainThread) {
|
|
@@ -935,6 +1226,32 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
|
|
|
935
1226
|
);
|
|
936
1227
|
return isLoaderWithWorkerResultDeserializer(loader) ? loader.deserializeWorkerResult(result, options, context) : result;
|
|
937
1228
|
}
|
|
1229
|
+
async function* parseWithWorkerInBatches(loader, inputIterator, options, context, parseOnMainThread, signal) {
|
|
1230
|
+
const workerSignal = signal || getWorkerAbortSignal(options);
|
|
1231
|
+
const outputIterator = processOnWorkerInBatches(
|
|
1232
|
+
loader,
|
|
1233
|
+
inputIterator,
|
|
1234
|
+
{ ...getWorkerOptions(options), signal: workerSignal },
|
|
1235
|
+
{
|
|
1236
|
+
process: async (input, processOptions, _workerContext, parseContext) => {
|
|
1237
|
+
if (!parseOnMainThread) {
|
|
1238
|
+
throw new Error("Worker not set up to parse on main thread");
|
|
1239
|
+
}
|
|
1240
|
+
const mainThreadContext = context ? { ...context, ...parseContext || {} } : void 0;
|
|
1241
|
+
return await callParseOnMainThread(
|
|
1242
|
+
parseOnMainThread,
|
|
1243
|
+
input,
|
|
1244
|
+
processOptions,
|
|
1245
|
+
mainThreadContext
|
|
1246
|
+
);
|
|
1247
|
+
}
|
|
1248
|
+
},
|
|
1249
|
+
getSerializableLoaderContext(context)
|
|
1250
|
+
);
|
|
1251
|
+
for await (const batch of outputIterator) {
|
|
1252
|
+
yield loader.deserializeWorkerBatch ? loader.deserializeWorkerBatch(batch, options, context) : batch;
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
938
1255
|
function getWorkerAbortSignal(options) {
|
|
939
1256
|
var _a;
|
|
940
1257
|
const parquetSignal = (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.signal;
|
|
@@ -949,12 +1266,19 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
|
|
|
949
1266
|
}
|
|
950
1267
|
return parseOnMainThread(input, void 0, options, context);
|
|
951
1268
|
}
|
|
952
|
-
function getWorkerOptions(options = {}) {
|
|
1269
|
+
function getWorkerOptions(options = {}, loader) {
|
|
953
1270
|
const serializedOptions = JSON.parse(JSON.stringify(options));
|
|
954
|
-
|
|
1271
|
+
const workerOptions = {
|
|
955
1272
|
...serializedOptions.core,
|
|
956
1273
|
...serializedOptions
|
|
957
1274
|
};
|
|
1275
|
+
if (workerOptions.worker === "auto") {
|
|
1276
|
+
workerOptions.worker = true;
|
|
1277
|
+
}
|
|
1278
|
+
if (loader) {
|
|
1279
|
+
workerOptions._workerLoaderId = loader.id;
|
|
1280
|
+
}
|
|
1281
|
+
return workerOptions;
|
|
958
1282
|
}
|
|
959
1283
|
function getSerializableLoaderContext(context) {
|
|
960
1284
|
if (!context) {
|
|
@@ -2029,6 +2353,7 @@ var DEFAULT_LOADER_OPTIONS = {
|
|
|
2029
2353
|
CDN: "https://unpkg.com/@loaders.gl",
|
|
2030
2354
|
worker: true,
|
|
2031
2355
|
// By default, use worker if provided by loader.
|
|
2356
|
+
workerThreshold: 0.1,
|
|
2032
2357
|
maxConcurrency: 3,
|
|
2033
2358
|
// How many worker instances should be created for each loader.
|
|
2034
2359
|
maxMobileConcurrency: 1,
|
|
@@ -2104,6 +2429,7 @@ var CORE_LOADER_OPTION_KEYS = [
|
|
|
2104
2429
|
"useLocalLibraries",
|
|
2105
2430
|
"CDN",
|
|
2106
2431
|
"worker",
|
|
2432
|
+
"workerThreshold",
|
|
2107
2433
|
"maxConcurrency",
|
|
2108
2434
|
"maxMobileConcurrency",
|
|
2109
2435
|
"reuseWorkers",
|
|
@@ -2144,7 +2470,10 @@ function normalizeOptions(options, loader, loaders, url) {
|
|
|
2144
2470
|
loaders = loaders || [];
|
|
2145
2471
|
loaders = Array.isArray(loaders) ? loaders : [loaders];
|
|
2146
2472
|
validateOptions(options, loaders);
|
|
2147
|
-
|
|
2473
|
+
validateWorkerThreshold(options);
|
|
2474
|
+
const normalizedOptions = normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
|
|
2475
|
+
validateWorkerThreshold(normalizedOptions);
|
|
2476
|
+
return normalizedOptions;
|
|
2148
2477
|
}
|
|
2149
2478
|
function normalizeLoaderOptions(options) {
|
|
2150
2479
|
const normalized = cloneLoaderOptions(options);
|
|
@@ -2168,6 +2497,13 @@ function validateOptions(options, loaders) {
|
|
|
2168
2497
|
validateOptionsObject(idOptions, loader.id, loaderOptions, deprecatedOptions, loaders);
|
|
2169
2498
|
}
|
|
2170
2499
|
}
|
|
2500
|
+
function validateWorkerThreshold(options) {
|
|
2501
|
+
var _a;
|
|
2502
|
+
const threshold = (_a = options.core) == null ? void 0 : _a.workerThreshold;
|
|
2503
|
+
if (threshold !== void 0 && (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)) {
|
|
2504
|
+
throw new Error("core.workerThreshold must be a finite number between 0 and 1");
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2171
2507
|
function validateOptionsObject(options, id, defaultOptions, deprecatedOptions, loaders) {
|
|
2172
2508
|
const loaderName = id || "Top level";
|
|
2173
2509
|
const prefix = id ? `${id}.` : "";
|
|
@@ -2872,7 +3208,8 @@ function getFetchFunction(options, context) {
|
|
|
2872
3208
|
if (typeof fetchOption === "function") {
|
|
2873
3209
|
fetchFunction = fetchOption;
|
|
2874
3210
|
} else if (isObject(fetchOption)) {
|
|
2875
|
-
|
|
3211
|
+
const baseFetch = (context == null ? void 0 : context.fetch) || fetchFile;
|
|
3212
|
+
fetchFunction = (url, requestOptions) => baseFetch(url, mergeFetchOptions(fetchOption, requestOptions));
|
|
2876
3213
|
} else if (context == null ? void 0 : context.fetch) {
|
|
2877
3214
|
fetchFunction = context.fetch;
|
|
2878
3215
|
} else {
|
|
@@ -3167,6 +3504,9 @@ async function parseToOutputIterator(loader, data, options, context) {
|
|
|
3167
3504
|
const inputIterator = await getAsyncIterableFromData(data, options);
|
|
3168
3505
|
const transformedIterator = await applyInputTransforms(inputIterator, ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a.transforms) || []);
|
|
3169
3506
|
if (loader.parseInBatches) {
|
|
3507
|
+
if (loader.id === "csv" && isBrowser && canParseWithWorker(loader, options)) {
|
|
3508
|
+
return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
|
|
3509
|
+
}
|
|
3170
3510
|
return loader.parseInBatches(transformedIterator, options, context);
|
|
3171
3511
|
}
|
|
3172
3512
|
return parseChunkInBatches(transformedIterator, loader, options, context);
|
|
@@ -3337,7 +3677,10 @@ var coreApi = Object.freeze({
|
|
|
3337
3677
|
// dist/lib/loader-utils/loader-context.js
|
|
3338
3678
|
function getLoaderContext(context, options, parentContext) {
|
|
3339
3679
|
if (parentContext) {
|
|
3340
|
-
return
|
|
3680
|
+
return {
|
|
3681
|
+
...parentContext,
|
|
3682
|
+
fetch: getFetchFunction(options, parentContext)
|
|
3683
|
+
};
|
|
3341
3684
|
}
|
|
3342
3685
|
const newContext = {
|
|
3343
3686
|
...context,
|
|
@@ -3408,7 +3751,7 @@ async function parseWithLoader(loader, data, options, context) {
|
|
|
3408
3751
|
context.response = { headers, ok, redirected, status, statusText, type, url };
|
|
3409
3752
|
}
|
|
3410
3753
|
const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
|
|
3411
|
-
if (
|
|
3754
|
+
if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
|
|
3412
3755
|
data = await getArrayBufferFromData(data, options);
|
|
3413
3756
|
return await parseWithWorker(loaderWithParser, data, options, context, parse);
|
|
3414
3757
|
}
|
|
@@ -3651,7 +3994,7 @@ function makeStream(source, options) {
|
|
|
3651
3994
|
}
|
|
3652
3995
|
|
|
3653
3996
|
// dist/null-loader.js
|
|
3654
|
-
var VERSION4 = true ? "5.0.0-alpha.
|
|
3997
|
+
var VERSION4 = true ? "5.0.0-alpha.6" : "latest";
|
|
3655
3998
|
var NullWorkerLoader = {
|
|
3656
3999
|
dataType: null,
|
|
3657
4000
|
batchType: null,
|