@loaders.gl/scan 5.0.0-alpha.5 → 5.0.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/delta.cjs +189 -62
- package/dist/delta.cjs.map +3 -3
- package/dist/dist.min.js +34 -12
- package/dist/hudi.cjs +189 -62
- package/dist/hudi.cjs.map +3 -3
- package/dist/iceberg.cjs +189 -62
- package/dist/iceberg.cjs.map +3 -3
- package/dist/index.cjs +29 -12
- package/dist/index.cjs.map +1 -1
- package/package.json +8 -8
package/dist/dist.min.js
CHANGED
|
@@ -15672,6 +15672,12 @@ return true;`);
|
|
|
15672
15672
|
// ../sql/src/sql-utils.ts
|
|
15673
15673
|
init_Arrow_dom();
|
|
15674
15674
|
|
|
15675
|
+
// ../../node_modules/@math.gl/types/dist/float16.js
|
|
15676
|
+
var NativeFloat16ArrayConstructor = globalThis.Float16Array;
|
|
15677
|
+
function getFloat16ArrayConstructor() {
|
|
15678
|
+
return NativeFloat16ArrayConstructor ?? Uint16Array;
|
|
15679
|
+
}
|
|
15680
|
+
|
|
15675
15681
|
// ../schema-utils/src/lib/schema/data-type.ts
|
|
15676
15682
|
function getDataTypeFromValue(value, defaultNumberType = "float32") {
|
|
15677
15683
|
if (value instanceof Date) {
|
|
@@ -15703,6 +15709,9 @@ return true;`);
|
|
|
15703
15709
|
return { type: "null", nullable: true };
|
|
15704
15710
|
}
|
|
15705
15711
|
function getDataTypeFromTypedArray(array) {
|
|
15712
|
+
if (typeof Float16Array !== "undefined" && array instanceof Float16Array) {
|
|
15713
|
+
return "float16";
|
|
15714
|
+
}
|
|
15706
15715
|
switch (array.constructor) {
|
|
15707
15716
|
case Int8Array:
|
|
15708
15717
|
return "int8";
|
|
@@ -15744,6 +15753,8 @@ return true;`);
|
|
|
15744
15753
|
return Float32Array;
|
|
15745
15754
|
case "float64":
|
|
15746
15755
|
return Float64Array;
|
|
15756
|
+
case "float16":
|
|
15757
|
+
return getFloat16ArrayConstructor();
|
|
15747
15758
|
default:
|
|
15748
15759
|
break;
|
|
15749
15760
|
}
|
|
@@ -15800,6 +15811,13 @@ return true;`);
|
|
|
15800
15811
|
if (isArrowViewType(arrowType, "Utf8View")) {
|
|
15801
15812
|
return "utf8-view";
|
|
15802
15813
|
}
|
|
15814
|
+
if (isOptionalArrowType(arrowType, "LargeList")) {
|
|
15815
|
+
const largeListType = arrowType;
|
|
15816
|
+
return {
|
|
15817
|
+
type: "large-list",
|
|
15818
|
+
children: [serializeArrowField(largeListType.valueField)]
|
|
15819
|
+
};
|
|
15820
|
+
}
|
|
15803
15821
|
switch (arrowType.constructor) {
|
|
15804
15822
|
case Null2:
|
|
15805
15823
|
return "null";
|
|
@@ -15938,12 +15956,6 @@ return true;`);
|
|
|
15938
15956
|
type: "list",
|
|
15939
15957
|
children: [serializeArrowField(listField)]
|
|
15940
15958
|
};
|
|
15941
|
-
case LargeList2:
|
|
15942
|
-
const largeListType = arrowType;
|
|
15943
|
-
return {
|
|
15944
|
-
type: "large-list",
|
|
15945
|
-
children: [serializeArrowField(largeListType.valueField)]
|
|
15946
|
-
};
|
|
15947
15959
|
case FixedSizeList2:
|
|
15948
15960
|
const fixedSizeList = arrowType;
|
|
15949
15961
|
return {
|
|
@@ -15996,7 +16008,13 @@ return true;`);
|
|
|
15996
16008
|
}
|
|
15997
16009
|
case "large-list": {
|
|
15998
16010
|
const field = deserializeArrowField(dataType.children[0], options);
|
|
15999
|
-
|
|
16011
|
+
const LargeList3 = getOptionalArrowConstructor("LargeList");
|
|
16012
|
+
if (!LargeList3) {
|
|
16013
|
+
throw new Error(
|
|
16014
|
+
"LargeList requires an Apache Arrow runtime that exports the LargeList constructor"
|
|
16015
|
+
);
|
|
16016
|
+
}
|
|
16017
|
+
return new LargeList3(field);
|
|
16000
16018
|
}
|
|
16001
16019
|
case "fixed-size-list": {
|
|
16002
16020
|
const child = deserializeArrowField(dataType.children[0], options);
|
|
@@ -16092,11 +16110,11 @@ return true;`);
|
|
|
16092
16110
|
throw new Error("array type not supported");
|
|
16093
16111
|
}
|
|
16094
16112
|
}
|
|
16095
|
-
function
|
|
16096
|
-
return Arrow_dom_exports;
|
|
16113
|
+
function getOptionalArrowConstructor(typeName) {
|
|
16114
|
+
return Arrow_dom_exports[typeName];
|
|
16097
16115
|
}
|
|
16098
16116
|
function isArrowViewType(arrowType, viewTypeName) {
|
|
16099
|
-
const constructor =
|
|
16117
|
+
const constructor = getOptionalArrowConstructor(viewTypeName);
|
|
16100
16118
|
if (constructor && arrowType instanceof constructor) {
|
|
16101
16119
|
return true;
|
|
16102
16120
|
}
|
|
@@ -16104,12 +16122,16 @@ return true;`);
|
|
|
16104
16122
|
const guard = viewTypeName === "BinaryView" ? guards.isBinaryView : guards.isUtf8View;
|
|
16105
16123
|
return Boolean(guard?.(arrowType));
|
|
16106
16124
|
}
|
|
16125
|
+
function isOptionalArrowType(arrowType, typeName) {
|
|
16126
|
+
const constructor = getOptionalArrowConstructor(typeName);
|
|
16127
|
+
return Boolean(constructor && arrowType instanceof constructor);
|
|
16128
|
+
}
|
|
16107
16129
|
function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
|
|
16108
16130
|
const viewTypeMode = options?.viewTypes || "never";
|
|
16109
16131
|
if (viewTypeMode === "never") {
|
|
16110
16132
|
return fallbackType;
|
|
16111
16133
|
}
|
|
16112
|
-
const constructor =
|
|
16134
|
+
const constructor = getOptionalArrowConstructor(viewTypeName);
|
|
16113
16135
|
if (constructor) {
|
|
16114
16136
|
return new constructor();
|
|
16115
16137
|
}
|
|
@@ -16121,7 +16143,7 @@ return true;`);
|
|
|
16121
16143
|
return fallbackType;
|
|
16122
16144
|
}
|
|
16123
16145
|
function makeRequiredArrowViewType(viewTypeName) {
|
|
16124
|
-
const constructor =
|
|
16146
|
+
const constructor = getOptionalArrowConstructor(viewTypeName);
|
|
16125
16147
|
if (!constructor) {
|
|
16126
16148
|
throw new Error(
|
|
16127
16149
|
`${viewTypeName} requires apache-arrow 21.2.0 or later; the installed runtime does not support it`
|
package/dist/hudi.cjs
CHANGED
|
@@ -424,6 +424,10 @@ var init_worker_thread = __esm({
|
|
|
424
424
|
name;
|
|
425
425
|
source;
|
|
426
426
|
url;
|
|
427
|
+
/** Lazily resolves the classic worker URL used as a fallback. */
|
|
428
|
+
getUrl;
|
|
429
|
+
/** Creates a browser Worker directly. */
|
|
430
|
+
loadWorker;
|
|
427
431
|
terminated = false;
|
|
428
432
|
worker;
|
|
429
433
|
onMessage;
|
|
@@ -434,11 +438,13 @@ var init_worker_thread = __esm({
|
|
|
434
438
|
return typeof Worker !== "undefined" && isBrowser2 || typeof NodeWorker !== "undefined" && !isBrowser2;
|
|
435
439
|
}
|
|
436
440
|
constructor(props) {
|
|
437
|
-
const { name, source, url } = props;
|
|
438
|
-
assert(source || url);
|
|
441
|
+
const { name, source, url, getUrl, loadWorker } = props;
|
|
442
|
+
assert(source || url || getUrl || loadWorker);
|
|
439
443
|
this.name = name;
|
|
440
444
|
this.source = source;
|
|
441
445
|
this.url = url;
|
|
446
|
+
this.getUrl = getUrl;
|
|
447
|
+
this.loadWorker = loadWorker;
|
|
442
448
|
this.onMessage = NOOP;
|
|
443
449
|
this.onError = (error) => console.log(error);
|
|
444
450
|
this.worker = isBrowser2 ? this._createBrowserWorker() : this._createNodeWorker();
|
|
@@ -484,7 +490,7 @@ var init_worker_thread = __esm({
|
|
|
484
490
|
*/
|
|
485
491
|
_getErrorFromErrorEvent(event) {
|
|
486
492
|
let message = "Failed to load ";
|
|
487
|
-
message += `worker ${this.name} from ${this.url}. `;
|
|
493
|
+
message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
|
|
488
494
|
if (event.message) {
|
|
489
495
|
message += `${event.message} in `;
|
|
490
496
|
}
|
|
@@ -497,8 +503,26 @@ var init_worker_thread = __esm({
|
|
|
497
503
|
* Creates a worker thread on the browser
|
|
498
504
|
*/
|
|
499
505
|
_createBrowserWorker() {
|
|
500
|
-
|
|
506
|
+
var _a;
|
|
507
|
+
if (this.loadWorker) {
|
|
508
|
+
try {
|
|
509
|
+
const worker2 = this.loadWorker();
|
|
510
|
+
if (worker2) {
|
|
511
|
+
return this._initializeBrowserWorker(worker2);
|
|
512
|
+
}
|
|
513
|
+
} catch (error) {
|
|
514
|
+
if (!this.source && !this.url && !this.getUrl) {
|
|
515
|
+
throw error;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
const url = this.url || ((_a = this.getUrl) == null ? void 0 : _a.call(this));
|
|
520
|
+
this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
|
|
501
521
|
const worker = new Worker(this._loadableURL, { name: this.name });
|
|
522
|
+
return this._initializeBrowserWorker(worker);
|
|
523
|
+
}
|
|
524
|
+
/** Attaches protocol handlers to a browser worker. */
|
|
525
|
+
_initializeBrowserWorker(worker) {
|
|
502
526
|
worker.onmessage = (event) => {
|
|
503
527
|
if (!event.data) {
|
|
504
528
|
this.onError(new Error("No data received"));
|
|
@@ -557,6 +581,10 @@ var init_worker_pool = __esm({
|
|
|
557
581
|
source;
|
|
558
582
|
// | Function;
|
|
559
583
|
url;
|
|
584
|
+
/** Lazily resolves a classic worker fallback URL. */
|
|
585
|
+
getUrl;
|
|
586
|
+
/** Creates a browser Worker directly. */
|
|
587
|
+
loadWorker;
|
|
560
588
|
maxConcurrency = 1;
|
|
561
589
|
maxMobileConcurrency = 1;
|
|
562
590
|
onDebug = () => {
|
|
@@ -565,6 +593,7 @@ var init_worker_pool = __esm({
|
|
|
565
593
|
props = {};
|
|
566
594
|
jobQueue = [];
|
|
567
595
|
idleQueue = [];
|
|
596
|
+
activeJobs = /* @__PURE__ */ new Set();
|
|
568
597
|
count = 0;
|
|
569
598
|
isDestroyed = false;
|
|
570
599
|
/** Checks if workers are supported on this platform */
|
|
@@ -578,13 +607,19 @@ var init_worker_pool = __esm({
|
|
|
578
607
|
constructor(props) {
|
|
579
608
|
this.source = props.source;
|
|
580
609
|
this.url = props.url;
|
|
610
|
+
this.getUrl = props.getUrl;
|
|
611
|
+
this.loadWorker = props.loadWorker;
|
|
581
612
|
this.setProps(props);
|
|
582
613
|
}
|
|
583
614
|
/**
|
|
584
|
-
* Terminates all workers in the pool
|
|
615
|
+
* Terminates all workers in the pool and aborts active jobs.
|
|
616
|
+
* @param reason Optional error delivered to active jobs.
|
|
585
617
|
* @note Can free up significant memory
|
|
586
618
|
*/
|
|
587
|
-
destroy() {
|
|
619
|
+
destroy(reason = new Error("Worker pool was destroyed")) {
|
|
620
|
+
this.activeJobs.forEach((job) => job.abort(reason));
|
|
621
|
+
const queuedJobs = this.jobQueue.splice(0);
|
|
622
|
+
queuedJobs.forEach((queuedJob) => queuedJob.onReject(reason));
|
|
588
623
|
this.idleQueue.forEach((worker) => worker.destroy());
|
|
589
624
|
this.isDestroyed = true;
|
|
590
625
|
}
|
|
@@ -607,8 +642,11 @@ var init_worker_pool = __esm({
|
|
|
607
642
|
}
|
|
608
643
|
}
|
|
609
644
|
async startJob(name, onMessage2 = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
|
|
610
|
-
|
|
611
|
-
|
|
645
|
+
if (this.isDestroyed) {
|
|
646
|
+
return Promise.reject(new Error("Worker pool was destroyed"));
|
|
647
|
+
}
|
|
648
|
+
const startPromise = new Promise((onStart, onReject) => {
|
|
649
|
+
this.jobQueue.push({ name, onMessage: onMessage2, onError, onStart, onReject });
|
|
612
650
|
return this;
|
|
613
651
|
});
|
|
614
652
|
this._startQueuedJob();
|
|
@@ -636,6 +674,7 @@ var init_worker_pool = __esm({
|
|
|
636
674
|
backlog: this.jobQueue.length
|
|
637
675
|
});
|
|
638
676
|
const job = new WorkerJob(queuedJob.name, workerThread);
|
|
677
|
+
this.activeJobs.add(job);
|
|
639
678
|
workerThread.ref();
|
|
640
679
|
workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
|
|
641
680
|
workerThread.onError = (error) => queuedJob.onError(job, error);
|
|
@@ -644,6 +683,7 @@ var init_worker_pool = __esm({
|
|
|
644
683
|
await job.result;
|
|
645
684
|
} catch {
|
|
646
685
|
} finally {
|
|
686
|
+
this.activeJobs.delete(job);
|
|
647
687
|
this.returnWorkerToQueue(workerThread);
|
|
648
688
|
}
|
|
649
689
|
}
|
|
@@ -687,7 +727,13 @@ var init_worker_pool = __esm({
|
|
|
687
727
|
if (this.count < this._getMaxConcurrency()) {
|
|
688
728
|
this.count++;
|
|
689
729
|
const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
|
|
690
|
-
return new WorkerThread({
|
|
730
|
+
return new WorkerThread({
|
|
731
|
+
name,
|
|
732
|
+
source: this.source,
|
|
733
|
+
url: this.url,
|
|
734
|
+
getUrl: this.getUrl,
|
|
735
|
+
loadWorker: this.loadWorker
|
|
736
|
+
});
|
|
691
737
|
}
|
|
692
738
|
return null;
|
|
693
739
|
}
|
|
@@ -699,7 +745,22 @@ var init_worker_pool = __esm({
|
|
|
699
745
|
});
|
|
700
746
|
|
|
701
747
|
// ../worker-utils/src/lib/worker-farm/worker-farm.ts
|
|
702
|
-
|
|
748
|
+
function getWorkerPoolKey(options) {
|
|
749
|
+
const { name, source, url, urlKey, loadWorker } = options;
|
|
750
|
+
const sourceKey = source ? `source:${source}` : "";
|
|
751
|
+
const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
|
|
752
|
+
const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
|
|
753
|
+
return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
|
|
754
|
+
}
|
|
755
|
+
function getLoadWorkerId(loadWorker) {
|
|
756
|
+
let loadWorkerId = loadWorkerIds.get(loadWorker);
|
|
757
|
+
if (!loadWorkerId) {
|
|
758
|
+
loadWorkerId = nextLoadWorkerId++;
|
|
759
|
+
loadWorkerIds.set(loadWorker, loadWorkerId);
|
|
760
|
+
}
|
|
761
|
+
return loadWorkerId;
|
|
762
|
+
}
|
|
763
|
+
var DEFAULT_PROPS, loadWorkerIds, nextLoadWorkerId, WorkerFarm;
|
|
703
764
|
var init_worker_farm = __esm({
|
|
704
765
|
"../worker-utils/src/lib/worker-farm/worker-farm.ts"() {
|
|
705
766
|
"use strict";
|
|
@@ -712,6 +773,8 @@ var init_worker_farm = __esm({
|
|
|
712
773
|
onDebug: () => {
|
|
713
774
|
}
|
|
714
775
|
};
|
|
776
|
+
loadWorkerIds = /* @__PURE__ */ new WeakMap();
|
|
777
|
+
nextLoadWorkerId = 1;
|
|
715
778
|
WorkerFarm = class _WorkerFarm {
|
|
716
779
|
props;
|
|
717
780
|
workerPools = /* @__PURE__ */ new Map();
|
|
@@ -734,12 +797,13 @@ var init_worker_farm = __esm({
|
|
|
734
797
|
this.workerPools = /* @__PURE__ */ new Map();
|
|
735
798
|
}
|
|
736
799
|
/**
|
|
737
|
-
* Terminate all workers in the farm
|
|
800
|
+
* Terminate all workers in the farm and abort active jobs.
|
|
801
|
+
* @param reason Optional error delivered to active jobs.
|
|
738
802
|
* @note Can free up significant memory
|
|
739
803
|
*/
|
|
740
|
-
destroy() {
|
|
804
|
+
destroy(reason) {
|
|
741
805
|
for (const workerPool of this.workerPools.values()) {
|
|
742
|
-
workerPool.destroy();
|
|
806
|
+
workerPool.destroy(reason);
|
|
743
807
|
}
|
|
744
808
|
this.workerPools = /* @__PURE__ */ new Map();
|
|
745
809
|
}
|
|
@@ -755,24 +819,28 @@ var init_worker_farm = __esm({
|
|
|
755
819
|
}
|
|
756
820
|
/**
|
|
757
821
|
* Returns a worker pool for the specified worker
|
|
758
|
-
* @param options
|
|
759
|
-
* @param options.name
|
|
760
|
-
* @param options.url
|
|
761
|
-
* @param options.source
|
|
822
|
+
* @param options Worker target used to identify and initialize the pool.
|
|
823
|
+
* @param options.name Worker name used as the base pool identity.
|
|
824
|
+
* @param options.url Worker script URL.
|
|
825
|
+
* @param options.source Inline worker source.
|
|
826
|
+
* @param options.loadWorker Built-in browser worker factory.
|
|
762
827
|
* @example
|
|
763
828
|
* const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
|
|
764
829
|
*/
|
|
765
830
|
getWorkerPool(options) {
|
|
766
|
-
const { name, source, url } = options;
|
|
767
|
-
|
|
831
|
+
const { name, source, url, getUrl, loadWorker } = options;
|
|
832
|
+
const workerPoolKey = getWorkerPoolKey(options);
|
|
833
|
+
let workerPool = this.workerPools.get(workerPoolKey);
|
|
768
834
|
if (!workerPool) {
|
|
769
835
|
workerPool = new WorkerPool({
|
|
770
836
|
name,
|
|
771
837
|
source,
|
|
772
|
-
url
|
|
838
|
+
url,
|
|
839
|
+
getUrl,
|
|
840
|
+
loadWorker
|
|
773
841
|
});
|
|
774
842
|
workerPool.setProps(this._getWorkerPoolProps());
|
|
775
|
-
this.workerPools.set(
|
|
843
|
+
this.workerPools.set(workerPoolKey, workerPool);
|
|
776
844
|
}
|
|
777
845
|
return workerPool;
|
|
778
846
|
}
|
|
@@ -793,35 +861,42 @@ function getWorkerName(worker) {
|
|
|
793
861
|
const warning = worker.version !== VERSION2 ? ` (worker-utils@${VERSION2})` : "";
|
|
794
862
|
return `${worker.name}@${worker.version}${warning}`;
|
|
795
863
|
}
|
|
796
|
-
function
|
|
864
|
+
function getCustomWorkerURL(worker, options = {}) {
|
|
797
865
|
var _a;
|
|
798
866
|
const workerOptions = options[worker.id] || {};
|
|
799
|
-
const workerFile = isBrowser2 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
867
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
800
868
|
let url = workerOptions.workerUrl;
|
|
801
|
-
if (!url && typeof worker.worker === "string") {
|
|
802
|
-
url = worker.worker;
|
|
803
|
-
}
|
|
804
869
|
if (!url && worker.id === "compression") {
|
|
805
870
|
url = options.workerUrl;
|
|
806
871
|
}
|
|
807
872
|
const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
|
|
808
|
-
if (workerType === "test") {
|
|
873
|
+
if (!url && workerType === "test") {
|
|
809
874
|
if (isBrowser2) {
|
|
810
875
|
url = `modules/${worker.module}/dist/${workerFile}`;
|
|
811
876
|
} else {
|
|
812
877
|
url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
|
|
813
878
|
}
|
|
814
879
|
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
880
|
+
return url || null;
|
|
881
|
+
}
|
|
882
|
+
function getWorkerURL(worker, options = {}) {
|
|
883
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
884
|
+
const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
|
|
885
|
+
const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
|
|
886
|
+
assert(url);
|
|
887
|
+
return url;
|
|
888
|
+
}
|
|
889
|
+
function getDefaultWorkerURL(worker, warn = false) {
|
|
890
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
891
|
+
let version2 = worker.version;
|
|
892
|
+
if (version2 === "latest") {
|
|
893
|
+
version2 = NPM_TAG;
|
|
894
|
+
}
|
|
895
|
+
const versionTag = version2 ? `@${version2}` : "";
|
|
896
|
+
const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
|
|
897
|
+
if (warn) {
|
|
822
898
|
warnIfUsingNpmTagFallback(worker, url);
|
|
823
899
|
}
|
|
824
|
-
assert(url);
|
|
825
900
|
return url;
|
|
826
901
|
}
|
|
827
902
|
function warnIfUsingNpmTagFallback(worker, url) {
|
|
@@ -855,19 +930,16 @@ function canProcessOnWorker(worker, options) {
|
|
|
855
930
|
return false;
|
|
856
931
|
}
|
|
857
932
|
const workerOptions = options == null ? void 0 : options[worker.id];
|
|
858
|
-
return Boolean(
|
|
933
|
+
return Boolean(
|
|
934
|
+
(worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
|
|
935
|
+
);
|
|
859
936
|
}
|
|
860
937
|
async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
|
|
861
938
|
var _a, _b, _c;
|
|
862
939
|
throwIfAborted(options.signal);
|
|
863
940
|
const name = getWorkerName(worker);
|
|
864
941
|
const workerFarm = WorkerFarm.getWorkerFarm(options);
|
|
865
|
-
const
|
|
866
|
-
const workerPoolProps = { name, source };
|
|
867
|
-
if (!source) {
|
|
868
|
-
workerPoolProps.url = getWorkerURL(worker, options);
|
|
869
|
-
}
|
|
870
|
-
const workerPool = workerFarm.getWorkerPool(workerPoolProps);
|
|
942
|
+
const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
|
|
871
943
|
const jobName = options.jobName || worker.name;
|
|
872
944
|
const job = await workerPool.startJob(
|
|
873
945
|
jobName,
|
|
@@ -910,6 +982,24 @@ function createAbortError2(message) {
|
|
|
910
982
|
error.name = "AbortError";
|
|
911
983
|
return error;
|
|
912
984
|
}
|
|
985
|
+
function getWorkerPoolTarget(worker, options, name) {
|
|
986
|
+
if (options.source) {
|
|
987
|
+
return { name, source: options.source };
|
|
988
|
+
}
|
|
989
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
990
|
+
if (customWorkerUrl) {
|
|
991
|
+
return { name, url: customWorkerUrl };
|
|
992
|
+
}
|
|
993
|
+
if (isBrowser2 && worker.loadWorker) {
|
|
994
|
+
return {
|
|
995
|
+
name,
|
|
996
|
+
loadWorker: worker.loadWorker,
|
|
997
|
+
getUrl: () => getWorkerURL(worker, options),
|
|
998
|
+
urlKey: getDefaultWorkerURL(worker)
|
|
999
|
+
};
|
|
1000
|
+
}
|
|
1001
|
+
return { name, url: getWorkerURL(worker, options) };
|
|
1002
|
+
}
|
|
913
1003
|
async function onMessage(context, job, type, payload) {
|
|
914
1004
|
switch (type) {
|
|
915
1005
|
case "done":
|
|
@@ -942,6 +1032,7 @@ var init_process_on_worker = __esm({
|
|
|
942
1032
|
init_worker_farm();
|
|
943
1033
|
init_get_worker_url();
|
|
944
1034
|
init_get_transfer_list();
|
|
1035
|
+
init_globals2();
|
|
945
1036
|
}
|
|
946
1037
|
});
|
|
947
1038
|
|
|
@@ -956,13 +1047,13 @@ var init_src = __esm({
|
|
|
956
1047
|
|
|
957
1048
|
// ../loader-utils/src/lib/worker-loader-utils/parse-with-worker.ts
|
|
958
1049
|
function canParseWithWorker(loader, options) {
|
|
959
|
-
var _a
|
|
1050
|
+
var _a;
|
|
960
1051
|
const workerOptions = getWorkerOptions(options);
|
|
961
1052
|
const nodeWorkers = workerOptions._nodeWorkers;
|
|
962
1053
|
if (!isBrowser2 && !nodeWorkers) {
|
|
963
1054
|
return false;
|
|
964
1055
|
}
|
|
965
|
-
if (loader.id === "
|
|
1056
|
+
if (loader.id === "las" && (((_a = options == null ? void 0 : options.las) == null ? void 0 : _a.shape) ?? "arrow-table") === "arrow-table") {
|
|
966
1057
|
return false;
|
|
967
1058
|
}
|
|
968
1059
|
if (loader.id === "csv" && !shouldParseCSVWithWorker(options)) {
|
|
@@ -975,7 +1066,7 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
|
|
|
975
1066
|
const result = await processOnWorker(
|
|
976
1067
|
loader,
|
|
977
1068
|
data,
|
|
978
|
-
{ ...getWorkerOptions(options), signal: workerSignal },
|
|
1069
|
+
{ ...getWorkerOptions(options, loader), signal: workerSignal },
|
|
979
1070
|
{
|
|
980
1071
|
process: async (input, processOptions, _workerContext, parseContext) => {
|
|
981
1072
|
if (!parseOnMainThread) {
|
|
@@ -1008,12 +1099,19 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
|
|
|
1008
1099
|
}
|
|
1009
1100
|
return parseOnMainThread(input, void 0, options, context);
|
|
1010
1101
|
}
|
|
1011
|
-
function getWorkerOptions(options = {}) {
|
|
1102
|
+
function getWorkerOptions(options = {}, loader) {
|
|
1012
1103
|
const serializedOptions = JSON.parse(JSON.stringify(options));
|
|
1013
|
-
|
|
1104
|
+
const workerOptions = {
|
|
1014
1105
|
...serializedOptions.core,
|
|
1015
1106
|
...serializedOptions
|
|
1016
1107
|
};
|
|
1108
|
+
if (workerOptions.worker === "auto") {
|
|
1109
|
+
workerOptions.worker = true;
|
|
1110
|
+
}
|
|
1111
|
+
if (loader) {
|
|
1112
|
+
workerOptions._workerLoaderId = loader.id;
|
|
1113
|
+
}
|
|
1114
|
+
return workerOptions;
|
|
1017
1115
|
}
|
|
1018
1116
|
function getSerializableLoaderContext(context) {
|
|
1019
1117
|
if (!context) {
|
|
@@ -2899,6 +2997,9 @@ function getDataTypeFromArray(array) {
|
|
|
2899
2997
|
return { type: "null", nullable: true };
|
|
2900
2998
|
}
|
|
2901
2999
|
function getDataTypeFromTypedArray(array) {
|
|
3000
|
+
if (typeof Float16Array !== "undefined" && array instanceof Float16Array) {
|
|
3001
|
+
return "float16";
|
|
3002
|
+
}
|
|
2902
3003
|
switch (array.constructor) {
|
|
2903
3004
|
case Int8Array:
|
|
2904
3005
|
return "int8";
|
|
@@ -2940,15 +3041,19 @@ function getArrayTypeFromDataType(type, nullable) {
|
|
|
2940
3041
|
return Float32Array;
|
|
2941
3042
|
case "float64":
|
|
2942
3043
|
return Float64Array;
|
|
3044
|
+
case "float16":
|
|
3045
|
+
return (0, import_types.getFloat16ArrayConstructor)();
|
|
2943
3046
|
default:
|
|
2944
3047
|
break;
|
|
2945
3048
|
}
|
|
2946
3049
|
}
|
|
2947
3050
|
return Array;
|
|
2948
3051
|
}
|
|
3052
|
+
var import_types;
|
|
2949
3053
|
var init_data_type = __esm({
|
|
2950
3054
|
"../schema-utils/src/lib/schema/data-type.ts"() {
|
|
2951
3055
|
"use strict";
|
|
3056
|
+
import_types = require("@math.gl/types");
|
|
2952
3057
|
}
|
|
2953
3058
|
});
|
|
2954
3059
|
|
|
@@ -2997,6 +3102,13 @@ function serializeArrowType(arrowType) {
|
|
|
2997
3102
|
if (isArrowViewType(arrowType, "Utf8View")) {
|
|
2998
3103
|
return "utf8-view";
|
|
2999
3104
|
}
|
|
3105
|
+
if (isOptionalArrowType(arrowType, "LargeList")) {
|
|
3106
|
+
const largeListType = arrowType;
|
|
3107
|
+
return {
|
|
3108
|
+
type: "large-list",
|
|
3109
|
+
children: [serializeArrowField(largeListType.valueField)]
|
|
3110
|
+
};
|
|
3111
|
+
}
|
|
3000
3112
|
switch (arrowType.constructor) {
|
|
3001
3113
|
case arrow.Null:
|
|
3002
3114
|
return "null";
|
|
@@ -3135,12 +3247,6 @@ function serializeArrowType(arrowType) {
|
|
|
3135
3247
|
type: "list",
|
|
3136
3248
|
children: [serializeArrowField(listField)]
|
|
3137
3249
|
};
|
|
3138
|
-
case arrow.LargeList:
|
|
3139
|
-
const largeListType = arrowType;
|
|
3140
|
-
return {
|
|
3141
|
-
type: "large-list",
|
|
3142
|
-
children: [serializeArrowField(largeListType.valueField)]
|
|
3143
|
-
};
|
|
3144
3250
|
case arrow.FixedSizeList:
|
|
3145
3251
|
const fixedSizeList = arrowType;
|
|
3146
3252
|
return {
|
|
@@ -3193,7 +3299,13 @@ function deserializeArrowType(dataType, options) {
|
|
|
3193
3299
|
}
|
|
3194
3300
|
case "large-list": {
|
|
3195
3301
|
const field = deserializeArrowField(dataType.children[0], options);
|
|
3196
|
-
|
|
3302
|
+
const LargeList = getOptionalArrowConstructor("LargeList");
|
|
3303
|
+
if (!LargeList) {
|
|
3304
|
+
throw new Error(
|
|
3305
|
+
"LargeList requires an Apache Arrow runtime that exports the LargeList constructor"
|
|
3306
|
+
);
|
|
3307
|
+
}
|
|
3308
|
+
return new LargeList(field);
|
|
3197
3309
|
}
|
|
3198
3310
|
case "fixed-size-list": {
|
|
3199
3311
|
const child = deserializeArrowField(dataType.children[0], options);
|
|
@@ -3289,11 +3401,11 @@ function deserializeArrowType(dataType, options) {
|
|
|
3289
3401
|
throw new Error("array type not supported");
|
|
3290
3402
|
}
|
|
3291
3403
|
}
|
|
3292
|
-
function
|
|
3293
|
-
return arrow;
|
|
3404
|
+
function getOptionalArrowConstructor(typeName) {
|
|
3405
|
+
return arrow[typeName];
|
|
3294
3406
|
}
|
|
3295
3407
|
function isArrowViewType(arrowType, viewTypeName) {
|
|
3296
|
-
const constructor =
|
|
3408
|
+
const constructor = getOptionalArrowConstructor(viewTypeName);
|
|
3297
3409
|
if (constructor && arrowType instanceof constructor) {
|
|
3298
3410
|
return true;
|
|
3299
3411
|
}
|
|
@@ -3301,12 +3413,16 @@ function isArrowViewType(arrowType, viewTypeName) {
|
|
|
3301
3413
|
const guard = viewTypeName === "BinaryView" ? guards.isBinaryView : guards.isUtf8View;
|
|
3302
3414
|
return Boolean(guard == null ? void 0 : guard(arrowType));
|
|
3303
3415
|
}
|
|
3416
|
+
function isOptionalArrowType(arrowType, typeName) {
|
|
3417
|
+
const constructor = getOptionalArrowConstructor(typeName);
|
|
3418
|
+
return Boolean(constructor && arrowType instanceof constructor);
|
|
3419
|
+
}
|
|
3304
3420
|
function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
|
|
3305
3421
|
const viewTypeMode = (options == null ? void 0 : options.viewTypes) || "never";
|
|
3306
3422
|
if (viewTypeMode === "never") {
|
|
3307
3423
|
return fallbackType;
|
|
3308
3424
|
}
|
|
3309
|
-
const constructor =
|
|
3425
|
+
const constructor = getOptionalArrowConstructor(viewTypeName);
|
|
3310
3426
|
if (constructor) {
|
|
3311
3427
|
return new constructor();
|
|
3312
3428
|
}
|
|
@@ -3318,7 +3434,7 @@ function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
|
|
|
3318
3434
|
return fallbackType;
|
|
3319
3435
|
}
|
|
3320
3436
|
function makeRequiredArrowViewType(viewTypeName) {
|
|
3321
|
-
const constructor =
|
|
3437
|
+
const constructor = getOptionalArrowConstructor(viewTypeName);
|
|
3322
3438
|
if (!constructor) {
|
|
3323
3439
|
throw new Error(
|
|
3324
3440
|
`${viewTypeName} requires apache-arrow 21.2.0 or later; the installed runtime does not support it`
|
|
@@ -13208,7 +13324,7 @@ var init_parquet_loader_options = __esm({
|
|
|
13208
13324
|
PARQUET_LOADER_DEFAULT_OPTIONS = {
|
|
13209
13325
|
columns: void 0,
|
|
13210
13326
|
preserveBinary: false,
|
|
13211
|
-
shape: "
|
|
13327
|
+
shape: "arrow-table"
|
|
13212
13328
|
};
|
|
13213
13329
|
}
|
|
13214
13330
|
});
|
|
@@ -13237,6 +13353,15 @@ var init_parquet_loader_base = __esm({
|
|
|
13237
13353
|
}
|
|
13238
13354
|
});
|
|
13239
13355
|
|
|
13356
|
+
// ../parquet/src/parquet-source-worker-factory.ts
|
|
13357
|
+
var PARQUET_SOURCE_WORKER_LOAD_WORKER;
|
|
13358
|
+
var init_parquet_source_worker_factory = __esm({
|
|
13359
|
+
"../parquet/src/parquet-source-worker-factory.ts"() {
|
|
13360
|
+
"use strict";
|
|
13361
|
+
PARQUET_SOURCE_WORKER_LOAD_WORKER = void 0;
|
|
13362
|
+
}
|
|
13363
|
+
});
|
|
13364
|
+
|
|
13240
13365
|
// ../parquet/src/parquet-source-worker-url.ts
|
|
13241
13366
|
function resolveParquetSourceWorkerUrl() {
|
|
13242
13367
|
if (!import_meta.url) {
|
|
@@ -13249,7 +13374,7 @@ function resolveParquetSourceWorkerUrl() {
|
|
|
13249
13374
|
}
|
|
13250
13375
|
}
|
|
13251
13376
|
function getParquetSourceWorkerFile() {
|
|
13252
|
-
return "
|
|
13377
|
+
return "../dist/parquet-source-worker.js";
|
|
13253
13378
|
}
|
|
13254
13379
|
var import_meta, PARQUET_SOURCE_WORKER_URL;
|
|
13255
13380
|
var init_parquet_source_worker_url = __esm({
|
|
@@ -13277,12 +13402,14 @@ var init_parquet_source_worker_client = __esm({
|
|
|
13277
13402
|
"use strict";
|
|
13278
13403
|
init_src2();
|
|
13279
13404
|
init_parquet_loader_base();
|
|
13405
|
+
init_parquet_source_worker_factory();
|
|
13280
13406
|
init_parquet_source_worker_url();
|
|
13281
13407
|
PARQUET_SOURCE_WORKER = {
|
|
13282
13408
|
...PARQUET_LOADER_BASE,
|
|
13283
13409
|
id: "parquet-source",
|
|
13284
13410
|
name: "ParquetSource",
|
|
13285
|
-
worker: PARQUET_SOURCE_WORKER_URL
|
|
13411
|
+
worker: PARQUET_SOURCE_WORKER_URL,
|
|
13412
|
+
loadWorker: PARQUET_SOURCE_WORKER_LOAD_WORKER
|
|
13286
13413
|
};
|
|
13287
13414
|
}
|
|
13288
13415
|
});
|