@loaders.gl/scan 5.0.0-alpha.4 → 5.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/delta.cjs +266 -94
- package/dist/delta.cjs.map +3 -3
- package/dist/dist.min.js +29 -0
- package/dist/hudi.cjs +266 -94
- package/dist/hudi.cjs.map +3 -3
- package/dist/iceberg.cjs +264 -92
- package/dist/iceberg.cjs.map +3 -3
- package/dist/index.cjs +29 -0
- package/dist/index.cjs.map +1 -1
- package/package.json +8 -8
package/dist/iceberg.cjs
CHANGED
|
@@ -424,6 +424,10 @@ var init_worker_thread = __esm({
|
|
|
424
424
|
name;
|
|
425
425
|
source;
|
|
426
426
|
url;
|
|
427
|
+
/** Lazily resolves the classic worker URL used as a fallback. */
|
|
428
|
+
getUrl;
|
|
429
|
+
/** Creates a browser Worker directly. */
|
|
430
|
+
loadWorker;
|
|
427
431
|
terminated = false;
|
|
428
432
|
worker;
|
|
429
433
|
onMessage;
|
|
@@ -434,11 +438,13 @@ var init_worker_thread = __esm({
|
|
|
434
438
|
return typeof Worker !== "undefined" && isBrowser2 || typeof NodeWorker !== "undefined" && !isBrowser2;
|
|
435
439
|
}
|
|
436
440
|
constructor(props) {
|
|
437
|
-
const { name, source, url } = props;
|
|
438
|
-
assert(source || url);
|
|
441
|
+
const { name, source, url, getUrl, loadWorker } = props;
|
|
442
|
+
assert(source || url || getUrl || loadWorker);
|
|
439
443
|
this.name = name;
|
|
440
444
|
this.source = source;
|
|
441
445
|
this.url = url;
|
|
446
|
+
this.getUrl = getUrl;
|
|
447
|
+
this.loadWorker = loadWorker;
|
|
442
448
|
this.onMessage = NOOP;
|
|
443
449
|
this.onError = (error) => console.log(error);
|
|
444
450
|
this.worker = isBrowser2 ? this._createBrowserWorker() : this._createNodeWorker();
|
|
@@ -484,7 +490,7 @@ var init_worker_thread = __esm({
|
|
|
484
490
|
*/
|
|
485
491
|
_getErrorFromErrorEvent(event) {
|
|
486
492
|
let message = "Failed to load ";
|
|
487
|
-
message += `worker ${this.name} from ${this.url}. `;
|
|
493
|
+
message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
|
|
488
494
|
if (event.message) {
|
|
489
495
|
message += `${event.message} in `;
|
|
490
496
|
}
|
|
@@ -497,8 +503,26 @@ var init_worker_thread = __esm({
|
|
|
497
503
|
* Creates a worker thread on the browser
|
|
498
504
|
*/
|
|
499
505
|
_createBrowserWorker() {
|
|
500
|
-
|
|
506
|
+
var _a;
|
|
507
|
+
if (this.loadWorker) {
|
|
508
|
+
try {
|
|
509
|
+
const worker2 = this.loadWorker();
|
|
510
|
+
if (worker2) {
|
|
511
|
+
return this._initializeBrowserWorker(worker2);
|
|
512
|
+
}
|
|
513
|
+
} catch (error) {
|
|
514
|
+
if (!this.source && !this.url && !this.getUrl) {
|
|
515
|
+
throw error;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
const url = this.url || ((_a = this.getUrl) == null ? void 0 : _a.call(this));
|
|
520
|
+
this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
|
|
501
521
|
const worker = new Worker(this._loadableURL, { name: this.name });
|
|
522
|
+
return this._initializeBrowserWorker(worker);
|
|
523
|
+
}
|
|
524
|
+
/** Attaches protocol handlers to a browser worker. */
|
|
525
|
+
_initializeBrowserWorker(worker) {
|
|
502
526
|
worker.onmessage = (event) => {
|
|
503
527
|
if (!event.data) {
|
|
504
528
|
this.onError(new Error("No data received"));
|
|
@@ -557,6 +581,10 @@ var init_worker_pool = __esm({
|
|
|
557
581
|
source;
|
|
558
582
|
// | Function;
|
|
559
583
|
url;
|
|
584
|
+
/** Lazily resolves a classic worker fallback URL. */
|
|
585
|
+
getUrl;
|
|
586
|
+
/** Creates a browser Worker directly. */
|
|
587
|
+
loadWorker;
|
|
560
588
|
maxConcurrency = 1;
|
|
561
589
|
maxMobileConcurrency = 1;
|
|
562
590
|
onDebug = () => {
|
|
@@ -578,6 +606,8 @@ var init_worker_pool = __esm({
|
|
|
578
606
|
constructor(props) {
|
|
579
607
|
this.source = props.source;
|
|
580
608
|
this.url = props.url;
|
|
609
|
+
this.getUrl = props.getUrl;
|
|
610
|
+
this.loadWorker = props.loadWorker;
|
|
581
611
|
this.setProps(props);
|
|
582
612
|
}
|
|
583
613
|
/**
|
|
@@ -687,7 +717,13 @@ var init_worker_pool = __esm({
|
|
|
687
717
|
if (this.count < this._getMaxConcurrency()) {
|
|
688
718
|
this.count++;
|
|
689
719
|
const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
|
|
690
|
-
return new WorkerThread({
|
|
720
|
+
return new WorkerThread({
|
|
721
|
+
name,
|
|
722
|
+
source: this.source,
|
|
723
|
+
url: this.url,
|
|
724
|
+
getUrl: this.getUrl,
|
|
725
|
+
loadWorker: this.loadWorker
|
|
726
|
+
});
|
|
691
727
|
}
|
|
692
728
|
return null;
|
|
693
729
|
}
|
|
@@ -699,7 +735,22 @@ var init_worker_pool = __esm({
|
|
|
699
735
|
});
|
|
700
736
|
|
|
701
737
|
// ../worker-utils/src/lib/worker-farm/worker-farm.ts
|
|
702
|
-
|
|
738
|
+
function getWorkerPoolKey(options) {
|
|
739
|
+
const { name, source, url, urlKey, loadWorker } = options;
|
|
740
|
+
const sourceKey = source ? `source:${source}` : "";
|
|
741
|
+
const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
|
|
742
|
+
const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
|
|
743
|
+
return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
|
|
744
|
+
}
|
|
745
|
+
function getLoadWorkerId(loadWorker) {
|
|
746
|
+
let loadWorkerId = loadWorkerIds.get(loadWorker);
|
|
747
|
+
if (!loadWorkerId) {
|
|
748
|
+
loadWorkerId = nextLoadWorkerId++;
|
|
749
|
+
loadWorkerIds.set(loadWorker, loadWorkerId);
|
|
750
|
+
}
|
|
751
|
+
return loadWorkerId;
|
|
752
|
+
}
|
|
753
|
+
var DEFAULT_PROPS, loadWorkerIds, nextLoadWorkerId, WorkerFarm;
|
|
703
754
|
var init_worker_farm = __esm({
|
|
704
755
|
"../worker-utils/src/lib/worker-farm/worker-farm.ts"() {
|
|
705
756
|
"use strict";
|
|
@@ -712,6 +763,8 @@ var init_worker_farm = __esm({
|
|
|
712
763
|
onDebug: () => {
|
|
713
764
|
}
|
|
714
765
|
};
|
|
766
|
+
loadWorkerIds = /* @__PURE__ */ new WeakMap();
|
|
767
|
+
nextLoadWorkerId = 1;
|
|
715
768
|
WorkerFarm = class _WorkerFarm {
|
|
716
769
|
props;
|
|
717
770
|
workerPools = /* @__PURE__ */ new Map();
|
|
@@ -755,24 +808,28 @@ var init_worker_farm = __esm({
|
|
|
755
808
|
}
|
|
756
809
|
/**
|
|
757
810
|
* Returns a worker pool for the specified worker
|
|
758
|
-
* @param options
|
|
759
|
-
* @param options.name
|
|
760
|
-
* @param options.url
|
|
761
|
-
* @param options.source
|
|
811
|
+
* @param options Worker target used to identify and initialize the pool.
|
|
812
|
+
* @param options.name Worker name used as the base pool identity.
|
|
813
|
+
* @param options.url Worker script URL.
|
|
814
|
+
* @param options.source Inline worker source.
|
|
815
|
+
* @param options.loadWorker Built-in browser worker factory.
|
|
762
816
|
* @example
|
|
763
817
|
* const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
|
|
764
818
|
*/
|
|
765
819
|
getWorkerPool(options) {
|
|
766
|
-
const { name, source, url } = options;
|
|
767
|
-
|
|
820
|
+
const { name, source, url, getUrl, loadWorker } = options;
|
|
821
|
+
const workerPoolKey = getWorkerPoolKey(options);
|
|
822
|
+
let workerPool = this.workerPools.get(workerPoolKey);
|
|
768
823
|
if (!workerPool) {
|
|
769
824
|
workerPool = new WorkerPool({
|
|
770
825
|
name,
|
|
771
826
|
source,
|
|
772
|
-
url
|
|
827
|
+
url,
|
|
828
|
+
getUrl,
|
|
829
|
+
loadWorker
|
|
773
830
|
});
|
|
774
831
|
workerPool.setProps(this._getWorkerPoolProps());
|
|
775
|
-
this.workerPools.set(
|
|
832
|
+
this.workerPools.set(workerPoolKey, workerPool);
|
|
776
833
|
}
|
|
777
834
|
return workerPool;
|
|
778
835
|
}
|
|
@@ -793,35 +850,42 @@ function getWorkerName(worker) {
|
|
|
793
850
|
const warning = worker.version !== VERSION2 ? ` (worker-utils@${VERSION2})` : "";
|
|
794
851
|
return `${worker.name}@${worker.version}${warning}`;
|
|
795
852
|
}
|
|
796
|
-
function
|
|
853
|
+
function getCustomWorkerURL(worker, options = {}) {
|
|
797
854
|
var _a;
|
|
798
855
|
const workerOptions = options[worker.id] || {};
|
|
799
|
-
const workerFile = isBrowser2 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
856
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
800
857
|
let url = workerOptions.workerUrl;
|
|
801
|
-
if (!url && typeof worker.worker === "string") {
|
|
802
|
-
url = worker.worker;
|
|
803
|
-
}
|
|
804
858
|
if (!url && worker.id === "compression") {
|
|
805
859
|
url = options.workerUrl;
|
|
806
860
|
}
|
|
807
861
|
const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
|
|
808
|
-
if (workerType === "test") {
|
|
862
|
+
if (!url && workerType === "test") {
|
|
809
863
|
if (isBrowser2) {
|
|
810
864
|
url = `modules/${worker.module}/dist/${workerFile}`;
|
|
811
865
|
} else {
|
|
812
866
|
url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
|
|
813
867
|
}
|
|
814
868
|
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
869
|
+
return url || null;
|
|
870
|
+
}
|
|
871
|
+
function getWorkerURL(worker, options = {}) {
|
|
872
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
873
|
+
const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
|
|
874
|
+
const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
|
|
875
|
+
assert(url);
|
|
876
|
+
return url;
|
|
877
|
+
}
|
|
878
|
+
function getDefaultWorkerURL(worker, warn = false) {
|
|
879
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
880
|
+
let version2 = worker.version;
|
|
881
|
+
if (version2 === "latest") {
|
|
882
|
+
version2 = NPM_TAG;
|
|
883
|
+
}
|
|
884
|
+
const versionTag = version2 ? `@${version2}` : "";
|
|
885
|
+
const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
|
|
886
|
+
if (warn) {
|
|
822
887
|
warnIfUsingNpmTagFallback(worker, url);
|
|
823
888
|
}
|
|
824
|
-
assert(url);
|
|
825
889
|
return url;
|
|
826
890
|
}
|
|
827
891
|
function warnIfUsingNpmTagFallback(worker, url) {
|
|
@@ -855,19 +919,16 @@ function canProcessOnWorker(worker, options) {
|
|
|
855
919
|
return false;
|
|
856
920
|
}
|
|
857
921
|
const workerOptions = options == null ? void 0 : options[worker.id];
|
|
858
|
-
return Boolean(
|
|
922
|
+
return Boolean(
|
|
923
|
+
(worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
|
|
924
|
+
);
|
|
859
925
|
}
|
|
860
926
|
async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
|
|
861
927
|
var _a, _b, _c;
|
|
862
928
|
throwIfAborted(options.signal);
|
|
863
929
|
const name = getWorkerName(worker);
|
|
864
930
|
const workerFarm = WorkerFarm.getWorkerFarm(options);
|
|
865
|
-
const
|
|
866
|
-
const workerPoolProps = { name, source };
|
|
867
|
-
if (!source) {
|
|
868
|
-
workerPoolProps.url = getWorkerURL(worker, options);
|
|
869
|
-
}
|
|
870
|
-
const workerPool = workerFarm.getWorkerPool(workerPoolProps);
|
|
931
|
+
const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
|
|
871
932
|
const jobName = options.jobName || worker.name;
|
|
872
933
|
const job = await workerPool.startJob(
|
|
873
934
|
jobName,
|
|
@@ -910,6 +971,24 @@ function createAbortError2(message) {
|
|
|
910
971
|
error.name = "AbortError";
|
|
911
972
|
return error;
|
|
912
973
|
}
|
|
974
|
+
function getWorkerPoolTarget(worker, options, name) {
|
|
975
|
+
if (options.source) {
|
|
976
|
+
return { name, source: options.source };
|
|
977
|
+
}
|
|
978
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
979
|
+
if (customWorkerUrl) {
|
|
980
|
+
return { name, url: customWorkerUrl };
|
|
981
|
+
}
|
|
982
|
+
if (isBrowser2 && worker.loadWorker) {
|
|
983
|
+
return {
|
|
984
|
+
name,
|
|
985
|
+
loadWorker: worker.loadWorker,
|
|
986
|
+
getUrl: () => getWorkerURL(worker, options),
|
|
987
|
+
urlKey: getDefaultWorkerURL(worker)
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
return { name, url: getWorkerURL(worker, options) };
|
|
991
|
+
}
|
|
913
992
|
async function onMessage(context, job, type, payload) {
|
|
914
993
|
switch (type) {
|
|
915
994
|
case "done":
|
|
@@ -942,6 +1021,7 @@ var init_process_on_worker = __esm({
|
|
|
942
1021
|
init_worker_farm();
|
|
943
1022
|
init_get_worker_url();
|
|
944
1023
|
init_get_transfer_list();
|
|
1024
|
+
init_globals2();
|
|
945
1025
|
}
|
|
946
1026
|
});
|
|
947
1027
|
|
|
@@ -975,7 +1055,7 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
|
|
|
975
1055
|
const result = await processOnWorker(
|
|
976
1056
|
loader,
|
|
977
1057
|
data,
|
|
978
|
-
{ ...getWorkerOptions(options), signal: workerSignal },
|
|
1058
|
+
{ ...getWorkerOptions(options, loader), signal: workerSignal },
|
|
979
1059
|
{
|
|
980
1060
|
process: async (input, processOptions, _workerContext, parseContext) => {
|
|
981
1061
|
if (!parseOnMainThread) {
|
|
@@ -1008,12 +1088,19 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
|
|
|
1008
1088
|
}
|
|
1009
1089
|
return parseOnMainThread(input, void 0, options, context);
|
|
1010
1090
|
}
|
|
1011
|
-
function getWorkerOptions(options = {}) {
|
|
1091
|
+
function getWorkerOptions(options = {}, loader) {
|
|
1012
1092
|
const serializedOptions = JSON.parse(JSON.stringify(options));
|
|
1013
|
-
|
|
1093
|
+
const workerOptions = {
|
|
1014
1094
|
...serializedOptions.core,
|
|
1015
1095
|
...serializedOptions
|
|
1016
1096
|
};
|
|
1097
|
+
if (workerOptions.worker === "auto") {
|
|
1098
|
+
workerOptions.worker = true;
|
|
1099
|
+
}
|
|
1100
|
+
if (loader) {
|
|
1101
|
+
workerOptions._workerLoaderId = loader.id;
|
|
1102
|
+
}
|
|
1103
|
+
return workerOptions;
|
|
1017
1104
|
}
|
|
1018
1105
|
function getSerializableLoaderContext(context) {
|
|
1019
1106
|
if (!context) {
|
|
@@ -6829,6 +6916,12 @@ function serializeArrowType(arrowType) {
|
|
|
6829
6916
|
type: "list",
|
|
6830
6917
|
children: [serializeArrowField(listField)]
|
|
6831
6918
|
};
|
|
6919
|
+
case arrow2.LargeList:
|
|
6920
|
+
const largeListType = arrowType;
|
|
6921
|
+
return {
|
|
6922
|
+
type: "large-list",
|
|
6923
|
+
children: [serializeArrowField(largeListType.valueField)]
|
|
6924
|
+
};
|
|
6832
6925
|
case arrow2.FixedSizeList:
|
|
6833
6926
|
const fixedSizeList = arrowType;
|
|
6834
6927
|
return {
|
|
@@ -6842,6 +6935,15 @@ function serializeArrowType(arrowType) {
|
|
|
6842
6935
|
type: "struct",
|
|
6843
6936
|
children: structType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
6844
6937
|
};
|
|
6938
|
+
case arrow2.SparseUnion:
|
|
6939
|
+
case arrow2.DenseUnion:
|
|
6940
|
+
const unionType = arrowType;
|
|
6941
|
+
return {
|
|
6942
|
+
type: unionType instanceof arrow2.DenseUnion ? "dense-union" : "sparse-union",
|
|
6943
|
+
typeIds: unionType.typeIds,
|
|
6944
|
+
children: unionType.children.map((arrowField) => serializeArrowField(arrowField)),
|
|
6945
|
+
typeIdToChildIndex: { ...unionType.typeIdToChildIndex }
|
|
6946
|
+
};
|
|
6845
6947
|
case arrow2.Dictionary:
|
|
6846
6948
|
const dictionaryType = arrowType;
|
|
6847
6949
|
return {
|
|
@@ -6870,6 +6972,10 @@ function deserializeArrowType(dataType, options) {
|
|
|
6870
6972
|
const field = deserializeArrowField(dataType.children[0], options);
|
|
6871
6973
|
return new arrow2.List(field);
|
|
6872
6974
|
}
|
|
6975
|
+
case "large-list": {
|
|
6976
|
+
const field = deserializeArrowField(dataType.children[0], options);
|
|
6977
|
+
return new arrow2.LargeList(field);
|
|
6978
|
+
}
|
|
6873
6979
|
case "fixed-size-list": {
|
|
6874
6980
|
const child = deserializeArrowField(dataType.children[0], options);
|
|
6875
6981
|
return new arrow2.FixedSizeList(dataType.listSize, child);
|
|
@@ -6882,6 +6988,13 @@ function deserializeArrowType(dataType, options) {
|
|
|
6882
6988
|
);
|
|
6883
6989
|
return new arrow2.Struct(children);
|
|
6884
6990
|
}
|
|
6991
|
+
case "sparse-union":
|
|
6992
|
+
case "dense-union": {
|
|
6993
|
+
const children = dataType.children.map(
|
|
6994
|
+
(arrowField) => deserializeArrowField(arrowField, options)
|
|
6995
|
+
);
|
|
6996
|
+
return dataType.type === "dense-union" ? new arrow2.DenseUnion(dataType.typeIds, children) : new arrow2.SparseUnion(dataType.typeIds, children);
|
|
6997
|
+
}
|
|
6885
6998
|
case "dictionary": {
|
|
6886
6999
|
return new arrow2.Dictionary(
|
|
6887
7000
|
deserializeArrowType(dataType.dictionary, options),
|
|
@@ -7519,8 +7632,85 @@ var init_deprecated = __esm({
|
|
|
7519
7632
|
}
|
|
7520
7633
|
});
|
|
7521
7634
|
|
|
7522
|
-
// ../schema
|
|
7635
|
+
// ../schema/src/metadata.ts
|
|
7636
|
+
function getMetadataValue(metadata, key) {
|
|
7637
|
+
return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
|
|
7638
|
+
}
|
|
7639
|
+
function setMetadataValue(metadata, key, value) {
|
|
7640
|
+
if (metadata instanceof Map) {
|
|
7641
|
+
metadata.set(key, value);
|
|
7642
|
+
} else {
|
|
7643
|
+
metadata[key] = value;
|
|
7644
|
+
}
|
|
7645
|
+
}
|
|
7646
|
+
function getGeoMetadata(metadata) {
|
|
7647
|
+
if (!metadata) {
|
|
7648
|
+
return null;
|
|
7649
|
+
}
|
|
7650
|
+
const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
|
|
7651
|
+
const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
|
|
7652
|
+
if (!geoMetadata) {
|
|
7653
|
+
return null;
|
|
7654
|
+
}
|
|
7655
|
+
for (const column of Object.values(geoMetadata.columns || {})) {
|
|
7656
|
+
if (column && typeof column === "object" && typeof column.encoding === "string") {
|
|
7657
|
+
column.encoding = column.encoding.toLowerCase();
|
|
7658
|
+
}
|
|
7659
|
+
}
|
|
7660
|
+
return geoMetadata;
|
|
7661
|
+
}
|
|
7662
|
+
function unpackGeoMetadata(metadata) {
|
|
7663
|
+
const geoMetadata = getGeoMetadata(metadata);
|
|
7664
|
+
if (!geoMetadata) {
|
|
7665
|
+
return;
|
|
7666
|
+
}
|
|
7667
|
+
const { version: version2, primary_column: primaryColumn, columns } = geoMetadata;
|
|
7668
|
+
if (version2) {
|
|
7669
|
+
setMetadataValue(metadata, "geo.version", version2);
|
|
7670
|
+
}
|
|
7671
|
+
if (primaryColumn) {
|
|
7672
|
+
setMetadataValue(metadata, "geo.primary_column", primaryColumn);
|
|
7673
|
+
}
|
|
7674
|
+
setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
|
|
7675
|
+
}
|
|
7676
|
+
function unpackJSONStringMetadata(metadata, metadataKey) {
|
|
7677
|
+
const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
|
|
7678
|
+
const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
|
|
7679
|
+
for (const [key, value] of Object.entries(json || {})) {
|
|
7680
|
+
setMetadataValue(
|
|
7681
|
+
metadata,
|
|
7682
|
+
`${metadataKey}.${key}`,
|
|
7683
|
+
typeof value === "string" ? value : JSON.stringify(value)
|
|
7684
|
+
);
|
|
7685
|
+
}
|
|
7686
|
+
}
|
|
7687
|
+
function parseJSONStringMetadata(stringifiedMetadata) {
|
|
7688
|
+
if (!stringifiedMetadata) {
|
|
7689
|
+
return null;
|
|
7690
|
+
}
|
|
7691
|
+
try {
|
|
7692
|
+
const metadata = JSON.parse(stringifiedMetadata);
|
|
7693
|
+
return metadata && typeof metadata === "object" ? metadata : null;
|
|
7694
|
+
} catch {
|
|
7695
|
+
return null;
|
|
7696
|
+
}
|
|
7697
|
+
}
|
|
7698
|
+
var init_metadata = __esm({
|
|
7699
|
+
"../schema/src/metadata.ts"() {
|
|
7700
|
+
"use strict";
|
|
7701
|
+
}
|
|
7702
|
+
});
|
|
7703
|
+
|
|
7704
|
+
// ../schema/src/index.ts
|
|
7523
7705
|
var init_src4 = __esm({
|
|
7706
|
+
"../schema/src/index.ts"() {
|
|
7707
|
+
"use strict";
|
|
7708
|
+
init_metadata();
|
|
7709
|
+
}
|
|
7710
|
+
});
|
|
7711
|
+
|
|
7712
|
+
// ../schema-utils/src/index.ts
|
|
7713
|
+
var init_src5 = __esm({
|
|
7524
7714
|
"../schema-utils/src/index.ts"() {
|
|
7525
7715
|
"use strict";
|
|
7526
7716
|
init_convert_arrow_schema();
|
|
@@ -8116,35 +8306,26 @@ var init_arrow_table_transport = __esm({
|
|
|
8116
8306
|
"../arrow/src/lib/utils/arrow-table-transport.ts"() {
|
|
8117
8307
|
"use strict";
|
|
8118
8308
|
arrow7 = __toESM(require("apache-arrow"), 1);
|
|
8119
|
-
|
|
8309
|
+
init_src5();
|
|
8120
8310
|
init_split_arrow_buffers();
|
|
8121
8311
|
}
|
|
8122
8312
|
});
|
|
8123
8313
|
|
|
8124
8314
|
// ../gis/src/lib/geoarrow/metadata-utils.ts
|
|
8125
|
-
function getMetadataValue(metadata, key) {
|
|
8126
|
-
return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
|
|
8127
|
-
}
|
|
8128
|
-
function setMetadataValue(metadata, key, value) {
|
|
8129
|
-
if (metadata instanceof Map) {
|
|
8130
|
-
metadata.set(key, value);
|
|
8131
|
-
} else {
|
|
8132
|
-
metadata[key] = value;
|
|
8133
|
-
}
|
|
8134
|
-
}
|
|
8135
8315
|
var init_metadata_utils = __esm({
|
|
8136
8316
|
"../gis/src/lib/geoarrow/metadata-utils.ts"() {
|
|
8137
8317
|
"use strict";
|
|
8318
|
+
init_src4();
|
|
8138
8319
|
}
|
|
8139
8320
|
});
|
|
8140
8321
|
|
|
8141
8322
|
// ../gis/src/lib/geoarrow/geoparquet-metadata.ts
|
|
8142
|
-
function
|
|
8323
|
+
function getGeoMetadata2(metadata) {
|
|
8143
8324
|
if (!metadata) {
|
|
8144
8325
|
return null;
|
|
8145
8326
|
}
|
|
8146
8327
|
const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
|
|
8147
|
-
const geoMetadata = stringifiedGeoMetadata &&
|
|
8328
|
+
const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata2(stringifiedGeoMetadata);
|
|
8148
8329
|
if (!geoMetadata) {
|
|
8149
8330
|
return null;
|
|
8150
8331
|
}
|
|
@@ -8155,32 +8336,7 @@ function getGeoMetadata(metadata) {
|
|
|
8155
8336
|
}
|
|
8156
8337
|
return geoMetadata;
|
|
8157
8338
|
}
|
|
8158
|
-
function
|
|
8159
|
-
const geoMetadata = getGeoMetadata(metadata);
|
|
8160
|
-
if (!geoMetadata) {
|
|
8161
|
-
return;
|
|
8162
|
-
}
|
|
8163
|
-
const { version: version2, primary_column, columns } = geoMetadata;
|
|
8164
|
-
if (version2) {
|
|
8165
|
-
setMetadataValue(metadata, "geo.version", version2);
|
|
8166
|
-
}
|
|
8167
|
-
if (primary_column) {
|
|
8168
|
-
setMetadataValue(metadata, "geo.primary_column", primary_column);
|
|
8169
|
-
}
|
|
8170
|
-
setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
|
|
8171
|
-
}
|
|
8172
|
-
function unpackJSONStringMetadata(metadata, metadataKey) {
|
|
8173
|
-
const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
|
|
8174
|
-
const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
|
|
8175
|
-
for (const [key, value] of Object.entries(json || {})) {
|
|
8176
|
-
setMetadataValue(
|
|
8177
|
-
metadata,
|
|
8178
|
-
`${metadataKey}.${key}`,
|
|
8179
|
-
typeof value === "string" ? value : JSON.stringify(value)
|
|
8180
|
-
);
|
|
8181
|
-
}
|
|
8182
|
-
}
|
|
8183
|
-
function parseJSONStringMetadata(stringifiedMetadata) {
|
|
8339
|
+
function parseJSONStringMetadata2(stringifiedMetadata) {
|
|
8184
8340
|
if (!stringifiedMetadata) {
|
|
8185
8341
|
return null;
|
|
8186
8342
|
}
|
|
@@ -8202,16 +8358,15 @@ var init_geoparquet_metadata = __esm({
|
|
|
8202
8358
|
});
|
|
8203
8359
|
|
|
8204
8360
|
// ../gis/src/index.ts
|
|
8205
|
-
var
|
|
8361
|
+
var init_src6 = __esm({
|
|
8206
8362
|
"../gis/src/index.ts"() {
|
|
8207
8363
|
"use strict";
|
|
8208
|
-
init_metadata_utils();
|
|
8209
8364
|
init_geoparquet_metadata();
|
|
8210
8365
|
}
|
|
8211
8366
|
});
|
|
8212
8367
|
|
|
8213
8368
|
// ../arrow/src/index.ts
|
|
8214
|
-
var
|
|
8369
|
+
var init_src7 = __esm({
|
|
8215
8370
|
"../arrow/src/index.ts"() {
|
|
8216
8371
|
"use strict";
|
|
8217
8372
|
init_indexed_arrow_table();
|
|
@@ -8455,10 +8610,11 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
|
|
|
8455
8610
|
if (!extensionName) {
|
|
8456
8611
|
return null;
|
|
8457
8612
|
}
|
|
8458
|
-
|
|
8613
|
+
const isBinaryGeospatialEncoding = normalizedEncoding === "wkb" || normalizedEncoding === "geometry" || normalizedEncoding === "geography";
|
|
8614
|
+
if (isBinaryGeospatialEncoding && !isBinaryFieldType(field.type)) {
|
|
8459
8615
|
return null;
|
|
8460
8616
|
}
|
|
8461
|
-
if (
|
|
8617
|
+
if (!isBinaryGeospatialEncoding && (isBinaryFieldType(field.type) || isStringFieldType(field.type))) {
|
|
8462
8618
|
return null;
|
|
8463
8619
|
}
|
|
8464
8620
|
const extensionMetadata = {};
|
|
@@ -8472,18 +8628,34 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
|
|
|
8472
8628
|
if (columnMetadata.edges && columnMetadata.edges !== "planar") {
|
|
8473
8629
|
extensionMetadata.edges = columnMetadata.edges;
|
|
8474
8630
|
}
|
|
8631
|
+
if (normalizedEncoding === "geography" && !extensionMetadata.edges) {
|
|
8632
|
+
extensionMetadata.edges = "spherical";
|
|
8633
|
+
}
|
|
8634
|
+
for (const [metadataKey, metadataValue] of Object.entries(columnMetadata)) {
|
|
8635
|
+
if (metadataValue !== void 0 && metadataValue !== null && !["encoding", "geometry_types", "crs", "edges"].includes(metadataKey)) {
|
|
8636
|
+
extensionMetadata[metadataKey] = metadataValue;
|
|
8637
|
+
}
|
|
8638
|
+
}
|
|
8475
8639
|
return Object.keys(extensionMetadata).length > 0 ? { extensionName, extensionMetadata } : { extensionName };
|
|
8476
8640
|
}
|
|
8641
|
+
function isBinaryFieldType(fieldType) {
|
|
8642
|
+
return typeof fieldType === "string" && (fieldType === "binary" || fieldType === "binary-view");
|
|
8643
|
+
}
|
|
8644
|
+
function isStringFieldType(fieldType) {
|
|
8645
|
+
return typeof fieldType === "string" && (fieldType === "utf8" || fieldType === "utf8-view");
|
|
8646
|
+
}
|
|
8477
8647
|
var arrow8, GEOARROW_EXTENSION_NAME_KEY, GEOARROW_EXTENSION_METADATA_KEY, GEOPARQUET_TO_GEOARROW_ENCODINGS;
|
|
8478
8648
|
var init_geospatial_metadata = __esm({
|
|
8479
8649
|
"../parquet/src/lib/geo/geospatial-metadata.ts"() {
|
|
8480
8650
|
"use strict";
|
|
8481
8651
|
arrow8 = __toESM(require("apache-arrow"), 1);
|
|
8482
|
-
|
|
8652
|
+
init_src4();
|
|
8483
8653
|
GEOARROW_EXTENSION_NAME_KEY = "ARROW:extension:name";
|
|
8484
8654
|
GEOARROW_EXTENSION_METADATA_KEY = "ARROW:extension:metadata";
|
|
8485
8655
|
GEOPARQUET_TO_GEOARROW_ENCODINGS = {
|
|
8486
8656
|
wkb: "geoarrow.wkb",
|
|
8657
|
+
geometry: "geoarrow.wkb",
|
|
8658
|
+
geography: "geoarrow.wkb",
|
|
8487
8659
|
point: "geoarrow.point",
|
|
8488
8660
|
linestring: "geoarrow.linestring",
|
|
8489
8661
|
polygon: "geoarrow.polygon",
|
|
@@ -8509,7 +8681,7 @@ async function getSchemaFromParquetReader(reader) {
|
|
|
8509
8681
|
var init_get_parquet_schema = __esm({
|
|
8510
8682
|
"../parquet/src/lib/parsers/get-parquet-schema.ts"() {
|
|
8511
8683
|
"use strict";
|
|
8512
|
-
|
|
8684
|
+
init_src4();
|
|
8513
8685
|
init_convert_schema_from_parquet();
|
|
8514
8686
|
init_geospatial_metadata();
|
|
8515
8687
|
}
|
|
@@ -8621,7 +8793,7 @@ function doLongitudeIntervalsIntersect(firstWest, firstEast, secondWest, secondE
|
|
|
8621
8793
|
var init_geoparquet_covering = __esm({
|
|
8622
8794
|
"../parquet/src/lib/geo/geoparquet-covering.ts"() {
|
|
8623
8795
|
"use strict";
|
|
8624
|
-
|
|
8796
|
+
init_src4();
|
|
8625
8797
|
}
|
|
8626
8798
|
});
|
|
8627
8799
|
|
|
@@ -15345,7 +15517,7 @@ var init_parse_bson = __esm({
|
|
|
15345
15517
|
});
|
|
15346
15518
|
|
|
15347
15519
|
// ../bson/src/index.ts
|
|
15348
|
-
var
|
|
15520
|
+
var init_src8 = __esm({
|
|
15349
15521
|
"../bson/src/index.ts"() {
|
|
15350
15522
|
"use strict";
|
|
15351
15523
|
init_bson_writer();
|
|
@@ -15754,7 +15926,7 @@ var PARQUET_LOGICAL_TYPES, kMillisPerDay;
|
|
|
15754
15926
|
var init_types = __esm({
|
|
15755
15927
|
"../parquet/src/parquetjs/schema/types.ts"() {
|
|
15756
15928
|
"use strict";
|
|
15757
|
-
|
|
15929
|
+
init_src8();
|
|
15758
15930
|
init_binary_utils();
|
|
15759
15931
|
PARQUET_LOGICAL_TYPES = {
|
|
15760
15932
|
BOOLEAN: {
|
|
@@ -16465,7 +16637,7 @@ function resolveParquetSourceWorkerUrl() {
|
|
|
16465
16637
|
}
|
|
16466
16638
|
}
|
|
16467
16639
|
function getParquetSourceWorkerFile() {
|
|
16468
|
-
return "
|
|
16640
|
+
return "../dist/parquet-source-worker.js";
|
|
16469
16641
|
}
|
|
16470
16642
|
var import_meta, PARQUET_SOURCE_WORKER_URL;
|
|
16471
16643
|
var init_parquet_source_worker_url = __esm({
|
|
@@ -23175,10 +23347,10 @@ var _preloadParquetSourceLoader, _createMetadataDataSource, ParquetSourceLoaderB
|
|
|
23175
23347
|
var init_parquet_source_loader = __esm({
|
|
23176
23348
|
"../parquet/src/parquet-source-loader.ts"() {
|
|
23177
23349
|
"use strict";
|
|
23178
|
-
|
|
23350
|
+
init_src7();
|
|
23179
23351
|
init_src2();
|
|
23180
|
-
init_src4();
|
|
23181
23352
|
init_src5();
|
|
23353
|
+
init_src6();
|
|
23182
23354
|
init_get_parquet_schema();
|
|
23183
23355
|
init_geoparquet_covering();
|
|
23184
23356
|
init_parquet_bloom_filter();
|
|
@@ -23361,7 +23533,7 @@ var init_parquet_source_loader = __esm({
|
|
|
23361
23533
|
);
|
|
23362
23534
|
const projectedColumns = normalizeColumns(readOptions.columns, initialization.schema);
|
|
23363
23535
|
const predicateColumns = physicalPlan.predicate ? getParquetPredicateColumns(physicalPlan.predicate) : [];
|
|
23364
|
-
const geoMetadata = readOptions.bbox ?
|
|
23536
|
+
const geoMetadata = readOptions.bbox ? getGeoMetadata2(initialization.schema.metadata) : void 0;
|
|
23365
23537
|
const geometryColumn = readOptions.bbox ? readOptions.geometryColumn || (geoMetadata == null ? void 0 : geoMetadata.primary_column) : void 0;
|
|
23366
23538
|
const filterColumns = [
|
|
23367
23539
|
.../* @__PURE__ */ new Set([...predicateColumns, ...geometryColumn ? [geometryColumn] : []])
|