@loaders.gl/scan 5.0.0-alpha.4 → 5.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/delta.cjs +266 -94
- package/dist/delta.cjs.map +3 -3
- package/dist/dist.min.js +29 -0
- package/dist/hudi.cjs +266 -94
- package/dist/hudi.cjs.map +3 -3
- package/dist/iceberg.cjs +264 -92
- package/dist/iceberg.cjs.map +3 -3
- package/dist/index.cjs +29 -0
- package/dist/index.cjs.map +1 -1
- package/package.json +8 -8
package/dist/delta.cjs
CHANGED
|
@@ -424,6 +424,10 @@ var init_worker_thread = __esm({
|
|
|
424
424
|
name;
|
|
425
425
|
source;
|
|
426
426
|
url;
|
|
427
|
+
/** Lazily resolves the classic worker URL used as a fallback. */
|
|
428
|
+
getUrl;
|
|
429
|
+
/** Creates a browser Worker directly. */
|
|
430
|
+
loadWorker;
|
|
427
431
|
terminated = false;
|
|
428
432
|
worker;
|
|
429
433
|
onMessage;
|
|
@@ -434,11 +438,13 @@ var init_worker_thread = __esm({
|
|
|
434
438
|
return typeof Worker !== "undefined" && isBrowser2 || typeof NodeWorker !== "undefined" && !isBrowser2;
|
|
435
439
|
}
|
|
436
440
|
constructor(props) {
|
|
437
|
-
const { name, source, url } = props;
|
|
438
|
-
assert(source || url);
|
|
441
|
+
const { name, source, url, getUrl, loadWorker } = props;
|
|
442
|
+
assert(source || url || getUrl || loadWorker);
|
|
439
443
|
this.name = name;
|
|
440
444
|
this.source = source;
|
|
441
445
|
this.url = url;
|
|
446
|
+
this.getUrl = getUrl;
|
|
447
|
+
this.loadWorker = loadWorker;
|
|
442
448
|
this.onMessage = NOOP;
|
|
443
449
|
this.onError = (error) => console.log(error);
|
|
444
450
|
this.worker = isBrowser2 ? this._createBrowserWorker() : this._createNodeWorker();
|
|
@@ -484,7 +490,7 @@ var init_worker_thread = __esm({
|
|
|
484
490
|
*/
|
|
485
491
|
_getErrorFromErrorEvent(event) {
|
|
486
492
|
let message = "Failed to load ";
|
|
487
|
-
message += `worker ${this.name} from ${this.url}. `;
|
|
493
|
+
message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
|
|
488
494
|
if (event.message) {
|
|
489
495
|
message += `${event.message} in `;
|
|
490
496
|
}
|
|
@@ -497,8 +503,26 @@ var init_worker_thread = __esm({
|
|
|
497
503
|
* Creates a worker thread on the browser
|
|
498
504
|
*/
|
|
499
505
|
_createBrowserWorker() {
|
|
500
|
-
|
|
506
|
+
var _a;
|
|
507
|
+
if (this.loadWorker) {
|
|
508
|
+
try {
|
|
509
|
+
const worker2 = this.loadWorker();
|
|
510
|
+
if (worker2) {
|
|
511
|
+
return this._initializeBrowserWorker(worker2);
|
|
512
|
+
}
|
|
513
|
+
} catch (error) {
|
|
514
|
+
if (!this.source && !this.url && !this.getUrl) {
|
|
515
|
+
throw error;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
const url = this.url || ((_a = this.getUrl) == null ? void 0 : _a.call(this));
|
|
520
|
+
this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
|
|
501
521
|
const worker = new Worker(this._loadableURL, { name: this.name });
|
|
522
|
+
return this._initializeBrowserWorker(worker);
|
|
523
|
+
}
|
|
524
|
+
/** Attaches protocol handlers to a browser worker. */
|
|
525
|
+
_initializeBrowserWorker(worker) {
|
|
502
526
|
worker.onmessage = (event) => {
|
|
503
527
|
if (!event.data) {
|
|
504
528
|
this.onError(new Error("No data received"));
|
|
@@ -557,6 +581,10 @@ var init_worker_pool = __esm({
|
|
|
557
581
|
source;
|
|
558
582
|
// | Function;
|
|
559
583
|
url;
|
|
584
|
+
/** Lazily resolves a classic worker fallback URL. */
|
|
585
|
+
getUrl;
|
|
586
|
+
/** Creates a browser Worker directly. */
|
|
587
|
+
loadWorker;
|
|
560
588
|
maxConcurrency = 1;
|
|
561
589
|
maxMobileConcurrency = 1;
|
|
562
590
|
onDebug = () => {
|
|
@@ -578,6 +606,8 @@ var init_worker_pool = __esm({
|
|
|
578
606
|
constructor(props) {
|
|
579
607
|
this.source = props.source;
|
|
580
608
|
this.url = props.url;
|
|
609
|
+
this.getUrl = props.getUrl;
|
|
610
|
+
this.loadWorker = props.loadWorker;
|
|
581
611
|
this.setProps(props);
|
|
582
612
|
}
|
|
583
613
|
/**
|
|
@@ -687,7 +717,13 @@ var init_worker_pool = __esm({
|
|
|
687
717
|
if (this.count < this._getMaxConcurrency()) {
|
|
688
718
|
this.count++;
|
|
689
719
|
const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
|
|
690
|
-
return new WorkerThread({
|
|
720
|
+
return new WorkerThread({
|
|
721
|
+
name,
|
|
722
|
+
source: this.source,
|
|
723
|
+
url: this.url,
|
|
724
|
+
getUrl: this.getUrl,
|
|
725
|
+
loadWorker: this.loadWorker
|
|
726
|
+
});
|
|
691
727
|
}
|
|
692
728
|
return null;
|
|
693
729
|
}
|
|
@@ -699,7 +735,22 @@ var init_worker_pool = __esm({
|
|
|
699
735
|
});
|
|
700
736
|
|
|
701
737
|
// ../worker-utils/src/lib/worker-farm/worker-farm.ts
|
|
702
|
-
|
|
738
|
+
function getWorkerPoolKey(options) {
|
|
739
|
+
const { name, source, url, urlKey, loadWorker } = options;
|
|
740
|
+
const sourceKey = source ? `source:${source}` : "";
|
|
741
|
+
const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
|
|
742
|
+
const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
|
|
743
|
+
return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
|
|
744
|
+
}
|
|
745
|
+
function getLoadWorkerId(loadWorker) {
|
|
746
|
+
let loadWorkerId = loadWorkerIds.get(loadWorker);
|
|
747
|
+
if (!loadWorkerId) {
|
|
748
|
+
loadWorkerId = nextLoadWorkerId++;
|
|
749
|
+
loadWorkerIds.set(loadWorker, loadWorkerId);
|
|
750
|
+
}
|
|
751
|
+
return loadWorkerId;
|
|
752
|
+
}
|
|
753
|
+
var DEFAULT_PROPS, loadWorkerIds, nextLoadWorkerId, WorkerFarm;
|
|
703
754
|
var init_worker_farm = __esm({
|
|
704
755
|
"../worker-utils/src/lib/worker-farm/worker-farm.ts"() {
|
|
705
756
|
"use strict";
|
|
@@ -712,6 +763,8 @@ var init_worker_farm = __esm({
|
|
|
712
763
|
onDebug: () => {
|
|
713
764
|
}
|
|
714
765
|
};
|
|
766
|
+
loadWorkerIds = /* @__PURE__ */ new WeakMap();
|
|
767
|
+
nextLoadWorkerId = 1;
|
|
715
768
|
WorkerFarm = class _WorkerFarm {
|
|
716
769
|
props;
|
|
717
770
|
workerPools = /* @__PURE__ */ new Map();
|
|
@@ -755,24 +808,28 @@ var init_worker_farm = __esm({
|
|
|
755
808
|
}
|
|
756
809
|
/**
|
|
757
810
|
* Returns a worker pool for the specified worker
|
|
758
|
-
* @param options
|
|
759
|
-
* @param options.name
|
|
760
|
-
* @param options.url
|
|
761
|
-
* @param options.source
|
|
811
|
+
* @param options Worker target used to identify and initialize the pool.
|
|
812
|
+
* @param options.name Worker name used as the base pool identity.
|
|
813
|
+
* @param options.url Worker script URL.
|
|
814
|
+
* @param options.source Inline worker source.
|
|
815
|
+
* @param options.loadWorker Built-in browser worker factory.
|
|
762
816
|
* @example
|
|
763
817
|
* const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
|
|
764
818
|
*/
|
|
765
819
|
getWorkerPool(options) {
|
|
766
|
-
const { name, source, url } = options;
|
|
767
|
-
|
|
820
|
+
const { name, source, url, getUrl, loadWorker } = options;
|
|
821
|
+
const workerPoolKey = getWorkerPoolKey(options);
|
|
822
|
+
let workerPool = this.workerPools.get(workerPoolKey);
|
|
768
823
|
if (!workerPool) {
|
|
769
824
|
workerPool = new WorkerPool({
|
|
770
825
|
name,
|
|
771
826
|
source,
|
|
772
|
-
url
|
|
827
|
+
url,
|
|
828
|
+
getUrl,
|
|
829
|
+
loadWorker
|
|
773
830
|
});
|
|
774
831
|
workerPool.setProps(this._getWorkerPoolProps());
|
|
775
|
-
this.workerPools.set(
|
|
832
|
+
this.workerPools.set(workerPoolKey, workerPool);
|
|
776
833
|
}
|
|
777
834
|
return workerPool;
|
|
778
835
|
}
|
|
@@ -793,35 +850,42 @@ function getWorkerName(worker) {
|
|
|
793
850
|
const warning = worker.version !== VERSION2 ? ` (worker-utils@${VERSION2})` : "";
|
|
794
851
|
return `${worker.name}@${worker.version}${warning}`;
|
|
795
852
|
}
|
|
796
|
-
function
|
|
853
|
+
function getCustomWorkerURL(worker, options = {}) {
|
|
797
854
|
var _a;
|
|
798
855
|
const workerOptions = options[worker.id] || {};
|
|
799
|
-
const workerFile = isBrowser2 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
856
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
800
857
|
let url = workerOptions.workerUrl;
|
|
801
|
-
if (!url && typeof worker.worker === "string") {
|
|
802
|
-
url = worker.worker;
|
|
803
|
-
}
|
|
804
858
|
if (!url && worker.id === "compression") {
|
|
805
859
|
url = options.workerUrl;
|
|
806
860
|
}
|
|
807
861
|
const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
|
|
808
|
-
if (workerType === "test") {
|
|
862
|
+
if (!url && workerType === "test") {
|
|
809
863
|
if (isBrowser2) {
|
|
810
864
|
url = `modules/${worker.module}/dist/${workerFile}`;
|
|
811
865
|
} else {
|
|
812
866
|
url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
|
|
813
867
|
}
|
|
814
868
|
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
869
|
+
return url || null;
|
|
870
|
+
}
|
|
871
|
+
function getWorkerURL(worker, options = {}) {
|
|
872
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
873
|
+
const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
|
|
874
|
+
const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
|
|
875
|
+
assert(url);
|
|
876
|
+
return url;
|
|
877
|
+
}
|
|
878
|
+
function getDefaultWorkerURL(worker, warn = false) {
|
|
879
|
+
const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
|
|
880
|
+
let version2 = worker.version;
|
|
881
|
+
if (version2 === "latest") {
|
|
882
|
+
version2 = NPM_TAG;
|
|
883
|
+
}
|
|
884
|
+
const versionTag = version2 ? `@${version2}` : "";
|
|
885
|
+
const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
|
|
886
|
+
if (warn) {
|
|
822
887
|
warnIfUsingNpmTagFallback(worker, url);
|
|
823
888
|
}
|
|
824
|
-
assert(url);
|
|
825
889
|
return url;
|
|
826
890
|
}
|
|
827
891
|
function warnIfUsingNpmTagFallback(worker, url) {
|
|
@@ -855,19 +919,16 @@ function canProcessOnWorker(worker, options) {
|
|
|
855
919
|
return false;
|
|
856
920
|
}
|
|
857
921
|
const workerOptions = options == null ? void 0 : options[worker.id];
|
|
858
|
-
return Boolean(
|
|
922
|
+
return Boolean(
|
|
923
|
+
(worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
|
|
924
|
+
);
|
|
859
925
|
}
|
|
860
926
|
async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
|
|
861
927
|
var _a, _b, _c;
|
|
862
928
|
throwIfAborted(options.signal);
|
|
863
929
|
const name = getWorkerName(worker);
|
|
864
930
|
const workerFarm = WorkerFarm.getWorkerFarm(options);
|
|
865
|
-
const
|
|
866
|
-
const workerPoolProps = { name, source };
|
|
867
|
-
if (!source) {
|
|
868
|
-
workerPoolProps.url = getWorkerURL(worker, options);
|
|
869
|
-
}
|
|
870
|
-
const workerPool = workerFarm.getWorkerPool(workerPoolProps);
|
|
931
|
+
const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
|
|
871
932
|
const jobName = options.jobName || worker.name;
|
|
872
933
|
const job = await workerPool.startJob(
|
|
873
934
|
jobName,
|
|
@@ -910,6 +971,24 @@ function createAbortError2(message) {
|
|
|
910
971
|
error.name = "AbortError";
|
|
911
972
|
return error;
|
|
912
973
|
}
|
|
974
|
+
function getWorkerPoolTarget(worker, options, name) {
|
|
975
|
+
if (options.source) {
|
|
976
|
+
return { name, source: options.source };
|
|
977
|
+
}
|
|
978
|
+
const customWorkerUrl = getCustomWorkerURL(worker, options);
|
|
979
|
+
if (customWorkerUrl) {
|
|
980
|
+
return { name, url: customWorkerUrl };
|
|
981
|
+
}
|
|
982
|
+
if (isBrowser2 && worker.loadWorker) {
|
|
983
|
+
return {
|
|
984
|
+
name,
|
|
985
|
+
loadWorker: worker.loadWorker,
|
|
986
|
+
getUrl: () => getWorkerURL(worker, options),
|
|
987
|
+
urlKey: getDefaultWorkerURL(worker)
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
return { name, url: getWorkerURL(worker, options) };
|
|
991
|
+
}
|
|
913
992
|
async function onMessage(context, job, type, payload) {
|
|
914
993
|
switch (type) {
|
|
915
994
|
case "done":
|
|
@@ -942,6 +1021,7 @@ var init_process_on_worker = __esm({
|
|
|
942
1021
|
init_worker_farm();
|
|
943
1022
|
init_get_worker_url();
|
|
944
1023
|
init_get_transfer_list();
|
|
1024
|
+
init_globals2();
|
|
945
1025
|
}
|
|
946
1026
|
});
|
|
947
1027
|
|
|
@@ -975,7 +1055,7 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
|
|
|
975
1055
|
const result = await processOnWorker(
|
|
976
1056
|
loader,
|
|
977
1057
|
data,
|
|
978
|
-
{ ...getWorkerOptions(options), signal: workerSignal },
|
|
1058
|
+
{ ...getWorkerOptions(options, loader), signal: workerSignal },
|
|
979
1059
|
{
|
|
980
1060
|
process: async (input, processOptions, _workerContext, parseContext) => {
|
|
981
1061
|
if (!parseOnMainThread) {
|
|
@@ -1008,12 +1088,19 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
|
|
|
1008
1088
|
}
|
|
1009
1089
|
return parseOnMainThread(input, void 0, options, context);
|
|
1010
1090
|
}
|
|
1011
|
-
function getWorkerOptions(options = {}) {
|
|
1091
|
+
function getWorkerOptions(options = {}, loader) {
|
|
1012
1092
|
const serializedOptions = JSON.parse(JSON.stringify(options));
|
|
1013
|
-
|
|
1093
|
+
const workerOptions = {
|
|
1014
1094
|
...serializedOptions.core,
|
|
1015
1095
|
...serializedOptions
|
|
1016
1096
|
};
|
|
1097
|
+
if (workerOptions.worker === "auto") {
|
|
1098
|
+
workerOptions.worker = true;
|
|
1099
|
+
}
|
|
1100
|
+
if (loader) {
|
|
1101
|
+
workerOptions._workerLoaderId = loader.id;
|
|
1102
|
+
}
|
|
1103
|
+
return workerOptions;
|
|
1017
1104
|
}
|
|
1018
1105
|
function getSerializableLoaderContext(context) {
|
|
1019
1106
|
if (!context) {
|
|
@@ -3135,6 +3222,12 @@ function serializeArrowType(arrowType) {
|
|
|
3135
3222
|
type: "list",
|
|
3136
3223
|
children: [serializeArrowField(listField)]
|
|
3137
3224
|
};
|
|
3225
|
+
case arrow.LargeList:
|
|
3226
|
+
const largeListType = arrowType;
|
|
3227
|
+
return {
|
|
3228
|
+
type: "large-list",
|
|
3229
|
+
children: [serializeArrowField(largeListType.valueField)]
|
|
3230
|
+
};
|
|
3138
3231
|
case arrow.FixedSizeList:
|
|
3139
3232
|
const fixedSizeList = arrowType;
|
|
3140
3233
|
return {
|
|
@@ -3148,6 +3241,15 @@ function serializeArrowType(arrowType) {
|
|
|
3148
3241
|
type: "struct",
|
|
3149
3242
|
children: structType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
3150
3243
|
};
|
|
3244
|
+
case arrow.SparseUnion:
|
|
3245
|
+
case arrow.DenseUnion:
|
|
3246
|
+
const unionType = arrowType;
|
|
3247
|
+
return {
|
|
3248
|
+
type: unionType instanceof arrow.DenseUnion ? "dense-union" : "sparse-union",
|
|
3249
|
+
typeIds: unionType.typeIds,
|
|
3250
|
+
children: unionType.children.map((arrowField) => serializeArrowField(arrowField)),
|
|
3251
|
+
typeIdToChildIndex: { ...unionType.typeIdToChildIndex }
|
|
3252
|
+
};
|
|
3151
3253
|
case arrow.Dictionary:
|
|
3152
3254
|
const dictionaryType = arrowType;
|
|
3153
3255
|
return {
|
|
@@ -3176,6 +3278,10 @@ function deserializeArrowType(dataType, options) {
|
|
|
3176
3278
|
const field = deserializeArrowField(dataType.children[0], options);
|
|
3177
3279
|
return new arrow.List(field);
|
|
3178
3280
|
}
|
|
3281
|
+
case "large-list": {
|
|
3282
|
+
const field = deserializeArrowField(dataType.children[0], options);
|
|
3283
|
+
return new arrow.LargeList(field);
|
|
3284
|
+
}
|
|
3179
3285
|
case "fixed-size-list": {
|
|
3180
3286
|
const child = deserializeArrowField(dataType.children[0], options);
|
|
3181
3287
|
return new arrow.FixedSizeList(dataType.listSize, child);
|
|
@@ -3188,6 +3294,13 @@ function deserializeArrowType(dataType, options) {
|
|
|
3188
3294
|
);
|
|
3189
3295
|
return new arrow.Struct(children);
|
|
3190
3296
|
}
|
|
3297
|
+
case "sparse-union":
|
|
3298
|
+
case "dense-union": {
|
|
3299
|
+
const children = dataType.children.map(
|
|
3300
|
+
(arrowField) => deserializeArrowField(arrowField, options)
|
|
3301
|
+
);
|
|
3302
|
+
return dataType.type === "dense-union" ? new arrow.DenseUnion(dataType.typeIds, children) : new arrow.SparseUnion(dataType.typeIds, children);
|
|
3303
|
+
}
|
|
3191
3304
|
case "dictionary": {
|
|
3192
3305
|
return new arrow.Dictionary(
|
|
3193
3306
|
deserializeArrowType(dataType.dictionary, options),
|
|
@@ -3825,8 +3938,85 @@ var init_deprecated = __esm({
|
|
|
3825
3938
|
}
|
|
3826
3939
|
});
|
|
3827
3940
|
|
|
3828
|
-
// ../schema
|
|
3941
|
+
// ../schema/src/metadata.ts
|
|
3942
|
+
function getMetadataValue(metadata, key) {
|
|
3943
|
+
return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
|
|
3944
|
+
}
|
|
3945
|
+
function setMetadataValue(metadata, key, value) {
|
|
3946
|
+
if (metadata instanceof Map) {
|
|
3947
|
+
metadata.set(key, value);
|
|
3948
|
+
} else {
|
|
3949
|
+
metadata[key] = value;
|
|
3950
|
+
}
|
|
3951
|
+
}
|
|
3952
|
+
function getGeoMetadata(metadata) {
|
|
3953
|
+
if (!metadata) {
|
|
3954
|
+
return null;
|
|
3955
|
+
}
|
|
3956
|
+
const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
|
|
3957
|
+
const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
|
|
3958
|
+
if (!geoMetadata) {
|
|
3959
|
+
return null;
|
|
3960
|
+
}
|
|
3961
|
+
for (const column of Object.values(geoMetadata.columns || {})) {
|
|
3962
|
+
if (column && typeof column === "object" && typeof column.encoding === "string") {
|
|
3963
|
+
column.encoding = column.encoding.toLowerCase();
|
|
3964
|
+
}
|
|
3965
|
+
}
|
|
3966
|
+
return geoMetadata;
|
|
3967
|
+
}
|
|
3968
|
+
function unpackGeoMetadata(metadata) {
|
|
3969
|
+
const geoMetadata = getGeoMetadata(metadata);
|
|
3970
|
+
if (!geoMetadata) {
|
|
3971
|
+
return;
|
|
3972
|
+
}
|
|
3973
|
+
const { version: version2, primary_column: primaryColumn, columns } = geoMetadata;
|
|
3974
|
+
if (version2) {
|
|
3975
|
+
setMetadataValue(metadata, "geo.version", version2);
|
|
3976
|
+
}
|
|
3977
|
+
if (primaryColumn) {
|
|
3978
|
+
setMetadataValue(metadata, "geo.primary_column", primaryColumn);
|
|
3979
|
+
}
|
|
3980
|
+
setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
|
|
3981
|
+
}
|
|
3982
|
+
function unpackJSONStringMetadata(metadata, metadataKey) {
|
|
3983
|
+
const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
|
|
3984
|
+
const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
|
|
3985
|
+
for (const [key, value] of Object.entries(json || {})) {
|
|
3986
|
+
setMetadataValue(
|
|
3987
|
+
metadata,
|
|
3988
|
+
`${metadataKey}.${key}`,
|
|
3989
|
+
typeof value === "string" ? value : JSON.stringify(value)
|
|
3990
|
+
);
|
|
3991
|
+
}
|
|
3992
|
+
}
|
|
3993
|
+
function parseJSONStringMetadata(stringifiedMetadata) {
|
|
3994
|
+
if (!stringifiedMetadata) {
|
|
3995
|
+
return null;
|
|
3996
|
+
}
|
|
3997
|
+
try {
|
|
3998
|
+
const metadata = JSON.parse(stringifiedMetadata);
|
|
3999
|
+
return metadata && typeof metadata === "object" ? metadata : null;
|
|
4000
|
+
} catch {
|
|
4001
|
+
return null;
|
|
4002
|
+
}
|
|
4003
|
+
}
|
|
4004
|
+
var init_metadata = __esm({
|
|
4005
|
+
"../schema/src/metadata.ts"() {
|
|
4006
|
+
"use strict";
|
|
4007
|
+
}
|
|
4008
|
+
});
|
|
4009
|
+
|
|
4010
|
+
// ../schema/src/index.ts
|
|
3829
4011
|
var init_src3 = __esm({
|
|
4012
|
+
"../schema/src/index.ts"() {
|
|
4013
|
+
"use strict";
|
|
4014
|
+
init_metadata();
|
|
4015
|
+
}
|
|
4016
|
+
});
|
|
4017
|
+
|
|
4018
|
+
// ../schema-utils/src/index.ts
|
|
4019
|
+
var init_src4 = __esm({
|
|
3830
4020
|
"../schema-utils/src/index.ts"() {
|
|
3831
4021
|
"use strict";
|
|
3832
4022
|
init_convert_arrow_schema();
|
|
@@ -4815,35 +5005,26 @@ var init_arrow_table_transport = __esm({
|
|
|
4815
5005
|
"../arrow/src/lib/utils/arrow-table-transport.ts"() {
|
|
4816
5006
|
"use strict";
|
|
4817
5007
|
arrow6 = __toESM(require("apache-arrow"), 1);
|
|
4818
|
-
|
|
5008
|
+
init_src4();
|
|
4819
5009
|
init_split_arrow_buffers();
|
|
4820
5010
|
}
|
|
4821
5011
|
});
|
|
4822
5012
|
|
|
4823
5013
|
// ../gis/src/lib/geoarrow/metadata-utils.ts
|
|
4824
|
-
function getMetadataValue(metadata, key) {
|
|
4825
|
-
return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
|
|
4826
|
-
}
|
|
4827
|
-
function setMetadataValue(metadata, key, value) {
|
|
4828
|
-
if (metadata instanceof Map) {
|
|
4829
|
-
metadata.set(key, value);
|
|
4830
|
-
} else {
|
|
4831
|
-
metadata[key] = value;
|
|
4832
|
-
}
|
|
4833
|
-
}
|
|
4834
5014
|
var init_metadata_utils = __esm({
|
|
4835
5015
|
"../gis/src/lib/geoarrow/metadata-utils.ts"() {
|
|
4836
5016
|
"use strict";
|
|
5017
|
+
init_src3();
|
|
4837
5018
|
}
|
|
4838
5019
|
});
|
|
4839
5020
|
|
|
4840
5021
|
// ../gis/src/lib/geoarrow/geoparquet-metadata.ts
|
|
4841
|
-
function
|
|
5022
|
+
function getGeoMetadata2(metadata) {
|
|
4842
5023
|
if (!metadata) {
|
|
4843
5024
|
return null;
|
|
4844
5025
|
}
|
|
4845
5026
|
const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
|
|
4846
|
-
const geoMetadata = stringifiedGeoMetadata &&
|
|
5027
|
+
const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata2(stringifiedGeoMetadata);
|
|
4847
5028
|
if (!geoMetadata) {
|
|
4848
5029
|
return null;
|
|
4849
5030
|
}
|
|
@@ -4854,32 +5035,7 @@ function getGeoMetadata(metadata) {
|
|
|
4854
5035
|
}
|
|
4855
5036
|
return geoMetadata;
|
|
4856
5037
|
}
|
|
4857
|
-
function
|
|
4858
|
-
const geoMetadata = getGeoMetadata(metadata);
|
|
4859
|
-
if (!geoMetadata) {
|
|
4860
|
-
return;
|
|
4861
|
-
}
|
|
4862
|
-
const { version: version2, primary_column, columns } = geoMetadata;
|
|
4863
|
-
if (version2) {
|
|
4864
|
-
setMetadataValue(metadata, "geo.version", version2);
|
|
4865
|
-
}
|
|
4866
|
-
if (primary_column) {
|
|
4867
|
-
setMetadataValue(metadata, "geo.primary_column", primary_column);
|
|
4868
|
-
}
|
|
4869
|
-
setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
|
|
4870
|
-
}
|
|
4871
|
-
function unpackJSONStringMetadata(metadata, metadataKey) {
|
|
4872
|
-
const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
|
|
4873
|
-
const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
|
|
4874
|
-
for (const [key, value] of Object.entries(json || {})) {
|
|
4875
|
-
setMetadataValue(
|
|
4876
|
-
metadata,
|
|
4877
|
-
`${metadataKey}.${key}`,
|
|
4878
|
-
typeof value === "string" ? value : JSON.stringify(value)
|
|
4879
|
-
);
|
|
4880
|
-
}
|
|
4881
|
-
}
|
|
4882
|
-
function parseJSONStringMetadata(stringifiedMetadata) {
|
|
5038
|
+
function parseJSONStringMetadata2(stringifiedMetadata) {
|
|
4883
5039
|
if (!stringifiedMetadata) {
|
|
4884
5040
|
return null;
|
|
4885
5041
|
}
|
|
@@ -4901,16 +5057,15 @@ var init_geoparquet_metadata = __esm({
|
|
|
4901
5057
|
});
|
|
4902
5058
|
|
|
4903
5059
|
// ../gis/src/index.ts
|
|
4904
|
-
var
|
|
5060
|
+
var init_src5 = __esm({
|
|
4905
5061
|
"../gis/src/index.ts"() {
|
|
4906
5062
|
"use strict";
|
|
4907
|
-
init_metadata_utils();
|
|
4908
5063
|
init_geoparquet_metadata();
|
|
4909
5064
|
}
|
|
4910
5065
|
});
|
|
4911
5066
|
|
|
4912
5067
|
// ../arrow/src/index.ts
|
|
4913
|
-
var
|
|
5068
|
+
var init_src6 = __esm({
|
|
4914
5069
|
"../arrow/src/index.ts"() {
|
|
4915
5070
|
"use strict";
|
|
4916
5071
|
init_indexed_arrow_table();
|
|
@@ -5154,10 +5309,11 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
|
|
|
5154
5309
|
if (!extensionName) {
|
|
5155
5310
|
return null;
|
|
5156
5311
|
}
|
|
5157
|
-
|
|
5312
|
+
const isBinaryGeospatialEncoding = normalizedEncoding === "wkb" || normalizedEncoding === "geometry" || normalizedEncoding === "geography";
|
|
5313
|
+
if (isBinaryGeospatialEncoding && !isBinaryFieldType(field.type)) {
|
|
5158
5314
|
return null;
|
|
5159
5315
|
}
|
|
5160
|
-
if (
|
|
5316
|
+
if (!isBinaryGeospatialEncoding && (isBinaryFieldType(field.type) || isStringFieldType(field.type))) {
|
|
5161
5317
|
return null;
|
|
5162
5318
|
}
|
|
5163
5319
|
const extensionMetadata = {};
|
|
@@ -5171,18 +5327,34 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
|
|
|
5171
5327
|
if (columnMetadata.edges && columnMetadata.edges !== "planar") {
|
|
5172
5328
|
extensionMetadata.edges = columnMetadata.edges;
|
|
5173
5329
|
}
|
|
5330
|
+
if (normalizedEncoding === "geography" && !extensionMetadata.edges) {
|
|
5331
|
+
extensionMetadata.edges = "spherical";
|
|
5332
|
+
}
|
|
5333
|
+
for (const [metadataKey, metadataValue] of Object.entries(columnMetadata)) {
|
|
5334
|
+
if (metadataValue !== void 0 && metadataValue !== null && !["encoding", "geometry_types", "crs", "edges"].includes(metadataKey)) {
|
|
5335
|
+
extensionMetadata[metadataKey] = metadataValue;
|
|
5336
|
+
}
|
|
5337
|
+
}
|
|
5174
5338
|
return Object.keys(extensionMetadata).length > 0 ? { extensionName, extensionMetadata } : { extensionName };
|
|
5175
5339
|
}
|
|
5340
|
+
function isBinaryFieldType(fieldType) {
|
|
5341
|
+
return typeof fieldType === "string" && (fieldType === "binary" || fieldType === "binary-view");
|
|
5342
|
+
}
|
|
5343
|
+
function isStringFieldType(fieldType) {
|
|
5344
|
+
return typeof fieldType === "string" && (fieldType === "utf8" || fieldType === "utf8-view");
|
|
5345
|
+
}
|
|
5176
5346
|
var arrow7, GEOARROW_EXTENSION_NAME_KEY, GEOARROW_EXTENSION_METADATA_KEY, GEOPARQUET_TO_GEOARROW_ENCODINGS;
|
|
5177
5347
|
var init_geospatial_metadata = __esm({
|
|
5178
5348
|
"../parquet/src/lib/geo/geospatial-metadata.ts"() {
|
|
5179
5349
|
"use strict";
|
|
5180
5350
|
arrow7 = __toESM(require("apache-arrow"), 1);
|
|
5181
|
-
|
|
5351
|
+
init_src3();
|
|
5182
5352
|
GEOARROW_EXTENSION_NAME_KEY = "ARROW:extension:name";
|
|
5183
5353
|
GEOARROW_EXTENSION_METADATA_KEY = "ARROW:extension:metadata";
|
|
5184
5354
|
GEOPARQUET_TO_GEOARROW_ENCODINGS = {
|
|
5185
5355
|
wkb: "geoarrow.wkb",
|
|
5356
|
+
geometry: "geoarrow.wkb",
|
|
5357
|
+
geography: "geoarrow.wkb",
|
|
5186
5358
|
point: "geoarrow.point",
|
|
5187
5359
|
linestring: "geoarrow.linestring",
|
|
5188
5360
|
polygon: "geoarrow.polygon",
|
|
@@ -5208,7 +5380,7 @@ async function getSchemaFromParquetReader(reader) {
|
|
|
5208
5380
|
var init_get_parquet_schema = __esm({
|
|
5209
5381
|
"../parquet/src/lib/parsers/get-parquet-schema.ts"() {
|
|
5210
5382
|
"use strict";
|
|
5211
|
-
|
|
5383
|
+
init_src3();
|
|
5212
5384
|
init_convert_schema_from_parquet();
|
|
5213
5385
|
init_geospatial_metadata();
|
|
5214
5386
|
}
|
|
@@ -5320,7 +5492,7 @@ function doLongitudeIntervalsIntersect(firstWest, firstEast, secondWest, secondE
|
|
|
5320
5492
|
var init_geoparquet_covering = __esm({
|
|
5321
5493
|
"../parquet/src/lib/geo/geoparquet-covering.ts"() {
|
|
5322
5494
|
"use strict";
|
|
5323
|
-
|
|
5495
|
+
init_src3();
|
|
5324
5496
|
}
|
|
5325
5497
|
});
|
|
5326
5498
|
|
|
@@ -12044,7 +12216,7 @@ var init_parse_bson = __esm({
|
|
|
12044
12216
|
});
|
|
12045
12217
|
|
|
12046
12218
|
// ../bson/src/index.ts
|
|
12047
|
-
var
|
|
12219
|
+
var init_src7 = __esm({
|
|
12048
12220
|
"../bson/src/index.ts"() {
|
|
12049
12221
|
"use strict";
|
|
12050
12222
|
init_bson_writer();
|
|
@@ -12453,7 +12625,7 @@ var PARQUET_LOGICAL_TYPES, kMillisPerDay;
|
|
|
12453
12625
|
var init_types = __esm({
|
|
12454
12626
|
"../parquet/src/parquetjs/schema/types.ts"() {
|
|
12455
12627
|
"use strict";
|
|
12456
|
-
|
|
12628
|
+
init_src7();
|
|
12457
12629
|
init_binary_utils();
|
|
12458
12630
|
PARQUET_LOGICAL_TYPES = {
|
|
12459
12631
|
BOOLEAN: {
|
|
@@ -13164,7 +13336,7 @@ function resolveParquetSourceWorkerUrl() {
|
|
|
13164
13336
|
}
|
|
13165
13337
|
}
|
|
13166
13338
|
function getParquetSourceWorkerFile() {
|
|
13167
|
-
return "
|
|
13339
|
+
return "../dist/parquet-source-worker.js";
|
|
13168
13340
|
}
|
|
13169
13341
|
var import_meta, PARQUET_SOURCE_WORKER_URL;
|
|
13170
13342
|
var init_parquet_source_worker_url = __esm({
|
|
@@ -16474,7 +16646,7 @@ var init_default_codecs = __esm({
|
|
|
16474
16646
|
});
|
|
16475
16647
|
|
|
16476
16648
|
// ../compression/src/index.ts
|
|
16477
|
-
var
|
|
16649
|
+
var init_src8 = __esm({
|
|
16478
16650
|
"../compression/src/index.ts"() {
|
|
16479
16651
|
"use strict";
|
|
16480
16652
|
init_compression();
|
|
@@ -16587,7 +16759,7 @@ var PARQUET_COMPRESSION_METHODS, ParquetSnappyDecompressor;
|
|
|
16587
16759
|
var init_compression2 = __esm({
|
|
16588
16760
|
"../parquet/src/parquetjs/compression.ts"() {
|
|
16589
16761
|
"use strict";
|
|
16590
|
-
|
|
16762
|
+
init_src8();
|
|
16591
16763
|
init_snappy_decompressor_hysnappy();
|
|
16592
16764
|
init_src2();
|
|
16593
16765
|
init_binary_utils();
|
|
@@ -22749,10 +22921,10 @@ var _preloadParquetSourceLoader, _createMetadataDataSource, ParquetSourceLoaderB
|
|
|
22749
22921
|
var init_parquet_source_loader = __esm({
|
|
22750
22922
|
"../parquet/src/parquet-source-loader.ts"() {
|
|
22751
22923
|
"use strict";
|
|
22752
|
-
|
|
22924
|
+
init_src6();
|
|
22753
22925
|
init_src2();
|
|
22754
|
-
init_src3();
|
|
22755
22926
|
init_src4();
|
|
22927
|
+
init_src5();
|
|
22756
22928
|
init_get_parquet_schema();
|
|
22757
22929
|
init_geoparquet_covering();
|
|
22758
22930
|
init_parquet_bloom_filter();
|
|
@@ -22935,7 +23107,7 @@ var init_parquet_source_loader = __esm({
|
|
|
22935
23107
|
);
|
|
22936
23108
|
const projectedColumns = normalizeColumns(readOptions.columns, initialization.schema);
|
|
22937
23109
|
const predicateColumns = physicalPlan.predicate ? getParquetPredicateColumns(physicalPlan.predicate) : [];
|
|
22938
|
-
const geoMetadata = readOptions.bbox ?
|
|
23110
|
+
const geoMetadata = readOptions.bbox ? getGeoMetadata2(initialization.schema.metadata) : void 0;
|
|
22939
23111
|
const geometryColumn = readOptions.bbox ? readOptions.geometryColumn || (geoMetadata == null ? void 0 : geoMetadata.primary_column) : void 0;
|
|
22940
23112
|
const filterColumns = [
|
|
22941
23113
|
.../* @__PURE__ */ new Set([...predicateColumns, ...geometryColumn ? [geometryColumn] : []])
|