@genome-spy/core 0.77.0 → 0.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle/{browser-KWU9rWZT.js → browser-CETrb2cm.js} +53 -33
- package/dist/bundle/esm-BdLYkz-m.js +248 -0
- package/dist/bundle/esm-BwiDsqSb.js +1367 -0
- package/dist/bundle/esm-CDFd1cjk.js +441 -0
- package/dist/bundle/{esm-CRMf_I9V.js → esm-CTUHLDbv.js} +30 -30
- package/dist/bundle/esm-Cx-EbkOj.js +1221 -0
- package/dist/bundle/esm-DlYGqi79.js +128 -0
- package/dist/bundle/{esm-BygJiwh0.js → esm-k9p3oHkt.js} +133 -158
- package/dist/bundle/{esm-CT3ygiMq.js → esm-zAZJQO6D.js} +226 -212
- package/dist/bundle/index.es.js +14102 -10810
- package/dist/bundle/index.js +109 -95
- package/dist/bundle/{parquetRead-DG_-F5j5.js → parquetRead-Cad1SOVV.js} +473 -399
- package/dist/schema.json +23788 -11049
- package/dist/src/config/axisConfig.d.ts +2 -2
- package/dist/src/config/axisConfig.d.ts.map +1 -1
- package/dist/src/config/axisConfig.js +28 -44
- package/dist/src/config/configLayers.d.ts +45 -0
- package/dist/src/config/configLayers.d.ts.map +1 -0
- package/dist/src/config/configLayers.js +110 -0
- package/dist/src/config/defaultConfig.d.ts.map +1 -1
- package/dist/src/config/defaultConfig.js +8 -1
- package/dist/src/config/defaults/legendDefaults.d.ts +14 -0
- package/dist/src/config/defaults/legendDefaults.d.ts.map +1 -0
- package/dist/src/config/defaults/legendDefaults.js +46 -0
- package/dist/src/config/defaults/titleDefaults.d.ts.map +1 -1
- package/dist/src/config/defaults/titleDefaults.js +26 -18
- package/dist/src/config/legendConfig.d.ts +11 -0
- package/dist/src/config/legendConfig.d.ts.map +1 -0
- package/dist/src/config/legendConfig.js +63 -0
- package/dist/src/config/styleUtils.d.ts +8 -2
- package/dist/src/config/styleUtils.d.ts.map +1 -1
- package/dist/src/config/styleUtils.js +25 -1
- package/dist/src/config/themes.d.ts.map +1 -1
- package/dist/src/config/themes.js +21 -2
- package/dist/src/config/titleConfig.d.ts.map +1 -1
- package/dist/src/config/titleConfig.js +2 -18
- package/dist/src/data/collector.d.ts.map +1 -1
- package/dist/src/data/collector.js +40 -18
- package/dist/src/data/flowInit.d.ts +6 -0
- package/dist/src/data/flowInit.d.ts.map +1 -1
- package/dist/src/data/flowInit.js +1 -1
- package/dist/src/data/flowNode.d.ts +32 -0
- package/dist/src/data/flowNode.d.ts.map +1 -1
- package/dist/src/data/flowNode.js +59 -0
- package/dist/src/data/sources/lazy/bamSource.d.ts +0 -1
- package/dist/src/data/sources/lazy/bamSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/bamSource.js +39 -30
- package/dist/src/data/sources/lazy/bigBedSource.d.ts +0 -10
- package/dist/src/data/sources/lazy/bigBedSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/bigBedSource.js +127 -62
- package/dist/src/data/sources/lazy/bigWigSource.d.ts +2 -2
- package/dist/src/data/sources/lazy/bigWigSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/bigWigSource.js +234 -81
- package/dist/src/data/sources/lazy/gff3Source.d.ts +7 -3
- package/dist/src/data/sources/lazy/gff3Source.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/gff3Source.js +7 -8
- package/dist/src/data/sources/lazy/indexedFastaSource.d.ts +1 -1
- package/dist/src/data/sources/lazy/indexedFastaSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/indexedFastaSource.js +28 -19
- package/dist/src/data/sources/lazy/legendEntriesSource.d.ts +24 -0
- package/dist/src/data/sources/lazy/legendEntriesSource.d.ts.map +1 -0
- package/dist/src/data/sources/lazy/legendEntriesSource.js +217 -0
- package/dist/src/data/sources/lazy/legendGradientSource.d.ts +30 -0
- package/dist/src/data/sources/lazy/legendGradientSource.d.ts.map +1 -0
- package/dist/src/data/sources/lazy/legendGradientSource.js +388 -0
- package/dist/src/data/sources/lazy/mockLazySource.d.ts +4 -1
- package/dist/src/data/sources/lazy/mockLazySource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/mockLazySource.js +49 -4
- package/dist/src/data/sources/lazy/registerCoreLazySources.js +2 -0
- package/dist/src/data/sources/lazy/singleAxisWindowedSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/singleAxisWindowedSource.js +3 -4
- package/dist/src/data/sources/lazy/tabixSource.d.ts +9 -4
- package/dist/src/data/sources/lazy/tabixSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/tabixSource.js +201 -70
- package/dist/src/data/sources/lazy/tabixTsvSource.d.ts +2 -3
- package/dist/src/data/sources/lazy/tabixTsvSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/tabixTsvSource.js +14 -12
- package/dist/src/data/sources/lazy/vcfSource.d.ts +7 -3
- package/dist/src/data/sources/lazy/vcfSource.d.ts.map +1 -1
- package/dist/src/data/sources/lazy/vcfSource.js +7 -8
- package/dist/src/data/sources/urlDescriptor.d.ts +165 -0
- package/dist/src/data/sources/urlDescriptor.d.ts.map +1 -0
- package/dist/src/data/sources/urlDescriptor.js +473 -0
- package/dist/src/data/sources/urlDescriptorController.d.ts +25 -0
- package/dist/src/data/sources/urlDescriptorController.d.ts.map +1 -0
- package/dist/src/data/sources/urlDescriptorController.js +72 -0
- package/dist/src/data/sources/urlDescriptorState.d.ts +47 -0
- package/dist/src/data/sources/urlDescriptorState.d.ts.map +1 -0
- package/dist/src/data/sources/urlDescriptorState.js +129 -0
- package/dist/src/data/sources/urlSource.d.ts.map +1 -1
- package/dist/src/data/sources/urlSource.js +101 -61
- package/dist/src/data/transforms/packLegendLabels.d.ts +21 -0
- package/dist/src/data/transforms/packLegendLabels.d.ts.map +1 -0
- package/dist/src/data/transforms/packLegendLabels.js +189 -0
- package/dist/src/data/transforms/transformFactory.d.ts.map +1 -1
- package/dist/src/data/transforms/transformFactory.js +4 -0
- package/dist/src/data/transforms/truncateText.d.ts +27 -0
- package/dist/src/data/transforms/truncateText.d.ts.map +1 -0
- package/dist/src/data/transforms/truncateText.js +94 -0
- package/dist/src/debug/dataflowDebugSnapshot.d.ts +58 -0
- package/dist/src/debug/dataflowDebugSnapshot.d.ts.map +1 -0
- package/dist/src/debug/dataflowDebugSnapshot.js +159 -0
- package/dist/src/debug/markDebugSnapshot.d.ts +54 -0
- package/dist/src/debug/markDebugSnapshot.d.ts.map +1 -0
- package/dist/src/debug/markDebugSnapshot.js +100 -0
- package/dist/src/debug/paramDebugSnapshot.d.ts +53 -0
- package/dist/src/debug/paramDebugSnapshot.d.ts.map +1 -0
- package/dist/src/debug/paramDebugSnapshot.js +86 -0
- package/dist/src/debug/resolutionDebugSnapshot.d.ts +155 -0
- package/dist/src/debug/resolutionDebugSnapshot.d.ts.map +1 -0
- package/dist/src/debug/resolutionDebugSnapshot.js +291 -0
- package/dist/src/debug/valuePreview.d.ts +9 -0
- package/dist/src/debug/valuePreview.d.ts.map +1 -0
- package/dist/src/debug/valuePreview.js +57 -0
- package/dist/src/debug/viewDebugSnapshot.d.ts +131 -0
- package/dist/src/debug/viewDebugSnapshot.d.ts.map +1 -0
- package/dist/src/debug/viewDebugSnapshot.js +390 -0
- package/dist/src/embedFactory.d.ts.map +1 -1
- package/dist/src/embedFactory.js +6 -1
- package/dist/src/encoder/encoder.d.ts +2 -2
- package/dist/src/encoder/encoder.d.ts.map +1 -1
- package/dist/src/encoder/encoder.js +5 -4
- package/dist/src/fonts/bmFontManager.d.ts +1 -1
- package/dist/src/fonts/bmFontManager.d.ts.map +1 -1
- package/dist/src/fonts/bmFontManager.js +45 -10
- package/dist/src/fonts/textMetrics.d.ts +69 -0
- package/dist/src/fonts/textMetrics.d.ts.map +1 -0
- package/dist/src/fonts/textMetrics.js +73 -0
- package/dist/src/genomeSpy/headlessBootstrap.d.ts.map +1 -1
- package/dist/src/genomeSpy/headlessBootstrap.js +8 -0
- package/dist/src/genomeSpy/interactionController.d.ts +4 -1
- package/dist/src/genomeSpy/interactionController.d.ts.map +1 -1
- package/dist/src/genomeSpy/interactionController.js +57 -13
- package/dist/src/genomeSpy/renderCoordinator.d.ts.map +1 -1
- package/dist/src/genomeSpy/renderCoordinator.js +25 -3
- package/dist/src/genomeSpy/viewDataInit.d.ts +14 -0
- package/dist/src/genomeSpy/viewDataInit.d.ts.map +1 -1
- package/dist/src/genomeSpy/viewDataInit.js +45 -8
- package/dist/src/genomeSpyBase.d.ts +6 -0
- package/dist/src/genomeSpyBase.d.ts.map +1 -1
- package/dist/src/genomeSpyBase.js +25 -4
- package/dist/src/gl/glslScaleGenerator.d.ts +17 -0
- package/dist/src/gl/glslScaleGenerator.d.ts.map +1 -1
- package/dist/src/gl/glslScaleGenerator.js +39 -2
- package/dist/src/gl/includes/common.glsl.js +1 -1
- package/dist/src/gl/vertexRangeIndex.d.ts.map +1 -1
- package/dist/src/gl/vertexRangeIndex.js +4 -2
- package/dist/src/gl/webGLHelper.d.ts +1 -1
- package/dist/src/gl/webGLHelper.d.ts.map +1 -1
- package/dist/src/gl/webGLHelper.js +13 -8
- package/dist/src/marks/__snapshots__/shaderSnapshot.test.js.snap +140 -3
- package/dist/src/marks/mark.d.ts +47 -4
- package/dist/src/marks/mark.d.ts.map +1 -1
- package/dist/src/marks/mark.js +158 -54
- package/dist/src/marks/point.d.ts.map +1 -1
- package/dist/src/marks/point.js +4 -0
- package/dist/src/marks/point.vertex.glsl.js +1 -1
- package/dist/src/marks/text.d.ts +1 -1
- package/dist/src/marks/text.d.ts.map +1 -1
- package/dist/src/marks/text.js +2 -7
- package/dist/src/marks/text.vertex.glsl.js +1 -1
- package/dist/src/paramRuntime/paramUtils.d.ts +43 -9
- package/dist/src/paramRuntime/paramUtils.d.ts.map +1 -1
- package/dist/src/paramRuntime/paramUtils.js +61 -1
- package/dist/src/paramRuntime/viewParamRuntime.d.ts +32 -0
- package/dist/src/paramRuntime/viewParamRuntime.d.ts.map +1 -1
- package/dist/src/paramRuntime/viewParamRuntime.js +63 -0
- package/dist/src/scales/axisResolution.d.ts +35 -0
- package/dist/src/scales/axisResolution.d.ts.map +1 -1
- package/dist/src/scales/axisResolution.js +115 -7
- package/dist/src/scales/domainExpressions.d.ts +21 -0
- package/dist/src/scales/domainExpressions.d.ts.map +1 -0
- package/dist/src/scales/domainExpressions.js +43 -0
- package/dist/src/scales/domainPlanner.d.ts +12 -1
- package/dist/src/scales/domainPlanner.d.ts.map +1 -1
- package/dist/src/scales/domainPlanner.js +55 -36
- package/dist/src/scales/legendResolution.d.ts +83 -0
- package/dist/src/scales/legendResolution.d.ts.map +1 -0
- package/dist/src/scales/legendResolution.js +461 -0
- package/dist/src/scales/scaleInstanceManager.d.ts +1 -0
- package/dist/src/scales/scaleInstanceManager.d.ts.map +1 -1
- package/dist/src/scales/scaleInstanceManager.js +5 -0
- package/dist/src/scales/scalePropsResolver.d.ts +6 -1
- package/dist/src/scales/scalePropsResolver.d.ts.map +1 -1
- package/dist/src/scales/scalePropsResolver.js +35 -10
- package/dist/src/scales/scaleResolution.d.ts +52 -0
- package/dist/src/scales/scaleResolution.d.ts.map +1 -1
- package/dist/src/scales/scaleResolution.js +195 -16
- package/dist/src/scales/scaleRules.d.ts +10 -0
- package/dist/src/scales/scaleRules.d.ts.map +1 -1
- package/dist/src/scales/scaleRules.js +38 -1
- package/dist/src/scales/viewLevelGuideConfig.d.ts +53 -0
- package/dist/src/scales/viewLevelGuideConfig.d.ts.map +1 -0
- package/dist/src/scales/viewLevelGuideConfig.js +224 -0
- package/dist/src/scales/viewLevelScaleConfig.d.ts +45 -0
- package/dist/src/scales/viewLevelScaleConfig.d.ts.map +1 -0
- package/dist/src/scales/viewLevelScaleConfig.js +149 -0
- package/dist/src/spec/axis.d.ts +109 -3
- package/dist/src/spec/channel.d.ts +23 -4
- package/dist/src/spec/config.d.ts +59 -4
- package/dist/src/spec/data.d.ts +177 -17
- package/dist/src/spec/legend.d.ts +246 -0
- package/dist/src/spec/mark.d.ts +16 -4
- package/dist/src/spec/scale.d.ts +19 -6
- package/dist/src/spec/title.d.ts +58 -1
- package/dist/src/spec/transform.d.ts +149 -0
- package/dist/src/spec/view.d.ts +50 -6
- package/dist/src/styles/genome-spy.css +4 -1
- package/dist/src/styles/genome-spy.css.d.ts +1 -1
- package/dist/src/styles/genome-spy.css.d.ts.map +1 -1
- package/dist/src/styles/genome-spy.css.js +4 -1
- package/dist/src/types/embedApi.d.ts +262 -6
- package/dist/src/types/rendering.d.ts +19 -3
- package/dist/src/types/viewContext.d.ts +18 -2
- package/dist/src/utils/arrayUtils.d.ts +11 -0
- package/dist/src/utils/arrayUtils.d.ts.map +1 -1
- package/dist/src/utils/arrayUtils.js +23 -0
- package/dist/src/utils/suspension.d.ts +17 -0
- package/dist/src/utils/suspension.d.ts.map +1 -0
- package/dist/src/utils/suspension.js +41 -0
- package/dist/src/utils/ui/tooltip.d.ts +4 -0
- package/dist/src/utils/ui/tooltip.d.ts.map +1 -1
- package/dist/src/utils/ui/tooltip.js +35 -10
- package/dist/src/view/axisGridView.d.ts.map +1 -1
- package/dist/src/view/axisGridView.js +1 -4
- package/dist/src/view/axisView.d.ts +18 -2
- package/dist/src/view/axisView.d.ts.map +1 -1
- package/dist/src/view/axisView.js +180 -75
- package/dist/src/view/concatView.d.ts +10 -2
- package/dist/src/view/concatView.d.ts.map +1 -1
- package/dist/src/view/concatView.js +46 -9
- package/dist/src/view/containerMutationHelper.d.ts +20 -1
- package/dist/src/view/containerMutationHelper.d.ts.map +1 -1
- package/dist/src/view/containerMutationHelper.js +203 -32
- package/dist/src/view/facetView.d.ts +1 -1
- package/dist/src/view/gridView/gridChild.d.ts +54 -4
- package/dist/src/view/gridView/gridChild.d.ts.map +1 -1
- package/dist/src/view/gridView/gridChild.js +301 -120
- package/dist/src/view/gridView/gridChildLegends.d.ts +57 -0
- package/dist/src/view/gridView/gridChildLegends.d.ts.map +1 -0
- package/dist/src/view/gridView/gridChildLegends.js +503 -0
- package/dist/src/view/gridView/gridView.d.ts +25 -0
- package/dist/src/view/gridView/gridView.d.ts.map +1 -1
- package/dist/src/view/gridView/gridView.js +454 -78
- package/dist/src/view/gridView/legendLayout.d.ts +26 -0
- package/dist/src/view/gridView/legendLayout.d.ts.map +1 -0
- package/dist/src/view/gridView/legendLayout.js +111 -0
- package/dist/src/view/gridView/scrollbar.d.ts.map +1 -1
- package/dist/src/view/gridView/scrollbar.js +1 -4
- package/dist/src/view/gridView/selectionRect.d.ts.map +1 -1
- package/dist/src/view/gridView/selectionRect.js +1 -4
- package/dist/src/view/gridView/separatorView.d.ts.map +1 -1
- package/dist/src/view/gridView/separatorView.js +1 -4
- package/dist/src/view/layerView.d.ts +9 -2
- package/dist/src/view/layerView.d.ts.map +1 -1
- package/dist/src/view/layerView.js +18 -1
- package/dist/src/view/layout/flexLayout.d.ts +20 -4
- package/dist/src/view/layout/flexLayout.d.ts.map +1 -1
- package/dist/src/view/layout/flexLayout.js +331 -31
- package/dist/src/view/layout/rectangle.d.ts +14 -0
- package/dist/src/view/layout/rectangle.d.ts.map +1 -1
- package/dist/src/view/layout/rectangle.js +40 -0
- package/dist/src/view/legend/legendEntries.d.ts +20 -0
- package/dist/src/view/legend/legendEntries.d.ts.map +1 -0
- package/dist/src/view/legend/legendEntries.js +21 -0
- package/dist/src/view/legendView.d.ts +134 -0
- package/dist/src/view/legendView.d.ts.map +1 -0
- package/dist/src/view/legendView.js +1611 -0
- package/dist/src/view/renderingContext/bufferedViewRenderingContext.d.ts.map +1 -1
- package/dist/src/view/renderingContext/bufferedViewRenderingContext.js +26 -4
- package/dist/src/view/renderingContext/clipOptions.d.ts +44 -0
- package/dist/src/view/renderingContext/clipOptions.d.ts.map +1 -0
- package/dist/src/view/renderingContext/clipOptions.js +140 -0
- package/dist/src/view/renderingContext/simpleViewRenderingContext.d.ts.map +1 -1
- package/dist/src/view/renderingContext/simpleViewRenderingContext.js +12 -1
- package/dist/src/view/resolutionPlanner.d.ts +2 -1
- package/dist/src/view/resolutionPlanner.d.ts.map +1 -1
- package/dist/src/view/resolutionPlanner.js +89 -25
- package/dist/src/view/testUtils.d.ts +4 -2
- package/dist/src/view/testUtils.d.ts.map +1 -1
- package/dist/src/view/testUtils.js +29 -7
- package/dist/src/view/titleView.d.ts +37 -0
- package/dist/src/view/titleView.d.ts.map +1 -0
- package/dist/src/view/titleView.js +584 -0
- package/dist/src/view/unitView.d.ts +3 -3
- package/dist/src/view/unitView.d.ts.map +1 -1
- package/dist/src/view/unitView.js +3 -2
- package/dist/src/view/view.d.ts +25 -24
- package/dist/src/view/view.d.ts.map +1 -1
- package/dist/src/view/view.js +121 -16
- package/dist/src/view/viewFactory.d.ts +2 -5
- package/dist/src/view/viewFactory.d.ts.map +1 -1
- package/dist/src/view/viewFactory.js +1 -2
- package/dist/src/view/viewIdentityRegistry.d.ts +37 -0
- package/dist/src/view/viewIdentityRegistry.d.ts.map +1 -0
- package/dist/src/view/viewIdentityRegistry.js +71 -0
- package/dist/src/view/viewMutationAcidTestUtils.d.ts +112 -0
- package/dist/src/view/viewMutationAcidTestUtils.d.ts.map +1 -0
- package/dist/src/view/viewMutationAcidTestUtils.js +234 -0
- package/dist/src/view/viewMutationApi.d.ts +42 -0
- package/dist/src/view/viewMutationApi.d.ts.map +1 -0
- package/dist/src/view/viewMutationApi.js +811 -0
- package/dist/src/view/viewSelectors.d.ts +10 -0
- package/dist/src/view/viewSelectors.d.ts.map +1 -1
- package/dist/src/view/viewSelectors.js +23 -1
- package/package.json +4 -4
- package/dist/bundle/esm-0dYHNV_D.js +0 -121
- package/dist/bundle/esm-C49STiCR.js +0 -1248
- package/dist/bundle/esm-CscjKVDc.js +0 -1426
- package/dist/bundle/esm-CuMSzCHy.js +0 -298
- package/dist/bundle/esm-DMXpJXM4.js +0 -369
- package/dist/src/view/title.d.ts +0 -13
- package/dist/src/view/title.d.ts.map +0 -1
- package/dist/src/view/title.js +0 -154
- /package/dist/bundle/{AbortablePromiseCache-3gHJdF3E.js → AbortablePromiseCache-BTmAcN-t.js} +0 -0
- /package/dist/bundle/{esm-CuVa5T98.js → esm-VvpZ9hsq.js} +0 -0
- /package/dist/bundle/{chunk-DmhlhrBa.js → rolldown-runtime-Dy4uBu1J.js} +0 -0
|
@@ -179,6 +179,11 @@ var f = new TextDecoder(), p = {
|
|
|
179
179
|
view: new DataView(e.buffer, e.byteOffset, e.byteLength),
|
|
180
180
|
offset: 0
|
|
181
181
|
});
|
|
182
|
+
},
|
|
183
|
+
uuidFromBytes(e) {
|
|
184
|
+
if (!e) return;
|
|
185
|
+
let t = Array.from(e, (e) => e.toString(16).padStart(2, "0")).join("");
|
|
186
|
+
return t.slice(0, 8) + "-" + t.slice(8, 12) + "-" + t.slice(12, 16) + "-" + t.slice(16, 20) + "-" + t.slice(20, 32);
|
|
182
187
|
}
|
|
183
188
|
};
|
|
184
189
|
function m(e, t, n, r) {
|
|
@@ -190,7 +195,7 @@ function m(e, t, n, r) {
|
|
|
190
195
|
} else return h(e, r);
|
|
191
196
|
}
|
|
192
197
|
function h(e, t) {
|
|
193
|
-
let { element: n, parsers: r, utf8: i = !0, schemaPath: a } = t, { type: o, converted_type: s, logical_type: c } = n;
|
|
198
|
+
let { element: n, parsers: r, utf8: i = !0, schemaPath: a } = t, { type: o, converted_type: s, logical_type: c } = n, l = n.repetition_type !== "REQUIRED";
|
|
194
199
|
if (a?.some((e) => e.element.logical_type?.type === "VARIANT") && o === "BYTE_ARRAY" && s !== "UTF8" && c?.type !== "STRING") return e;
|
|
195
200
|
if (s === "DECIMAL") {
|
|
196
201
|
let t = 10 ** -(n.scale || 0), r = Array(e.length);
|
|
@@ -206,17 +211,18 @@ function h(e, t) {
|
|
|
206
211
|
if (s === "INTERVAL") throw Error("parquet interval not supported");
|
|
207
212
|
if (c?.type === "GEOMETRY") return e.map((e) => r.geometryFromBytes(e));
|
|
208
213
|
if (c?.type === "GEOGRAPHY") return e.map((e) => r.geographyFromBytes(e));
|
|
214
|
+
if (c?.type === "UUID") return e.map((e) => r.uuidFromBytes(e));
|
|
209
215
|
if (s === "UTF8" || c?.type === "STRING" || i && o === "BYTE_ARRAY") return e.map((e) => r.stringFromBytes(e));
|
|
210
216
|
if (s === "UINT_64" || c?.type === "INTEGER" && c.bitWidth === 64 && !c.isSigned) {
|
|
211
217
|
if (e instanceof BigInt64Array) return new BigUint64Array(e.buffer, e.byteOffset, e.length);
|
|
212
|
-
let t = new BigUint64Array(e.length);
|
|
213
|
-
for (let n = 0; n < t.length; n++) t[n] =
|
|
218
|
+
let t = l ? Array(e.length) : new BigUint64Array(e.length);
|
|
219
|
+
for (let n = 0; n < t.length; n++) t[n] = e[n];
|
|
214
220
|
return t;
|
|
215
221
|
}
|
|
216
222
|
if (s === "UINT_32" || c?.type === "INTEGER" && c.bitWidth === 32 && !c.isSigned) {
|
|
217
223
|
if (e instanceof Int32Array) return new Uint32Array(e.buffer, e.byteOffset, e.length);
|
|
218
|
-
let t = new Uint32Array(e.length);
|
|
219
|
-
for (let n = 0; n < t.length; n++) t[n] = e[n];
|
|
224
|
+
let t = l ? Array(e.length) : new Uint32Array(e.length);
|
|
225
|
+
for (let n = 0; n < t.length; n++) t[n] = e[n] < 0 ? 4294967296 + e[n] : e[n];
|
|
220
226
|
return t;
|
|
221
227
|
}
|
|
222
228
|
if (c?.type === "FLOAT16") return Array.from(e).map(v);
|
|
@@ -277,94 +283,72 @@ function ee(e) {
|
|
|
277
283
|
}
|
|
278
284
|
return n(e), t;
|
|
279
285
|
}
|
|
280
|
-
function
|
|
286
|
+
function te(e) {
|
|
281
287
|
let t = 0;
|
|
282
288
|
for (let { element: n } of e) n.repetition_type === "REPEATED" && t++;
|
|
283
289
|
return t;
|
|
284
290
|
}
|
|
285
|
-
function
|
|
291
|
+
function x(e) {
|
|
286
292
|
let t = 0;
|
|
287
293
|
for (let { element: n } of e.slice(1)) n.repetition_type !== "REQUIRED" && t++;
|
|
288
294
|
return t;
|
|
289
295
|
}
|
|
290
|
-
function
|
|
296
|
+
function ne(e) {
|
|
291
297
|
if (!e || e.element.converted_type !== "LIST" || e.children.length > 1) return !1;
|
|
292
298
|
let t = e.children[0];
|
|
293
299
|
return !(t.children.length > 1 || t.element.repetition_type !== "REPEATED");
|
|
294
300
|
}
|
|
295
|
-
function
|
|
301
|
+
function re(e) {
|
|
296
302
|
if (!e || e.element.converted_type !== "MAP" || e.children.length > 1) return !1;
|
|
297
303
|
let t = e.children[0];
|
|
298
304
|
return !(t.children.length !== 2 || t.element.repetition_type !== "REPEATED" || t.children.find((e) => e.element.name === "key")?.element.repetition_type === "REPEATED" || t.children.find((e) => e.element.name === "value")?.element.repetition_type === "REPEATED");
|
|
299
305
|
}
|
|
300
|
-
function
|
|
306
|
+
function S(e) {
|
|
301
307
|
if (e.length !== 2) return !1;
|
|
302
308
|
let [, t] = e;
|
|
303
309
|
return !(t.element.repetition_type === "REPEATED" || t.children.length);
|
|
304
310
|
}
|
|
305
311
|
//#endregion
|
|
306
312
|
//#region ../../node_modules/hyparquet/src/thrift.js
|
|
307
|
-
var
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
FALSE: 2,
|
|
311
|
-
BYTE: 3,
|
|
312
|
-
I16: 4,
|
|
313
|
-
I32: 5,
|
|
314
|
-
I64: 6,
|
|
315
|
-
DOUBLE: 7,
|
|
316
|
-
BINARY: 8,
|
|
317
|
-
LIST: 9,
|
|
318
|
-
SET: 10,
|
|
319
|
-
MAP: 11,
|
|
320
|
-
STRUCT: 12,
|
|
321
|
-
UUID: 13
|
|
322
|
-
};
|
|
323
|
-
function w(e) {
|
|
324
|
-
let t = 0, n = {};
|
|
313
|
+
var ie = 0, ae = 1, oe = 2, se = 3, ce = 4, le = 5, ue = 6, de = 7, fe = 8, pe = 9, me = 12;
|
|
314
|
+
function C(e) {
|
|
315
|
+
let t = {}, n = 0;
|
|
325
316
|
for (; e.offset < e.view.byteLength;) {
|
|
326
|
-
let
|
|
327
|
-
if (
|
|
328
|
-
|
|
317
|
+
let r = e.view.getUint8(e.offset++), i = r & 15;
|
|
318
|
+
if (i === ie) break;
|
|
319
|
+
let a = r >> 4;
|
|
320
|
+
n = a ? n + a : ge(e), t[`field_${n}`] = w(e, i);
|
|
329
321
|
}
|
|
330
|
-
return
|
|
322
|
+
return t;
|
|
331
323
|
}
|
|
332
|
-
function
|
|
324
|
+
function w(e, t) {
|
|
333
325
|
switch (t) {
|
|
334
|
-
case
|
|
335
|
-
case
|
|
336
|
-
case
|
|
337
|
-
case
|
|
338
|
-
case
|
|
339
|
-
case
|
|
340
|
-
case
|
|
326
|
+
case ae: return !0;
|
|
327
|
+
case oe: return !1;
|
|
328
|
+
case se: return e.view.getInt8(e.offset++);
|
|
329
|
+
case ce:
|
|
330
|
+
case le: return ge(e);
|
|
331
|
+
case ue: return E(e);
|
|
332
|
+
case de: {
|
|
341
333
|
let t = e.view.getFloat64(e.offset, !0);
|
|
342
334
|
return e.offset += 8, t;
|
|
343
335
|
}
|
|
344
|
-
case
|
|
345
|
-
let t =
|
|
336
|
+
case fe: {
|
|
337
|
+
let t = T(e), n = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
346
338
|
return e.offset += t, n;
|
|
347
339
|
}
|
|
348
|
-
case
|
|
340
|
+
case pe: {
|
|
349
341
|
let t = e.view.getUint8(e.offset++), n = t & 15, r = t >> 4;
|
|
350
|
-
r === 15 && (r =
|
|
351
|
-
let i = n ===
|
|
352
|
-
for (let t = 0; t < r; t++) a[t] = i ?
|
|
342
|
+
r === 15 && (r = T(e));
|
|
343
|
+
let i = n === ae || n === oe, a = Array(r);
|
|
344
|
+
for (let t = 0; t < r; t++) a[t] = i ? w(e, se) === 1 : w(e, n);
|
|
353
345
|
return a;
|
|
354
346
|
}
|
|
355
|
-
case
|
|
356
|
-
let t = {}, n = 0;
|
|
357
|
-
for (;;) {
|
|
358
|
-
let [r, i, a] = ae(e, n);
|
|
359
|
-
if (n = a, r === C.STOP) break;
|
|
360
|
-
t[`field_${i}`] = T(e, r);
|
|
361
|
-
}
|
|
362
|
-
return t;
|
|
363
|
-
}
|
|
347
|
+
case me: return C(e);
|
|
364
348
|
default: throw Error(`thrift unhandled type: ${t}`);
|
|
365
349
|
}
|
|
366
350
|
}
|
|
367
|
-
function
|
|
351
|
+
function T(e) {
|
|
368
352
|
let t = 0, n = 0;
|
|
369
353
|
for (;;) {
|
|
370
354
|
let r = e.view.getUint8(e.offset++);
|
|
@@ -372,7 +356,7 @@ function E(e) {
|
|
|
372
356
|
n += 7;
|
|
373
357
|
}
|
|
374
358
|
}
|
|
375
|
-
function
|
|
359
|
+
function he(e) {
|
|
376
360
|
let t = 0n, n = 0n;
|
|
377
361
|
for (;;) {
|
|
378
362
|
let r = e.view.getUint8(e.offset++);
|
|
@@ -380,31 +364,17 @@ function ie(e) {
|
|
|
380
364
|
n += 7n;
|
|
381
365
|
}
|
|
382
366
|
}
|
|
383
|
-
function
|
|
384
|
-
let t =
|
|
367
|
+
function ge(e) {
|
|
368
|
+
let t = T(e);
|
|
385
369
|
return t >>> 1 ^ -(t & 1);
|
|
386
370
|
}
|
|
387
|
-
function
|
|
388
|
-
let t =
|
|
371
|
+
function E(e) {
|
|
372
|
+
let t = he(e);
|
|
389
373
|
return t >> 1n ^ -(t & 1n);
|
|
390
374
|
}
|
|
391
|
-
function ae(e, t) {
|
|
392
|
-
let n = e.view.getUint8(e.offset++), r = n & 15;
|
|
393
|
-
if (r === C.STOP) return [
|
|
394
|
-
0,
|
|
395
|
-
0,
|
|
396
|
-
t
|
|
397
|
-
];
|
|
398
|
-
let i = n >> 4, a = i ? t + i : D(e);
|
|
399
|
-
return [
|
|
400
|
-
r,
|
|
401
|
-
a,
|
|
402
|
-
a
|
|
403
|
-
];
|
|
404
|
-
}
|
|
405
375
|
//#endregion
|
|
406
376
|
//#region ../../node_modules/hyparquet/src/geoparquet.js
|
|
407
|
-
function
|
|
377
|
+
function _e(e, t) {
|
|
408
378
|
let n = /* @__PURE__ */ new Map(), r = t?.find(({ key: e }) => e === "geo")?.value, i = (r && JSON.parse(r)?.columns) ?? {};
|
|
409
379
|
for (let [e, t] of Object.entries(i)) {
|
|
410
380
|
if (t.encoding !== "WKB") continue;
|
|
@@ -415,21 +385,21 @@ function oe(e, t) {
|
|
|
415
385
|
});
|
|
416
386
|
}
|
|
417
387
|
for (let t = 1; t < e.length; t++) {
|
|
418
|
-
let
|
|
419
|
-
if (
|
|
420
|
-
t +=
|
|
388
|
+
let { logical_type: r, name: i, num_children: a, type: o } = e[t];
|
|
389
|
+
if (a) {
|
|
390
|
+
t += a;
|
|
421
391
|
continue;
|
|
422
392
|
}
|
|
423
|
-
|
|
393
|
+
o === "BYTE_ARRAY" && !r && (e[t].logical_type = n.get(i));
|
|
424
394
|
}
|
|
425
395
|
}
|
|
426
396
|
//#endregion
|
|
427
397
|
//#region ../../node_modules/hyparquet/src/metadata.js
|
|
428
|
-
var
|
|
429
|
-
function
|
|
430
|
-
return e &&
|
|
398
|
+
var ve = 1 << 19, ye = new TextDecoder();
|
|
399
|
+
function D(e) {
|
|
400
|
+
return e && ye.decode(e);
|
|
431
401
|
}
|
|
432
|
-
async function
|
|
402
|
+
async function be(e, { parsers: t, initialFetchSize: n = ve, geoparquet: r = !0 } = {}) {
|
|
433
403
|
if (!e || !(e.byteLength >= 0)) throw Error("parquet expected AsyncBuffer");
|
|
434
404
|
let i = Math.max(0, e.byteLength - n), a = await e.slice(i, e.byteLength), o = new DataView(a);
|
|
435
405
|
if (o.getUint32(a.byteLength - 4, !0) !== 827474256) throw Error("parquet file invalid (footer != PAR1)");
|
|
@@ -437,16 +407,16 @@ async function le(e, { parsers: t, initialFetchSize: n = se, geoparquet: r = !0
|
|
|
437
407
|
if (s > e.byteLength - 8) throw Error(`parquet metadata length ${s} exceeds available buffer ${e.byteLength - 8}`);
|
|
438
408
|
if (s + 8 > n) {
|
|
439
409
|
let n = e.byteLength - s - 8, o = await e.slice(n, i), c = new ArrayBuffer(s + 8), l = new Uint8Array(c);
|
|
440
|
-
return l.set(new Uint8Array(o)), l.set(new Uint8Array(a), i - n),
|
|
410
|
+
return l.set(new Uint8Array(o)), l.set(new Uint8Array(a), i - n), xe(c, {
|
|
441
411
|
parsers: t,
|
|
442
412
|
geoparquet: r
|
|
443
413
|
});
|
|
444
|
-
} else return
|
|
414
|
+
} else return xe(a, {
|
|
445
415
|
parsers: t,
|
|
446
416
|
geoparquet: r
|
|
447
417
|
});
|
|
448
418
|
}
|
|
449
|
-
function
|
|
419
|
+
function xe(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
450
420
|
if (!(o instanceof ArrayBuffer)) throw Error("parquet expected ArrayBuffer");
|
|
451
421
|
let l = new DataView(o);
|
|
452
422
|
if (s = {
|
|
@@ -456,40 +426,40 @@ function ue(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
|
456
426
|
if (l.getUint32(l.byteLength - 4, !0) !== 827474256) throw Error("parquet file invalid (footer != PAR1)");
|
|
457
427
|
let u = l.byteLength - 8, d = l.getUint32(u, !0);
|
|
458
428
|
if (d > l.byteLength - 8) throw Error(`parquet metadata length ${d} exceeds available buffer ${l.byteLength - 8}`);
|
|
459
|
-
let f =
|
|
429
|
+
let f = C({
|
|
460
430
|
view: l,
|
|
461
431
|
offset: u - d
|
|
462
432
|
}), m = f.field_1, h = f.field_2.map((t) => ({
|
|
463
433
|
type: e[t.field_1],
|
|
464
434
|
type_length: t.field_2,
|
|
465
435
|
repetition_type: n[t.field_3],
|
|
466
|
-
name:
|
|
436
|
+
name: D(t.field_4),
|
|
467
437
|
num_children: t.field_5,
|
|
468
438
|
converted_type: r[t.field_6],
|
|
469
439
|
scale: t.field_7,
|
|
470
440
|
precision: t.field_8,
|
|
471
441
|
field_id: t.field_9,
|
|
472
|
-
logical_type:
|
|
442
|
+
logical_type: Ce(t.field_10)
|
|
473
443
|
})), g = h.filter((e) => e.type), _ = f.field_3, v = f.field_4.map((n) => ({
|
|
474
444
|
columns: n.field_1.map((n, r) => ({
|
|
475
|
-
file_path:
|
|
445
|
+
file_path: D(n.field_1),
|
|
476
446
|
file_offset: n.field_2,
|
|
477
447
|
meta_data: n.field_3 && {
|
|
478
448
|
type: e[n.field_3.field_1],
|
|
479
449
|
encodings: n.field_3.field_2?.map((e) => t[e]),
|
|
480
|
-
path_in_schema: n.field_3.field_3.map(
|
|
450
|
+
path_in_schema: n.field_3.field_3.map(D),
|
|
481
451
|
codec: i[n.field_3.field_4],
|
|
482
452
|
num_values: n.field_3.field_5,
|
|
483
453
|
total_uncompressed_size: n.field_3.field_6,
|
|
484
454
|
total_compressed_size: n.field_3.field_7,
|
|
485
455
|
key_value_metadata: n.field_3.field_8?.map((e) => ({
|
|
486
|
-
key:
|
|
487
|
-
value:
|
|
456
|
+
key: D(e.field_1),
|
|
457
|
+
value: D(e.field_2)
|
|
488
458
|
})),
|
|
489
459
|
data_page_offset: n.field_3.field_9,
|
|
490
460
|
index_page_offset: n.field_3.field_10,
|
|
491
461
|
dictionary_page_offset: n.field_3.field_11,
|
|
492
|
-
statistics:
|
|
462
|
+
statistics: Te(n.field_3.field_12, g[r], s),
|
|
493
463
|
encoding_stats: n.field_3.field_13?.map((e) => ({
|
|
494
464
|
page_type: a[e.field_1],
|
|
495
465
|
encoding: t[e.field_2],
|
|
@@ -534,10 +504,10 @@ function ue(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
|
534
504
|
total_compressed_size: n.field_6,
|
|
535
505
|
ordinal: n.field_7
|
|
536
506
|
})), y = f.field_5?.map((e) => ({
|
|
537
|
-
key:
|
|
538
|
-
value:
|
|
539
|
-
})), b =
|
|
540
|
-
return c &&
|
|
507
|
+
key: D(e.field_1),
|
|
508
|
+
value: D(e.field_2)
|
|
509
|
+
})), b = D(f.field_6);
|
|
510
|
+
return c && _e(h, y), {
|
|
541
511
|
version: m,
|
|
542
512
|
schema: h,
|
|
543
513
|
num_rows: _,
|
|
@@ -547,10 +517,10 @@ function ue(o, { parsers: s, geoparquet: c = !0 } = {}) {
|
|
|
547
517
|
metadata_length: d
|
|
548
518
|
};
|
|
549
519
|
}
|
|
550
|
-
function
|
|
520
|
+
function Se({ schema: e }) {
|
|
551
521
|
return b(e, [])[0];
|
|
552
522
|
}
|
|
553
|
-
function
|
|
523
|
+
function Ce(e) {
|
|
554
524
|
return e?.field_1 ? { type: "STRING" } : e?.field_2 ? { type: "MAP" } : e?.field_3 ? { type: "LIST" } : e?.field_4 ? { type: "ENUM" } : e?.field_5 ? {
|
|
555
525
|
type: "DECIMAL",
|
|
556
526
|
scale: e.field_5.field_1,
|
|
@@ -558,11 +528,11 @@ function de(e) {
|
|
|
558
528
|
} : e?.field_6 ? { type: "DATE" } : e?.field_7 ? {
|
|
559
529
|
type: "TIME",
|
|
560
530
|
isAdjustedToUTC: e.field_7.field_1,
|
|
561
|
-
unit:
|
|
531
|
+
unit: we(e.field_7.field_2)
|
|
562
532
|
} : e?.field_8 ? {
|
|
563
533
|
type: "TIMESTAMP",
|
|
564
534
|
isAdjustedToUTC: e.field_8.field_1,
|
|
565
|
-
unit:
|
|
535
|
+
unit: we(e.field_8.field_2)
|
|
566
536
|
} : e?.field_10 ? {
|
|
567
537
|
type: "INTEGER",
|
|
568
538
|
bitWidth: e.field_10.field_1,
|
|
@@ -572,168 +542,271 @@ function de(e) {
|
|
|
572
542
|
specification_version: e.field_16.field_1
|
|
573
543
|
} : e?.field_17 ? {
|
|
574
544
|
type: "GEOMETRY",
|
|
575
|
-
crs:
|
|
545
|
+
crs: D(e.field_17.field_1)
|
|
576
546
|
} : e?.field_18 ? {
|
|
577
547
|
type: "GEOGRAPHY",
|
|
578
|
-
crs:
|
|
548
|
+
crs: D(e.field_18.field_1),
|
|
579
549
|
algorithm: o[e.field_18.field_2]
|
|
580
550
|
} : e;
|
|
581
551
|
}
|
|
582
|
-
function
|
|
552
|
+
function we(e) {
|
|
583
553
|
if (e.field_1) return "MILLIS";
|
|
584
554
|
if (e.field_2) return "MICROS";
|
|
585
555
|
if (e.field_3) return "NANOS";
|
|
586
556
|
throw Error("parquet time unit required");
|
|
587
557
|
}
|
|
588
|
-
function
|
|
558
|
+
function Te(e, t, n) {
|
|
589
559
|
return e && {
|
|
590
|
-
max:
|
|
591
|
-
min:
|
|
560
|
+
max: O(e.field_1, t, n),
|
|
561
|
+
min: O(e.field_2, t, n),
|
|
592
562
|
null_count: e.field_3,
|
|
593
563
|
distinct_count: e.field_4,
|
|
594
|
-
max_value:
|
|
595
|
-
min_value:
|
|
564
|
+
max_value: O(e.field_5, t, n),
|
|
565
|
+
min_value: O(e.field_6, t, n),
|
|
596
566
|
is_max_value_exact: e.field_7,
|
|
597
567
|
is_min_value_exact: e.field_8
|
|
598
568
|
};
|
|
599
569
|
}
|
|
600
|
-
function
|
|
570
|
+
function O(e, t, n) {
|
|
601
571
|
let { type: r, converted_type: i, logical_type: a } = t;
|
|
602
572
|
if (e === void 0) return e;
|
|
603
573
|
if (r === "BOOLEAN") return e[0] === 1;
|
|
604
574
|
if (r === "BYTE_ARRAY") return n.stringFromBytes(e);
|
|
605
575
|
let o = new DataView(e.buffer, e.byteOffset, e.byteLength);
|
|
606
|
-
return r === "FLOAT" && o.byteLength === 4 ? o.getFloat32(0, !0) : r === "DOUBLE" && o.byteLength === 8 ? o.getFloat64(0, !0) : r === "INT32" && i === "DATE" ? n.dateFromDays(o.getInt32(0, !0)) : r === "INT64" && i === "TIMESTAMP_MILLIS" ? n.timestampFromMilliseconds(o.getBigInt64(0, !0)) : r === "INT64" && i === "TIMESTAMP_MICROS" ? n.timestampFromMicroseconds(o.getBigInt64(0, !0)) : r === "INT64" && a?.type === "TIMESTAMP" && a?.unit === "NANOS" ? n.timestampFromNanoseconds(o.getBigInt64(0, !0)) : r === "INT64" && a?.type === "TIMESTAMP" && a?.unit === "MICROS" ? n.timestampFromMicroseconds(o.getBigInt64(0, !0)) : r === "INT64" && a?.type === "TIMESTAMP" ? n.timestampFromMilliseconds(o.getBigInt64(0, !0)) : r === "INT32" && o.byteLength === 4 ? o.getInt32(0, !0) : r === "INT64" && o.byteLength === 8 ? o.getBigInt64(0, !0) : i === "DECIMAL" ? g(e) * 10 ** -(t.scale || 0) : a?.type === "FLOAT16" ? v(e) : e;
|
|
576
|
+
return r === "FLOAT" && o.byteLength === 4 ? o.getFloat32(0, !0) : r === "DOUBLE" && o.byteLength === 8 ? o.getFloat64(0, !0) : r === "INT32" && i === "DATE" ? n.dateFromDays(o.getInt32(0, !0)) : r === "INT64" && i === "TIMESTAMP_MILLIS" ? n.timestampFromMilliseconds(o.getBigInt64(0, !0)) : r === "INT64" && i === "TIMESTAMP_MICROS" ? n.timestampFromMicroseconds(o.getBigInt64(0, !0)) : r === "INT64" && a?.type === "TIMESTAMP" && a?.unit === "NANOS" ? n.timestampFromNanoseconds(o.getBigInt64(0, !0)) : r === "INT64" && a?.type === "TIMESTAMP" && a?.unit === "MICROS" ? n.timestampFromMicroseconds(o.getBigInt64(0, !0)) : r === "INT64" && a?.type === "TIMESTAMP" ? n.timestampFromMilliseconds(o.getBigInt64(0, !0)) : r === "INT32" && o.byteLength === 4 ? o.getInt32(0, !0) : r === "INT64" && o.byteLength === 8 ? o.getBigInt64(0, !0) : i === "DECIMAL" ? g(e) * 10 ** -(t.scale || 0) : a?.type === "FLOAT16" ? v(e) : a?.type === "UUID" ? n.uuidFromBytes(e) : e;
|
|
607
577
|
}
|
|
608
578
|
//#endregion
|
|
609
|
-
//#region ../../node_modules/hyparquet/src/
|
|
579
|
+
//#region ../../node_modules/hyparquet/src/xxhash.js
|
|
580
|
+
var k = 18446744073709551615n, A = 11400714785074694791n, j = 14029467366897019727n, Ee = 1609587929392839161n, De = 9650029242287828579n, Oe = 2870177450012600261n;
|
|
581
|
+
function M(e, t) {
|
|
582
|
+
return (e << t | e >> 64n - t) & k;
|
|
583
|
+
}
|
|
610
584
|
function N(e, t) {
|
|
585
|
+
return e = e + t * j & k, e = M(e, 31n), e * A & k;
|
|
586
|
+
}
|
|
587
|
+
function P(e, t) {
|
|
588
|
+
return e ^= N(0n, t), e * A + De & k;
|
|
589
|
+
}
|
|
590
|
+
function F(e, t = 0n) {
|
|
591
|
+
let n = new DataView(e.buffer, e.byteOffset, e.byteLength), r = e.byteLength, i = 0, a;
|
|
592
|
+
if (r >= 32) {
|
|
593
|
+
let e = t + A + j & k, o = t + j & k, s = t, c = t - A & k;
|
|
594
|
+
for (; i + 32 <= r;) e = N(e, n.getBigUint64(i, !0)), i += 8, o = N(o, n.getBigUint64(i, !0)), i += 8, s = N(s, n.getBigUint64(i, !0)), i += 8, c = N(c, n.getBigUint64(i, !0)), i += 8;
|
|
595
|
+
a = M(e, 1n) + M(o, 7n) + M(s, 12n) + M(c, 18n) & k, a = P(a, e), a = P(a, o), a = P(a, s), a = P(a, c);
|
|
596
|
+
} else a = t + Oe & k;
|
|
597
|
+
for (a = a + BigInt(r) & k; i + 8 <= r;) a ^= N(0n, n.getBigUint64(i, !0)), a = M(a, 27n) * A + De & k, i += 8;
|
|
598
|
+
for (i + 4 <= r && (a ^= BigInt(n.getUint32(i, !0)) * A & k, a = M(a, 23n) * j + Ee & k, i += 4); i < r;) a ^= BigInt(n.getUint8(i)) * Oe & k, a = M(a, 11n) * A & k, i += 1;
|
|
599
|
+
return a ^= a >> 33n, a = a * j & k, a ^= a >> 29n, a = a * Ee & k, a ^= a >> 32n, a;
|
|
600
|
+
}
|
|
601
|
+
//#endregion
|
|
602
|
+
//#region ../../node_modules/hyparquet/src/bloom.js
|
|
603
|
+
var ke = new TextEncoder(), Ae = new Uint32Array([
|
|
604
|
+
1203114875,
|
|
605
|
+
1150766481,
|
|
606
|
+
2284105051,
|
|
607
|
+
2729912477,
|
|
608
|
+
1884591559,
|
|
609
|
+
770785867,
|
|
610
|
+
2667333959,
|
|
611
|
+
1550580529
|
|
612
|
+
]);
|
|
613
|
+
function je(e, t) {
|
|
614
|
+
return Number((e >> 32n) * BigInt(t) >> 32n);
|
|
615
|
+
}
|
|
616
|
+
function Me(e) {
|
|
617
|
+
let t = /* @__PURE__ */ new Uint32Array(8), n = Number(e & 4294967295n) | 0;
|
|
618
|
+
for (let e = 0; e < 8; e++) t[e] = 1 << (Math.imul(n, Ae[e]) >>> 27);
|
|
619
|
+
return t;
|
|
620
|
+
}
|
|
621
|
+
function Ne(e, t) {
|
|
622
|
+
let n = je(t, e.length >> 3) << 3, r = Me(t);
|
|
623
|
+
for (let t = 0; t < 8; t++) if ((e[n + t] & r[t]) === 0) return !1;
|
|
624
|
+
return !0;
|
|
625
|
+
}
|
|
626
|
+
function Pe(e, t) {
|
|
627
|
+
if (e == null) return;
|
|
628
|
+
let { type: n, converted_type: r, logical_type: i } = t;
|
|
629
|
+
if (n === "BOOLEAN") return typeof e == "boolean" ? F(new Uint8Array([+!!e])) : void 0;
|
|
630
|
+
if (n === "FLOAT") {
|
|
631
|
+
if (typeof e != "number") return;
|
|
632
|
+
let t = /* @__PURE__ */ new ArrayBuffer(4);
|
|
633
|
+
return new DataView(t).setFloat32(0, e, !0), F(new Uint8Array(t));
|
|
634
|
+
}
|
|
635
|
+
if (n === "DOUBLE") {
|
|
636
|
+
if (typeof e != "number") return;
|
|
637
|
+
let t = /* @__PURE__ */ new ArrayBuffer(8);
|
|
638
|
+
return new DataView(t).setFloat64(0, e, !0), F(new Uint8Array(t));
|
|
639
|
+
}
|
|
640
|
+
if (n === "INT32") {
|
|
641
|
+
if (r === "DATE" || r === "DECIMAL" || r === "TIME_MILLIS" || i?.type === "DATE" || i?.type === "TIME" || i?.type === "DECIMAL" || typeof e != "number" || !Number.isInteger(e)) return;
|
|
642
|
+
let t = /* @__PURE__ */ new ArrayBuffer(4);
|
|
643
|
+
return new DataView(t).setInt32(0, e | 0, !0), F(new Uint8Array(t));
|
|
644
|
+
}
|
|
645
|
+
if (n === "INT64") {
|
|
646
|
+
if (r === "TIMESTAMP_MILLIS" || r === "TIMESTAMP_MICROS" || r === "TIME_MICROS" || r === "DECIMAL" || i?.type === "TIMESTAMP" || i?.type === "TIME" || i?.type === "DECIMAL") return;
|
|
647
|
+
let t;
|
|
648
|
+
if (typeof e == "bigint") t = e;
|
|
649
|
+
else if (typeof e == "number" && Number.isSafeInteger(e)) t = BigInt(e);
|
|
650
|
+
else return;
|
|
651
|
+
let n = /* @__PURE__ */ new ArrayBuffer(8);
|
|
652
|
+
return new DataView(n).setBigUint64(0, BigInt.asUintN(64, t), !0), F(new Uint8Array(n));
|
|
653
|
+
}
|
|
654
|
+
if (n === "BYTE_ARRAY") return r === "JSON" || r === "BSON" || r === "DECIMAL" || i?.type === "JSON" || i?.type === "BSON" || i?.type === "VARIANT" || i?.type === "GEOMETRY" || i?.type === "GEOGRAPHY" ? void 0 : typeof e == "string" ? F(ke.encode(e)) : e instanceof Uint8Array ? F(e) : void 0;
|
|
655
|
+
if (n === "FIXED_LEN_BYTE_ARRAY") return r === "DECIMAL" || r === "INTERVAL" || i?.type === "DECIMAL" || i?.type === "UUID" || i?.type === "FLOAT16" || i?.type === "GEOMETRY" || i?.type === "GEOGRAPHY" ? void 0 : e instanceof Uint8Array ? F(e) : void 0;
|
|
656
|
+
}
|
|
657
|
+
//#endregion
|
|
658
|
+
//#region ../../node_modules/hyparquet/src/utils.js
|
|
659
|
+
function Fe(e, t) {
|
|
611
660
|
let n = 1e4;
|
|
612
661
|
for (let r = 0; r < t.length; r += n) e.push(...t.slice(r, r + n));
|
|
613
662
|
}
|
|
614
|
-
function
|
|
663
|
+
function I(e, t, n = !0) {
|
|
615
664
|
if (n ? e === t : e == t) return !0;
|
|
616
|
-
if (e
|
|
617
|
-
if (
|
|
665
|
+
if (!e || !t || typeof e != "object" || typeof t != "object") return !1;
|
|
666
|
+
if (e instanceof Uint8Array && t instanceof Uint8Array) {
|
|
667
|
+
if (e.length !== t.length) return !1;
|
|
668
|
+
for (let n = 0; n < e.length; n++) if (e[n] !== t[n]) return !1;
|
|
669
|
+
return !0;
|
|
670
|
+
}
|
|
618
671
|
if (Array.isArray(e) && Array.isArray(t)) {
|
|
619
672
|
if (e.length !== t.length) return !1;
|
|
620
|
-
for (let r = 0; r < e.length; r++) if (!
|
|
673
|
+
for (let r = 0; r < e.length; r++) if (!I(e[r], t[r], n)) return !1;
|
|
621
674
|
return !0;
|
|
622
675
|
}
|
|
623
|
-
if (typeof e != "object") return !1;
|
|
624
676
|
let r = Object.keys(e);
|
|
625
677
|
if (r.length !== Object.keys(t).length) return !1;
|
|
626
|
-
for (let i of r) if (!
|
|
678
|
+
for (let i of r) if (!I(e[i], t[i], n)) return !1;
|
|
627
679
|
return !0;
|
|
628
680
|
}
|
|
629
|
-
function
|
|
681
|
+
function Ie(e) {
|
|
630
682
|
if (!e) return [];
|
|
631
683
|
if (e.length === 1) return e[0];
|
|
632
684
|
let t = [];
|
|
633
|
-
for (let n of e)
|
|
685
|
+
for (let n of e) Fe(t, n);
|
|
634
686
|
return t;
|
|
635
687
|
}
|
|
636
688
|
//#endregion
|
|
637
689
|
//#region ../../node_modules/hyparquet/src/filter.js
|
|
638
|
-
function
|
|
690
|
+
function L({ rowGroup: e, physicalColumns: t, filter: n, strict: r = !0, bloomFilters: i, schemaElements: a }) {
|
|
639
691
|
if (!n) return !1;
|
|
640
|
-
if ("$and" in n && Array.isArray(n.$and)) return n.$and.some((n) =>
|
|
692
|
+
if ("$and" in n && Array.isArray(n.$and)) return n.$and.some((n) => L({
|
|
641
693
|
rowGroup: e,
|
|
642
694
|
physicalColumns: t,
|
|
643
695
|
filter: n,
|
|
644
|
-
strict: r
|
|
696
|
+
strict: r,
|
|
697
|
+
bloomFilters: i,
|
|
698
|
+
schemaElements: a
|
|
645
699
|
}));
|
|
646
|
-
if ("$or" in n && Array.isArray(n.$or)) return n.$or.every((n) =>
|
|
700
|
+
if ("$or" in n && Array.isArray(n.$or)) return n.$or.every((n) => L({
|
|
647
701
|
rowGroup: e,
|
|
648
702
|
physicalColumns: t,
|
|
649
703
|
filter: n,
|
|
650
|
-
strict: r
|
|
704
|
+
strict: r,
|
|
705
|
+
bloomFilters: i,
|
|
706
|
+
schemaElements: a
|
|
651
707
|
}));
|
|
652
708
|
if ("$nor" in n && Array.isArray(n.$nor)) return !1;
|
|
653
|
-
for (let [
|
|
654
|
-
let n = t.indexOf(
|
|
709
|
+
for (let [o, s] of Object.entries(n)) {
|
|
710
|
+
let n = t.indexOf(o);
|
|
655
711
|
if (n === -1) continue;
|
|
656
|
-
let
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
712
|
+
let { min: c, max: l, min_value: u, max_value: d } = e.columns[n].meta_data?.statistics || {}, f = u === void 0 ? c : u, p = d === void 0 ? l : d, m = f !== void 0 && p !== void 0, h = i?.[o], g = a?.[o];
|
|
713
|
+
for (let [e, t] of Object.entries(s || {})) {
|
|
714
|
+
if (m && (e === "$gt" && p <= t || e === "$gte" && p < t || e === "$lt" && f >= t || e === "$lte" && f > t || e === "$eq" && (t < f || t > p) || e === "$ne" && I(f, p, r) && I(f, t, r) || e === "$in" && Array.isArray(t) && t.every((e) => e < f || e > p) || e === "$nin" && Array.isArray(t) && I(f, p, r) && t.includes(f))) return !0;
|
|
715
|
+
if (h && g) {
|
|
716
|
+
if (e === "$eq") {
|
|
717
|
+
let e = Pe(t, g);
|
|
718
|
+
if (e !== void 0 && !Ne(h.blocks, e)) return !0;
|
|
719
|
+
}
|
|
720
|
+
if (e === "$in" && Array.isArray(t) && t.length > 0) {
|
|
721
|
+
let e = !0;
|
|
722
|
+
for (let n of t) {
|
|
723
|
+
let t = Pe(n, g);
|
|
724
|
+
if (t === void 0 || Ne(h.blocks, t)) {
|
|
725
|
+
e = !1;
|
|
726
|
+
break;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
if (e) return !0;
|
|
730
|
+
}
|
|
731
|
+
}
|
|
661
732
|
}
|
|
662
733
|
}
|
|
663
734
|
return !1;
|
|
664
735
|
}
|
|
665
736
|
//#endregion
|
|
666
737
|
//#region ../../node_modules/hyparquet/src/plan.js
|
|
667
|
-
var
|
|
668
|
-
function
|
|
738
|
+
var Le = 1 << 21;
|
|
739
|
+
function Re({ metadata: e, rowStart: t = 0, rowEnd: n = Infinity, columns: r, filter: i, filterStrict: a = !0, useOffsetIndex: o = !1, bloomFiltersByGroup: s, schemaElements: c }) {
|
|
669
740
|
if (!e) throw Error("parquetPlan requires metadata");
|
|
670
|
-
let
|
|
671
|
-
for (let
|
|
672
|
-
let e = Number(
|
|
673
|
-
if (e > 0 &&
|
|
674
|
-
rowGroup:
|
|
675
|
-
physicalColumns:
|
|
741
|
+
let l = [], u = [], d = [], f = ee(Se(e)), p = 0, m = 0;
|
|
742
|
+
for (let h of e.row_groups) {
|
|
743
|
+
let e = Number(h.num_rows), g = p + e, _ = s?.[m];
|
|
744
|
+
if (e > 0 && g > t && p < n && !L({
|
|
745
|
+
rowGroup: h,
|
|
746
|
+
physicalColumns: f,
|
|
676
747
|
filter: i,
|
|
677
|
-
strict: a
|
|
748
|
+
strict: a,
|
|
749
|
+
bloomFilters: _,
|
|
750
|
+
schemaElements: c
|
|
678
751
|
})) {
|
|
679
|
-
let i = [], a = Infinity,
|
|
680
|
-
for (let e of
|
|
681
|
-
let
|
|
752
|
+
let i = [], a = Infinity, s = -Infinity;
|
|
753
|
+
for (let e of h.columns) {
|
|
754
|
+
let c = e.meta_data;
|
|
682
755
|
if (e.file_path) throw Error("parquet file_path not supported");
|
|
683
|
-
if (!
|
|
684
|
-
if (!r || r.includes(
|
|
685
|
-
let
|
|
686
|
-
if (
|
|
687
|
-
let
|
|
756
|
+
if (!c) throw Error("parquet column metadata is undefined");
|
|
757
|
+
if (!r || r.includes(c.path_in_schema[0])) {
|
|
758
|
+
let r = c.dictionary_page_offset || c.data_page_offset, l = Number(r), u = Number(r + c.total_compressed_size);
|
|
759
|
+
if (l < a && (a = l), u > s && (s = u), o && e.offset_index_offset && e.offset_index_length && (t > p || n < g)) {
|
|
760
|
+
let t = Number(e.offset_index_offset);
|
|
688
761
|
i.push({
|
|
689
|
-
columnMetadata:
|
|
762
|
+
columnMetadata: c,
|
|
690
763
|
offsetIndex: {
|
|
691
|
-
startByte:
|
|
692
|
-
endByte:
|
|
764
|
+
startByte: t,
|
|
765
|
+
endByte: t + e.offset_index_length
|
|
693
766
|
},
|
|
694
|
-
|
|
695
|
-
startByte:
|
|
696
|
-
endByte:
|
|
767
|
+
range: {
|
|
768
|
+
startByte: l,
|
|
769
|
+
endByte: u
|
|
697
770
|
}
|
|
698
771
|
});
|
|
699
772
|
} else i.push({
|
|
700
|
-
columnMetadata:
|
|
773
|
+
columnMetadata: c,
|
|
701
774
|
range: {
|
|
702
|
-
startByte:
|
|
703
|
-
endByte:
|
|
775
|
+
startByte: l,
|
|
776
|
+
endByte: u
|
|
704
777
|
}
|
|
705
778
|
});
|
|
706
779
|
}
|
|
707
780
|
}
|
|
708
|
-
let
|
|
709
|
-
|
|
781
|
+
let c = Math.max(t - p, 0), f = Math.min(n - p, e);
|
|
782
|
+
l.push({
|
|
710
783
|
chunks: i,
|
|
711
|
-
rowGroup:
|
|
712
|
-
groupStart:
|
|
784
|
+
rowGroup: h,
|
|
785
|
+
groupStart: p,
|
|
713
786
|
groupRows: e,
|
|
714
|
-
selectStart:
|
|
715
|
-
selectEnd:
|
|
787
|
+
selectStart: c,
|
|
788
|
+
selectEnd: f
|
|
716
789
|
});
|
|
717
|
-
let
|
|
718
|
-
for (let e of i) if ("offsetIndex" in e)
|
|
790
|
+
let m;
|
|
791
|
+
for (let e of i) if ("offsetIndex" in e) d.push(e.offsetIndex);
|
|
719
792
|
else {
|
|
720
793
|
let { range: t } = e;
|
|
721
|
-
r ?
|
|
794
|
+
r ? u.push(t) : m && t.endByte - m.startByte <= Le ? m.endByte = t.endByte : (m && u.push(m), m = { ...t });
|
|
722
795
|
}
|
|
723
|
-
|
|
796
|
+
m && u.push(m);
|
|
724
797
|
}
|
|
725
|
-
|
|
798
|
+
p = g, m++;
|
|
726
799
|
}
|
|
727
|
-
return isFinite(n) || (n =
|
|
800
|
+
return isFinite(n) || (n = p), u.push(...d), {
|
|
728
801
|
metadata: e,
|
|
729
802
|
rowStart: t,
|
|
730
803
|
rowEnd: n,
|
|
731
804
|
columns: r,
|
|
732
|
-
fetches:
|
|
733
|
-
groups:
|
|
805
|
+
fetches: u,
|
|
806
|
+
groups: l
|
|
734
807
|
};
|
|
735
808
|
}
|
|
736
|
-
function
|
|
809
|
+
function ze(e, { fetches: t }) {
|
|
737
810
|
let n = t.map(({ startByte: t, endByte: n }) => e.slice(t, n));
|
|
738
811
|
return {
|
|
739
812
|
byteLength: e.byteLength,
|
|
@@ -749,12 +822,12 @@ function ge(e, { fetches: t }) {
|
|
|
749
822
|
}
|
|
750
823
|
//#endregion
|
|
751
824
|
//#region ../../node_modules/hyparquet/src/variant.js
|
|
752
|
-
var
|
|
753
|
-
function
|
|
754
|
-
if (Array.isArray(e)) return e.map((e) =>
|
|
825
|
+
var R = new TextDecoder(), z = /* @__PURE__ */ new WeakMap();
|
|
826
|
+
function B(e, t = p) {
|
|
827
|
+
if (Array.isArray(e)) return e.map((e) => B(e, t));
|
|
755
828
|
if (typeof e != "object") return e;
|
|
756
829
|
if ("metadata" in e) {
|
|
757
|
-
let n =
|
|
830
|
+
let n = Be(e.metadata), r = e.typed_value && V(e.typed_value, n, t), i = e.value && W(H(e.value), n, t);
|
|
758
831
|
return r && i ? {
|
|
759
832
|
...i,
|
|
760
833
|
...r
|
|
@@ -762,35 +835,37 @@ function R(e, t = p) {
|
|
|
762
835
|
}
|
|
763
836
|
return e;
|
|
764
837
|
}
|
|
765
|
-
function
|
|
838
|
+
function V(e, t, n) {
|
|
839
|
+
if (e instanceof Date) return e;
|
|
766
840
|
if (e && typeof e == "object" && !Array.isArray(e) && !(e instanceof Uint8Array)) {
|
|
767
|
-
if ("typed_value" in e) return
|
|
768
|
-
if ("value" in e && e.value instanceof Uint8Array) return H(
|
|
841
|
+
if ("typed_value" in e && e.typed_value !== null && e.typed_value !== void 0) return V(e.typed_value, t, n);
|
|
842
|
+
if ("value" in e && e.value instanceof Uint8Array) return W(H(e.value), t, n);
|
|
843
|
+
if ("typed_value" in e || "value" in e) return null;
|
|
769
844
|
let r = {};
|
|
770
|
-
for (let [i, a] of Object.entries(e)) r[i] =
|
|
845
|
+
for (let [i, a] of Object.entries(e)) t.dictionary.includes(i) && (r[i] = V(a, t, n));
|
|
771
846
|
return r;
|
|
772
847
|
}
|
|
773
|
-
return e instanceof Uint8Array ? H(
|
|
848
|
+
return e instanceof Uint8Array ? W(H(e), t, n) : Array.isArray(e) ? e.map((e) => V(e, t, n)) : e;
|
|
774
849
|
}
|
|
775
|
-
function
|
|
850
|
+
function H(e) {
|
|
776
851
|
return {
|
|
777
852
|
view: new DataView(e.buffer, e.byteOffset, e.byteLength),
|
|
778
853
|
offset: 0
|
|
779
854
|
};
|
|
780
855
|
}
|
|
781
|
-
function
|
|
782
|
-
let t =
|
|
783
|
-
t || (t = /* @__PURE__ */ new Map(),
|
|
856
|
+
function Be(e) {
|
|
857
|
+
let t = z.get(e.buffer);
|
|
858
|
+
t || (t = /* @__PURE__ */ new Map(), z.set(e.buffer, t));
|
|
784
859
|
let n = `${e.byteOffset}:${e.byteLength}`, r = t.get(n);
|
|
785
860
|
if (r) return r;
|
|
786
|
-
let i =
|
|
861
|
+
let i = H(e), a = i.view.getUint8(i.offset++), o = a & 15;
|
|
787
862
|
if (o !== 1) throw Error(`parquet unsupported variant metadata version: ${o}`);
|
|
788
|
-
let s = (a >> 4 & 1) == 1, c = (a >> 6 & 3) + 1, l =
|
|
789
|
-
for (let e = 0; e < u.length; e++) u[e] =
|
|
863
|
+
let s = (a >> 4 & 1) == 1, c = (a >> 6 & 3) + 1, l = U(i, c), u = Array(l + 1);
|
|
864
|
+
for (let e = 0; e < u.length; e++) u[e] = U(i, c);
|
|
790
865
|
let d = i.offset, f = Array(l);
|
|
791
866
|
for (let t = 0; t < l; t++) {
|
|
792
867
|
let n = u[t], r = u[t + 1], i = new Uint8Array(e.buffer, e.byteOffset + d + n, r - n);
|
|
793
|
-
f[t] =
|
|
868
|
+
f[t] = R.decode(i);
|
|
794
869
|
}
|
|
795
870
|
let p = {
|
|
796
871
|
dictionary: f,
|
|
@@ -798,20 +873,20 @@ function _e(e) {
|
|
|
798
873
|
};
|
|
799
874
|
return t.set(n, p), p;
|
|
800
875
|
}
|
|
801
|
-
function
|
|
876
|
+
function U(e, t) {
|
|
802
877
|
let n = 0;
|
|
803
878
|
for (let r = 0; r < t; r++) n |= e.view.getUint8(e.offset + r) << r * 8;
|
|
804
879
|
return e.offset += t, n;
|
|
805
880
|
}
|
|
806
|
-
function
|
|
881
|
+
function W(e, t, n) {
|
|
807
882
|
let r = e.view.getUint8(e.offset++), i = r & 3, a = r >> 2;
|
|
808
|
-
if (i === 0) return
|
|
809
|
-
if (i === 2) return
|
|
810
|
-
if (i === 3) return
|
|
883
|
+
if (i === 0) return Ve(e, a, n);
|
|
884
|
+
if (i === 2) return He(e, a, t, n);
|
|
885
|
+
if (i === 3) return Ue(e, a, t, n);
|
|
811
886
|
let o = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, a);
|
|
812
|
-
return e.offset += a,
|
|
887
|
+
return e.offset += a, R.decode(o);
|
|
813
888
|
}
|
|
814
|
-
function
|
|
889
|
+
function Ve(e, t, n) {
|
|
815
890
|
switch (t) {
|
|
816
891
|
case 0: return null;
|
|
817
892
|
case 1: return !0;
|
|
@@ -836,9 +911,9 @@ function ve(e, t, n) {
|
|
|
836
911
|
let t = e.view.getFloat64(e.offset, !0);
|
|
837
912
|
return e.offset += 8, t;
|
|
838
913
|
}
|
|
839
|
-
case 8: return
|
|
840
|
-
case 9: return
|
|
841
|
-
case 10: return
|
|
914
|
+
case 8: return G(e, 4);
|
|
915
|
+
case 9: return G(e, 8);
|
|
916
|
+
case 10: return G(e, 16);
|
|
842
917
|
case 11: {
|
|
843
918
|
let t = e.view.getInt32(e.offset, !0);
|
|
844
919
|
return e.offset += 4, n.dateFromDays(t);
|
|
@@ -852,10 +927,10 @@ function ve(e, t, n) {
|
|
|
852
927
|
let t = e.view.getFloat32(e.offset, !0);
|
|
853
928
|
return e.offset += 4, t;
|
|
854
929
|
}
|
|
855
|
-
case 15: return
|
|
930
|
+
case 15: return We(e);
|
|
856
931
|
case 16: {
|
|
857
|
-
let t =
|
|
858
|
-
return
|
|
932
|
+
let t = We(e);
|
|
933
|
+
return R.decode(t);
|
|
859
934
|
}
|
|
860
935
|
case 17: {
|
|
861
936
|
let t = e.view.getBigInt64(e.offset, !0);
|
|
@@ -875,32 +950,32 @@ function ve(e, t, n) {
|
|
|
875
950
|
default: throw Error(`parquet unsupported variant primitive type: ${t}`);
|
|
876
951
|
}
|
|
877
952
|
}
|
|
878
|
-
function
|
|
879
|
-
let i = (t & 3) + 1, a = (t >> 2 & 3) + 1, o = t >> 4 & 1 ?
|
|
880
|
-
for (let t = 0; t < o; t++) s[t] =
|
|
953
|
+
function He(e, t, n, r) {
|
|
954
|
+
let i = (t & 3) + 1, a = (t >> 2 & 3) + 1, o = t >> 4 & 1 ? U(e, 4) : e.view.getUint8(e.offset++), s = Array(o);
|
|
955
|
+
for (let t = 0; t < o; t++) s[t] = U(e, a);
|
|
881
956
|
let c = Array(o + 1);
|
|
882
|
-
for (let t = 0; t < c.length; t++) c[t] =
|
|
957
|
+
for (let t = 0; t < c.length; t++) c[t] = U(e, i);
|
|
883
958
|
let l = {};
|
|
884
959
|
for (let t = 0; t < o; t++) {
|
|
885
960
|
let i = n.dictionary[s[t]];
|
|
886
|
-
l[i] =
|
|
961
|
+
l[i] = W({
|
|
887
962
|
view: e.view,
|
|
888
963
|
offset: e.offset + c[t]
|
|
889
964
|
}, n, r);
|
|
890
965
|
}
|
|
891
966
|
return e.offset += c[c.length - 1], l;
|
|
892
967
|
}
|
|
893
|
-
function
|
|
894
|
-
let i = t & 3, a = t >> 2 & 1, o = i + 1, s =
|
|
895
|
-
for (let t = 0; t < c.length; t++) c[t] =
|
|
968
|
+
function Ue(e, t, n, r) {
|
|
969
|
+
let i = t & 3, a = t >> 2 & 1, o = i + 1, s = U(e, a ? 4 : 1), c = Array(s + 1);
|
|
970
|
+
for (let t = 0; t < c.length; t++) c[t] = U(e, o);
|
|
896
971
|
let l = e.offset, u = Array(s);
|
|
897
|
-
for (let t = 0; t < s; t++) u[t] =
|
|
972
|
+
for (let t = 0; t < s; t++) u[t] = W({
|
|
898
973
|
view: e.view,
|
|
899
974
|
offset: l + c[t]
|
|
900
975
|
}, n, r);
|
|
901
976
|
return e.offset = l + c[c.length - 1], u;
|
|
902
977
|
}
|
|
903
|
-
function
|
|
978
|
+
function G(e, t) {
|
|
904
979
|
let n = e.view.getUint8(e.offset);
|
|
905
980
|
e.offset += 1;
|
|
906
981
|
let r;
|
|
@@ -912,7 +987,7 @@ function U(e, t) {
|
|
|
912
987
|
}
|
|
913
988
|
return Number(r) * 10 ** -n;
|
|
914
989
|
}
|
|
915
|
-
function
|
|
990
|
+
function We(e) {
|
|
916
991
|
let t = e.view.getUint32(e.offset, !0);
|
|
917
992
|
e.offset += 4;
|
|
918
993
|
let n = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
@@ -920,8 +995,8 @@ function W(e) {
|
|
|
920
995
|
}
|
|
921
996
|
//#endregion
|
|
922
997
|
//#region ../../node_modules/hyparquet/src/assemble.js
|
|
923
|
-
function
|
|
924
|
-
let a =
|
|
998
|
+
function Ge(e, t, n, r, i) {
|
|
999
|
+
let a = x(i);
|
|
925
1000
|
if (!t?.length && !n.length) {
|
|
926
1001
|
if (!a || !r.length) return r;
|
|
927
1002
|
t = Array(r.length).fill(a);
|
|
@@ -948,7 +1023,7 @@ function G(e, t, n, r, i) {
|
|
|
948
1023
|
}
|
|
949
1024
|
function K(e, t, n, r = 0) {
|
|
950
1025
|
let i = t.path.join("."), a = t.element.repetition_type === "OPTIONAL", o = a ? r + 1 : r;
|
|
951
|
-
if (
|
|
1026
|
+
if (ne(t)) {
|
|
952
1027
|
let s = t.children[0], c = o;
|
|
953
1028
|
s.children.length === 1 && (s = s.children[0], c++), K(e, s, n, c);
|
|
954
1029
|
let l = s.path.join("."), u = e.get(l);
|
|
@@ -956,14 +1031,14 @@ function K(e, t, n, r = 0) {
|
|
|
956
1031
|
a && q(u, r), e.set(i, u), e.delete(l);
|
|
957
1032
|
return;
|
|
958
1033
|
}
|
|
959
|
-
if (
|
|
1034
|
+
if (re(t)) {
|
|
960
1035
|
let s = t.children[0].element.name;
|
|
961
1036
|
K(e, t.children[0].children[0], n, o + 1), K(e, t.children[0].children[1], n, o + 1);
|
|
962
1037
|
let c = e.get(`${i}.${s}.key`), l = e.get(`${i}.${s}.value`);
|
|
963
1038
|
if (!c) throw Error("parquet map column missing keys");
|
|
964
1039
|
if (!l) throw Error("parquet map column missing values");
|
|
965
1040
|
if (c.length !== l.length) throw Error("parquet map column key/value length mismatch");
|
|
966
|
-
let u =
|
|
1041
|
+
let u = Ke(c, l, o);
|
|
967
1042
|
a && q(u, r), e.delete(`${i}.${s}.key`), e.delete(`${i}.${s}.value`), e.set(i, u);
|
|
968
1043
|
return;
|
|
969
1044
|
}
|
|
@@ -976,16 +1051,16 @@ function K(e, t, n, r = 0) {
|
|
|
976
1051
|
s[r.element.name] = t;
|
|
977
1052
|
}
|
|
978
1053
|
for (let n of t.children) e.delete(n.path.join("."));
|
|
979
|
-
let c =
|
|
980
|
-
t.element.logical_type?.type === "VARIANT" && (c =
|
|
1054
|
+
let c = qe(s, o);
|
|
1055
|
+
t.element.logical_type?.type === "VARIANT" && (c = B(c, n)), a && q(c, r), e.set(i, c);
|
|
981
1056
|
}
|
|
982
1057
|
}
|
|
983
1058
|
function q(e, t) {
|
|
984
1059
|
for (let n = 0; n < e.length; n++) t ? q(e[n], t - 1) : e[n] = e[n][0];
|
|
985
1060
|
}
|
|
986
|
-
function
|
|
1061
|
+
function Ke(e, t, n) {
|
|
987
1062
|
let r = [];
|
|
988
|
-
for (let i = 0; i < e.length; i++) if (n) r.push(
|
|
1063
|
+
for (let i = 0; i < e.length; i++) if (n) r.push(Ke(e[i], t[i], n - 1));
|
|
989
1064
|
else if (e[i]) {
|
|
990
1065
|
let n = {};
|
|
991
1066
|
for (let r = 0; r < e[i].length; r++) {
|
|
@@ -996,7 +1071,7 @@ function xe(e, t, n) {
|
|
|
996
1071
|
} else r.push(void 0);
|
|
997
1072
|
return r;
|
|
998
1073
|
}
|
|
999
|
-
function
|
|
1074
|
+
function qe(e, t) {
|
|
1000
1075
|
let n = Object.keys(e), r = e[n[0]]?.length, i = [];
|
|
1001
1076
|
for (let a = 0; a < r; a++) {
|
|
1002
1077
|
let o = {};
|
|
@@ -1004,20 +1079,20 @@ function Se(e, t) {
|
|
|
1004
1079
|
if (e[t].length !== r) throw Error("parquet struct parsing error");
|
|
1005
1080
|
o[t] = e[t][a];
|
|
1006
1081
|
}
|
|
1007
|
-
t ? i.push(
|
|
1082
|
+
t ? i.push(qe(o, t - 1)) : i.push(o);
|
|
1008
1083
|
}
|
|
1009
1084
|
return i;
|
|
1010
1085
|
}
|
|
1011
1086
|
//#endregion
|
|
1012
1087
|
//#region ../../node_modules/hyparquet/src/delta.js
|
|
1013
1088
|
function J(e, t, n) {
|
|
1014
|
-
let r = n instanceof Int32Array, i =
|
|
1015
|
-
|
|
1016
|
-
let o =
|
|
1089
|
+
let r = n instanceof Int32Array, i = T(e), a = T(e);
|
|
1090
|
+
T(e);
|
|
1091
|
+
let o = E(e), s = 0;
|
|
1017
1092
|
n[s++] = r ? Number(o) : o;
|
|
1018
1093
|
let c = i / a;
|
|
1019
1094
|
for (; s < t;) {
|
|
1020
|
-
let i =
|
|
1095
|
+
let i = E(e), l = new Uint8Array(a);
|
|
1021
1096
|
for (let t = 0; t < a; t++) l[t] = e.view.getUint8(e.offset++);
|
|
1022
1097
|
for (let u = 0; u < a && s < t; u++) {
|
|
1023
1098
|
let a = BigInt(l[u]);
|
|
@@ -1034,12 +1109,12 @@ function J(e, t, n) {
|
|
|
1034
1109
|
}
|
|
1035
1110
|
}
|
|
1036
1111
|
}
|
|
1037
|
-
function
|
|
1112
|
+
function Je(e, t, n) {
|
|
1038
1113
|
let r = new Int32Array(t);
|
|
1039
1114
|
J(e, t, r);
|
|
1040
1115
|
for (let i = 0; i < t; i++) n[i] = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, r[i]), e.offset += r[i];
|
|
1041
1116
|
}
|
|
1042
|
-
function
|
|
1117
|
+
function Ye(e, t, n) {
|
|
1043
1118
|
let r = new Int32Array(t);
|
|
1044
1119
|
J(e, t, r);
|
|
1045
1120
|
let i = new Int32Array(t);
|
|
@@ -1051,28 +1126,25 @@ function we(e, t, n) {
|
|
|
1051
1126
|
}
|
|
1052
1127
|
//#endregion
|
|
1053
1128
|
//#region ../../node_modules/hyparquet/src/encoding.js
|
|
1054
|
-
function Y(e) {
|
|
1055
|
-
return 32 - Math.clz32(e);
|
|
1056
|
-
}
|
|
1057
|
-
function X(e, t, n, r) {
|
|
1129
|
+
function Y(e, t, n, r) {
|
|
1058
1130
|
r === void 0 && (r = e.view.getUint32(e.offset, !0), e.offset += 4);
|
|
1059
1131
|
let i = e.offset, a = 0;
|
|
1060
1132
|
for (; a < n.length;) {
|
|
1061
|
-
let r =
|
|
1062
|
-
if (r & 1) a =
|
|
1133
|
+
let r = T(e);
|
|
1134
|
+
if (r & 1) a = Ze(e, r, t, n, a);
|
|
1063
1135
|
else {
|
|
1064
1136
|
let i = r >>> 1;
|
|
1065
|
-
|
|
1137
|
+
Xe(e, i, t, n, a), a += i;
|
|
1066
1138
|
}
|
|
1067
1139
|
}
|
|
1068
1140
|
e.offset = i + r;
|
|
1069
1141
|
}
|
|
1070
|
-
function
|
|
1142
|
+
function Xe(e, t, n, r, i) {
|
|
1071
1143
|
let a = n + 7 >> 3, o = 0;
|
|
1072
1144
|
for (let t = 0; t < a; t++) o |= e.view.getUint8(e.offset++) << (t << 3);
|
|
1073
1145
|
for (let e = 0; e < t; e++) r[i + e] = o;
|
|
1074
1146
|
}
|
|
1075
|
-
function
|
|
1147
|
+
function Ze(e, t, n, r, i) {
|
|
1076
1148
|
let a = t >> 1 << 3, o = (1 << n) - 1, s = 0;
|
|
1077
1149
|
if (e.offset < e.view.byteLength) s = e.view.getUint8(e.offset++);
|
|
1078
1150
|
else if (o) throw Error(`parquet bitpack offset ${e.offset} out of range`);
|
|
@@ -1080,8 +1152,8 @@ function Ee(e, t, n, r, i) {
|
|
|
1080
1152
|
for (; a;) l > 8 ? (l -= 8, c -= 8, s >>>= 8) : c - l < n ? (s |= e.view.getUint8(e.offset) << c, e.offset++, c += 8) : (i < r.length && (r[i++] = s >> l & o), a--, l += n);
|
|
1081
1153
|
return i;
|
|
1082
1154
|
}
|
|
1083
|
-
function
|
|
1084
|
-
let i =
|
|
1155
|
+
function Qe(e, t, n, r) {
|
|
1156
|
+
let i = $e(n, r), a = new Uint8Array(t * i);
|
|
1085
1157
|
for (let n = 0; n < i; n++) for (let r = 0; r < t; r++) a[r * i + n] = e.view.getUint8(e.offset++);
|
|
1086
1158
|
if (n === "FLOAT") return new Float32Array(a.buffer);
|
|
1087
1159
|
if (n === "DOUBLE") return new Float64Array(a.buffer);
|
|
@@ -1094,7 +1166,7 @@ function De(e, t, n, r) {
|
|
|
1094
1166
|
}
|
|
1095
1167
|
throw Error(`parquet byte_stream_split unsupported type: ${n}`);
|
|
1096
1168
|
}
|
|
1097
|
-
function
|
|
1169
|
+
function $e(e, t) {
|
|
1098
1170
|
switch (e) {
|
|
1099
1171
|
case "INT32":
|
|
1100
1172
|
case "FLOAT": return 4;
|
|
@@ -1108,21 +1180,21 @@ function Oe(e, t) {
|
|
|
1108
1180
|
}
|
|
1109
1181
|
//#endregion
|
|
1110
1182
|
//#region ../../node_modules/hyparquet/src/plain.js
|
|
1111
|
-
function
|
|
1183
|
+
function X(e, t, n, r) {
|
|
1112
1184
|
if (n === 0) return [];
|
|
1113
|
-
if (t === "BOOLEAN") return
|
|
1114
|
-
if (t === "INT32") return
|
|
1115
|
-
if (t === "INT64") return
|
|
1116
|
-
if (t === "INT96") return
|
|
1117
|
-
if (t === "FLOAT") return
|
|
1118
|
-
if (t === "DOUBLE") return
|
|
1119
|
-
if (t === "BYTE_ARRAY") return
|
|
1185
|
+
if (t === "BOOLEAN") return et(e, n);
|
|
1186
|
+
if (t === "INT32") return tt(e, n);
|
|
1187
|
+
if (t === "INT64") return nt(e, n);
|
|
1188
|
+
if (t === "INT96") return rt(e, n);
|
|
1189
|
+
if (t === "FLOAT") return it(e, n);
|
|
1190
|
+
if (t === "DOUBLE") return at(e, n);
|
|
1191
|
+
if (t === "BYTE_ARRAY") return ot(e, n);
|
|
1120
1192
|
if (t === "FIXED_LEN_BYTE_ARRAY") {
|
|
1121
1193
|
if (!r) throw Error("parquet missing fixed length");
|
|
1122
|
-
return
|
|
1194
|
+
return st(e, n, r);
|
|
1123
1195
|
} else throw Error(`parquet unhandled type: ${t}`);
|
|
1124
1196
|
}
|
|
1125
|
-
function
|
|
1197
|
+
function et(e, t) {
|
|
1126
1198
|
let n = Array(t);
|
|
1127
1199
|
for (let r = 0; r < t; r++) {
|
|
1128
1200
|
let t = e.offset + (r / 8 | 0), i = r % 8;
|
|
@@ -1130,15 +1202,15 @@ function ke(e, t) {
|
|
|
1130
1202
|
}
|
|
1131
1203
|
return e.offset += Math.ceil(t / 8), n;
|
|
1132
1204
|
}
|
|
1133
|
-
function
|
|
1134
|
-
let n = (e.view.byteOffset + e.offset) % 4 ? new Int32Array(
|
|
1205
|
+
function tt(e, t) {
|
|
1206
|
+
let n = (e.view.byteOffset + e.offset) % 4 ? new Int32Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 4)) : new Int32Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1135
1207
|
return e.offset += t * 4, n;
|
|
1136
1208
|
}
|
|
1137
|
-
function
|
|
1138
|
-
let n = (e.view.byteOffset + e.offset) % 8 ? new BigInt64Array(
|
|
1209
|
+
function nt(e, t) {
|
|
1210
|
+
let n = (e.view.byteOffset + e.offset) % 8 ? new BigInt64Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 8)) : new BigInt64Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1139
1211
|
return e.offset += t * 8, n;
|
|
1140
1212
|
}
|
|
1141
|
-
function
|
|
1213
|
+
function rt(e, t) {
|
|
1142
1214
|
let n = Array(t);
|
|
1143
1215
|
for (let r = 0; r < t; r++) {
|
|
1144
1216
|
let t = e.view.getBigInt64(e.offset + r * 12, !0), i = e.view.getInt32(e.offset + r * 12 + 8, !0);
|
|
@@ -1146,15 +1218,15 @@ function Me(e, t) {
|
|
|
1146
1218
|
}
|
|
1147
1219
|
return e.offset += t * 12, n;
|
|
1148
1220
|
}
|
|
1149
|
-
function
|
|
1150
|
-
let n = (e.view.byteOffset + e.offset) % 4 ? new Float32Array(
|
|
1221
|
+
function it(e, t) {
|
|
1222
|
+
let n = (e.view.byteOffset + e.offset) % 4 ? new Float32Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 4)) : new Float32Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1151
1223
|
return e.offset += t * 4, n;
|
|
1152
1224
|
}
|
|
1153
|
-
function
|
|
1154
|
-
let n = (e.view.byteOffset + e.offset) % 8 ? new Float64Array(
|
|
1225
|
+
function at(e, t) {
|
|
1226
|
+
let n = (e.view.byteOffset + e.offset) % 8 ? new Float64Array(Z(e.view.buffer, e.view.byteOffset + e.offset, t * 8)) : new Float64Array(e.view.buffer, e.view.byteOffset + e.offset, t);
|
|
1155
1227
|
return e.offset += t * 8, n;
|
|
1156
1228
|
}
|
|
1157
|
-
function
|
|
1229
|
+
function ot(e, t) {
|
|
1158
1230
|
let n = Array(t);
|
|
1159
1231
|
for (let r = 0; r < t; r++) {
|
|
1160
1232
|
let t = e.view.getUint32(e.offset, !0);
|
|
@@ -1162,28 +1234,28 @@ function Fe(e, t) {
|
|
|
1162
1234
|
}
|
|
1163
1235
|
return n;
|
|
1164
1236
|
}
|
|
1165
|
-
function
|
|
1237
|
+
function st(e, t, n) {
|
|
1166
1238
|
let r = Array(t);
|
|
1167
1239
|
for (let i = 0; i < t; i++) r[i] = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, n), e.offset += n;
|
|
1168
1240
|
return r;
|
|
1169
1241
|
}
|
|
1170
|
-
function
|
|
1242
|
+
function Z(e, t, n) {
|
|
1171
1243
|
let r = new ArrayBuffer(n);
|
|
1172
1244
|
return new Uint8Array(r).set(new Uint8Array(e, t, n)), r;
|
|
1173
1245
|
}
|
|
1174
1246
|
//#endregion
|
|
1175
1247
|
//#region ../../node_modules/hyparquet/src/snappy.js
|
|
1176
|
-
var
|
|
1248
|
+
var ct = [
|
|
1177
1249
|
0,
|
|
1178
1250
|
255,
|
|
1179
1251
|
65535,
|
|
1180
1252
|
16777215,
|
|
1181
1253
|
4294967295
|
|
1182
1254
|
];
|
|
1183
|
-
function
|
|
1255
|
+
function lt(e, t, n, r, i) {
|
|
1184
1256
|
for (let a = 0; a < i; a++) n[r + a] = e[t + a];
|
|
1185
1257
|
}
|
|
1186
|
-
function
|
|
1258
|
+
function ut(e, t) {
|
|
1187
1259
|
let n = e.byteLength, r = t.byteLength, i = 0, a = 0;
|
|
1188
1260
|
for (; i < n;) {
|
|
1189
1261
|
let t = e[i];
|
|
@@ -1211,34 +1283,34 @@ function ze(e, t) {
|
|
|
1211
1283
|
}
|
|
1212
1284
|
if (s === 0 || isNaN(s)) throw Error(`invalid offset ${s} pos ${i} inputLength ${n}`);
|
|
1213
1285
|
if (s > a) throw Error("cannot copy from before start of buffer");
|
|
1214
|
-
|
|
1286
|
+
lt(t, a - s, t, a, o), a += o;
|
|
1215
1287
|
} else {
|
|
1216
1288
|
let o = (r >>> 2) + 1;
|
|
1217
1289
|
if (o > 60) {
|
|
1218
1290
|
if (i + 3 >= n) throw Error("snappy error literal pos + 3 >= inputLength");
|
|
1219
1291
|
let t = o - 60;
|
|
1220
|
-
o = e[i] + (e[i + 1] << 8) + (e[i + 2] << 16) + (e[i + 3] << 24), o = (o &
|
|
1292
|
+
o = e[i] + (e[i + 1] << 8) + (e[i + 2] << 16) + (e[i + 3] << 24), o = (o & ct[t]) + 1, i += t;
|
|
1221
1293
|
}
|
|
1222
1294
|
if (i + o > n) throw Error("snappy error literal exceeds input length");
|
|
1223
|
-
|
|
1295
|
+
lt(e, i, t, a, o), i += o, a += o;
|
|
1224
1296
|
}
|
|
1225
1297
|
}
|
|
1226
1298
|
if (a !== r) throw Error("premature end of input");
|
|
1227
1299
|
}
|
|
1228
1300
|
//#endregion
|
|
1229
1301
|
//#region ../../node_modules/hyparquet/src/datapage.js
|
|
1230
|
-
function
|
|
1302
|
+
function dt(e, t, { type: n, element: r, schemaPath: i }) {
|
|
1231
1303
|
let a = new DataView(e.buffer, e.byteOffset, e.byteLength), o = {
|
|
1232
1304
|
view: a,
|
|
1233
1305
|
offset: 0
|
|
1234
|
-
}, s, c =
|
|
1235
|
-
if (t.encoding === "PLAIN") s =
|
|
1306
|
+
}, s, c = ft(o, t, i), { definitionLevels: l, numNulls: u } = pt(o, t, i), d = t.num_values - u;
|
|
1307
|
+
if (t.encoding === "PLAIN") s = X(o, n, d, r.type_length);
|
|
1236
1308
|
else if (t.encoding === "PLAIN_DICTIONARY" || t.encoding === "RLE_DICTIONARY" || t.encoding === "RLE") {
|
|
1237
1309
|
let e = n === "BOOLEAN" ? 1 : a.getUint8(o.offset++);
|
|
1238
|
-
e ? (s = Array(d), n === "BOOLEAN" ? (
|
|
1239
|
-
} else if (t.encoding === "BYTE_STREAM_SPLIT") s =
|
|
1310
|
+
e ? (s = Array(d), n === "BOOLEAN" ? (Y(o, e, s), s = s.map((e) => !!e)) : Y(o, e, s, a.byteLength - o.offset)) : s = new Uint8Array(d);
|
|
1311
|
+
} else if (t.encoding === "BYTE_STREAM_SPLIT") s = Qe(o, d, n, r.type_length);
|
|
1240
1312
|
else if (t.encoding === "DELTA_BINARY_PACKED") s = n === "INT32" ? new Int32Array(d) : new BigInt64Array(d), J(o, d, s);
|
|
1241
|
-
else if (t.encoding === "DELTA_LENGTH_BYTE_ARRAY") s = Array(d),
|
|
1313
|
+
else if (t.encoding === "DELTA_LENGTH_BYTE_ARRAY") s = Array(d), Je(o, d, s);
|
|
1242
1314
|
else throw Error(`parquet unsupported encoding: ${t.encoding}`);
|
|
1243
1315
|
return {
|
|
1244
1316
|
definitionLevels: l,
|
|
@@ -1246,24 +1318,24 @@ function Be(e, t, { type: n, element: r, schemaPath: i }) {
|
|
|
1246
1318
|
dataPage: s
|
|
1247
1319
|
};
|
|
1248
1320
|
}
|
|
1249
|
-
function
|
|
1321
|
+
function ft(e, t, n) {
|
|
1250
1322
|
if (n.length > 1) {
|
|
1251
|
-
let r =
|
|
1323
|
+
let r = te(n);
|
|
1252
1324
|
if (r) {
|
|
1253
1325
|
let n = Array(t.num_values);
|
|
1254
|
-
return
|
|
1326
|
+
return Y(e, $(r), n), n;
|
|
1255
1327
|
}
|
|
1256
1328
|
}
|
|
1257
1329
|
return [];
|
|
1258
1330
|
}
|
|
1259
|
-
function
|
|
1260
|
-
let r =
|
|
1331
|
+
function pt(e, t, n) {
|
|
1332
|
+
let r = x(n);
|
|
1261
1333
|
if (!r) return {
|
|
1262
1334
|
definitionLevels: [],
|
|
1263
1335
|
numNulls: 0
|
|
1264
1336
|
};
|
|
1265
1337
|
let i = Array(t.num_values);
|
|
1266
|
-
|
|
1338
|
+
Y(e, $(r), i);
|
|
1267
1339
|
let a = t.num_values;
|
|
1268
1340
|
for (let e of i) e === r && a--;
|
|
1269
1341
|
return a === 0 && (i.length = 0), {
|
|
@@ -1271,38 +1343,38 @@ function He(e, t, n) {
|
|
|
1271
1343
|
numNulls: a
|
|
1272
1344
|
};
|
|
1273
1345
|
}
|
|
1274
|
-
function
|
|
1346
|
+
function Q(e, t, n, r) {
|
|
1275
1347
|
let i, a = r?.[n];
|
|
1276
1348
|
if (n === "UNCOMPRESSED") i = e;
|
|
1277
1349
|
else if (a) i = a(e, t);
|
|
1278
|
-
else if (n === "SNAPPY") i = new Uint8Array(t),
|
|
1350
|
+
else if (n === "SNAPPY") i = new Uint8Array(t), ut(e, i);
|
|
1279
1351
|
else throw Error(`parquet unsupported compression codec: ${n}`);
|
|
1280
1352
|
if (i?.length !== t) throw Error(`parquet decompressed page length ${i?.length} does not match header ${t}`);
|
|
1281
1353
|
return i;
|
|
1282
1354
|
}
|
|
1283
|
-
function
|
|
1355
|
+
function mt(e, t, n) {
|
|
1284
1356
|
let r = {
|
|
1285
1357
|
view: new DataView(e.buffer, e.byteOffset, e.byteLength),
|
|
1286
1358
|
offset: 0
|
|
1287
1359
|
}, { type: i, element: a, schemaPath: o, codec: s, compressors: c } = n, l = t.data_page_header_v2;
|
|
1288
1360
|
if (!l) throw Error("parquet data page header v2 is undefined");
|
|
1289
|
-
let u =
|
|
1361
|
+
let u = ht(r, l, o);
|
|
1290
1362
|
r.offset = l.repetition_levels_byte_length;
|
|
1291
|
-
let d =
|
|
1292
|
-
l.is_compressed !== !1 && (p =
|
|
1363
|
+
let d = gt(r, l, o), f = t.uncompressed_page_size - l.definition_levels_byte_length - l.repetition_levels_byte_length, p = e.subarray(r.offset);
|
|
1364
|
+
l.is_compressed !== !1 && (p = Q(p, f, s, c));
|
|
1293
1365
|
let m = new DataView(p.buffer, p.byteOffset, p.byteLength), h = {
|
|
1294
1366
|
view: m,
|
|
1295
1367
|
offset: 0
|
|
1296
1368
|
}, g, _ = l.num_values - l.num_nulls;
|
|
1297
|
-
if (l.encoding === "PLAIN") g =
|
|
1298
|
-
else if (l.encoding === "RLE") g = Array(_),
|
|
1369
|
+
if (l.encoding === "PLAIN") g = X(h, i, _, a.type_length);
|
|
1370
|
+
else if (l.encoding === "RLE") g = Array(_), Y(h, 1, g), g = g.map((e) => !!e);
|
|
1299
1371
|
else if (l.encoding === "PLAIN_DICTIONARY" || l.encoding === "RLE_DICTIONARY") {
|
|
1300
1372
|
let e = m.getUint8(h.offset++);
|
|
1301
|
-
g = Array(_),
|
|
1373
|
+
g = Array(_), Y(h, e, g, f - 1);
|
|
1302
1374
|
} else if (l.encoding === "DELTA_BINARY_PACKED") g = i === "INT32" ? new Int32Array(_) : new BigInt64Array(_), J(h, _, g);
|
|
1303
|
-
else if (l.encoding === "DELTA_LENGTH_BYTE_ARRAY") g = Array(_),
|
|
1304
|
-
else if (l.encoding === "DELTA_BYTE_ARRAY") g = Array(_),
|
|
1305
|
-
else if (l.encoding === "BYTE_STREAM_SPLIT") g =
|
|
1375
|
+
else if (l.encoding === "DELTA_LENGTH_BYTE_ARRAY") g = Array(_), Je(h, _, g);
|
|
1376
|
+
else if (l.encoding === "DELTA_BYTE_ARRAY") g = Array(_), Ye(h, _, g);
|
|
1377
|
+
else if (l.encoding === "BYTE_STREAM_SPLIT") g = Qe(h, _, i, a.type_length);
|
|
1306
1378
|
else throw Error(`parquet unsupported encoding: ${l.encoding}`);
|
|
1307
1379
|
return {
|
|
1308
1380
|
definitionLevels: d,
|
|
@@ -1310,23 +1382,26 @@ function Ue(e, t, n) {
|
|
|
1310
1382
|
dataPage: g
|
|
1311
1383
|
};
|
|
1312
1384
|
}
|
|
1313
|
-
function
|
|
1314
|
-
let r =
|
|
1385
|
+
function ht(e, t, n) {
|
|
1386
|
+
let r = te(n);
|
|
1315
1387
|
if (!r) return [];
|
|
1316
1388
|
let i = Array(t.num_values);
|
|
1317
|
-
return
|
|
1389
|
+
return Y(e, $(r), i, t.repetition_levels_byte_length), i;
|
|
1318
1390
|
}
|
|
1319
|
-
function
|
|
1320
|
-
let r =
|
|
1391
|
+
function gt(e, t, n) {
|
|
1392
|
+
let r = x(n);
|
|
1321
1393
|
if (r) {
|
|
1322
1394
|
let n = Array(t.num_values);
|
|
1323
|
-
return
|
|
1395
|
+
return Y(e, $(r), n, t.definition_levels_byte_length), n;
|
|
1324
1396
|
}
|
|
1325
1397
|
}
|
|
1398
|
+
function $(e) {
|
|
1399
|
+
return 32 - Math.clz32(e);
|
|
1400
|
+
}
|
|
1326
1401
|
//#endregion
|
|
1327
1402
|
//#region ../../node_modules/hyparquet/src/column.js
|
|
1328
|
-
function
|
|
1329
|
-
let { pathInSchema: o, schemaPath: s } = i, c =
|
|
1403
|
+
function _t(e, { groupStart: t, selectStart: n, selectEnd: r }, i, a) {
|
|
1404
|
+
let { pathInSchema: o, schemaPath: s } = i, c = S(s), l = [], u, d, f = 0, p = 0, m = a && (() => {
|
|
1330
1405
|
d && a({
|
|
1331
1406
|
pathInSchema: o,
|
|
1332
1407
|
columnData: d,
|
|
@@ -1335,12 +1410,12 @@ function Ke(e, { groupStart: t, selectStart: n, selectEnd: r }, i, a) {
|
|
|
1335
1410
|
});
|
|
1336
1411
|
});
|
|
1337
1412
|
for (; (c ? f < r : e.offset < e.view.byteLength - 1) && !(e.offset >= e.view.byteLength - 1);) {
|
|
1338
|
-
let t =
|
|
1413
|
+
let t = yt(e);
|
|
1339
1414
|
if (t.type === "DICTIONARY_PAGE") {
|
|
1340
|
-
let { data: n } =
|
|
1415
|
+
let { data: n } = vt(e, t, i, u, void 0, 0);
|
|
1341
1416
|
n && (u = h(n, i));
|
|
1342
1417
|
} else {
|
|
1343
|
-
let r = d?.length || 0, a =
|
|
1418
|
+
let r = d?.length || 0, a = vt(e, t, i, u, d, n - f);
|
|
1344
1419
|
a.skipped ? (l.length || (p += a.skipped), f += a.skipped) : a.data && d === a.data ? f += a.data.length - r : a.data && a.data.length && (m?.(), l.push(a.data), f += a.data.length, d = a.data);
|
|
1345
1420
|
}
|
|
1346
1421
|
}
|
|
@@ -1349,41 +1424,41 @@ function Ke(e, { groupStart: t, selectStart: n, selectEnd: r }, i, a) {
|
|
|
1349
1424
|
skipped: p
|
|
1350
1425
|
};
|
|
1351
1426
|
}
|
|
1352
|
-
function
|
|
1427
|
+
function vt(e, t, n, r, i, a) {
|
|
1353
1428
|
let { type: o, element: s, schemaPath: c, codec: l, compressors: u } = n, d = new Uint8Array(e.view.buffer, e.view.byteOffset + e.offset, t.compressed_page_size);
|
|
1354
1429
|
if (e.offset += t.compressed_page_size, t.type === "DATA_PAGE") {
|
|
1355
1430
|
let e = t.data_page_header;
|
|
1356
1431
|
if (!e) throw Error("parquet data page header is undefined");
|
|
1357
|
-
if (a > e.num_values &&
|
|
1358
|
-
let { definitionLevels: o, repetitionLevels: s, dataPage: f } =
|
|
1432
|
+
if (a > e.num_values && S(c)) return { skipped: e.num_values };
|
|
1433
|
+
let { definitionLevels: o, repetitionLevels: s, dataPage: f } = dt(Q(d, Number(t.uncompressed_page_size), l, u), e, n), p = m(f, r, e.encoding, n);
|
|
1359
1434
|
return {
|
|
1360
1435
|
skipped: 0,
|
|
1361
|
-
data:
|
|
1436
|
+
data: Ge(Array.isArray(i) ? i : [], o, s, p, c)
|
|
1362
1437
|
};
|
|
1363
1438
|
} else if (t.type === "DATA_PAGE_V2") {
|
|
1364
1439
|
let e = t.data_page_header_v2;
|
|
1365
1440
|
if (!e) throw Error("parquet data page header v2 is undefined");
|
|
1366
1441
|
if (a > e.num_rows) return { skipped: e.num_values };
|
|
1367
|
-
let { definitionLevels: o, repetitionLevels: s, dataPage: l } =
|
|
1442
|
+
let { definitionLevels: o, repetitionLevels: s, dataPage: l } = mt(d, t, n), u = m(l, r, e.encoding, n);
|
|
1368
1443
|
return {
|
|
1369
1444
|
skipped: 0,
|
|
1370
|
-
data:
|
|
1445
|
+
data: Ge(Array.isArray(i) ? i : [], o, s, u, c)
|
|
1371
1446
|
};
|
|
1372
1447
|
} else if (t.type === "DICTIONARY_PAGE") {
|
|
1373
1448
|
let e = t.dictionary_page_header;
|
|
1374
1449
|
if (!e) throw Error("parquet dictionary page header is undefined");
|
|
1375
|
-
let n =
|
|
1450
|
+
let n = Q(d, Number(t.uncompressed_page_size), l, u);
|
|
1376
1451
|
return {
|
|
1377
1452
|
skipped: 0,
|
|
1378
|
-
data:
|
|
1453
|
+
data: X({
|
|
1379
1454
|
view: new DataView(n.buffer, n.byteOffset, n.byteLength),
|
|
1380
1455
|
offset: 0
|
|
1381
1456
|
}, o, e.num_values, s.type_length)
|
|
1382
1457
|
};
|
|
1383
1458
|
} else throw Error(`parquet unsupported page type: ${t.type}`);
|
|
1384
1459
|
}
|
|
1385
|
-
function
|
|
1386
|
-
let n =
|
|
1460
|
+
function yt(e) {
|
|
1461
|
+
let n = C(e);
|
|
1387
1462
|
return {
|
|
1388
1463
|
type: a[n.field_1],
|
|
1389
1464
|
uncompressed_page_size: n.field_2,
|
|
@@ -1423,71 +1498,67 @@ function Je(e) {
|
|
|
1423
1498
|
}
|
|
1424
1499
|
//#endregion
|
|
1425
1500
|
//#region ../../node_modules/hyparquet/src/indexes.js
|
|
1426
|
-
function
|
|
1427
|
-
let t =
|
|
1501
|
+
function bt(e) {
|
|
1502
|
+
let t = C(e);
|
|
1428
1503
|
return {
|
|
1429
|
-
page_locations: t.field_1.map(
|
|
1504
|
+
page_locations: t.field_1.map((e) => ({
|
|
1505
|
+
offset: e.field_1,
|
|
1506
|
+
compressed_page_size: e.field_2,
|
|
1507
|
+
first_row_index: e.field_3
|
|
1508
|
+
})),
|
|
1430
1509
|
unencoded_byte_array_data_bytes: t.field_2
|
|
1431
1510
|
};
|
|
1432
1511
|
}
|
|
1433
|
-
function Xe(e) {
|
|
1434
|
-
return {
|
|
1435
|
-
offset: e.field_1,
|
|
1436
|
-
compressed_page_size: e.field_2,
|
|
1437
|
-
first_row_index: e.field_3
|
|
1438
|
-
};
|
|
1439
|
-
}
|
|
1440
1512
|
//#endregion
|
|
1441
1513
|
//#region ../../node_modules/hyparquet/src/rowgroup.js
|
|
1442
|
-
function
|
|
1443
|
-
let
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
pathInSchema: l.path_in_schema,
|
|
1461
|
-
data: Promise.resolve(r.slice(c.range.startByte, c.range.endByte)).then((t) => Ke({
|
|
1514
|
+
function xt(e, { metadata: t }, n) {
|
|
1515
|
+
let r = [];
|
|
1516
|
+
for (let i of n.chunks) {
|
|
1517
|
+
let { data_page_offset: a, dictionary_page_offset: o, path_in_schema: s } = i.columnMetadata, c = b(t.schema, s), l = {
|
|
1518
|
+
pathInSchema: s,
|
|
1519
|
+
element: c[c.length - 1].element,
|
|
1520
|
+
schemaPath: c,
|
|
1521
|
+
parsers: {
|
|
1522
|
+
...p,
|
|
1523
|
+
...e.parsers
|
|
1524
|
+
},
|
|
1525
|
+
...e,
|
|
1526
|
+
...i.columnMetadata
|
|
1527
|
+
}, { startByte: u, endByte: d } = i.range;
|
|
1528
|
+
if (!("offsetIndex" in i)) {
|
|
1529
|
+
r.push({
|
|
1530
|
+
pathInSchema: s,
|
|
1531
|
+
data: Promise.resolve(e.file.slice(u, d)).then((t) => _t({
|
|
1462
1532
|
view: new DataView(t),
|
|
1463
1533
|
offset: 0
|
|
1464
|
-
}, n,
|
|
1534
|
+
}, n, l, e.onPage))
|
|
1465
1535
|
});
|
|
1466
1536
|
continue;
|
|
1467
1537
|
}
|
|
1468
|
-
|
|
1469
|
-
pathInSchema:
|
|
1470
|
-
data: Promise.resolve(
|
|
1471
|
-
let i =
|
|
1538
|
+
r.push({
|
|
1539
|
+
pathInSchema: s,
|
|
1540
|
+
data: Promise.resolve(e.file.slice(i.offsetIndex.startByte, i.offsetIndex.endByte)).then(async (t) => {
|
|
1541
|
+
let { selectStart: r, selectEnd: i } = n, s = bt({
|
|
1472
1542
|
view: new DataView(t),
|
|
1473
1543
|
offset: 0
|
|
1474
|
-
})
|
|
1544
|
+
}).page_locations, c = -1, f = o || a < s[0].offset;
|
|
1475
1545
|
for (let e = 0; e < s.length; e++) {
|
|
1476
|
-
let t = s[e],
|
|
1477
|
-
|
|
1546
|
+
let t = s[e], a = Number(t.first_row_index), o = e + 1 < s.length ? Number(s[e + 1].first_row_index) : n.groupRows;
|
|
1547
|
+
c < 0 && !f && o > r && (u = Number(t.offset), c = a), a < i && (d = Number(t.offset) + t.compressed_page_size);
|
|
1478
1548
|
}
|
|
1479
|
-
|
|
1480
|
-
|
|
1549
|
+
c < 0 && (c = 0);
|
|
1550
|
+
let p = await e.file.slice(u, d), { data: m, skipped: h } = _t({
|
|
1551
|
+
view: new DataView(p),
|
|
1481
1552
|
offset: 0
|
|
1482
|
-
},
|
|
1553
|
+
}, c ? {
|
|
1483
1554
|
...n,
|
|
1484
|
-
groupStart: n.groupStart +
|
|
1485
|
-
selectStart: n.selectStart -
|
|
1486
|
-
selectEnd: n.selectEnd -
|
|
1487
|
-
} : n,
|
|
1555
|
+
groupStart: n.groupStart + c,
|
|
1556
|
+
selectStart: n.selectStart - c,
|
|
1557
|
+
selectEnd: n.selectEnd - c
|
|
1558
|
+
} : n, l, e.onPage);
|
|
1488
1559
|
return {
|
|
1489
|
-
data:
|
|
1490
|
-
skipped:
|
|
1560
|
+
data: m,
|
|
1561
|
+
skipped: c + h
|
|
1491
1562
|
};
|
|
1492
1563
|
})
|
|
1493
1564
|
});
|
|
@@ -1495,10 +1566,10 @@ function Ze(e, { metadata: t }, n) {
|
|
|
1495
1566
|
return {
|
|
1496
1567
|
groupStart: n.groupStart,
|
|
1497
1568
|
groupRows: n.groupRows,
|
|
1498
|
-
asyncColumns:
|
|
1569
|
+
asyncColumns: r
|
|
1499
1570
|
};
|
|
1500
1571
|
}
|
|
1501
|
-
function
|
|
1572
|
+
function St(e, t, n) {
|
|
1502
1573
|
let { asyncColumns: r } = e;
|
|
1503
1574
|
n = {
|
|
1504
1575
|
...p,
|
|
@@ -1508,20 +1579,23 @@ function Qe(e, t, n) {
|
|
|
1508
1579
|
for (let e of t.children) if (e.children.length) {
|
|
1509
1580
|
let t = r.filter((t) => t.pathInSchema[0] === e.element.name);
|
|
1510
1581
|
if (!t.length) continue;
|
|
1511
|
-
let a = /* @__PURE__ */ new Map(), o = Promise.all(t.map((e) => e.data.then(({ data: t }) => {
|
|
1512
|
-
a.set(e.pathInSchema.join("."), pe(t));
|
|
1513
|
-
}))).then(() => {
|
|
1514
|
-
K(a, e, n);
|
|
1515
|
-
let t = a.get(e.path.join("."));
|
|
1516
|
-
if (!t) throw Error("parquet column data not assembled");
|
|
1517
|
-
return {
|
|
1518
|
-
data: [t],
|
|
1519
|
-
skipped: 0
|
|
1520
|
-
};
|
|
1521
|
-
});
|
|
1522
1582
|
i.push({
|
|
1523
1583
|
pathInSchema: e.path,
|
|
1524
|
-
data:
|
|
1584
|
+
data: (async () => {
|
|
1585
|
+
let r = await Promise.all(t.map((e) => e.data)), i = /* @__PURE__ */ new Map(), a = Infinity;
|
|
1586
|
+
for (let e = 0; e < t.length; e++) {
|
|
1587
|
+
let n = Ie(r[e].data);
|
|
1588
|
+
i.set(t[e].pathInSchema.join("."), n), a = Math.min(a, n.length);
|
|
1589
|
+
}
|
|
1590
|
+
for (let [e, t] of i) t.length > a && i.set(e, t.slice(0, a));
|
|
1591
|
+
K(i, e, n);
|
|
1592
|
+
let o = i.get(e.element.name);
|
|
1593
|
+
if (!o) throw Error("parquet column data not assembled");
|
|
1594
|
+
return {
|
|
1595
|
+
data: [o],
|
|
1596
|
+
skipped: 0
|
|
1597
|
+
};
|
|
1598
|
+
})()
|
|
1525
1599
|
});
|
|
1526
1600
|
} else {
|
|
1527
1601
|
let t = r.find((t) => t.pathInSchema[0] === e.element.name);
|
|
@@ -1534,13 +1608,13 @@ function Qe(e, t, n) {
|
|
|
1534
1608
|
}
|
|
1535
1609
|
//#endregion
|
|
1536
1610
|
//#region src/data/formats/parquetRead.js
|
|
1537
|
-
var
|
|
1538
|
-
function
|
|
1611
|
+
var Ct = /* @__PURE__ */ new Map(), wt = 200;
|
|
1612
|
+
function Tt(e) {
|
|
1539
1613
|
if (!e.metadata) throw Error("parquet requires metadata");
|
|
1540
|
-
let t =
|
|
1541
|
-
return e.file =
|
|
1614
|
+
let t = Re(e);
|
|
1615
|
+
return e.file = ze(e.file, t), t.groups.map((n) => xt(e, t, n));
|
|
1542
1616
|
}
|
|
1543
|
-
function
|
|
1617
|
+
function Et(e) {
|
|
1544
1618
|
if (!e) return [];
|
|
1545
1619
|
if (e.length === 1) return e[0];
|
|
1546
1620
|
let t = 0;
|
|
@@ -1552,13 +1626,13 @@ function nt(e) {
|
|
|
1552
1626
|
}
|
|
1553
1627
|
return n;
|
|
1554
1628
|
}
|
|
1555
|
-
function
|
|
1556
|
-
let t = e.join(""), n =
|
|
1629
|
+
function Dt(e) {
|
|
1630
|
+
let t = e.join(""), n = Ct.get(t);
|
|
1557
1631
|
if (n) return n;
|
|
1558
1632
|
let r = e.map((e, t) => JSON.stringify(e) + ": columnData[" + t + "][row - columnSkipped[" + t + "]]").join(",\n"), i = Function("groupData", "selectStart", "selectCount", "columnData", "columnSkipped", "for (let selectRow = 0; selectRow < selectCount; selectRow++) {\n const row = selectStart + selectRow;\n groupData[selectRow] = {\n" + r + "\n };\n}\nreturn groupData;");
|
|
1559
|
-
return
|
|
1633
|
+
return Ct.set(t, i), i;
|
|
1560
1634
|
}
|
|
1561
|
-
function
|
|
1635
|
+
function Ot(e, t, n, r, i, a) {
|
|
1562
1636
|
for (let o = 0; o < n; o++) {
|
|
1563
1637
|
let n = t + o, s = {};
|
|
1564
1638
|
for (let e = 0; e < r.length; e++) s[r[e]] = i[e][n - a[e]];
|
|
@@ -1566,22 +1640,22 @@ function it(e, t, n, r, i, a) {
|
|
|
1566
1640
|
}
|
|
1567
1641
|
return e;
|
|
1568
1642
|
}
|
|
1569
|
-
async function
|
|
1643
|
+
async function kt({ asyncColumns: e }, t, n) {
|
|
1570
1644
|
let r = await Promise.all(e.map((e) => e.data)), i = e.length, a = Array(i), o = Array(i), s = Array(i);
|
|
1571
|
-
for (let t = 0; t < i; t++) a[t] = e[t].pathInSchema[0], o[t] =
|
|
1645
|
+
for (let t = 0; t < i; t++) a[t] = e[t].pathInSchema[0], o[t] = Et(r[t].data), s[t] = r[t].skipped;
|
|
1572
1646
|
let c = n - t, l = Array(c);
|
|
1573
|
-
return i >
|
|
1647
|
+
return i > wt ? Ot(l, t, c, a, o, s) : Dt(a)(l, t, c, o, s);
|
|
1574
1648
|
}
|
|
1575
|
-
async function
|
|
1649
|
+
async function At(e) {
|
|
1576
1650
|
if ("rowFormat" in e) throw Error("parquetRead supports only object rows; use rowFormat: \"object\" implicitly");
|
|
1577
1651
|
if ("filter" in e || "filterStrict" in e) throw Error("parquetRead does not support filtering");
|
|
1578
|
-
e.metadata ??= await
|
|
1579
|
-
let { rowStart: t = 0, rowEnd: n, onChunk: r, onComplete: i } = e, a =
|
|
1652
|
+
e.metadata ??= await be(e.file, e);
|
|
1653
|
+
let { rowStart: t = 0, rowEnd: n, onChunk: r, onComplete: i } = e, a = Tt(e);
|
|
1580
1654
|
if (!i && !r) {
|
|
1581
1655
|
for (let { asyncColumns: e } of a) for (let { data: t } of e) await t;
|
|
1582
1656
|
return;
|
|
1583
1657
|
}
|
|
1584
|
-
let o =
|
|
1658
|
+
let o = Se(e.metadata), s = a.map((t) => St(t, o, e.parsers));
|
|
1585
1659
|
if (r) for (let e of s) for (let t of e.asyncColumns) t.data.then((n) => {
|
|
1586
1660
|
let i = e.groupStart + n.skipped;
|
|
1587
1661
|
for (let e of n.data) r({
|
|
@@ -1593,17 +1667,17 @@ async function ot(e) {
|
|
|
1593
1667
|
});
|
|
1594
1668
|
if (i) {
|
|
1595
1669
|
let e = [];
|
|
1596
|
-
for (let r of s)
|
|
1670
|
+
for (let r of s) Fe(e, await kt(r, Math.max(t - r.groupStart, 0), Math.min((n ?? Infinity) - r.groupStart, r.groupRows)));
|
|
1597
1671
|
i(e);
|
|
1598
1672
|
} else for (let { asyncColumns: e } of s) for (let { data: t } of e) await t;
|
|
1599
1673
|
}
|
|
1600
|
-
function
|
|
1674
|
+
function jt(e) {
|
|
1601
1675
|
return new Promise((t, n) => {
|
|
1602
|
-
|
|
1676
|
+
At({
|
|
1603
1677
|
...e,
|
|
1604
1678
|
onComplete: t
|
|
1605
1679
|
}).catch(n);
|
|
1606
1680
|
});
|
|
1607
1681
|
}
|
|
1608
1682
|
//#endregion
|
|
1609
|
-
export {
|
|
1683
|
+
export { jt as parquetReadObjects };
|