@gscdump/engine 3.4.4 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -18
- package/dist/THIRD-PARTY-LICENSES.md +2 -2
- package/dist/adapters/duckdb-node.d.mts +4 -4
- package/dist/adapters/duckdb-node.mjs +103 -77
- package/dist/adapters/filesystem.d.mts +6 -7
- package/dist/adapters/filesystem.mjs +11 -6
- package/dist/adapters/hyparquet.d.mts +9 -10
- package/dist/adapters/hyparquet.mjs +84 -44
- package/dist/adapters/node-harness.d.mts +3 -4
- package/dist/adapters/parquet-attach.d.mts +4 -5
- package/dist/adapters/parquet-attach.mjs +7 -2
- package/dist/adapters/r2-manifest.d.mts +4 -5
- package/dist/adapters/r2.d.mts +2 -3
- package/dist/adapters/snapshot-attach.d.mts +7 -8
- package/dist/adapters/snapshot-attach.mjs +7 -2
- package/dist/analysis-range.d.mts +2 -3
- package/dist/analysis-types.d.mts +4 -5
- package/dist/analyzer/define.d.mts +6 -7
- package/dist/analyzer/dispatch.d.mts +2 -3
- package/dist/analyzer/registry.d.mts +4 -5
- package/dist/analyzer/types.d.mts +11 -12
- package/dist/arrow-utils.d.mts +4 -5
- package/dist/arrow-utils.mjs +10 -0
- package/dist/coerce.d.mts +1 -2
- package/dist/compaction.d.mts +2 -3
- package/dist/drizzle-schema.d.mts +14 -15
- package/dist/duckdb.d.mts +5 -6
- package/dist/duckdb.mjs +9 -5
- package/dist/engine.d.mts +1 -2
- package/dist/entities/empty-types.d.mts +4 -5
- package/dist/entities/indexing-metadata.d.mts +5 -6
- package/dist/entities/inspection.d.mts +9 -10
- package/dist/entities/query-dim.d.mts +7 -7
- package/dist/entities/sitemap-shared.d.mts +26 -27
- package/dist/entities/sitemap-write.d.mts +1 -2
- package/dist/entities/sitemap.d.mts +1 -2
- package/dist/entity-keys.d.mts +28 -29
- package/dist/errors.d.mts +5 -6
- package/dist/iceberg/append-sink.d.mts +2 -3
- package/dist/iceberg/catalog.d.mts +10 -10
- package/dist/iceberg/catalog.mjs +2 -2
- package/dist/iceberg/overwrite-writer.d.mts +17 -18
- package/dist/iceberg/schema.d.mts +16 -16
- package/dist/ingest-accumulator.d.mts +8 -9
- package/dist/ingest.d.mts +15 -12
- package/dist/ingest.mjs +22 -6
- package/dist/layout.d.mts +4 -5
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/datapage.mjs +1 -1
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/indexes.mjs +1 -1
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/metadata.mjs +1 -1
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/types.d.mts +2 -3
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/index.d.mts +2 -3
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/assemble.mjs +3 -144
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/bloom.mjs +3 -58
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/column.mjs +8 -129
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/convert.mjs +3 -118
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/datapage.mjs +6 -144
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/delta.mjs +2 -65
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/encoding.mjs +2 -76
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/filter.mjs +4 -212
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/geoparquet.mjs +1 -24
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/index.mjs +3 -3
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/indexes.mjs +5 -32
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/metadata.mjs +7 -241
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs +1 -5
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/plain.mjs +1 -79
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/plan.mjs +6 -375
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/read.mjs +6 -77
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/rowgroup.mjs +7 -175
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/scan.mjs +5 -73
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs +1 -22
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/snappy.mjs +1 -68
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/thrift.mjs +1 -86
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/types.d.mts +2 -28
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/utils.mjs +1 -31
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/variant.mjs +3 -228
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/wkb.mjs +1 -94
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/index.d.mts +2 -52
- package/dist/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/types/node.d.mts +2 -2
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/assemble.mjs +144 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/bloom.mjs +124 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/column.mjs +129 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/constants.mjs +80 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/convert.mjs +118 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/datapage.mjs +144 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/delta.mjs +65 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/encoding.mjs +76 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/filter.mjs +212 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/geoparquet.mjs +24 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/index.mjs +8 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/indexes.mjs +32 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/metadata.mjs +241 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/node.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/plain.mjs +79 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/plan.mjs +375 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/query.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/read.mjs +81 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/rowgroup.mjs +175 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/scan.mjs +74 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/schema.mjs +75 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/snappy.mjs +68 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/thrift.mjs +86 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/types.d.mts +25 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/utils.mjs +32 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/variant.mjs +228 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/wkb.mjs +94 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/src/xxhash.mjs +68 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/index.d.mts +58 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/node.d.mts +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/scan.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.30.0/node_modules/hyparquet/types/utils.d.mts +1 -0
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/index.mjs +1 -1
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/read.mjs +1 -1
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/sql/icebergDataSource.mjs +1 -1
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/sql/icebergQuery.mjs +1 -1
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/types.d.mts +17 -18
- package/dist/node_modules/.pnpm/icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/create.d.mts +1 -0
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/index.d.mts +1 -1
- package/dist/node_modules/.pnpm/icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/manifest.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/metadata.d.mts +1 -0
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/read.d.mts +1 -1
- package/dist/node_modules/.pnpm/icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/s3.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/sql/icebergDataSource.d.mts +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/sql/icebergQuery.d.mts +2 -0
- package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/write/write.d.mts +2 -3
- package/dist/node_modules/.pnpm/squirreling@0.16.4/node_modules/squirreling/src/ast.d.mts +1 -0
- package/dist/parquet-plan.d.mts +5 -6
- package/dist/period/index.d.mts +8 -8
- package/dist/profile.d.mts +3 -3
- package/dist/report/define.d.mts +1 -2
- package/dist/report/hash.d.mts +3 -4
- package/dist/report/registry.d.mts +3 -4
- package/dist/report/types.d.mts +22 -23
- package/dist/resolver/adapter.d.mts +2 -3
- package/dist/resolver/canonical-source.d.mts +1 -2
- package/dist/resolver/compile.d.mts +12 -8
- package/dist/resolver/compile.mjs +7 -3
- package/dist/resolver/datasets.d.mts +9 -9
- package/dist/resolver/extras-overlay.d.mts +2 -3
- package/dist/resolver/filter-utils.d.mts +7 -7
- package/dist/resolver/fragments.d.mts +3 -4
- package/dist/resolver/pg-adapter.d.mts +7 -8
- package/dist/resolver/run-query.d.mts +16 -17
- package/dist/resolver/run-query.mjs +16 -15
- package/dist/resolver/schema-drift.d.mts +2 -3
- package/dist/resolver/types.d.mts +8 -9
- package/dist/rollups/canonical.d.mts +4 -5
- package/dist/rollups/core.d.mts +12 -13
- package/dist/rollups/defaults.d.mts +2 -3
- package/dist/rollups/hourly.d.mts +3 -4
- package/dist/rollups/indexing.d.mts +5 -6
- package/dist/rollups/traffic.d.mts +5 -6
- package/dist/rollups/windows.d.mts +7 -8
- package/dist/schedule.d.mts +6 -7
- package/dist/schema.d.mts +9 -9
- package/dist/scope.d.mts +6 -7
- package/dist/sink.d.mts +9 -10
- package/dist/sinks/in-memory-sink.d.mts +3 -4
- package/dist/snapshot.d.mts +2 -3
- package/dist/source/attached-table.d.mts +4 -5
- package/dist/source/create-sql-query-source.d.mts +2 -3
- package/dist/source/index.d.mts +6 -6
- package/dist/source/source-types.d.mts +7 -8
- package/dist/sql-bind.d.mts +3 -4
- package/dist/sql-fragments.d.mts +5 -6
- package/dist/storage.d.mts +37 -37
- package/dist/sync-config.d.mts +20 -21
- package/dist/vendor/hysnappy-purejs.d.mts +2 -3
- package/dist/vendor/hysnappy-purejs.mjs +1 -1
- package/package.json +12 -10
- package/dist/node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/sql/icebergDataSource.d.mts +0 -2
- package/dist/node_modules/.pnpm/icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/sql/icebergQuery.d.mts +0 -2
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/create.d.mts → hyparquet@1.30.0/node_modules/hyparquet/types/indexes.d.mts} +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird → hyparquet@1.30.0/node_modules/hyparquet}/types/metadata.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/manifest.d.mts → hyparquet@1.30.0/node_modules/hyparquet/types/query.d.mts} +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/types/s3.d.mts → hyparquet@1.30.0/node_modules/hyparquet/types/read.d.mts} +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3/node_modules/squirreling/src/ast.d.mts → hyparquet@1.30.0/node_modules/hyparquet/types/snappy.d.mts} +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/avro/avro.metadata.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/avro/avro.read.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/avro/avro.write.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/catalog/file.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/catalog/loadTable.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/catalog/rest.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/create.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/delete.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/fetch.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/json.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/manifest.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/metadata.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/prune.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/puffin/deletion-vector.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/puffin/puffin.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/puffin/roaring.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/s3.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/schema.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/sigv4.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/sql/whereFilter.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/utils.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/commit.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/conversions.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/delete-file.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/geospatial.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/manifest-list.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/manifest.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/parquet.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/partition.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/rewrite.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/serde.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/snapshot.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/sort.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/stage-deletion-vector.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/stage-position-delete.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/stage.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/stats.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/transform.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/write.mjs +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/catalog/file.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/catalog/rest.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{icebird@0.8.27_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7 → icebird@0.8.28_patch_hash_651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/fetch.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/backend/batch.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/backend/batchAdapters.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/backend/dataSource.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/accumulator.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/aggregates.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/batchResults.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/batches.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/execute.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/fold.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/join.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/rowScope.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/scanColumn.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/sort.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/streamingAggregate.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/utils.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/window.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/execute/yield.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/alias.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/batch.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/binary.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/date.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/evaluate.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/math.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/regexp.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/scalar.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/expression/strings.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/index.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/index.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/expression.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/extractTables.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/functions.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/joins.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/parse.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/primary.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/state.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/tokenize.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/parse/types.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/plan/columns.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/plan/plan.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/plan/types.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/bbox.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/equality.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/operations.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/pointRelations.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/primitives.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/segments.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/spatial.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/spatial/wkt.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/types.d.mts +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/validation/aggregates.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/validation/executionErrors.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/validation/functions.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/validation/keywords.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/validation/parseErrors.mjs +0 -0
- /package/dist/node_modules/.pnpm/{squirreling@0.16.3 → squirreling@0.16.4}/node_modules/squirreling/src/validation/tables.mjs +0 -0
|
@@ -1,74 +1,6 @@
|
|
|
1
|
-
import
|
|
1
|
+
import "./metadata.mjs";
|
|
2
2
|
import "./utils.mjs";
|
|
3
|
-
import
|
|
4
|
-
import
|
|
5
|
-
import
|
|
6
|
-
|
|
7
|
-
const prepared = await prepareParquetOptions(options);
|
|
8
|
-
return {
|
|
9
|
-
options: prepared,
|
|
10
|
-
plan: parquetPlan(prepared)
|
|
11
|
-
};
|
|
12
|
-
}
|
|
13
|
-
async function prepareParquetOptions(options) {
|
|
14
|
-
const metadata = options.metadata ?? await parquetMetadataAsync(options.file, options);
|
|
15
|
-
const schemaColumns = parquetSchema(metadata).children.map((child) => child.element.name);
|
|
16
|
-
const missingFilterColumns = columnsNeededForFilter(options.filter).filter((column) => !schemaColumns.includes(column));
|
|
17
|
-
if (missingFilterColumns.length) throw new Error(`parquet filter columns not found: ${missingFilterColumns.join(", ")}`);
|
|
18
|
-
if (options.columns) {
|
|
19
|
-
const missingColumns = options.columns.filter((column) => !schemaColumns.includes(column));
|
|
20
|
-
if (missingColumns.length) throw new Error(`parquet column not found: ${missingColumns[0]}`);
|
|
21
|
-
}
|
|
22
|
-
let prepared = {
|
|
23
|
-
...options,
|
|
24
|
-
metadata
|
|
25
|
-
};
|
|
26
|
-
prepared = await withBloomFilters(prepared);
|
|
27
|
-
prepared = await withPageIndexes(prepared);
|
|
28
|
-
return prepared;
|
|
29
|
-
}
|
|
30
|
-
function readParquetPlan(options, plan) {
|
|
31
|
-
const readOptions = {
|
|
32
|
-
...options,
|
|
33
|
-
file: prefetchAsyncBuffer(options.file, plan)
|
|
34
|
-
};
|
|
35
|
-
return plan.groups.map((group) => readRowGroup(readOptions, plan, group));
|
|
36
|
-
}
|
|
37
|
-
async function withBloomFilters(options) {
|
|
38
|
-
if (!options.useBloomFilters || !options.filter || !options.metadata) return options;
|
|
39
|
-
const schemaTree = parquetSchema(options.metadata);
|
|
40
|
-
const schemaElements = {};
|
|
41
|
-
for (const child of schemaTree.children) schemaElements[child.element.name] = child.element;
|
|
42
|
-
const bloomFiltersByGroup = await prefetchBloomFilters({
|
|
43
|
-
file: options.file,
|
|
44
|
-
metadata: options.metadata,
|
|
45
|
-
filter: options.filter,
|
|
46
|
-
filterStrict: options.filterStrict
|
|
47
|
-
});
|
|
48
|
-
return {
|
|
49
|
-
...options,
|
|
50
|
-
bloomFiltersByGroup,
|
|
51
|
-
schemaElements
|
|
52
|
-
};
|
|
53
|
-
}
|
|
54
|
-
async function withPageIndexes(options) {
|
|
55
|
-
if (!options.usePageIndex || !options.filter || !options.metadata) return options;
|
|
56
|
-
const { pageRangesByGroup, pageLocationsByGroup } = await prefetchPageIndexes({
|
|
57
|
-
file: options.file,
|
|
58
|
-
metadata: options.metadata,
|
|
59
|
-
filter: options.filter,
|
|
60
|
-
filterStrict: options.filterStrict,
|
|
61
|
-
rowStart: options.rowStart,
|
|
62
|
-
rowEnd: options.rowEnd,
|
|
63
|
-
columns: options.columns,
|
|
64
|
-
bloomFiltersByGroup: options.bloomFiltersByGroup,
|
|
65
|
-
schemaElements: options.schemaElements,
|
|
66
|
-
parsers: options.parsers
|
|
67
|
-
});
|
|
68
|
-
return {
|
|
69
|
-
...options,
|
|
70
|
-
pageRangesByGroup,
|
|
71
|
-
pageLocationsByGroup
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
export { prepareParquetRead, readParquetPlan };
|
|
3
|
+
import "./filter.mjs";
|
|
4
|
+
import "./plan.mjs";
|
|
5
|
+
import "./rowgroup.mjs";
|
|
6
|
+
export {};
|
|
@@ -26,20 +26,6 @@ function getSchemaPath(schema, name) {
|
|
|
26
26
|
}
|
|
27
27
|
return path;
|
|
28
28
|
}
|
|
29
|
-
function getPhysicalColumns(schemaTree) {
|
|
30
|
-
const columns = [];
|
|
31
|
-
function traverse(node) {
|
|
32
|
-
if (node.children.length) for (const child of node.children) traverse(child);
|
|
33
|
-
else columns.push(node.path.join("."));
|
|
34
|
-
}
|
|
35
|
-
traverse(schemaTree);
|
|
36
|
-
return columns;
|
|
37
|
-
}
|
|
38
|
-
function getMaxRepetitionLevel(schemaPath) {
|
|
39
|
-
let maxLevel = 0;
|
|
40
|
-
for (const { element } of schemaPath) if (element.repetition_type === "REPEATED") maxLevel++;
|
|
41
|
-
return maxLevel;
|
|
42
|
-
}
|
|
43
29
|
function getMaxDefinitionLevel(schemaPath) {
|
|
44
30
|
let maxLevel = 0;
|
|
45
31
|
for (const { element } of schemaPath.slice(1)) if (element.repetition_type !== "REQUIRED") maxLevel++;
|
|
@@ -65,11 +51,4 @@ function isMapLike(schema) {
|
|
|
65
51
|
if (firstChild.children.find((child) => child.element.name === "value")?.element.repetition_type === "REPEATED") return false;
|
|
66
52
|
return true;
|
|
67
53
|
}
|
|
68
|
-
|
|
69
|
-
if (schemaPath.length !== 2) return false;
|
|
70
|
-
const [, column] = schemaPath;
|
|
71
|
-
if (column.element.repetition_type === "REPEATED") return false;
|
|
72
|
-
if (column.children.length) return false;
|
|
73
|
-
return true;
|
|
74
|
-
}
|
|
75
|
-
export { getMaxDefinitionLevel, getMaxRepetitionLevel, getPhysicalColumns, getSchemaPath, isFlatColumn, isListLike, isMapLike };
|
|
54
|
+
export { getMaxDefinitionLevel, getSchemaPath, isListLike, isMapLike };
|
|
@@ -1,68 +1 @@
|
|
|
1
|
-
|
|
2
|
-
0,
|
|
3
|
-
255,
|
|
4
|
-
65535,
|
|
5
|
-
16777215,
|
|
6
|
-
4294967295
|
|
7
|
-
];
|
|
8
|
-
function copyBytes(fromArray, fromPos, toArray, toPos, length) {
|
|
9
|
-
for (let i = 0; i < length; i++) toArray[toPos + i] = fromArray[fromPos + i];
|
|
10
|
-
}
|
|
11
|
-
function snappyUncompress(input, output) {
|
|
12
|
-
const inputLength = input.byteLength;
|
|
13
|
-
const outputLength = output.byteLength;
|
|
14
|
-
let pos = 0;
|
|
15
|
-
let outPos = 0;
|
|
16
|
-
while (pos < inputLength) {
|
|
17
|
-
const c = input[pos];
|
|
18
|
-
pos++;
|
|
19
|
-
if (c < 128) break;
|
|
20
|
-
}
|
|
21
|
-
if (outputLength && pos >= inputLength) throw new Error("invalid snappy length header");
|
|
22
|
-
while (pos < inputLength) {
|
|
23
|
-
const c = input[pos];
|
|
24
|
-
let len = 0;
|
|
25
|
-
pos++;
|
|
26
|
-
if (pos >= inputLength) throw new Error("missing eof marker");
|
|
27
|
-
if ((c & 3) === 0) {
|
|
28
|
-
let len = (c >>> 2) + 1;
|
|
29
|
-
if (len > 60) {
|
|
30
|
-
if (pos + 3 >= inputLength) throw new Error("snappy error literal pos + 3 >= inputLength");
|
|
31
|
-
const lengthSize = len - 60;
|
|
32
|
-
len = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
|
|
33
|
-
len = (len & WORD_MASK[lengthSize]) + 1;
|
|
34
|
-
pos += lengthSize;
|
|
35
|
-
}
|
|
36
|
-
if (pos + len > inputLength) throw new Error("snappy error literal exceeds input length");
|
|
37
|
-
copyBytes(input, pos, output, outPos, len);
|
|
38
|
-
pos += len;
|
|
39
|
-
outPos += len;
|
|
40
|
-
} else {
|
|
41
|
-
let offset = 0;
|
|
42
|
-
switch (c & 3) {
|
|
43
|
-
case 1:
|
|
44
|
-
len = (c >>> 2 & 7) + 4;
|
|
45
|
-
offset = input[pos] + (c >>> 5 << 8);
|
|
46
|
-
pos++;
|
|
47
|
-
break;
|
|
48
|
-
case 2:
|
|
49
|
-
if (inputLength <= pos + 1) throw new Error("snappy error end of input");
|
|
50
|
-
len = (c >>> 2) + 1;
|
|
51
|
-
offset = input[pos] + (input[pos + 1] << 8);
|
|
52
|
-
pos += 2;
|
|
53
|
-
break;
|
|
54
|
-
case 3:
|
|
55
|
-
if (inputLength <= pos + 3) throw new Error("snappy error end of input");
|
|
56
|
-
len = (c >>> 2) + 1;
|
|
57
|
-
offset = input[pos] + (input[pos + 1] << 8) + (input[pos + 2] << 16) + (input[pos + 3] << 24);
|
|
58
|
-
pos += 4;
|
|
59
|
-
}
|
|
60
|
-
if (offset === 0 || isNaN(offset)) throw new Error(`invalid offset ${offset} pos ${pos} inputLength ${inputLength}`);
|
|
61
|
-
if (offset > outPos) throw new Error("cannot copy from before start of buffer");
|
|
62
|
-
copyBytes(output, outPos - offset, output, outPos, len);
|
|
63
|
-
outPos += len;
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
if (outPos !== outputLength) throw new Error("premature end of input");
|
|
67
|
-
}
|
|
68
|
-
export { snappyUncompress };
|
|
1
|
+
export {};
|
|
@@ -1,86 +1 @@
|
|
|
1
|
-
|
|
2
|
-
const TRUE = 1;
|
|
3
|
-
const FALSE = 2;
|
|
4
|
-
const BYTE = 3;
|
|
5
|
-
const I16 = 4;
|
|
6
|
-
const I32 = 5;
|
|
7
|
-
const I64 = 6;
|
|
8
|
-
const DOUBLE = 7;
|
|
9
|
-
const BINARY = 8;
|
|
10
|
-
const LIST = 9;
|
|
11
|
-
const STRUCT = 12;
|
|
12
|
-
function deserializeTCompactProtocol(reader) {
|
|
13
|
-
const value = {};
|
|
14
|
-
let fid = 0;
|
|
15
|
-
while (reader.offset < reader.view.byteLength) {
|
|
16
|
-
const byte = reader.view.getUint8(reader.offset++);
|
|
17
|
-
const type = byte & 15;
|
|
18
|
-
if (type === STOP) break;
|
|
19
|
-
const delta = byte >> 4;
|
|
20
|
-
fid = delta ? fid + delta : readZigZag(reader);
|
|
21
|
-
value[`field_${fid}`] = readElement(reader, type);
|
|
22
|
-
}
|
|
23
|
-
return value;
|
|
24
|
-
}
|
|
25
|
-
function readElement(reader, type) {
|
|
26
|
-
switch (type) {
|
|
27
|
-
case TRUE: return true;
|
|
28
|
-
case FALSE: return false;
|
|
29
|
-
case BYTE: return reader.view.getInt8(reader.offset++);
|
|
30
|
-
case I16:
|
|
31
|
-
case I32: return readZigZag(reader);
|
|
32
|
-
case I64: return readZigZagBigInt(reader);
|
|
33
|
-
case DOUBLE: {
|
|
34
|
-
const value = reader.view.getFloat64(reader.offset, true);
|
|
35
|
-
reader.offset += 8;
|
|
36
|
-
return value;
|
|
37
|
-
}
|
|
38
|
-
case BINARY: {
|
|
39
|
-
const stringLength = readVarInt(reader);
|
|
40
|
-
const strBytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, stringLength);
|
|
41
|
-
reader.offset += stringLength;
|
|
42
|
-
return strBytes;
|
|
43
|
-
}
|
|
44
|
-
case LIST: {
|
|
45
|
-
const byte = reader.view.getUint8(reader.offset++);
|
|
46
|
-
const elemType = byte & 15;
|
|
47
|
-
let listSize = byte >> 4;
|
|
48
|
-
if (listSize === 15) listSize = readVarInt(reader);
|
|
49
|
-
const boolType = elemType === TRUE || elemType === FALSE;
|
|
50
|
-
const values = new Array(listSize);
|
|
51
|
-
for (let i = 0; i < listSize; i++) values[i] = boolType ? readElement(reader, BYTE) === 1 : readElement(reader, elemType);
|
|
52
|
-
return values;
|
|
53
|
-
}
|
|
54
|
-
case STRUCT: return deserializeTCompactProtocol(reader);
|
|
55
|
-
default: throw new Error(`thrift unhandled type: ${type}`);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
function readVarInt(reader) {
|
|
59
|
-
let result = 0;
|
|
60
|
-
let shift = 0;
|
|
61
|
-
while (true) {
|
|
62
|
-
const byte = reader.view.getUint8(reader.offset++);
|
|
63
|
-
result |= (byte & 127) << shift;
|
|
64
|
-
if (!(byte & 128)) return result;
|
|
65
|
-
shift += 7;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
function readVarBigInt(reader) {
|
|
69
|
-
let result = 0n;
|
|
70
|
-
let shift = 0n;
|
|
71
|
-
while (true) {
|
|
72
|
-
const byte = reader.view.getUint8(reader.offset++);
|
|
73
|
-
result |= BigInt(byte & 127) << shift;
|
|
74
|
-
if (!(byte & 128)) return result;
|
|
75
|
-
shift += 7n;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
function readZigZag(reader) {
|
|
79
|
-
const zigzag = readVarInt(reader);
|
|
80
|
-
return zigzag >>> 1 ^ -(zigzag & 1);
|
|
81
|
-
}
|
|
82
|
-
function readZigZagBigInt(reader) {
|
|
83
|
-
const zigzag = readVarBigInt(reader);
|
|
84
|
-
return zigzag >> 1n ^ -(zigzag & 1n);
|
|
85
|
-
}
|
|
86
|
-
export { deserializeTCompactProtocol, readVarInt, readZigZag, readZigZagBigInt };
|
|
1
|
+
export {};
|
|
@@ -1,34 +1,8 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Parquet query options for filtering data
|
|
3
|
-
*/
|
|
4
|
-
type ParquetQueryFilter = ParquetQueryColumnsFilter | {
|
|
5
|
-
$and: ParquetQueryFilter[];
|
|
6
|
-
} | {
|
|
7
|
-
$or: ParquetQueryFilter[];
|
|
8
|
-
} | {
|
|
9
|
-
$nor: ParquetQueryFilter[];
|
|
10
|
-
};
|
|
11
|
-
type ParquetQueryColumnsFilter = {
|
|
12
|
-
[key: string]: ParquetQueryOperator;
|
|
13
|
-
};
|
|
14
|
-
type ParquetQueryValue = string | number | bigint | boolean | object | null | undefined;
|
|
15
|
-
type ParquetQueryOperator = {
|
|
16
|
-
$gt?: ParquetQueryValue;
|
|
17
|
-
$gte?: ParquetQueryValue;
|
|
18
|
-
$lt?: ParquetQueryValue;
|
|
19
|
-
$lte?: ParquetQueryValue;
|
|
20
|
-
$eq?: ParquetQueryValue;
|
|
21
|
-
$ne?: ParquetQueryValue;
|
|
22
|
-
$in?: ParquetQueryValue[];
|
|
23
|
-
$nin?: ParquetQueryValue[];
|
|
24
|
-
$not?: ParquetQueryOperator;
|
|
25
|
-
};
|
|
26
1
|
/**
|
|
27
2
|
* File-like object that can read slices of a file asynchronously.
|
|
28
3
|
*/
|
|
29
|
-
interface AsyncBuffer {
|
|
4
|
+
export interface AsyncBuffer {
|
|
30
5
|
byteLength: number;
|
|
31
6
|
slice(start: number, end?: number): Awaitable<ArrayBuffer>;
|
|
32
7
|
}
|
|
33
|
-
type Awaitable<T> = T | Promise<T>;
|
|
34
|
-
export { AsyncBuffer, Awaitable, ParquetQueryFilter, ParquetQueryOperator, ParquetQueryValue };
|
|
8
|
+
export type Awaitable<T> = T | Promise<T>;
|
|
@@ -16,34 +16,4 @@ function toJson(obj) {
|
|
|
16
16
|
}
|
|
17
17
|
return obj;
|
|
18
18
|
}
|
|
19
|
-
|
|
20
|
-
const chunk = 1e4;
|
|
21
|
-
for (let i = 0; i < bbb.length; i += chunk) aaa.push(...bbb.slice(i, i + chunk));
|
|
22
|
-
}
|
|
23
|
-
function equals(a, b, strict = true) {
|
|
24
|
-
if (strict ? a === b : a == b) return true;
|
|
25
|
-
if (!a || !b || typeof a !== "object" || typeof b !== "object") return false;
|
|
26
|
-
if (a instanceof Uint8Array && b instanceof Uint8Array) {
|
|
27
|
-
if (a.length !== b.length) return false;
|
|
28
|
-
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
|
|
29
|
-
return true;
|
|
30
|
-
}
|
|
31
|
-
if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
|
|
32
|
-
if (Array.isArray(a) && Array.isArray(b)) {
|
|
33
|
-
if (a.length !== b.length) return false;
|
|
34
|
-
for (let i = 0; i < a.length; i++) if (!equals(a[i], b[i], strict)) return false;
|
|
35
|
-
return true;
|
|
36
|
-
}
|
|
37
|
-
const aKeys = Object.keys(a);
|
|
38
|
-
if (aKeys.length !== Object.keys(b).length) return false;
|
|
39
|
-
for (const k of aKeys) if (!equals(a[k], b[k], strict)) return false;
|
|
40
|
-
return true;
|
|
41
|
-
}
|
|
42
|
-
function flatten(chunks) {
|
|
43
|
-
if (!chunks) return [];
|
|
44
|
-
if (chunks.length === 1) return chunks[0];
|
|
45
|
-
const output = [];
|
|
46
|
-
for (const chunk of chunks) concat(output, chunk);
|
|
47
|
-
return output;
|
|
48
|
-
}
|
|
49
|
-
export { concat, equals, flatten, toJson };
|
|
19
|
+
export { toJson };
|
|
@@ -1,228 +1,3 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
function decodeVariantColumn(value, parsers = DEFAULT_PARSERS) {
|
|
5
|
-
if (Array.isArray(value)) return value.map((entry) => decodeVariantColumn(entry, parsers));
|
|
6
|
-
if (typeof value !== "object") return value;
|
|
7
|
-
if ("metadata" in value) {
|
|
8
|
-
const metadata = parseVariantMetadata(value.metadata);
|
|
9
|
-
const shreddedFields = value.typed_value && decodeTypedValue(value.typed_value, metadata, parsers);
|
|
10
|
-
const binaryValue = value.value && readVariant(makeReader(value.value), metadata, parsers);
|
|
11
|
-
if (shreddedFields && binaryValue) return {
|
|
12
|
-
...binaryValue,
|
|
13
|
-
...shreddedFields
|
|
14
|
-
};
|
|
15
|
-
return shreddedFields ?? binaryValue;
|
|
16
|
-
}
|
|
17
|
-
return value;
|
|
18
|
-
}
|
|
19
|
-
function decodeTypedValue(typedValue, metadata, parsers) {
|
|
20
|
-
if (typedValue instanceof Date) return typedValue;
|
|
21
|
-
if (typedValue && typeof typedValue === "object" && !Array.isArray(typedValue) && !(typedValue instanceof Uint8Array)) {
|
|
22
|
-
if ("typed_value" in typedValue && typedValue.typed_value !== null && typedValue.typed_value !== void 0) return decodeTypedValue(typedValue.typed_value, metadata, parsers);
|
|
23
|
-
if ("value" in typedValue && typedValue.value instanceof Uint8Array) return readVariant(makeReader(typedValue.value), metadata, parsers);
|
|
24
|
-
if ("typed_value" in typedValue || "value" in typedValue) return null;
|
|
25
|
-
const result = {};
|
|
26
|
-
for (const [key, field] of Object.entries(typedValue)) {
|
|
27
|
-
if (!metadata.dictionary.includes(key)) continue;
|
|
28
|
-
result[key] = decodeTypedValue(field, metadata, parsers);
|
|
29
|
-
}
|
|
30
|
-
return result;
|
|
31
|
-
}
|
|
32
|
-
if (typedValue instanceof Uint8Array) return readVariant(makeReader(typedValue), metadata, parsers);
|
|
33
|
-
if (Array.isArray(typedValue)) return typedValue.map((element) => decodeTypedValue(element, metadata, parsers));
|
|
34
|
-
return typedValue;
|
|
35
|
-
}
|
|
36
|
-
function makeReader(bytes) {
|
|
37
|
-
return {
|
|
38
|
-
view: new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength),
|
|
39
|
-
offset: 0
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
function parseVariantMetadata(bytes) {
|
|
43
|
-
let bufferCache = metadataCache.get(bytes.buffer);
|
|
44
|
-
if (!bufferCache) {
|
|
45
|
-
bufferCache = /* @__PURE__ */ new Map();
|
|
46
|
-
metadataCache.set(bytes.buffer, bufferCache);
|
|
47
|
-
}
|
|
48
|
-
const key = `${bytes.byteOffset}:${bytes.byteLength}`;
|
|
49
|
-
const cached = bufferCache.get(key);
|
|
50
|
-
if (cached) return cached;
|
|
51
|
-
const reader = makeReader(bytes);
|
|
52
|
-
const header = reader.view.getUint8(reader.offset++);
|
|
53
|
-
const version = header & 15;
|
|
54
|
-
if (version !== 1) throw new Error(`parquet unsupported variant metadata version: ${version}`);
|
|
55
|
-
const sorted = (header >> 4 & 1) === 1;
|
|
56
|
-
const offsetSize = (header >> 6 & 3) + 1;
|
|
57
|
-
const dictionarySize = readUnsigned(reader, offsetSize);
|
|
58
|
-
const offsets = new Array(dictionarySize + 1);
|
|
59
|
-
for (let i = 0; i < offsets.length; i++) offsets[i] = readUnsigned(reader, offsetSize);
|
|
60
|
-
const base = reader.offset;
|
|
61
|
-
const dictionary = new Array(dictionarySize);
|
|
62
|
-
for (let i = 0; i < dictionarySize; i++) {
|
|
63
|
-
const start = offsets[i];
|
|
64
|
-
const end = offsets[i + 1];
|
|
65
|
-
const strBytes = new Uint8Array(bytes.buffer, bytes.byteOffset + base + start, end - start);
|
|
66
|
-
dictionary[i] = decoder.decode(strBytes);
|
|
67
|
-
}
|
|
68
|
-
const metadata = {
|
|
69
|
-
dictionary,
|
|
70
|
-
sorted
|
|
71
|
-
};
|
|
72
|
-
bufferCache.set(key, metadata);
|
|
73
|
-
return metadata;
|
|
74
|
-
}
|
|
75
|
-
function readUnsigned(reader, byteWidth) {
|
|
76
|
-
let value = 0;
|
|
77
|
-
for (let i = 0; i < byteWidth; i++) value |= reader.view.getUint8(reader.offset + i) << i * 8;
|
|
78
|
-
reader.offset += byteWidth;
|
|
79
|
-
return value;
|
|
80
|
-
}
|
|
81
|
-
function readVariant(reader, metadata, parsers) {
|
|
82
|
-
const typeByte = reader.view.getUint8(reader.offset++);
|
|
83
|
-
const basicType = typeByte & 3;
|
|
84
|
-
const header = typeByte >> 2;
|
|
85
|
-
if (basicType === 0) return readVariantPrimitive(reader, header, parsers);
|
|
86
|
-
if (basicType === 2) return readVariantObject(reader, header, metadata, parsers);
|
|
87
|
-
if (basicType === 3) return readVariantArray(reader, header, metadata, parsers);
|
|
88
|
-
const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, header);
|
|
89
|
-
reader.offset += header;
|
|
90
|
-
return decoder.decode(bytes);
|
|
91
|
-
}
|
|
92
|
-
function readVariantPrimitive(reader, typeId, parsers) {
|
|
93
|
-
switch (typeId) {
|
|
94
|
-
case 0: return null;
|
|
95
|
-
case 1: return true;
|
|
96
|
-
case 2: return false;
|
|
97
|
-
case 3: {
|
|
98
|
-
const value = reader.view.getInt8(reader.offset);
|
|
99
|
-
reader.offset += 1;
|
|
100
|
-
return value;
|
|
101
|
-
}
|
|
102
|
-
case 4: {
|
|
103
|
-
const value = reader.view.getInt16(reader.offset, true);
|
|
104
|
-
reader.offset += 2;
|
|
105
|
-
return value;
|
|
106
|
-
}
|
|
107
|
-
case 5: {
|
|
108
|
-
const value = reader.view.getInt32(reader.offset, true);
|
|
109
|
-
reader.offset += 4;
|
|
110
|
-
return value;
|
|
111
|
-
}
|
|
112
|
-
case 6: {
|
|
113
|
-
const value = reader.view.getBigInt64(reader.offset, true);
|
|
114
|
-
reader.offset += 8;
|
|
115
|
-
return value;
|
|
116
|
-
}
|
|
117
|
-
case 7: {
|
|
118
|
-
const value = reader.view.getFloat64(reader.offset, true);
|
|
119
|
-
reader.offset += 8;
|
|
120
|
-
return value;
|
|
121
|
-
}
|
|
122
|
-
case 8: return readVariantDecimal(reader, 4);
|
|
123
|
-
case 9: return readVariantDecimal(reader, 8);
|
|
124
|
-
case 10: return readVariantDecimal(reader, 16);
|
|
125
|
-
case 11: {
|
|
126
|
-
const value = reader.view.getInt32(reader.offset, true);
|
|
127
|
-
reader.offset += 4;
|
|
128
|
-
return parsers.dateFromDays(value);
|
|
129
|
-
}
|
|
130
|
-
case 12:
|
|
131
|
-
case 13: {
|
|
132
|
-
const value = reader.view.getBigInt64(reader.offset, true);
|
|
133
|
-
reader.offset += 8;
|
|
134
|
-
return parsers.timestampFromMicroseconds(value);
|
|
135
|
-
}
|
|
136
|
-
case 14: {
|
|
137
|
-
const value = reader.view.getFloat32(reader.offset, true);
|
|
138
|
-
reader.offset += 4;
|
|
139
|
-
return value;
|
|
140
|
-
}
|
|
141
|
-
case 15: return readVariantBinary(reader);
|
|
142
|
-
case 16: {
|
|
143
|
-
const bytes = readVariantBinary(reader);
|
|
144
|
-
return decoder.decode(bytes);
|
|
145
|
-
}
|
|
146
|
-
case 17: {
|
|
147
|
-
const value = reader.view.getBigInt64(reader.offset, true);
|
|
148
|
-
reader.offset += 8;
|
|
149
|
-
return value;
|
|
150
|
-
}
|
|
151
|
-
case 18:
|
|
152
|
-
case 19: {
|
|
153
|
-
const value = reader.view.getBigInt64(reader.offset, true);
|
|
154
|
-
reader.offset += 8;
|
|
155
|
-
return parsers.timestampFromNanoseconds(value);
|
|
156
|
-
}
|
|
157
|
-
case 20: {
|
|
158
|
-
const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
|
|
159
|
-
reader.offset += 16;
|
|
160
|
-
const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
161
|
-
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
162
|
-
}
|
|
163
|
-
default: throw new Error(`parquet unsupported variant primitive type: ${typeId}`);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
function readVariantObject(reader, header, metadata, parsers) {
|
|
167
|
-
const offsetWidth = (header & 3) + 1;
|
|
168
|
-
const idWidth = (header >> 2 & 3) + 1;
|
|
169
|
-
const numElements = header >> 4 & 1 ? readUnsigned(reader, 4) : reader.view.getUint8(reader.offset++);
|
|
170
|
-
const fieldIds = new Array(numElements);
|
|
171
|
-
for (let i = 0; i < numElements; i++) fieldIds[i] = readUnsigned(reader, idWidth);
|
|
172
|
-
const offsets = new Array(numElements + 1);
|
|
173
|
-
for (let i = 0; i < offsets.length; i++) offsets[i] = readUnsigned(reader, offsetWidth);
|
|
174
|
-
const out = {};
|
|
175
|
-
for (let i = 0; i < numElements; i++) {
|
|
176
|
-
const key = metadata.dictionary[fieldIds[i]];
|
|
177
|
-
out[key] = readVariant({
|
|
178
|
-
view: reader.view,
|
|
179
|
-
offset: reader.offset + offsets[i]
|
|
180
|
-
}, metadata, parsers);
|
|
181
|
-
}
|
|
182
|
-
reader.offset += offsets[offsets.length - 1];
|
|
183
|
-
return out;
|
|
184
|
-
}
|
|
185
|
-
function readVariantArray(reader, header, metadata, parsers) {
|
|
186
|
-
const fieldOffsetSize = header & 3;
|
|
187
|
-
const isLarge = header >> 2 & 1;
|
|
188
|
-
const offsetWidth = fieldOffsetSize + 1;
|
|
189
|
-
const numElements = readUnsigned(reader, isLarge ? 4 : 1);
|
|
190
|
-
const offsets = new Array(numElements + 1);
|
|
191
|
-
for (let i = 0; i < offsets.length; i++) offsets[i] = readUnsigned(reader, offsetWidth);
|
|
192
|
-
const valuesStart = reader.offset;
|
|
193
|
-
const result = new Array(numElements);
|
|
194
|
-
for (let i = 0; i < numElements; i++) {
|
|
195
|
-
const valueReader = {
|
|
196
|
-
view: reader.view,
|
|
197
|
-
offset: valuesStart + offsets[i]
|
|
198
|
-
};
|
|
199
|
-
result[i] = readVariant(valueReader, metadata, parsers);
|
|
200
|
-
}
|
|
201
|
-
reader.offset = valuesStart + offsets[offsets.length - 1];
|
|
202
|
-
return result;
|
|
203
|
-
}
|
|
204
|
-
function readVariantDecimal(reader, width) {
|
|
205
|
-
const scale = reader.view.getUint8(reader.offset);
|
|
206
|
-
reader.offset += 1;
|
|
207
|
-
let unscaled;
|
|
208
|
-
if (width === 4) {
|
|
209
|
-
unscaled = BigInt(reader.view.getInt32(reader.offset, true));
|
|
210
|
-
reader.offset += 4;
|
|
211
|
-
} else if (width === 8) {
|
|
212
|
-
unscaled = reader.view.getBigInt64(reader.offset, true);
|
|
213
|
-
reader.offset += 8;
|
|
214
|
-
} else {
|
|
215
|
-
const low = reader.view.getBigUint64(reader.offset, true);
|
|
216
|
-
unscaled = reader.view.getBigInt64(reader.offset + 8, true) << 64n | low;
|
|
217
|
-
reader.offset += 16;
|
|
218
|
-
}
|
|
219
|
-
return Number(unscaled) * 10 ** -scale;
|
|
220
|
-
}
|
|
221
|
-
function readVariantBinary(reader) {
|
|
222
|
-
const length = reader.view.getUint32(reader.offset, true);
|
|
223
|
-
reader.offset += 4;
|
|
224
|
-
const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
|
|
225
|
-
reader.offset += length;
|
|
226
|
-
return bytes;
|
|
227
|
-
}
|
|
228
|
-
export { decodeVariantColumn };
|
|
1
|
+
import "./convert.mjs";
|
|
2
|
+
new TextDecoder();
|
|
3
|
+
export {};
|
|
@@ -1,94 +1 @@
|
|
|
1
|
-
|
|
2
|
-
const flags = getFlags(reader);
|
|
3
|
-
if (flags.type === 1) return {
|
|
4
|
-
type: "Point",
|
|
5
|
-
coordinates: readPosition(reader, flags)
|
|
6
|
-
};
|
|
7
|
-
else if (flags.type === 2) return {
|
|
8
|
-
type: "LineString",
|
|
9
|
-
coordinates: readLine(reader, flags)
|
|
10
|
-
};
|
|
11
|
-
else if (flags.type === 3) return {
|
|
12
|
-
type: "Polygon",
|
|
13
|
-
coordinates: readPolygon(reader, flags)
|
|
14
|
-
};
|
|
15
|
-
else if (flags.type === 4) {
|
|
16
|
-
const points = [];
|
|
17
|
-
for (let i = 0; i < flags.count; i++) points.push(readPosition(reader, getFlags(reader)));
|
|
18
|
-
return {
|
|
19
|
-
type: "MultiPoint",
|
|
20
|
-
coordinates: points
|
|
21
|
-
};
|
|
22
|
-
} else if (flags.type === 5) {
|
|
23
|
-
const lines = [];
|
|
24
|
-
for (let i = 0; i < flags.count; i++) lines.push(readLine(reader, getFlags(reader)));
|
|
25
|
-
return {
|
|
26
|
-
type: "MultiLineString",
|
|
27
|
-
coordinates: lines
|
|
28
|
-
};
|
|
29
|
-
} else if (flags.type === 6) {
|
|
30
|
-
const polygons = [];
|
|
31
|
-
for (let i = 0; i < flags.count; i++) polygons.push(readPolygon(reader, getFlags(reader)));
|
|
32
|
-
return {
|
|
33
|
-
type: "MultiPolygon",
|
|
34
|
-
coordinates: polygons
|
|
35
|
-
};
|
|
36
|
-
} else if (flags.type === 7) {
|
|
37
|
-
const geometries = [];
|
|
38
|
-
for (let i = 0; i < flags.count; i++) geometries.push(wkbToGeojson(reader));
|
|
39
|
-
return {
|
|
40
|
-
type: "GeometryCollection",
|
|
41
|
-
geometries
|
|
42
|
-
};
|
|
43
|
-
} else throw new Error(`Unsupported geometry type: ${flags.type}`);
|
|
44
|
-
}
|
|
45
|
-
function getFlags(reader) {
|
|
46
|
-
const { view } = reader;
|
|
47
|
-
const littleEndian = view.getUint8(reader.offset++) === 1;
|
|
48
|
-
const rawType = view.getUint32(reader.offset, littleEndian);
|
|
49
|
-
reader.offset += 4;
|
|
50
|
-
const type = rawType % 1e3;
|
|
51
|
-
const flags = Math.floor(rawType / 1e3);
|
|
52
|
-
let count = 0;
|
|
53
|
-
if (type > 1 && type <= 7) {
|
|
54
|
-
count = view.getUint32(reader.offset, littleEndian);
|
|
55
|
-
reader.offset += 4;
|
|
56
|
-
}
|
|
57
|
-
let dim = 2;
|
|
58
|
-
if (flags) dim++;
|
|
59
|
-
if (flags === 3) dim++;
|
|
60
|
-
return {
|
|
61
|
-
littleEndian,
|
|
62
|
-
type,
|
|
63
|
-
dim,
|
|
64
|
-
count
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
function readPosition(reader, flags) {
|
|
68
|
-
const points = [];
|
|
69
|
-
for (let i = 0; i < flags.dim; i++) {
|
|
70
|
-
const coord = reader.view.getFloat64(reader.offset, flags.littleEndian);
|
|
71
|
-
reader.offset += 8;
|
|
72
|
-
points.push(coord);
|
|
73
|
-
}
|
|
74
|
-
return points;
|
|
75
|
-
}
|
|
76
|
-
function readLine(reader, flags) {
|
|
77
|
-
const points = [];
|
|
78
|
-
for (let i = 0; i < flags.count; i++) points.push(readPosition(reader, flags));
|
|
79
|
-
return points;
|
|
80
|
-
}
|
|
81
|
-
function readPolygon(reader, flags) {
|
|
82
|
-
const { view } = reader;
|
|
83
|
-
const rings = [];
|
|
84
|
-
for (let r = 0; r < flags.count; r++) {
|
|
85
|
-
const count = view.getUint32(reader.offset, flags.littleEndian);
|
|
86
|
-
reader.offset += 4;
|
|
87
|
-
rings.push(readLine(reader, {
|
|
88
|
-
...flags,
|
|
89
|
-
count
|
|
90
|
-
}));
|
|
91
|
-
}
|
|
92
|
-
return rings;
|
|
93
|
-
}
|
|
94
|
-
export { wkbToGeojson };
|
|
1
|
+
export {};
|