@gscdump/engine 1.3.2 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/duckdb-node.d.mts +7 -0
- package/dist/adapters/duckdb-node.mjs +94 -0
- package/dist/adapters/filesystem.d.mts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/hyparquet.d.mts +4 -3
- package/dist/adapters/hyparquet.mjs +7 -3
- package/dist/adapters/node-harness.d.mts +38 -0
- package/dist/adapters/node-harness.mjs +47 -0
- package/dist/adapters/node.d.mts +4 -100
- package/dist/adapters/node.mjs +4 -240
- package/dist/adapters/parquet-attach.d.mts +23 -0
- package/dist/adapters/parquet-attach.mjs +21 -0
- package/dist/adapters/r2-manifest.d.mts +82 -0
- package/dist/adapters/r2-manifest.mjs +364 -0
- package/dist/adapters/r2.d.mts +1 -1
- package/dist/adapters/read-optional.mjs +15 -0
- package/dist/adapters/snapshot-attach.d.mts +38 -0
- package/dist/adapters/snapshot-attach.mjs +85 -0
- package/dist/analysis-types.d.mts +56 -1
- package/dist/analyzer/define.d.mts +48 -0
- package/dist/analyzer/define.mjs +60 -0
- package/dist/analyzer/dispatch.d.mts +17 -0
- package/dist/analyzer/index.d.mts +4 -14
- package/dist/analyzer/index.mjs +4 -112
- package/dist/analyzer/registry.d.mts +33 -0
- package/dist/analyzer/registry.mjs +49 -0
- package/dist/analyzer/types.d.mts +95 -0
- package/dist/analyzer/types.mjs +5 -0
- package/dist/coerce.d.mts +4 -0
- package/dist/compaction.d.mts +14 -0
- package/dist/{_chunks/parquet-plan.mjs → compaction.mjs} +2 -138
- package/dist/contracts.d.mts +1 -2
- package/dist/{_chunks/schema.d.mts → drizzle-schema.d.mts} +22 -54
- package/dist/{_chunks/schema.mjs → drizzle-schema.mjs} +2 -78
- package/dist/{_chunks/snapshot.d.mts → duckdb.d.mts} +1 -14
- package/dist/duckdb.mjs +207 -0
- package/dist/engine.d.mts +4 -0
- package/dist/engine.mjs +385 -0
- package/dist/entities.d.mts +3 -52
- package/dist/entities.mjs +867 -2
- package/dist/errors.d.mts +114 -1
- package/dist/gc.mjs +92 -0
- package/dist/iceberg/append-sink.d.mts +14 -0
- package/dist/iceberg/append-sink.mjs +142 -0
- package/dist/iceberg/catalog.d.mts +71 -0
- package/dist/iceberg/catalog.mjs +109 -0
- package/dist/iceberg/index.d.mts +4 -13
- package/dist/iceberg/index.mjs +3 -244
- package/dist/iceberg/overwrite-writer.d.mts +161 -0
- package/dist/iceberg/overwrite-writer.mjs +102 -0
- package/dist/iceberg/pyiceberg-runtime.mjs +34 -0
- package/dist/iceberg/schema.d.mts +69 -0
- package/dist/{_chunks/schema2.mjs → iceberg/schema.mjs} +3 -2
- package/dist/index.d.mts +14 -190
- package/dist/index.mjs +10 -645
- package/dist/ingest-accumulator.d.mts +1 -1
- package/dist/ingest.d.mts +1 -1
- package/dist/layout.d.mts +31 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.bitreader.mjs +88 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.blocks.mjs +120 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.context.mjs +1805 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.contextmap.mjs +49 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.dictionary.mjs +984 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.huffman.mjs +266 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.mjs +340 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.prefix.mjs +109 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.streams.mjs +21 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/brotli.transform.mjs +190 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/gzip.huffman.mjs +121 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/gzip.mjs +176 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/lz4.mjs +52 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bloom.mjs +93 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs +91 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/column.mjs +186 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/datapage.mjs +113 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/delta.mjs +111 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/dictionary.mjs +99 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/dremel.mjs +116 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/encoding.mjs +51 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/geospatial.mjs +84 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/index.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/indexes.mjs +31 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/metadata.mjs +138 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs +8 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/parquet-writer.mjs +139 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/plain.mjs +68 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/schema.mjs +254 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/snappy.mjs +125 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/splitstream.mjs +38 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/thrift.mjs +79 -0
- package/dist/{_chunks/libs/hyparquet-writer.d.mts → node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/types.d.mts} +2 -2
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/unconvert.mjs +247 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/variant.mjs +321 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/wkb.mjs +79 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/write-rows.mjs +122 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/write.mjs +5 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/bytewriter.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/index.d.mts +10 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/node.d.mts +3 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/parquet-writer.d.mts +2 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/schema.d.mts +2 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/wkb.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/write-rows.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet-writer@0.16.1_patch_hash_7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/types/write.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/assemble.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/bloom.mjs +69 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/column.mjs +8 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/constants.mjs +80 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/convert.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/datapage.mjs +6 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/delta.mjs +2 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/encoding.mjs +2 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/filter.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/geoparquet.mjs +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/index.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/indexes.mjs +5 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/metadata.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/node.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/plain.mjs +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/plan.mjs +5 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/query.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/read.mjs +6 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/rowgroup.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/schema.mjs +54 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/snappy.mjs +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/thrift.mjs +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/types.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/utils.mjs +19 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/variant.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/wkb.mjs +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/src/xxhash.mjs +68 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/index.d.mts +7 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/indexes.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/metadata.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/node.d.mts +2 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/query.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/read.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/snappy.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.1/node_modules/hyparquet/types/utils.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/assemble.mjs +144 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/bloom.mjs +124 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/column.mjs +127 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/constants.mjs +75 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/convert.mjs +118 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/datapage.mjs +144 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/delta.mjs +65 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/encoding.mjs +76 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/filter.mjs +98 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/geoparquet.mjs +24 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/index.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/indexes.mjs +16 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs +235 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/node.mjs +7 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/plain.mjs +78 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/plan.mjs +147 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/query.mjs +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs +106 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/rowgroup.mjs +154 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/schema.mjs +75 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/snappy.mjs +70 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/thrift.mjs +86 -0
- package/dist/{_chunks/libs/hyparquet.d.mts → node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/types.d.mts} +6 -8
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/utils.mjs +31 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/variant.mjs +225 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/wkb.mjs +94 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/xxhash.mjs +68 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/index.d.mts +10 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/indexes.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/metadata.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/node.d.mts +3 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/query.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/read.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/snappy.d.mts +1 -0
- package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/utils.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/avro/avro.metadata.mjs +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/avro/avro.read.mjs +4 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/avro/avro.write.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/file.mjs +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/loadTable.mjs +3 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs +73 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/create.mjs +67 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/delete.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/fetch.mjs +8 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs +13 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/json.mjs +147 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/metadata.mjs +3 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/prune.mjs +4 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/puffin/deletion-vector.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/puffin/puffin.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/puffin/roaring.mjs +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/read.mjs +9 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/s3.mjs +3 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/schema.mjs +76 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/sigv4.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/sql/icebergDataSource.mjs +8 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/sql/icebergQuery.mjs +5 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/sql/whereFilter.mjs +1 -0
- package/dist/{_chunks/libs/icebird.d.mts → node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/types.d.mts} +5 -33
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/utils.mjs +8 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/commit.mjs +5 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/conversions.mjs +19 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/delete-file.mjs +3 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/geospatial.mjs +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/manifest-list.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/manifest.mjs +3 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/parquet.mjs +5 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/partition.mjs +75 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/rewrite.mjs +10 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/serde.mjs +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/snapshot.mjs +5 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/sort.mjs +3 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/stage-deletion-vector.mjs +11 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/stage-position-delete.mjs +8 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/stage.mjs +10 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/stats.mjs +4 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/transform.mjs +60 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs +45 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/catalog/file.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/catalog/rest.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/create.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/fetch.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/index.d.mts +13 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/manifest.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/metadata.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/read.d.mts +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/s3.d.mts +1 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/sql/icebergDataSource.d.mts +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/sql/icebergQuery.d.mts +2 -0
- package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/write/write.d.mts +32 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/ast.d.mts +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/backend/dataSource.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/accumulator.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/aggregates.mjs +10 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/execute.mjs +15 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/join.mjs +5 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/scanColumn.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/sort.mjs +5 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/streamingAggregate.mjs +9 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/utils.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/window.mjs +5 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/execute/yield.mjs +17 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/expression/alias.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/expression/binary.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/expression/date.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/expression/evaluate.mjs +15 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/expression/math.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/expression/regexp.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/expression/strings.mjs +3 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/index.d.mts +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/index.mjs +9 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/expression.mjs +6 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/extractTables.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/functions.mjs +5 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/joins.mjs +7 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/parse.mjs +10 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/primary.mjs +8 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/state.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/tokenize.mjs +3 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/parse/types.d.mts +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/plan/columns.mjs +3 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/plan/plan.mjs +7 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/plan/types.d.mts +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/bbox.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/equality.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/operations.mjs +5 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/pointRelations.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/primitives.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/segments.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/spatial.mjs +6 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/spatial/wkt.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/types.d.mts +3 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/validation/aggregates.mjs +3 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/validation/executionErrors.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/validation/functions.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/validation/keywords.mjs +1 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/validation/parseErrors.mjs +2 -0
- package/dist/node_modules/.pnpm/squirreling@0.15.0/node_modules/squirreling/src/validation/tables.mjs +2 -0
- package/dist/parquet-plan.d.mts +15 -0
- package/dist/parquet-plan.mjs +139 -0
- package/dist/parquet-pushdown.mjs +58 -0
- package/dist/period/index.d.mts +52 -1
- package/dist/planner.d.mts +2 -14
- package/dist/planner.mjs +2 -1
- package/dist/profile.d.mts +21 -0
- package/dist/profile.mjs +27 -0
- package/dist/query-dim.d.mts +54 -0
- package/dist/query-dim.mjs +91 -0
- package/dist/report/define.d.mts +8 -0
- package/dist/report/define.mjs +15 -0
- package/dist/report/hash.d.mts +13 -0
- package/dist/report/hash.mjs +29 -0
- package/dist/report/index.d.mts +4 -170
- package/dist/report/index.mjs +3 -55
- package/dist/report/registry.d.mts +18 -0
- package/dist/report/registry.mjs +14 -0
- package/dist/report/types.d.mts +139 -0
- package/dist/resolver/adapter.d.mts +13 -0
- package/dist/resolver/adapter.mjs +27 -0
- package/dist/resolver/canonical-source.d.mts +5 -0
- package/dist/resolver/canonical-source.mjs +14 -0
- package/dist/resolver/compile.d.mts +15 -0
- package/dist/resolver/compile.mjs +396 -0
- package/dist/resolver/datasets.d.mts +16 -0
- package/dist/resolver/datasets.mjs +159 -0
- package/dist/resolver/extras-overlay.d.mts +37 -0
- package/dist/resolver/extras-overlay.mjs +17 -0
- package/dist/resolver/filter-utils.d.mts +10 -0
- package/dist/resolver/filter-utils.mjs +59 -0
- package/dist/resolver/fragments.d.mts +60 -0
- package/dist/resolver/fragments.mjs +206 -0
- package/dist/resolver/index.d.mts +11 -371
- package/dist/resolver/index.mjs +10 -1
- package/dist/resolver/pg-adapter.d.mts +56 -0
- package/dist/resolver/pg-adapter.mjs +110 -0
- package/dist/resolver/run-query.d.mts +172 -0
- package/dist/resolver/run-query.mjs +241 -0
- package/dist/resolver/schema-drift.d.mts +15 -0
- package/dist/resolver/schema-drift.mjs +12 -0
- package/dist/rollups.d.mts +4 -4
- package/dist/rollups.mjs +3 -2
- package/dist/schedule.mjs +100 -0
- package/dist/schema.d.mts +34 -1
- package/dist/schema.mjs +78 -1
- package/dist/sink-node.d.mts +1 -159
- package/dist/sink-node.mjs +1 -133
- package/dist/sink.d.mts +125 -0
- package/dist/sinks/in-memory-sink.d.mts +25 -0
- package/dist/sinks/in-memory-sink.mjs +61 -0
- package/dist/sinks/index.d.mts +2 -0
- package/dist/sinks/index.mjs +2 -0
- package/dist/snapshot.d.mts +14 -0
- package/dist/source/attached-table.d.mts +45 -0
- package/dist/source/attached-table.mjs +55 -0
- package/dist/source/create-sql-query-source.d.mts +24 -0
- package/dist/source/create-sql-query-source.mjs +28 -0
- package/dist/source/index.d.mts +6 -66
- package/dist/source/index.mjs +9 -82
- package/dist/source/source-types.d.mts +49 -0
- package/dist/{_chunks/storage.d.mts → storage.d.mts} +4 -14
- package/dist/sync-config.d.mts +36 -0
- package/dist/sync-config.mjs +98 -0
- package/dist/vendor/hysnappy-purejs.mjs +12 -1
- package/package.json +4 -4
- package/dist/_chunks/analysis-types.d.mts +0 -57
- package/dist/_chunks/contracts.d.mts +0 -1
- package/dist/_chunks/engine.mjs +0 -731
- package/dist/_chunks/entities.mjs +0 -967
- package/dist/_chunks/errors.d.mts +0 -115
- package/dist/_chunks/index.d.mts +0 -53
- package/dist/_chunks/libs/hyparquet-compressors.mjs +0 -2796
- package/dist/_chunks/libs/hyparquet-writer.mjs +0 -2524
- package/dist/_chunks/libs/hyparquet.mjs +0 -2596
- package/dist/_chunks/libs/icebird.mjs +0 -564
- package/dist/_chunks/registry.d.mts +0 -213
- package/dist/_chunks/resolver.mjs +0 -1220
- package/dist/_chunks/sink.d.mts +0 -257
- /package/dist/{_chunks → analyzer}/dispatch.mjs +0 -0
- /package/dist/{_chunks/coerce.mjs → coerce.mjs} +0 -0
- /package/dist/{_chunks/layout.mjs → layout.mjs} +0 -0
- /package/dist/{_chunks/manifest-store-utils.mjs → manifest-store-utils.mjs} +0 -0
- /package/dist/{_chunks/libs/fzstd.mjs → node_modules/.pnpm/fzstd@0.1.1/node_modules/fzstd/esm/index.mjs} +0 -0
- /package/dist/{_chunks → resolver}/types.d.mts +0 -0
- /package/dist/{_chunks/schedule.d.mts → schedule.d.mts} +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TableName } from "@gscdump/contracts";
|
|
2
2
|
declare const pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
3
3
|
name: "pages";
|
|
4
4
|
schema: undefined;
|
|
@@ -79,7 +79,7 @@ declare const pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
79
79
|
generated: undefined;
|
|
80
80
|
}>;
|
|
81
81
|
};
|
|
82
|
-
dialect:
|
|
82
|
+
dialect: 'pg';
|
|
83
83
|
}>;
|
|
84
84
|
declare const queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
85
85
|
name: "queries";
|
|
@@ -161,7 +161,7 @@ declare const queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
161
161
|
generated: undefined;
|
|
162
162
|
}>;
|
|
163
163
|
};
|
|
164
|
-
dialect:
|
|
164
|
+
dialect: 'pg';
|
|
165
165
|
}>;
|
|
166
166
|
declare const countries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
167
167
|
name: "countries";
|
|
@@ -243,7 +243,7 @@ declare const countries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
243
243
|
generated: undefined;
|
|
244
244
|
}>;
|
|
245
245
|
};
|
|
246
|
-
dialect:
|
|
246
|
+
dialect: 'pg';
|
|
247
247
|
}>;
|
|
248
248
|
declare const page_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
249
249
|
name: "page_queries";
|
|
@@ -340,7 +340,7 @@ declare const page_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
340
340
|
generated: undefined;
|
|
341
341
|
}>;
|
|
342
342
|
};
|
|
343
|
-
dialect:
|
|
343
|
+
dialect: 'pg';
|
|
344
344
|
}>;
|
|
345
345
|
/**
|
|
346
346
|
* Per-`(site, search_type, date)` daily totals + device breakdown.
|
|
@@ -573,7 +573,7 @@ declare const dates: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
573
573
|
generated: undefined;
|
|
574
574
|
}>;
|
|
575
575
|
};
|
|
576
|
-
dialect:
|
|
576
|
+
dialect: 'pg';
|
|
577
577
|
}>;
|
|
578
578
|
declare const search_appearance: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
579
579
|
name: "search_appearance";
|
|
@@ -655,7 +655,7 @@ declare const search_appearance: import("drizzle-orm/pg-core").PgTableWithColumn
|
|
|
655
655
|
generated: undefined;
|
|
656
656
|
}>;
|
|
657
657
|
};
|
|
658
|
-
dialect:
|
|
658
|
+
dialect: 'pg';
|
|
659
659
|
}>;
|
|
660
660
|
declare const search_appearance_pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
661
661
|
name: "search_appearance_pages";
|
|
@@ -752,7 +752,7 @@ declare const search_appearance_pages: import("drizzle-orm/pg-core").PgTableWith
|
|
|
752
752
|
generated: undefined;
|
|
753
753
|
}>;
|
|
754
754
|
};
|
|
755
|
-
dialect:
|
|
755
|
+
dialect: 'pg';
|
|
756
756
|
}>;
|
|
757
757
|
declare const search_appearance_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
758
758
|
name: "search_appearance_queries";
|
|
@@ -849,7 +849,7 @@ declare const search_appearance_queries: import("drizzle-orm/pg-core").PgTableWi
|
|
|
849
849
|
generated: undefined;
|
|
850
850
|
}>;
|
|
851
851
|
};
|
|
852
|
-
dialect:
|
|
852
|
+
dialect: 'pg';
|
|
853
853
|
}>;
|
|
854
854
|
declare const search_appearance_page_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
855
855
|
name: "search_appearance_page_queries";
|
|
@@ -961,7 +961,7 @@ declare const search_appearance_page_queries: import("drizzle-orm/pg-core").PgTa
|
|
|
961
961
|
generated: undefined;
|
|
962
962
|
}>;
|
|
963
963
|
};
|
|
964
|
-
dialect:
|
|
964
|
+
dialect: 'pg';
|
|
965
965
|
}>;
|
|
966
966
|
declare const hourly_pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
967
967
|
name: "hourly_pages";
|
|
@@ -1058,7 +1058,7 @@ declare const hourly_pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
|
1058
1058
|
generated: undefined;
|
|
1059
1059
|
}>;
|
|
1060
1060
|
};
|
|
1061
|
-
dialect:
|
|
1061
|
+
dialect: 'pg';
|
|
1062
1062
|
}>;
|
|
1063
1063
|
declare const drizzleSchema: {
|
|
1064
1064
|
pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
@@ -1141,7 +1141,7 @@ declare const drizzleSchema: {
|
|
|
1141
1141
|
generated: undefined;
|
|
1142
1142
|
}>;
|
|
1143
1143
|
};
|
|
1144
|
-
dialect:
|
|
1144
|
+
dialect: 'pg';
|
|
1145
1145
|
}>;
|
|
1146
1146
|
queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1147
1147
|
name: "queries";
|
|
@@ -1223,7 +1223,7 @@ declare const drizzleSchema: {
|
|
|
1223
1223
|
generated: undefined;
|
|
1224
1224
|
}>;
|
|
1225
1225
|
};
|
|
1226
|
-
dialect:
|
|
1226
|
+
dialect: 'pg';
|
|
1227
1227
|
}>;
|
|
1228
1228
|
countries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1229
1229
|
name: "countries";
|
|
@@ -1305,7 +1305,7 @@ declare const drizzleSchema: {
|
|
|
1305
1305
|
generated: undefined;
|
|
1306
1306
|
}>;
|
|
1307
1307
|
};
|
|
1308
|
-
dialect:
|
|
1308
|
+
dialect: 'pg';
|
|
1309
1309
|
}>;
|
|
1310
1310
|
page_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1311
1311
|
name: "page_queries";
|
|
@@ -1402,7 +1402,7 @@ declare const drizzleSchema: {
|
|
|
1402
1402
|
generated: undefined;
|
|
1403
1403
|
}>;
|
|
1404
1404
|
};
|
|
1405
|
-
dialect:
|
|
1405
|
+
dialect: 'pg';
|
|
1406
1406
|
}>;
|
|
1407
1407
|
dates: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1408
1408
|
name: "dates";
|
|
@@ -1619,7 +1619,7 @@ declare const drizzleSchema: {
|
|
|
1619
1619
|
generated: undefined;
|
|
1620
1620
|
}>;
|
|
1621
1621
|
};
|
|
1622
|
-
dialect:
|
|
1622
|
+
dialect: 'pg';
|
|
1623
1623
|
}>;
|
|
1624
1624
|
search_appearance: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1625
1625
|
name: "search_appearance";
|
|
@@ -1701,7 +1701,7 @@ declare const drizzleSchema: {
|
|
|
1701
1701
|
generated: undefined;
|
|
1702
1702
|
}>;
|
|
1703
1703
|
};
|
|
1704
|
-
dialect:
|
|
1704
|
+
dialect: 'pg';
|
|
1705
1705
|
}>;
|
|
1706
1706
|
search_appearance_pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1707
1707
|
name: "search_appearance_pages";
|
|
@@ -1798,7 +1798,7 @@ declare const drizzleSchema: {
|
|
|
1798
1798
|
generated: undefined;
|
|
1799
1799
|
}>;
|
|
1800
1800
|
};
|
|
1801
|
-
dialect:
|
|
1801
|
+
dialect: 'pg';
|
|
1802
1802
|
}>;
|
|
1803
1803
|
search_appearance_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1804
1804
|
name: "search_appearance_queries";
|
|
@@ -1895,7 +1895,7 @@ declare const drizzleSchema: {
|
|
|
1895
1895
|
generated: undefined;
|
|
1896
1896
|
}>;
|
|
1897
1897
|
};
|
|
1898
|
-
dialect:
|
|
1898
|
+
dialect: 'pg';
|
|
1899
1899
|
}>;
|
|
1900
1900
|
search_appearance_page_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
1901
1901
|
name: "search_appearance_page_queries";
|
|
@@ -2007,7 +2007,7 @@ declare const drizzleSchema: {
|
|
|
2007
2007
|
generated: undefined;
|
|
2008
2008
|
}>;
|
|
2009
2009
|
};
|
|
2010
|
-
dialect:
|
|
2010
|
+
dialect: 'pg';
|
|
2011
2011
|
}>;
|
|
2012
2012
|
hourly_pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
|
|
2013
2013
|
name: "hourly_pages";
|
|
@@ -2104,7 +2104,7 @@ declare const drizzleSchema: {
|
|
|
2104
2104
|
generated: undefined;
|
|
2105
2105
|
}>;
|
|
2106
2106
|
};
|
|
2107
|
-
dialect:
|
|
2107
|
+
dialect: 'pg';
|
|
2108
2108
|
}>;
|
|
2109
2109
|
};
|
|
2110
2110
|
type DrizzleSchema = typeof drizzleSchema;
|
|
@@ -2113,36 +2113,4 @@ declare const TABLE_METADATA: Record<TableName, {
|
|
|
2113
2113
|
clusterKey: string[];
|
|
2114
2114
|
version: number;
|
|
2115
2115
|
}>;
|
|
2116
|
-
|
|
2117
|
-
declare function currentSchemaVersion(table: TableName): number;
|
|
2118
|
-
/**
|
|
2119
|
-
* DATE column names for a table. The single schema-derived source every read
|
|
2120
|
-
* path uses to build the legacy-VARCHAR date canonicalization (see
|
|
2121
|
-
* `dateReplaceClause` in `./sql-fragments`), so the engine codec and the CLI
|
|
2122
|
-
* `dump`/`export` commands agree on which columns to cast.
|
|
2123
|
-
*/
|
|
2124
|
-
declare function dateColumnsFor(table: TableName): string[];
|
|
2125
|
-
declare function allTables(): readonly TableName[];
|
|
2126
|
-
declare function inferTable(dimensions: readonly string[]): TableName;
|
|
2127
|
-
/**
|
|
2128
|
-
* Natural-key columns for a table: the `sortKey` dimensions that uniquely
|
|
2129
|
-
* identify a metric row (e.g. `['date', 'url']` for `pages`).
|
|
2130
|
-
*/
|
|
2131
|
-
declare function naturalKeyColumns(table: TableName): readonly string[];
|
|
2132
|
-
/**
|
|
2133
|
-
* Collapse rows that share a natural key (see {@link naturalKeyColumns}) to a
|
|
2134
|
-
* single survivor.
|
|
2135
|
-
*
|
|
2136
|
-
* Correct write and compaction inputs never collide on the natural key — each
|
|
2137
|
-
* (date, dimension) tuple is produced exactly once per slice — so on healthy
|
|
2138
|
-
* data this is a no-op. It exists as a recurrence guard: the 2026-04 monthly
|
|
2139
|
-
* compaction corruption folded a complete month back on top of its own daily
|
|
2140
|
-
* inputs, doubling every row. Absorbing duplicates at every write/compaction
|
|
2141
|
-
* boundary keeps that class of bug from inflating impressions downstream.
|
|
2142
|
-
*
|
|
2143
|
-
* Last occurrence wins. Observed duplicates are byte-identical, so the choice
|
|
2144
|
-
* of survivor is immaterial.
|
|
2145
|
-
*/
|
|
2146
|
-
declare function dedupeByNaturalKey(table: TableName, rows: readonly Row[]): Row[];
|
|
2147
|
-
declare function dimensionToColumn(dim: string, _table: TableName): string;
|
|
2148
|
-
export { type ColumnDef, type ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema$1 as TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
|
|
2116
|
+
export { DrizzleSchema, TABLE_METADATA, countries, dates, drizzleSchema, hourly_pages, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { date, doublePrecision,
|
|
1
|
+
import { date, doublePrecision, integer, pgTable, varchar } from "drizzle-orm/pg-core";
|
|
2
2
|
function metricCols() {
|
|
3
3
|
return {
|
|
4
4
|
clicks: integer("clicks").notNull(),
|
|
@@ -180,80 +180,4 @@ const TABLE_METADATA = {
|
|
|
180
180
|
version: 2
|
|
181
181
|
}
|
|
182
182
|
};
|
|
183
|
-
|
|
184
|
-
const t = sqlType.toLowerCase();
|
|
185
|
-
if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
|
|
186
|
-
if (t === "date" || t.startsWith("timestamp")) return "DATE";
|
|
187
|
-
if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
|
|
188
|
-
if (t === "bigint" || t === "int8") return "BIGINT";
|
|
189
|
-
if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
|
|
190
|
-
throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
|
|
191
|
-
}
|
|
192
|
-
function tableSchemaFrom(tableName) {
|
|
193
|
-
const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
|
|
194
|
-
name: col.name,
|
|
195
|
-
type: pgSqlTypeToColumnType(col.getSQLType()),
|
|
196
|
-
nullable: !col.notNull
|
|
197
|
-
}));
|
|
198
|
-
const meta = TABLE_METADATA[tableName];
|
|
199
|
-
return {
|
|
200
|
-
name: tableName,
|
|
201
|
-
columns,
|
|
202
|
-
sortKey: meta.sortKey,
|
|
203
|
-
version: meta.version
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
const METRIC_TABLES = [
|
|
207
|
-
"pages",
|
|
208
|
-
"queries",
|
|
209
|
-
"countries",
|
|
210
|
-
"page_queries",
|
|
211
|
-
"dates",
|
|
212
|
-
"search_appearance",
|
|
213
|
-
"search_appearance_pages",
|
|
214
|
-
"search_appearance_queries",
|
|
215
|
-
"search_appearance_page_queries",
|
|
216
|
-
"hourly_pages"
|
|
217
|
-
];
|
|
218
|
-
const SCHEMAS = Object.fromEntries(METRIC_TABLES.map((t) => [t, tableSchemaFrom(t)]));
|
|
219
|
-
function currentSchemaVersion(table) {
|
|
220
|
-
return SCHEMAS[table].version;
|
|
221
|
-
}
|
|
222
|
-
function dateColumnsFor(table) {
|
|
223
|
-
return SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
|
|
224
|
-
}
|
|
225
|
-
function allTables() {
|
|
226
|
-
return METRIC_TABLES;
|
|
227
|
-
}
|
|
228
|
-
function inferTable(dimensions) {
|
|
229
|
-
const dims = new Set(dimensions);
|
|
230
|
-
const hasPage = dims.has("page");
|
|
231
|
-
const hasQuery = dims.has("query");
|
|
232
|
-
if (dims.has("searchAppearance")) return hasPage && hasQuery ? "search_appearance_page_queries" : hasPage ? "search_appearance_pages" : hasQuery ? "search_appearance_queries" : "search_appearance";
|
|
233
|
-
if (hasPage && hasQuery) return "page_queries";
|
|
234
|
-
if (hasQuery) return "queries";
|
|
235
|
-
if (hasPage) return "pages";
|
|
236
|
-
if (dims.has("country")) return "countries";
|
|
237
|
-
if (dims.has("device")) return "dates";
|
|
238
|
-
return "dates";
|
|
239
|
-
}
|
|
240
|
-
function naturalKeyColumns(table) {
|
|
241
|
-
return TABLE_METADATA[table].sortKey;
|
|
242
|
-
}
|
|
243
|
-
function dedupeByNaturalKey(table, rows) {
|
|
244
|
-
const key = TABLE_METADATA[table].sortKey;
|
|
245
|
-
if (key.length === 0) return rows.slice();
|
|
246
|
-
const seen = /* @__PURE__ */ new Map();
|
|
247
|
-
for (const row of rows) {
|
|
248
|
-
const r = row;
|
|
249
|
-
const k = key.map((col) => `${r[col] ?? ""}`).join("\0");
|
|
250
|
-
seen.set(k, row);
|
|
251
|
-
}
|
|
252
|
-
return [...seen.values()];
|
|
253
|
-
}
|
|
254
|
-
function dimensionToColumn(dim, _table) {
|
|
255
|
-
if (dim === "page") return "url";
|
|
256
|
-
if (dim === "queryCanonical") return "query_canonical";
|
|
257
|
-
return dim;
|
|
258
|
-
}
|
|
259
|
-
export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
|
|
183
|
+
export { TABLE_METADATA, countries, dates, drizzleSchema, hourly_pages, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
|
|
@@ -24,17 +24,4 @@ interface DuckDBReadOptions {
|
|
|
24
24
|
}
|
|
25
25
|
declare function createDuckDBCodec(factory: DuckDBFactory, options?: DuckDBReadOptions): ParquetCodec;
|
|
26
26
|
declare function createDuckDBExecutor(factory: DuckDBFactory, options?: DuckDBReadOptions): QueryExecutor;
|
|
27
|
-
|
|
28
|
-
* Describes a hot/cold snapshot set. Produced by the snapshot builder,
|
|
29
|
-
* consumed by `attachSnapshotIndex`. Filenames are derived from `cold`
|
|
30
|
-
* via `cold-${yearMonth}.duckdb`; hot is always `hot.duckdb` when
|
|
31
|
-
* `hot: true`.
|
|
32
|
-
*/
|
|
33
|
-
interface SnapshotIndex {
|
|
34
|
-
version: 1;
|
|
35
|
-
builtAt: string;
|
|
36
|
-
cold: string[];
|
|
37
|
-
hot: boolean;
|
|
38
|
-
hotDays: number;
|
|
39
|
-
}
|
|
40
|
-
export { DuckDBFactory, DuckDBHandle, SnapshotIndex, createDuckDBCodec, createDuckDBExecutor };
|
|
27
|
+
export { DuckDBFactory, DuckDBHandle, DuckDBReadOptions, createDuckDBCodec, createDuckDBExecutor };
|
package/dist/duckdb.mjs
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { coerceRows } from "./coerce.mjs";
|
|
2
|
+
import { TABLE_METADATA } from "./drizzle-schema.mjs";
|
|
3
|
+
import { SCHEMAS, dateColumnsFor } from "./schema.mjs";
|
|
4
|
+
import { dateReplaceClause as dateReplaceClause$1 } from "./sql-fragments.mjs";
|
|
5
|
+
import { substituteNamedFiles } from "./parquet-plan.mjs";
|
|
6
|
+
import { sqlEscape } from "./sql-bind.mjs";
|
|
7
|
+
import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
|
|
8
|
+
const DEFAULT_BUFFER_READ_CONCURRENCY = 16;
|
|
9
|
+
const MAX_BUFFER_READ_CONCURRENCY = 64;
|
|
10
|
+
function bufferReadConcurrency(value, fileCount) {
|
|
11
|
+
return Math.max(1, Math.min(fileCount || 1, typeof value === "number" && Number.isFinite(value) ? Math.floor(value) : DEFAULT_BUFFER_READ_CONCURRENCY, MAX_BUFFER_READ_CONCURRENCY));
|
|
12
|
+
}
|
|
13
|
+
async function registerBufferedFiles(db, files, dataSource, registered, signal, maxConcurrency) {
|
|
14
|
+
const batchSize = bufferReadConcurrency(maxConcurrency, files.length);
|
|
15
|
+
for (let i = 0; i < files.length; i += batchSize) {
|
|
16
|
+
signal?.throwIfAborted();
|
|
17
|
+
const batch = files.slice(i, i + batchSize);
|
|
18
|
+
const buffers = await Promise.all(batch.map((file) => dataSource.read(file.key, void 0, signal)));
|
|
19
|
+
for (let j = 0; j < batch.length; j++) {
|
|
20
|
+
signal?.throwIfAborted();
|
|
21
|
+
const file = batch[j];
|
|
22
|
+
await db.registerFileBuffer(file.name, buffers[j]);
|
|
23
|
+
registered.push(file.name);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
async function encodeBytes(db, table, rows) {
|
|
28
|
+
const inName = db.makeTempPath("json");
|
|
29
|
+
const outName = db.makeTempPath("parquet");
|
|
30
|
+
const jsonBytes = encodeJsonBigintSafe(coerceRows(rows));
|
|
31
|
+
const registered = [];
|
|
32
|
+
await db.registerFileBuffer(inName, jsonBytes);
|
|
33
|
+
registered.push(inName);
|
|
34
|
+
try {
|
|
35
|
+
const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`;
|
|
36
|
+
await db.query(sql);
|
|
37
|
+
registered.push(outName);
|
|
38
|
+
return await db.copyFileToBuffer(outName);
|
|
39
|
+
} finally {
|
|
40
|
+
await db.dropFiles(registered);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
async function decodeBytes(db, bytes, table) {
|
|
44
|
+
const name = db.makeTempPath("parquet");
|
|
45
|
+
await db.registerFileBuffer(name, bytes);
|
|
46
|
+
try {
|
|
47
|
+
return await db.query(`SELECT * ${dateReplaceClause(table)} FROM read_parquet('${sqlEscape(name)}')`);
|
|
48
|
+
} finally {
|
|
49
|
+
await db.dropFiles([name]);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
function createDuckDBCodec(factory, options = {}) {
|
|
53
|
+
return {
|
|
54
|
+
async writeRows(ctx, rows, key, dataSource) {
|
|
55
|
+
const bytes = await encodeBytes(await factory.getDuckDB(), ctx.table, rows);
|
|
56
|
+
await dataSource.write(key, bytes);
|
|
57
|
+
return {
|
|
58
|
+
bytes: bytes.byteLength,
|
|
59
|
+
rowCount: rows.length
|
|
60
|
+
};
|
|
61
|
+
},
|
|
62
|
+
async readRows(ctx, key, dataSource) {
|
|
63
|
+
return decodeBytes(await factory.getDuckDB(), await dataSource.read(key), ctx.table);
|
|
64
|
+
},
|
|
65
|
+
async compactRows(ctx, inputKeys, outputKey, dataSource) {
|
|
66
|
+
const db = await factory.getDuckDB();
|
|
67
|
+
if (inputKeys.length === 0) {
|
|
68
|
+
const bytes = await encodeBytes(db, ctx.table, []);
|
|
69
|
+
await dataSource.write(outputKey, bytes);
|
|
70
|
+
return {
|
|
71
|
+
bytes: bytes.byteLength,
|
|
72
|
+
rowCount: 0
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
const inputUris = inputKeys.map((k) => dataSource.uri?.(k));
|
|
76
|
+
if (inputUris.every((u) => u !== void 0)) {
|
|
77
|
+
const outName = db.makeTempPath("parquet");
|
|
78
|
+
const fileList = inputUris.map((u) => `'${sqlEscape(u)}'`).join(", ");
|
|
79
|
+
try {
|
|
80
|
+
await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
|
|
81
|
+
const bytes = await db.copyFileToBuffer(outName);
|
|
82
|
+
const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
|
|
83
|
+
const rowCount = Number(countRows[0]?.n ?? 0);
|
|
84
|
+
await dataSource.write(outputKey, bytes);
|
|
85
|
+
return {
|
|
86
|
+
bytes: bytes.byteLength,
|
|
87
|
+
rowCount
|
|
88
|
+
};
|
|
89
|
+
} finally {
|
|
90
|
+
await db.dropFiles([outName]);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const inNames = [];
|
|
94
|
+
const outName = db.makeTempPath("parquet");
|
|
95
|
+
const registered = [];
|
|
96
|
+
const bufferedInputs = inputKeys.map((key) => {
|
|
97
|
+
const name = db.makeTempPath("parquet");
|
|
98
|
+
inNames.push(name);
|
|
99
|
+
return {
|
|
100
|
+
key,
|
|
101
|
+
name
|
|
102
|
+
};
|
|
103
|
+
});
|
|
104
|
+
try {
|
|
105
|
+
await registerBufferedFiles(db, bufferedInputs, dataSource, registered, void 0, options.bufferReadConcurrency);
|
|
106
|
+
const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
|
|
107
|
+
await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
|
|
108
|
+
registered.push(outName);
|
|
109
|
+
const bytes = await db.copyFileToBuffer(outName);
|
|
110
|
+
const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
|
|
111
|
+
const rowCount = Number(countRows[0]?.n ?? 0);
|
|
112
|
+
await dataSource.write(outputKey, bytes);
|
|
113
|
+
return {
|
|
114
|
+
bytes: bytes.byteLength,
|
|
115
|
+
rowCount
|
|
116
|
+
};
|
|
117
|
+
} finally {
|
|
118
|
+
await db.dropFiles(registered);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
const quoteCol = (c) => `"${c.replace(/"/g, "\"\"")}"`;
|
|
124
|
+
function dedupedMergeSql(table, fileListSql) {
|
|
125
|
+
const base = `SELECT * FROM read_parquet([${fileListSql}], union_by_name = true)`;
|
|
126
|
+
const sortKey = SCHEMAS[table].sortKey;
|
|
127
|
+
const clusterKey = TABLE_METADATA[table].clusterKey;
|
|
128
|
+
const dedup = sortKey.length === 0 ? base : `${base} QUALIFY row_number() OVER (PARTITION BY ${sortKey.map(quoteCol).join(", ")}) = 1`;
|
|
129
|
+
if (clusterKey.length === 0) return dedup;
|
|
130
|
+
return `${dedup} ORDER BY ${clusterKey.map(quoteCol).join(", ")}`;
|
|
131
|
+
}
|
|
132
|
+
function rewriteEmptyFileSets(sql, placeholders, defaultTable, placeholderTables) {
|
|
133
|
+
let out = sql;
|
|
134
|
+
for (const [name, keys] of Object.entries(placeholders)) {
|
|
135
|
+
if (keys.length > 0) continue;
|
|
136
|
+
const emptyFallback = `(SELECT * FROM ${emptyTableSchema(placeholderTables?.[name] ?? defaultTable)} WHERE FALSE)`;
|
|
137
|
+
const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${name}\\}\\}\\s*(?:,\\s*union_by_name\\s*=\\s*true\\s*)?\\)`, "g");
|
|
138
|
+
out = out.replace(pattern, emptyFallback);
|
|
139
|
+
}
|
|
140
|
+
return out;
|
|
141
|
+
}
|
|
142
|
+
function createDuckDBExecutor(factory, options = {}) {
|
|
143
|
+
return { async execute({ sql, params, fileKeys, placeholderTables, dataSource, table, signal, profiler }) {
|
|
144
|
+
signal?.throwIfAborted();
|
|
145
|
+
const db = await factory.getDuckDB();
|
|
146
|
+
const placeholders = {};
|
|
147
|
+
const registered = [];
|
|
148
|
+
const totalFiles = Object.values(fileKeys).reduce((n, keys) => n + keys.length, 0);
|
|
149
|
+
const endRegister = profiler?.start("files.register", { files: totalFiles });
|
|
150
|
+
try {
|
|
151
|
+
const bufferedByName = /* @__PURE__ */ new Map();
|
|
152
|
+
for (const [name, keys] of Object.entries(fileKeys)) {
|
|
153
|
+
const resolved = [];
|
|
154
|
+
for (let i = 0; i < keys.length; i++) {
|
|
155
|
+
const key = keys[i];
|
|
156
|
+
const uri = dataSource.uri?.(key);
|
|
157
|
+
if (uri !== void 0) resolved.push(uri);
|
|
158
|
+
else {
|
|
159
|
+
if (!bufferedByName.has(key)) bufferedByName.set(key, {
|
|
160
|
+
key,
|
|
161
|
+
name: key
|
|
162
|
+
});
|
|
163
|
+
resolved.push(key);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
placeholders[name] = resolved;
|
|
167
|
+
}
|
|
168
|
+
await registerBufferedFiles(db, [...bufferedByName.values()], dataSource, registered, signal, options.bufferReadConcurrency);
|
|
169
|
+
endRegister?.({ buffered: registered.length });
|
|
170
|
+
signal?.throwIfAborted();
|
|
171
|
+
const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);
|
|
172
|
+
const endQuery = profiler?.start("query.run");
|
|
173
|
+
const rows = await db.query(finalSql, params);
|
|
174
|
+
endQuery?.({ rows: rows.length });
|
|
175
|
+
return {
|
|
176
|
+
rows,
|
|
177
|
+
sql: finalSql
|
|
178
|
+
};
|
|
179
|
+
} finally {
|
|
180
|
+
if (registered.length > 0) await db.dropFiles(registered);
|
|
181
|
+
}
|
|
182
|
+
} };
|
|
183
|
+
}
|
|
184
|
+
function emptyTableSchema(table) {
|
|
185
|
+
return `(FROM (VALUES ${placeholderValues(table)}) t(${columnList(table)}))`;
|
|
186
|
+
}
|
|
187
|
+
function dateReplaceClause(table) {
|
|
188
|
+
if (!table) return "";
|
|
189
|
+
return dateReplaceClause$1(dateColumnsFor(table), "string");
|
|
190
|
+
}
|
|
191
|
+
function columnList(table) {
|
|
192
|
+
return SCHEMAS[table].columns.map((c) => c.name).join(", ");
|
|
193
|
+
}
|
|
194
|
+
function placeholderValues(table) {
|
|
195
|
+
return `(${SCHEMAS[table].columns.map((c) => defaultForType(c.type)).join(", ")})`;
|
|
196
|
+
}
|
|
197
|
+
function defaultForType(t) {
|
|
198
|
+
if (t === "VARCHAR") return "''";
|
|
199
|
+
if (t === "DATE") return "DATE '1970-01-01'";
|
|
200
|
+
if (t === "INTEGER" || t === "BIGINT") return "0";
|
|
201
|
+
if (t === "DOUBLE") return "CAST(0 AS DOUBLE)";
|
|
202
|
+
return "NULL";
|
|
203
|
+
}
|
|
204
|
+
function columnsJson(table) {
|
|
205
|
+
return `{${SCHEMAS[table].columns.map((c) => `'${c.name}': '${c.type}'`).join(", ")}}`;
|
|
206
|
+
}
|
|
207
|
+
export { createDuckDBCodec, createDuckDBExecutor };
|