duckdb 0.8.1-dev26.0 → 0.8.1-dev287.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +8 -8
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +5 -4
- package/src/duckdb/extension/json/buffered_json_reader.cpp +23 -14
- package/src/duckdb/extension/json/include/buffered_json_reader.hpp +6 -6
- package/src/duckdb/extension/json/include/json_common.hpp +13 -3
- package/src/duckdb/extension/json/include/json_executors.hpp +1 -1
- package/src/duckdb/extension/json/include/json_scan.hpp +3 -1
- package/src/duckdb/extension/json/json_functions/json_create.cpp +16 -16
- package/src/duckdb/extension/json/json_functions/json_merge_patch.cpp +3 -3
- package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +2 -2
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +5 -3
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +12 -12
- package/src/duckdb/extension/json/json_functions/read_json.cpp +2 -1
- package/src/duckdb/extension/json/json_functions.cpp +6 -3
- package/src/duckdb/extension/json/json_scan.cpp +40 -25
- package/src/duckdb/extension/parquet/column_reader.cpp +57 -52
- package/src/duckdb/extension/parquet/column_writer.cpp +57 -45
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/column_reader.hpp +5 -4
- package/src/duckdb/extension/parquet/include/column_writer.hpp +24 -0
- package/src/duckdb/extension/parquet/include/decode_utils.hpp +6 -0
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +4 -4
- package/src/duckdb/extension/parquet/include/parquet_rle_bp_decoder.hpp +4 -5
- package/src/duckdb/extension/parquet/include/resizable_buffer.hpp +4 -4
- package/src/duckdb/extension/parquet/include/row_number_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/parquet-extension.cpp +1 -1
- package/src/duckdb/extension/parquet/parquet_metadata.cpp +1 -1
- package/src/duckdb/extension/parquet/parquet_reader.cpp +18 -18
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +23 -26
- package/src/duckdb/extension/parquet/parquet_timestamp.cpp +2 -2
- package/src/duckdb/extension/parquet/parquet_writer.cpp +3 -3
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +3 -3
- package/src/duckdb/src/catalog/catalog.cpp +5 -17
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +113 -0
- package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +7 -0
- package/src/duckdb/src/catalog/catalog_entry.cpp +4 -4
- package/src/duckdb/src/catalog/catalog_search_path.cpp +49 -12
- package/src/duckdb/src/catalog/catalog_set.cpp +1 -1
- package/src/duckdb/src/catalog/default/default_types.cpp +9 -84
- package/src/duckdb/src/catalog/dependency_manager.cpp +2 -2
- package/src/duckdb/src/catalog/duck_catalog.cpp +1 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +301 -106
- package/src/duckdb/src/common/adbc/driver_manager.cpp +10 -22
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +12 -12
- package/src/duckdb/src/common/arrow/arrow_wrapper.cpp +7 -7
- package/src/duckdb/src/common/checksum.cpp +1 -1
- package/src/duckdb/src/common/compressed_file_system.cpp +6 -6
- package/src/duckdb/src/common/crypto/md5.cpp +9 -9
- package/src/duckdb/src/common/exception.cpp +4 -1
- package/src/duckdb/src/common/exception_format_value.cpp +19 -14
- package/src/duckdb/src/common/field_writer.cpp +1 -1
- package/src/duckdb/src/common/file_system.cpp +15 -2
- package/src/duckdb/src/common/fsst.cpp +11 -6
- package/src/duckdb/src/common/gzip_file_system.cpp +8 -8
- package/src/duckdb/src/common/hive_partitioning.cpp +1 -1
- package/src/duckdb/src/common/local_file_system.cpp +11 -11
- package/src/duckdb/src/common/multi_file_reader.cpp +3 -0
- package/src/duckdb/src/common/operator/cast_operators.cpp +1 -1
- package/src/duckdb/src/common/pipe_file_system.cpp +2 -2
- package/src/duckdb/src/common/radix_partitioning.cpp +2 -2
- package/src/duckdb/src/common/row_operations/row_heap_gather.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_heap_scatter.cpp +5 -5
- package/src/duckdb/src/common/row_operations/row_match.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_radix_scatter.cpp +2 -2
- package/src/duckdb/src/common/row_operations/row_scatter.cpp +4 -4
- package/src/duckdb/src/common/serializer/binary_deserializer.cpp +1 -1
- package/src/duckdb/src/common/serializer/binary_serializer.cpp +3 -3
- package/src/duckdb/src/common/serializer.cpp +1 -1
- package/src/duckdb/src/common/sort/comparators.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +7 -2
- package/src/duckdb/src/common/sort/partition_state.cpp +2 -2
- package/src/duckdb/src/common/types/bit.cpp +5 -5
- package/src/duckdb/src/common/types/blob.cpp +8 -8
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +4 -4
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +3 -3
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +1 -1
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/hash.cpp +2 -2
- package/src/duckdb/src/common/types/hyperloglog.cpp +22 -21
- package/src/duckdb/src/common/types/list_segment.cpp +77 -49
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +1 -1
- package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp +5 -4
- package/src/duckdb/src/common/types/row/tuple_data_allocator.cpp +5 -3
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +61 -24
- package/src/duckdb/src/common/types/string_heap.cpp +1 -1
- package/src/duckdb/src/common/types/value.cpp +3 -2
- package/src/duckdb/src/common/types/vector.cpp +98 -101
- package/src/duckdb/src/common/types/vector_cache.cpp +6 -6
- package/src/duckdb/src/common/types/vector_constants.cpp +2 -1
- package/src/duckdb/src/common/types.cpp +44 -33
- package/src/duckdb/src/common/vector_operations/boolean_operators.cpp +2 -2
- package/src/duckdb/src/common/vector_operations/is_distinct_from.cpp +12 -12
- package/src/duckdb/src/common/vector_operations/vector_hash.cpp +13 -11
- package/src/duckdb/src/common/vector_operations/vector_storage.cpp +1 -1
- package/src/duckdb/src/core_functions/aggregate/algebraic/avg.cpp +30 -33
- package/src/duckdb/src/core_functions/aggregate/algebraic/covar.cpp +0 -4
- package/src/duckdb/src/core_functions/aggregate/distributive/approx_count.cpp +32 -36
- package/src/duckdb/src/core_functions/aggregate/distributive/arg_min_max.cpp +53 -66
- package/src/duckdb/src/core_functions/aggregate/distributive/bitagg.cpp +48 -48
- package/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp +44 -44
- package/src/duckdb/src/core_functions/aggregate/distributive/bool.cpp +32 -32
- package/src/duckdb/src/core_functions/aggregate/distributive/entropy.cpp +34 -34
- package/src/duckdb/src/core_functions/aggregate/distributive/kurtosis.cpp +30 -31
- package/src/duckdb/src/core_functions/aggregate/distributive/minmax.cpp +91 -103
- package/src/duckdb/src/core_functions/aggregate/distributive/product.cpp +17 -17
- package/src/duckdb/src/core_functions/aggregate/distributive/skew.cpp +25 -27
- package/src/duckdb/src/core_functions/aggregate/distributive/string_agg.cpp +36 -37
- package/src/duckdb/src/core_functions/aggregate/distributive/sum.cpp +22 -22
- package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +48 -84
- package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +49 -51
- package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +97 -121
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +62 -99
- package/src/duckdb/src/core_functions/aggregate/nested/histogram.cpp +24 -26
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +22 -23
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +16 -18
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +22 -25
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +19 -24
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +18 -23
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +14 -18
- package/src/duckdb/src/core_functions/function_list.cpp +1 -0
- package/src/duckdb/src/core_functions/scalar/blob/base64.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +45 -45
- package/src/duckdb/src/core_functions/scalar/date/strftime.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/generic/current_setting.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/generic/least.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/generic/stats.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/generic/system_functions.cpp +14 -0
- package/src/duckdb/src/core_functions/scalar/list/array_slice.cpp +3 -3
- package/src/duckdb/src/core_functions/scalar/list/flatten.cpp +2 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +3 -3
- package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +6 -7
- package/src/duckdb/src/core_functions/scalar/list/list_sort.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/map/cardinality.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/map/map.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/map/map_concat.cpp +4 -1
- package/src/duckdb/src/core_functions/scalar/map/map_extract.cpp +5 -9
- package/src/duckdb/src/core_functions/scalar/math/numeric.cpp +3 -3
- package/src/duckdb/src/core_functions/scalar/string/starts_with.cpp +3 -2
- package/src/duckdb/src/core_functions/scalar/string/string_split.cpp +4 -4
- package/src/duckdb/src/core_functions/scalar/string/trim.cpp +1 -1
- package/src/duckdb/src/core_functions/scalar/union/union_extract.cpp +1 -1
- package/src/duckdb/src/execution/adaptive_filter.cpp +1 -1
- package/src/duckdb/src/execution/expression_executor/execute_case.cpp +10 -10
- package/src/duckdb/src/execution/expression_executor/execute_conjunction.cpp +7 -7
- package/src/duckdb/src/execution/expression_executor.cpp +28 -28
- package/src/duckdb/src/execution/index/art/art.cpp +30 -32
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +3 -4
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/nested_loop_join/nested_loop_join_inner.cpp +4 -4
- package/src/duckdb/src/execution/nested_loop_join/nested_loop_join_mark.cpp +2 -2
- package/src/duckdb/src/execution/operator/aggregate/distinct_aggregate_data.cpp +1 -1
- package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +1 -1
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +2 -2
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +25 -9
- package/src/duckdb/src/execution/operator/filter/physical_filter.cpp +1 -1
- package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +5 -2
- package/src/duckdb/src/execution/operator/helper/physical_set.cpp +5 -1
- package/src/duckdb/src/execution/operator/join/outer_join_marker.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/perfect_hash_join_executor.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +2 -2
- package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +4 -4
- package/src/duckdb/src/execution/operator/join/physical_index_join.cpp +3 -3
- package/src/duckdb/src/execution/operator/join/physical_nested_loop_join.cpp +3 -3
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +3 -3
- package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +2 -2
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +2 -2
- package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -2
- package/src/duckdb/src/execution/operator/projection/physical_projection.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp +3 -3
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_expression_scan.cpp +1 -1
- package/src/duckdb/src/execution/operator/scan/physical_positional_scan.cpp +2 -2
- package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp +1 -1
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan/plan_create_table.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan_generator.cpp +2 -2
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +1 -1
- package/src/duckdb/src/execution/reservoir_sample.cpp +2 -2
- package/src/duckdb/src/execution/window_segment_tree.cpp +8 -6
- package/src/duckdb/src/function/aggregate/distributive/count.cpp +158 -20
- package/src/duckdb/src/function/aggregate/distributive/first.cpp +66 -74
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +13 -13
- package/src/duckdb/src/function/cast/cast_function_set.cpp +1 -1
- package/src/duckdb/src/function/cast/decimal_cast.cpp +1 -1
- package/src/duckdb/src/function/cast/enum_casts.cpp +2 -2
- package/src/duckdb/src/function/cast/list_casts.cpp +2 -4
- package/src/duckdb/src/function/cast/string_cast.cpp +11 -11
- package/src/duckdb/src/function/cast/union_casts.cpp +2 -2
- package/src/duckdb/src/function/cast/vector_cast_helpers.cpp +3 -2
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +33 -23
- package/src/duckdb/src/function/scalar/generic/constant_or_null.cpp +1 -1
- package/src/duckdb/src/function/scalar/list/list_concat.cpp +2 -2
- package/src/duckdb/src/function/scalar/list/list_extract.cpp +3 -3
- package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +2 -3
- package/src/duckdb/src/function/scalar/string/concat.cpp +8 -7
- package/src/duckdb/src/function/scalar/string/contains.cpp +4 -4
- package/src/duckdb/src/function/scalar/string/like.cpp +5 -5
- package/src/duckdb/src/function/scalar/string/regexp/regexp_extract_all.cpp +4 -4
- package/src/duckdb/src/function/scalar/string/regexp/regexp_util.cpp +6 -2
- package/src/duckdb/src/function/scalar/string/regexp.cpp +3 -3
- package/src/duckdb/src/function/scalar/string/strip_accents.cpp +1 -1
- package/src/duckdb/src/function/scalar/struct/struct_extract.cpp +1 -1
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +25 -23
- package/src/duckdb/src/function/scalar_function.cpp +3 -3
- package/src/duckdb/src/function/table/arrow.cpp +6 -6
- package/src/duckdb/src/function/table/arrow_conversion.cpp +67 -61
- package/src/duckdb/src/function/table/checkpoint.cpp +1 -1
- package/src/duckdb/src/function/table/copy_csv.cpp +11 -7
- package/src/duckdb/src/function/table/glob.cpp +1 -1
- package/src/duckdb/src/function/table/pragma_last_profiling_output.cpp +1 -1
- package/src/duckdb/src/function/table/range.cpp +4 -4
- package/src/duckdb/src/function/table/read_csv.cpp +15 -20
- package/src/duckdb/src/function/table/repeat.cpp +2 -2
- package/src/duckdb/src/function/table/repeat_row.cpp +10 -3
- package/src/duckdb/src/function/table/system/duckdb_functions.cpp +1 -1
- package/src/duckdb/src/function/table/system/test_vector_types.cpp +82 -26
- package/src/duckdb/src/function/table/table_scan.cpp +2 -2
- package/src/duckdb/src/function/table/unnest.cpp +1 -1
- package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +2 -5
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +2 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +6 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry.hpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +8 -2
- package/src/duckdb/src/include/duckdb/catalog/default/builtin_types/types.hpp +97 -0
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +5 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +15 -4
- package/src/duckdb/src/include/duckdb/common/arrow/arrow_buffer.hpp +7 -2
- package/src/duckdb/src/include/duckdb/common/crypto/md5.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/exception.hpp +5 -2
- package/src/duckdb/src/include/duckdb/common/field_writer.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +11 -0
- package/src/duckdb/src/include/duckdb/common/fsst.hpp +2 -3
- package/src/duckdb/src/include/duckdb/common/radix.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp +4 -1
- package/src/duckdb/src/include/duckdb/common/serializer.hpp +4 -4
- package/src/duckdb/src/include/duckdb/common/typedefs.hpp +30 -0
- package/src/duckdb/src/include/duckdb/common/types/column/partitioned_column_data.hpp +12 -0
- package/src/duckdb/src/include/duckdb/common/types/hyperloglog.hpp +6 -2
- package/src/duckdb/src/include/duckdb/common/types/null_value.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +12 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +10 -10
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/vector.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types/vector_buffer.hpp +18 -6
- package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp +112 -76
- package/src/duckdb/src/include/duckdb/common/vector_operations/binary_executor.hpp +16 -15
- package/src/duckdb/src/include/duckdb/common/vector_operations/generic_executor.hpp +11 -11
- package/src/duckdb/src/include/duckdb/common/vector_operations/ternary_executor.hpp +23 -19
- package/src/duckdb/src/include/duckdb/common/vector_operations/unary_executor.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +20 -24
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +36 -39
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/stddev.hpp +57 -53
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +8 -9
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +16 -18
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp +7 -8
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +9 -0
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +2 -6
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +16 -36
- package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp +14 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +3 -3
- package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_result_collector.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_buffer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +5 -2
- package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +12 -12
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +8 -29
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +95 -0
- package/src/duckdb/src/include/duckdb/function/cast/default_casts.hpp +13 -2
- package/src/duckdb/src/include/duckdb/function/cast/vector_cast_helpers.hpp +4 -4
- package/src/duckdb/src/include/duckdb/function/compression_function.hpp +44 -0
- package/src/duckdb/src/include/duckdb/function/copy_function.hpp +6 -6
- package/src/duckdb/src/include/duckdb/function/function.hpp +3 -3
- package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +3 -1
- package/src/duckdb/src/include/duckdb/function/macro_function.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +3 -3
- package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/table_function.hpp +8 -7
- package/src/duckdb/src/include/duckdb/main/capi/cast/utils.hpp +1 -1
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/materialized_query_result.hpp +3 -0
- package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp +3 -0
- package/src/duckdb/src/include/duckdb/main/query_result.hpp +17 -0
- package/src/duckdb/src/include/duckdb/main/relation.hpp +12 -0
- package/src/duckdb/src/include/duckdb/main/settings.hpp +9 -0
- package/src/duckdb/src/include/duckdb/main/stream_query_result.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/base_expression.hpp +6 -12
- package/src/duckdb/src/include/duckdb/parser/constraint.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/bound_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/case_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/conjunction_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/parameter_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/subquery_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression_map.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/parsed_data/parse_info.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/parsed_expression.hpp +5 -1
- package/src/duckdb/src/include/duckdb/parser/parser.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/result_modifier.hpp +36 -5
- package/src/duckdb/src/include/duckdb/parser/sql_statement.hpp +7 -4
- package/src/duckdb/src/include/duckdb/parser/statement/select_statement.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/basetableref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/tableref/expressionlistref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/table_function_ref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref.hpp +4 -3
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +107 -91
- package/src/duckdb/src/include/duckdb/planner/bind_context.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/bound_constraint.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/bound_query_node.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp +34 -1
- package/src/duckdb/src/include/duckdb/planner/bound_tableref.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_between_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_case_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_cast_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_columnref_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_comparison_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_conjunction_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_constant_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_function_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_lambda_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_lambdaref_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_operator_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_reference_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_unnest_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression/bound_window_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression.hpp +5 -7
- package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/filter/conjunction_filter.hpp +6 -0
- package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +3 -0
- package/src/duckdb/src/include/duckdb/planner/filter/null_filter.hpp +6 -0
- package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +6 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_execute.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_explain.hpp +5 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +5 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_pivot.hpp +3 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_pragma.hpp +6 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_prepare.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/table_binding.hpp +26 -0
- package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +17 -0
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_pivotref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_analyze.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_compress.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_analyze.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_compress.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/index.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +12 -0
- package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +25 -2
- package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +12 -0
- package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +22 -0
- package/src/duckdb/src/include/duckdb/transaction/duck_transaction_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/transaction/transaction.hpp +2 -2
- package/src/duckdb/src/main/capi/appender-c.cpp +5 -5
- package/src/duckdb/src/main/capi/arrow-c.cpp +10 -10
- package/src/duckdb/src/main/capi/cast/from_decimal-c.cpp +1 -1
- package/src/duckdb/src/main/capi/cast/utils-c.cpp +1 -1
- package/src/duckdb/src/main/capi/config-c.cpp +3 -6
- package/src/duckdb/src/main/capi/data_chunk-c.cpp +17 -17
- package/src/duckdb/src/main/capi/duckdb-c.cpp +4 -4
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +4 -4
- package/src/duckdb/src/main/capi/logical_types-c.cpp +22 -21
- package/src/duckdb/src/main/capi/pending-c.cpp +6 -6
- package/src/duckdb/src/main/capi/prepared-c.cpp +10 -10
- package/src/duckdb/src/main/capi/replacement_scan-c.cpp +6 -6
- package/src/duckdb/src/main/capi/result-c.cpp +23 -23
- package/src/duckdb/src/main/capi/table_function-c.cpp +1 -1
- package/src/duckdb/src/main/client_context.cpp +3 -3
- package/src/duckdb/src/main/config.cpp +1 -0
- package/src/duckdb/src/main/database_manager.cpp +1 -1
- package/src/duckdb/src/main/error_manager.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_load.cpp +1 -1
- package/src/duckdb/src/main/relation/create_table_relation.cpp +1 -1
- package/src/duckdb/src/main/relation/create_view_relation.cpp +1 -1
- package/src/duckdb/src/main/relation/delete_relation.cpp +1 -1
- package/src/duckdb/src/main/relation/explain_relation.cpp +1 -1
- package/src/duckdb/src/main/relation/insert_relation.cpp +1 -1
- package/src/duckdb/src/main/relation/update_relation.cpp +1 -1
- package/src/duckdb/src/main/relation/write_csv_relation.cpp +1 -1
- package/src/duckdb/src/main/relation/write_parquet_relation.cpp +1 -1
- package/src/duckdb/src/main/relation.cpp +1 -1
- package/src/duckdb/src/main/settings/settings.cpp +22 -6
- package/src/duckdb/src/optimizer/deliminator.cpp +12 -12
- package/src/duckdb/src/optimizer/expression_heuristics.cpp +1 -0
- package/src/duckdb/src/optimizer/filter_combiner.cpp +3 -3
- package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +10 -10
- package/src/duckdb/src/optimizer/matcher/expression_matcher.cpp +1 -1
- package/src/duckdb/src/optimizer/pullup/pullup_projection.cpp +2 -2
- package/src/duckdb/src/optimizer/regex_range_filter.cpp +2 -4
- package/src/duckdb/src/optimizer/rule/distributivity.cpp +2 -2
- package/src/duckdb/src/optimizer/statistics/operator/propagate_filter.cpp +6 -6
- package/src/duckdb/src/optimizer/statistics/operator/propagate_get.cpp +2 -2
- package/src/duckdb/src/parallel/executor.cpp +1 -1
- package/src/duckdb/src/parser/base_expression.cpp +2 -5
- package/src/duckdb/src/parser/column_definition.cpp +5 -8
- package/src/duckdb/src/parser/expression/between_expression.cpp +4 -4
- package/src/duckdb/src/parser/expression/case_expression.cpp +6 -6
- package/src/duckdb/src/parser/expression/cast_expression.cpp +4 -4
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -3
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +4 -4
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -3
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +2 -2
- package/src/duckdb/src/parser/expression/constant_expression.cpp +2 -2
- package/src/duckdb/src/parser/expression/function_expression.cpp +10 -14
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +2 -2
- package/src/duckdb/src/parser/expression/operator_expression.cpp +4 -4
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +2 -2
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +3 -3
- package/src/duckdb/src/parser/expression/star_expression.cpp +9 -9
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +5 -5
- package/src/duckdb/src/parser/expression/window_expression.cpp +13 -24
- package/src/duckdb/src/parser/parsed_expression.cpp +34 -18
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +4 -4
- package/src/duckdb/src/parser/parser.cpp +4 -4
- package/src/duckdb/src/parser/query_node/select_node.cpp +6 -13
- package/src/duckdb/src/parser/query_node.cpp +7 -6
- package/src/duckdb/src/parser/result_modifier.cpp +25 -18
- package/src/duckdb/src/parser/statement/select_statement.cpp +3 -3
- package/src/duckdb/src/parser/tableref/basetableref.cpp +4 -4
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +1 -1
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +5 -5
- package/src/duckdb/src/parser/tableref/joinref.cpp +6 -6
- package/src/duckdb/src/parser/tableref/pivotref.cpp +10 -15
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -3
- package/src/duckdb/src/parser/tableref/table_function.cpp +3 -3
- package/src/duckdb/src/parser/tableref.cpp +12 -3
- package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +7 -7
- package/src/duckdb/src/parser/transform/expression/transform_bool_expr.cpp +4 -4
- package/src/duckdb/src/parser/transform/expression/transform_boolean_test.cpp +4 -4
- package/src/duckdb/src/parser/transform/expression/transform_case.cpp +8 -10
- package/src/duckdb/src/parser/transform/expression/transform_cast.cpp +7 -9
- package/src/duckdb/src/parser/transform/expression/transform_coalesce.cpp +3 -5
- package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp +22 -22
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +2 -2
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +42 -44
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +70 -75
- package/src/duckdb/src/parser/transform/expression/transform_grouping_function.cpp +4 -4
- package/src/duckdb/src/parser/transform/expression/transform_interval.cpp +7 -7
- package/src/duckdb/src/parser/transform/expression/transform_is_null.cpp +4 -5
- package/src/duckdb/src/parser/transform/expression/transform_lambda.cpp +5 -6
- package/src/duckdb/src/parser/transform/expression/transform_operator.cpp +28 -29
- package/src/duckdb/src/parser/transform/expression/transform_param_ref.cpp +13 -14
- package/src/duckdb/src/parser/transform/expression/transform_positional_reference.cpp +4 -4
- package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp +9 -10
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +28 -32
- package/src/duckdb/src/parser/transform/helpers/transform_groupby.cpp +18 -18
- package/src/duckdb/src/parser/transform/helpers/transform_sample.cpp +3 -3
- package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +27 -26
- package/src/duckdb/src/parser/transform/statement/transform_alter_sequence.cpp +11 -14
- package/src/duckdb/src/parser/transform/statement/transform_alter_table.cpp +14 -16
- package/src/duckdb/src/parser/transform/statement/transform_attach.cpp +8 -9
- package/src/duckdb/src/parser/transform/statement/transform_call.cpp +2 -5
- package/src/duckdb/src/parser/transform/statement/transform_checkpoint.cpp +4 -6
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +22 -23
- package/src/duckdb/src/parser/transform/statement/transform_create_database.cpp +3 -4
- package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp +14 -18
- package/src/duckdb/src/parser/transform/statement/transform_create_index.cpp +13 -15
- package/src/duckdb/src/parser/transform/statement/transform_create_schema.cpp +8 -10
- package/src/duckdb/src/parser/transform/statement/transform_create_sequence.cpp +8 -10
- package/src/duckdb/src/parser/transform/statement/transform_create_table.cpp +26 -28
- package/src/duckdb/src/parser/transform/statement/transform_create_table_as.cpp +8 -10
- package/src/duckdb/src/parser/transform/statement/transform_create_type.cpp +12 -15
- package/src/duckdb/src/parser/transform/statement/transform_create_view.cpp +13 -18
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +11 -13
- package/src/duckdb/src/parser/transform/statement/transform_detach.cpp +3 -4
- package/src/duckdb/src/parser/transform/statement/transform_drop.cpp +20 -22
- package/src/duckdb/src/parser/transform/statement/transform_explain.cpp +5 -7
- package/src/duckdb/src/parser/transform/statement/transform_export.cpp +5 -6
- package/src/duckdb/src/parser/transform/statement/transform_import.cpp +2 -3
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +21 -24
- package/src/duckdb/src/parser/transform/statement/transform_load.cpp +4 -5
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -7
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +7 -9
- package/src/duckdb/src/parser/transform/statement/transform_prepare.cpp +11 -19
- package/src/duckdb/src/parser/transform/statement/transform_rename.cpp +12 -14
- package/src/duckdb/src/parser/transform/statement/transform_select.cpp +12 -9
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +34 -34
- package/src/duckdb/src/parser/transform/statement/transform_set.cpp +18 -19
- package/src/duckdb/src/parser/transform/statement/transform_show.cpp +5 -7
- package/src/duckdb/src/parser/transform/statement/transform_show_select.cpp +4 -5
- package/src/duckdb/src/parser/transform/statement/transform_transaction.cpp +3 -5
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +10 -13
- package/src/duckdb/src/parser/transform/statement/transform_upsert.cpp +4 -4
- package/src/duckdb/src/parser/transform/statement/transform_use.cpp +2 -3
- package/src/duckdb/src/parser/transform/statement/transform_vacuum.cpp +6 -10
- package/src/duckdb/src/parser/transform/tableref/transform_base_tableref.cpp +18 -18
- package/src/duckdb/src/parser/transform/tableref/transform_from.cpp +5 -5
- package/src/duckdb/src/parser/transform/tableref/transform_join.cpp +11 -11
- package/src/duckdb/src/parser/transform/tableref/transform_pivot.cpp +30 -27
- package/src/duckdb/src/parser/transform/tableref/transform_subquery.cpp +5 -5
- package/src/duckdb/src/parser/transform/tableref/transform_table_function.cpp +13 -12
- package/src/duckdb/src/parser/transform/tableref/transform_tableref.cpp +8 -8
- package/src/duckdb/src/parser/transformer.cpp +46 -46
- package/src/duckdb/src/planner/bind_context.cpp +9 -10
- package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +1 -1
- package/src/duckdb/src/planner/binder/expression/bind_lambda.cpp +1 -1
- package/src/duckdb/src/planner/binder/expression/bind_positional_reference_expression.cpp +8 -3
- package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp +1 -1
- package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +6 -3
- package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +8 -8
- package/src/duckdb/src/planner/binder/query_node/plan_query_node.cpp +4 -4
- package/src/duckdb/src/planner/binder/statement/bind_create.cpp +1 -1
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +18 -1
- package/src/duckdb/src/planner/binder/statement/bind_insert.cpp +2 -2
- package/src/duckdb/src/planner/binder/statement/bind_update.cpp +2 -114
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +4 -2
- package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +17 -7
- package/src/duckdb/src/planner/binder.cpp +1 -1
- package/src/duckdb/src/planner/bound_result_modifier.cpp +16 -11
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +5 -5
- package/src/duckdb/src/planner/expression/bound_between_expression.cpp +5 -5
- package/src/duckdb/src/planner/expression/bound_case_expression.cpp +5 -5
- package/src/duckdb/src/planner/expression/bound_cast_expression.cpp +3 -3
- package/src/duckdb/src/planner/expression/bound_columnref_expression.cpp +2 -2
- package/src/duckdb/src/planner/expression/bound_comparison_expression.cpp +4 -4
- package/src/duckdb/src/planner/expression/bound_conjunction_expression.cpp +2 -2
- package/src/duckdb/src/planner/expression/bound_constant_expression.cpp +2 -2
- package/src/duckdb/src/planner/expression/bound_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression/bound_function_expression.cpp +3 -4
- package/src/duckdb/src/planner/expression/bound_lambda_expression.cpp +4 -5
- package/src/duckdb/src/planner/expression/bound_lambdaref_expression.cpp +2 -2
- package/src/duckdb/src/planner/expression/bound_operator_expression.cpp +3 -4
- package/src/duckdb/src/planner/expression/bound_parameter_expression.cpp +2 -2
- package/src/duckdb/src/planner/expression/bound_reference_expression.cpp +2 -2
- package/src/duckdb/src/planner/expression/bound_subquery_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression/bound_unnest_expression.cpp +3 -3
- package/src/duckdb/src/planner/expression/bound_window_expression.cpp +8 -21
- package/src/duckdb/src/planner/expression.cpp +15 -0
- package/src/duckdb/src/planner/expression_binder/base_select_binder.cpp +2 -2
- package/src/duckdb/src/planner/expression_binder.cpp +3 -2
- package/src/duckdb/src/planner/expression_iterator.cpp +2 -2
- package/src/duckdb/src/planner/filter/conjunction_filter.cpp +2 -2
- package/src/duckdb/src/planner/filter/constant_filter.cpp +1 -1
- package/src/duckdb/src/planner/logical_operator.cpp +3 -4
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +1 -1
- package/src/duckdb/src/planner/planner.cpp +5 -15
- package/src/duckdb/src/planner/table_filter.cpp +1 -1
- package/src/duckdb/src/storage/arena_allocator.cpp +2 -2
- package/src/duckdb/src/storage/buffer/block_handle.cpp +1 -1
- package/src/duckdb/src/storage/checkpoint/write_overflow_strings_to_disk.cpp +2 -2
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -3
- package/src/duckdb/src/storage/compression/bitpacking.cpp +8 -8
- package/src/duckdb/src/storage/compression/dictionary_compression.cpp +36 -36
- package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp +11 -11
- package/src/duckdb/src/storage/compression/fsst.cpp +34 -34
- package/src/duckdb/src/storage/compression/rle.cpp +8 -8
- package/src/duckdb/src/storage/compression/string_uncompressed.cpp +13 -13
- package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +11 -11
- package/src/duckdb/src/storage/data_table.cpp +8 -7
- package/src/duckdb/src/storage/index.cpp +1 -3
- package/src/duckdb/src/storage/local_storage.cpp +2 -1
- package/src/duckdb/src/storage/magic_bytes.cpp +1 -1
- package/src/duckdb/src/storage/single_file_block_manager.cpp +3 -3
- package/src/duckdb/src/storage/standard_buffer_manager.cpp +3 -3
- package/src/duckdb/src/storage/statistics/list_stats.cpp +1 -1
- package/src/duckdb/src/storage/statistics/numeric_stats.cpp +1 -1
- package/src/duckdb/src/storage/statistics/string_stats.cpp +15 -14
- package/src/duckdb/src/storage/table/chunk_info.cpp +2 -2
- package/src/duckdb/src/storage/table/column_segment.cpp +3 -3
- package/src/duckdb/src/storage/table/list_column_data.cpp +3 -3
- package/src/duckdb/src/storage/table/row_group.cpp +4 -4
- package/src/duckdb/src/storage/table/standard_column_data.cpp +1 -1
- package/src/duckdb/src/storage/table/update_segment.cpp +12 -12
- package/src/duckdb/src/storage/wal_replay.cpp +5 -6
- package/src/duckdb/src/transaction/cleanup_state.cpp +3 -3
- package/src/duckdb/src/transaction/commit_state.cpp +8 -8
- package/src/duckdb/src/transaction/duck_transaction.cpp +9 -7
- package/src/duckdb/src/transaction/duck_transaction_manager.cpp +16 -16
- package/src/duckdb/src/transaction/rollback_state.cpp +3 -3
- package/src/duckdb/src/verification/prepared_statement_verifier.cpp +1 -1
- package/src/duckdb/src/verification/statement_verifier.cpp +3 -4
- package/src/duckdb/third_party/hyperloglog/hyperloglog.hpp +2 -2
- package/src/duckdb/third_party/pcg/pcg_extras.hpp +1 -1
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +5 -5
@@ -244,14 +244,13 @@ unique_ptr<GlobalTableFunctionState> JSONGlobalTableFunctionState::Init(ClientCo
|
|
244
244
|
|
245
245
|
idx_t JSONGlobalTableFunctionState::MaxThreads() const {
|
246
246
|
auto &bind_data = state.bind_data;
|
247
|
-
if (bind_data.options.format == JSONFormat::NEWLINE_DELIMITED
|
248
|
-
bind_data.options.compression == FileCompressionType::UNCOMPRESSED) {
|
247
|
+
if (bind_data.options.format == JSONFormat::NEWLINE_DELIMITED) {
|
249
248
|
return state.system_threads;
|
250
249
|
}
|
251
250
|
|
252
251
|
if (!state.json_readers.empty() && state.json_readers[0]->IsOpen()) {
|
253
252
|
auto &reader = *state.json_readers[0];
|
254
|
-
if (reader.
|
253
|
+
if (reader.GetFormat() == JSONFormat::NEWLINE_DELIMITED) { // Auto-detected NDJSON
|
255
254
|
return state.system_threads;
|
256
255
|
}
|
257
256
|
}
|
@@ -298,6 +297,7 @@ idx_t JSONScanLocalState::ReadNext(JSONScanGlobalState &gstate) {
|
|
298
297
|
if (!ReadNextBuffer(gstate)) {
|
299
298
|
return scan_count;
|
300
299
|
}
|
300
|
+
D_ASSERT(buffer_size != 0);
|
301
301
|
if (current_buffer_handle->buffer_index != 0 && current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
|
302
302
|
ReconstructFirstObject(gstate);
|
303
303
|
scan_count++;
|
@@ -308,8 +308,8 @@ idx_t JSONScanLocalState::ReadNext(JSONScanGlobalState &gstate) {
|
|
308
308
|
return scan_count;
|
309
309
|
}
|
310
310
|
|
311
|
-
static inline const char *NextNewline(
|
312
|
-
return (
|
311
|
+
static inline const char *NextNewline(char *ptr, idx_t size) {
|
312
|
+
return char_ptr_cast(memchr(ptr, '\n', size));
|
313
313
|
}
|
314
314
|
|
315
315
|
static inline const char *PreviousNewline(const char *ptr) {
|
@@ -455,7 +455,21 @@ void JSONScanLocalState::ThrowInvalidAtEndError() {
|
|
455
455
|
throw InvalidInputException("Invalid JSON detected at the end of file \"%s\".", current_reader->GetFileName());
|
456
456
|
}
|
457
457
|
|
458
|
-
|
458
|
+
bool JSONScanLocalState::IsParallel(JSONScanGlobalState &gstate) const {
|
459
|
+
if (bind_data.files.size() >= gstate.system_threads) {
|
460
|
+
// More files than threads, just parallelize over the files
|
461
|
+
return false;
|
462
|
+
}
|
463
|
+
|
464
|
+
if (current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
|
465
|
+
// NDJSON can be read in parallel
|
466
|
+
return true;
|
467
|
+
}
|
468
|
+
|
469
|
+
return false;
|
470
|
+
}
|
471
|
+
|
472
|
+
static pair<JSONFormat, JSONRecordType> DetectFormatAndRecordType(char *const buffer_ptr, const idx_t buffer_size,
|
459
473
|
yyjson_alc *alc) {
|
460
474
|
// First we do the easy check whether it's NEWLINE_DELIMITED
|
461
475
|
auto line_end = NextNewline(buffer_ptr, buffer_size);
|
@@ -464,7 +478,7 @@ static pair<JSONFormat, JSONRecordType> DetectFormatAndRecordType(const char *co
|
|
464
478
|
SkipWhitespace(buffer_ptr, line_size, buffer_size);
|
465
479
|
|
466
480
|
yyjson_read_err error;
|
467
|
-
auto doc = JSONCommon::ReadDocumentUnsafe(
|
481
|
+
auto doc = JSONCommon::ReadDocumentUnsafe(buffer_ptr, line_size, JSONCommon::READ_FLAG, alc, &error);
|
468
482
|
if (error.code == YYJSON_READ_SUCCESS) { // We successfully read the line
|
469
483
|
if (yyjson_is_arr(doc->root) && line_size == buffer_size) {
|
470
484
|
// It's just one array, let's actually assume ARRAY, not NEWLINE_DELIMITED
|
@@ -500,8 +514,8 @@ static pair<JSONFormat, JSONRecordType> DetectFormatAndRecordType(const char *co
|
|
500
514
|
|
501
515
|
// It's definitely an ARRAY, but now we have to figure out if there's more than one top-level array
|
502
516
|
yyjson_read_err error;
|
503
|
-
auto doc =
|
504
|
-
|
517
|
+
auto doc =
|
518
|
+
JSONCommon::ReadDocumentUnsafe(buffer_ptr + buffer_offset, remaining, JSONCommon::READ_STOP_FLAG, alc, &error);
|
505
519
|
if (error.code == YYJSON_READ_SUCCESS) {
|
506
520
|
D_ASSERT(yyjson_is_arr(doc->root));
|
507
521
|
|
@@ -563,7 +577,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
|
|
563
577
|
} else {
|
564
578
|
buffer = gstate.allocator.Allocate(gstate.buffer_capacity);
|
565
579
|
}
|
566
|
-
buffer_ptr = (
|
580
|
+
buffer_ptr = char_ptr_cast(buffer.get());
|
567
581
|
|
568
582
|
idx_t buffer_index;
|
569
583
|
while (true) {
|
@@ -573,7 +587,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
|
|
573
587
|
if (is_last && gstate.bind_data.type != JSONScanType::SAMPLE) {
|
574
588
|
current_reader->CloseJSONFile();
|
575
589
|
}
|
576
|
-
if (
|
590
|
+
if (IsParallel(gstate)) {
|
577
591
|
// If this threads' current reader is still the one at gstate.file_index,
|
578
592
|
// this thread can end the parallel scan
|
579
593
|
lock_guard<mutex> guard(gstate.lock);
|
@@ -599,7 +613,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
|
|
599
613
|
current_reader = gstate.json_readers[gstate.file_index].get();
|
600
614
|
if (current_reader->IsOpen()) {
|
601
615
|
// Can only be open from auto detection, so these should be known
|
602
|
-
if (!
|
616
|
+
if (!IsParallel(gstate)) {
|
603
617
|
batch_index = gstate.batch_index++;
|
604
618
|
gstate.file_index++;
|
605
619
|
}
|
@@ -609,15 +623,15 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
|
|
609
623
|
current_reader->OpenJSONFile();
|
610
624
|
batch_index = gstate.batch_index++;
|
611
625
|
if (current_reader->GetFormat() != JSONFormat::AUTO_DETECT) {
|
612
|
-
if (!
|
626
|
+
if (!IsParallel(gstate)) {
|
613
627
|
gstate.file_index++;
|
614
628
|
}
|
615
629
|
continue;
|
616
630
|
}
|
617
631
|
|
618
|
-
// If we have
|
632
|
+
// If we have less files than threads, we auto-detect within the lock,
|
619
633
|
// so other threads may join a parallel NDJSON scan
|
620
|
-
if (gstate.json_readers.size() <
|
634
|
+
if (gstate.json_readers.size() < gstate.system_threads) {
|
621
635
|
if (ReadAndAutoDetect(gstate, buffer_index, false)) {
|
622
636
|
continue;
|
623
637
|
}
|
@@ -637,7 +651,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
|
|
637
651
|
D_ASSERT(buffer_size != 0); // We should have read something if we got here
|
638
652
|
|
639
653
|
idx_t readers = 1;
|
640
|
-
if (current_reader->
|
654
|
+
if (current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
|
641
655
|
readers = is_last ? 1 : 2;
|
642
656
|
}
|
643
657
|
|
@@ -650,7 +664,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
|
|
650
664
|
lines_or_objects_in_buffer = 0;
|
651
665
|
|
652
666
|
// YYJSON needs this
|
653
|
-
memset(
|
667
|
+
memset(buffer_ptr + buffer_size, 0, YYJSON_PADDING_SIZE);
|
654
668
|
|
655
669
|
return true;
|
656
670
|
}
|
@@ -680,7 +694,7 @@ bool JSONScanLocalState::ReadAndAutoDetect(JSONScanGlobalState &gstate, idx_t &b
|
|
680
694
|
throw InvalidInputException("Expected file \"%s\" to contain records, detected non-record JSON instead.",
|
681
695
|
current_reader->GetFileName());
|
682
696
|
}
|
683
|
-
if (!already_incremented_file_idx && !
|
697
|
+
if (!already_incremented_file_idx && !IsParallel(gstate)) {
|
684
698
|
gstate.file_index++;
|
685
699
|
}
|
686
700
|
return false;
|
@@ -739,13 +753,14 @@ void JSONScanLocalState::ReadNextBufferNoSeek(JSONScanGlobalState &gstate, idx_t
|
|
739
753
|
lock_guard<mutex> reader_guard(current_reader->lock);
|
740
754
|
buffer_index = current_reader->GetBufferIndex();
|
741
755
|
|
742
|
-
if (current_reader->IsOpen()) {
|
756
|
+
if (current_reader->IsOpen() && !current_reader->IsDone()) {
|
743
757
|
read_size = current_reader->GetFileHandle().Read(buffer_ptr + prev_buffer_remainder, request_size,
|
744
758
|
gstate.bind_data.type == JSONScanType::SAMPLE);
|
759
|
+
is_last = read_size < request_size;
|
745
760
|
} else {
|
746
761
|
read_size = 0;
|
762
|
+
is_last = false;
|
747
763
|
}
|
748
|
-
is_last = read_size < request_size;
|
749
764
|
|
750
765
|
if (!gstate.bind_data.ignore_errors && read_size == 0 && prev_buffer_remainder != 0) {
|
751
766
|
ThrowInvalidAtEndError();
|
@@ -796,13 +811,13 @@ void JSONScanLocalState::ReconstructFirstObject(JSONScanGlobalState &gstate) {
|
|
796
811
|
D_ASSERT(current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED);
|
797
812
|
|
798
813
|
// Spinlock until the previous batch index has also read its buffer
|
799
|
-
JSONBufferHandle
|
814
|
+
optional_ptr<JSONBufferHandle> previous_buffer_handle;
|
800
815
|
while (!previous_buffer_handle) {
|
801
816
|
previous_buffer_handle = current_reader->GetBuffer(current_buffer_handle->buffer_index - 1);
|
802
817
|
}
|
803
818
|
|
804
819
|
// First we find the newline in the previous block
|
805
|
-
auto prev_buffer_ptr = (
|
820
|
+
auto prev_buffer_ptr = char_ptr_cast(previous_buffer_handle->buffer.get()) + previous_buffer_handle->buffer_size;
|
806
821
|
auto part1_ptr = PreviousNewline(prev_buffer_ptr);
|
807
822
|
auto part1_size = prev_buffer_ptr - part1_ptr;
|
808
823
|
|
@@ -825,7 +840,7 @@ void JSONScanLocalState::ReconstructFirstObject(JSONScanGlobalState &gstate) {
|
|
825
840
|
|
826
841
|
// And copy the remainder of the line to the reconstruct buffer
|
827
842
|
memcpy(reconstruct_ptr + part1_size, buffer_ptr, part2_size);
|
828
|
-
memset(
|
843
|
+
memset(reconstruct_ptr + line_size, 0, YYJSON_PADDING_SIZE);
|
829
844
|
buffer_offset += part2_size;
|
830
845
|
|
831
846
|
// We copied the object, so we are no longer reading the previous buffer
|
@@ -833,7 +848,7 @@ void JSONScanLocalState::ReconstructFirstObject(JSONScanGlobalState &gstate) {
|
|
833
848
|
current_reader->RemoveBuffer(current_buffer_handle->buffer_index - 1);
|
834
849
|
}
|
835
850
|
|
836
|
-
ParseJSON((
|
851
|
+
ParseJSON(char_ptr_cast(reconstruct_ptr), line_size, line_size);
|
837
852
|
}
|
838
853
|
|
839
854
|
void JSONScanLocalState::ParseNextChunk() {
|
@@ -867,7 +882,7 @@ void JSONScanLocalState::ParseNextChunk() {
|
|
867
882
|
}
|
868
883
|
|
869
884
|
idx_t json_size = json_end - json_start;
|
870
|
-
ParseJSON(
|
885
|
+
ParseJSON(json_start, json_size, remaining);
|
871
886
|
buffer_offset += json_size;
|
872
887
|
|
873
888
|
if (format == JSONFormat::ARRAY) {
|
@@ -95,7 +95,11 @@ const uint64_t ParquetDecodeUtils::BITPACK_MASKS[] = {0,
|
|
95
95
|
576460752303423487,
|
96
96
|
1152921504606846975,
|
97
97
|
2305843009213693951,
|
98
|
-
4611686018427387903
|
98
|
+
4611686018427387903,
|
99
|
+
9223372036854775807,
|
100
|
+
18446744073709551615ULL};
|
101
|
+
|
102
|
+
const uint64_t ParquetDecodeUtils::BITPACK_MASKS_SIZE = sizeof(ParquetDecodeUtils::BITPACK_MASKS) / sizeof(uint64_t);
|
99
103
|
|
100
104
|
const uint8_t ParquetDecodeUtils::BITPACK_DLEN = 8;
|
101
105
|
|
@@ -270,7 +274,7 @@ void ColumnReader::ResetPage() {
|
|
270
274
|
void ColumnReader::PreparePageV2(PageHeader &page_hdr) {
|
271
275
|
D_ASSERT(page_hdr.type == PageType::DATA_PAGE_V2);
|
272
276
|
|
273
|
-
auto &trans =
|
277
|
+
auto &trans = reinterpret_cast<ThriftFileTransport &>(*protocol->getTransport());
|
274
278
|
|
275
279
|
AllocateBlock(page_hdr.uncompressed_page_size + 1);
|
276
280
|
bool uncompressed = false;
|
@@ -284,22 +288,22 @@ void ColumnReader::PreparePageV2(PageHeader &page_hdr) {
|
|
284
288
|
uncompressed = true;
|
285
289
|
}
|
286
290
|
if (uncompressed) {
|
287
|
-
trans.read(
|
291
|
+
trans.read(block->ptr, page_hdr.compressed_page_size);
|
288
292
|
return;
|
289
293
|
}
|
290
294
|
|
291
295
|
// copy repeats & defines as-is because FOR SOME REASON they are uncompressed
|
292
296
|
auto uncompressed_bytes = page_hdr.data_page_header_v2.repetition_levels_byte_length +
|
293
297
|
page_hdr.data_page_header_v2.definition_levels_byte_length;
|
294
|
-
trans.read(
|
298
|
+
trans.read(block->ptr, uncompressed_bytes);
|
295
299
|
|
296
300
|
auto compressed_bytes = page_hdr.compressed_page_size - uncompressed_bytes;
|
297
301
|
|
298
302
|
AllocateCompressed(compressed_bytes);
|
299
|
-
trans.read(
|
303
|
+
trans.read(compressed_buffer.ptr, compressed_bytes);
|
300
304
|
|
301
|
-
DecompressInternal(chunk->meta_data.codec,
|
302
|
-
|
305
|
+
DecompressInternal(chunk->meta_data.codec, compressed_buffer.ptr, compressed_bytes, block->ptr + uncompressed_bytes,
|
306
|
+
page_hdr.uncompressed_page_size - uncompressed_bytes);
|
303
307
|
}
|
304
308
|
|
305
309
|
void ColumnReader::AllocateBlock(idx_t size) {
|
@@ -315,7 +319,7 @@ void ColumnReader::AllocateCompressed(idx_t size) {
|
|
315
319
|
}
|
316
320
|
|
317
321
|
void ColumnReader::PreparePage(PageHeader &page_hdr) {
|
318
|
-
auto &trans =
|
322
|
+
auto &trans = reinterpret_cast<ThriftFileTransport &>(*protocol->getTransport());
|
319
323
|
|
320
324
|
AllocateBlock(page_hdr.uncompressed_page_size + 1);
|
321
325
|
if (chunk->meta_data.codec == CompressionCodec::UNCOMPRESSED) {
|
@@ -329,24 +333,24 @@ void ColumnReader::PreparePage(PageHeader &page_hdr) {
|
|
329
333
|
AllocateCompressed(page_hdr.compressed_page_size + 1);
|
330
334
|
trans.read((uint8_t *)compressed_buffer.ptr, page_hdr.compressed_page_size);
|
331
335
|
|
332
|
-
DecompressInternal(chunk->meta_data.codec,
|
333
|
-
|
336
|
+
DecompressInternal(chunk->meta_data.codec, compressed_buffer.ptr, page_hdr.compressed_page_size, block->ptr,
|
337
|
+
page_hdr.uncompressed_page_size);
|
334
338
|
}
|
335
339
|
|
336
|
-
void ColumnReader::DecompressInternal(CompressionCodec::type codec,
|
337
|
-
idx_t dst_size) {
|
340
|
+
void ColumnReader::DecompressInternal(CompressionCodec::type codec, const_data_ptr_t src, idx_t src_size,
|
341
|
+
data_ptr_t dst, idx_t dst_size) {
|
338
342
|
switch (codec) {
|
339
343
|
case CompressionCodec::UNCOMPRESSED:
|
340
344
|
throw InternalException("Parquet data unexpectedly uncompressed");
|
341
345
|
case CompressionCodec::GZIP: {
|
342
346
|
MiniZStream s;
|
343
|
-
s.Decompress(src, src_size, dst, dst_size);
|
347
|
+
s.Decompress(const_char_ptr_cast(src), src_size, char_ptr_cast(dst), dst_size);
|
344
348
|
break;
|
345
349
|
}
|
346
350
|
case CompressionCodec::SNAPPY: {
|
347
351
|
{
|
348
352
|
size_t uncompressed_size = 0;
|
349
|
-
auto res = duckdb_snappy::GetUncompressedLength(src, src_size, &uncompressed_size);
|
353
|
+
auto res = duckdb_snappy::GetUncompressedLength(const_char_ptr_cast(src), src_size, &uncompressed_size);
|
350
354
|
if (!res) {
|
351
355
|
throw std::runtime_error("Snappy decompression failure");
|
352
356
|
}
|
@@ -354,7 +358,7 @@ void ColumnReader::DecompressInternal(CompressionCodec::type codec, const char *
|
|
354
358
|
throw std::runtime_error("Snappy decompression failure: Uncompressed data size mismatch");
|
355
359
|
}
|
356
360
|
}
|
357
|
-
auto res = duckdb_snappy::RawUncompress(src, src_size, dst);
|
361
|
+
auto res = duckdb_snappy::RawUncompress(const_char_ptr_cast(src), src_size, char_ptr_cast(dst));
|
358
362
|
if (!res) {
|
359
363
|
throw std::runtime_error("Snappy decompression failure");
|
360
364
|
}
|
@@ -395,8 +399,7 @@ void ColumnReader::PrepareDataPage(PageHeader &page_hdr) {
|
|
395
399
|
if (HasRepeats()) {
|
396
400
|
uint32_t rep_length = is_v1 ? block->read<uint32_t>() : v2_header.repetition_levels_byte_length;
|
397
401
|
block->available(rep_length);
|
398
|
-
repeated_decoder =
|
399
|
-
make_uniq<RleBpDecoder>((const uint8_t *)block->ptr, rep_length, RleBpDecoder::ComputeBitWidth(max_repeat));
|
402
|
+
repeated_decoder = make_uniq<RleBpDecoder>(block->ptr, rep_length, RleBpDecoder::ComputeBitWidth(max_repeat));
|
400
403
|
block->inc(rep_length);
|
401
404
|
} else if (is_v2 && v2_header.repetition_levels_byte_length > 0) {
|
402
405
|
block->inc(v2_header.repetition_levels_byte_length);
|
@@ -405,8 +408,7 @@ void ColumnReader::PrepareDataPage(PageHeader &page_hdr) {
|
|
405
408
|
if (HasDefines()) {
|
406
409
|
uint32_t def_length = is_v1 ? block->read<uint32_t>() : v2_header.definition_levels_byte_length;
|
407
410
|
block->available(def_length);
|
408
|
-
defined_decoder =
|
409
|
-
make_uniq<RleBpDecoder>((const uint8_t *)block->ptr, def_length, RleBpDecoder::ComputeBitWidth(max_define));
|
411
|
+
defined_decoder = make_uniq<RleBpDecoder>(block->ptr, def_length, RleBpDecoder::ComputeBitWidth(max_define));
|
410
412
|
block->inc(def_length);
|
411
413
|
} else if (is_v2 && v2_header.definition_levels_byte_length > 0) {
|
412
414
|
block->inc(v2_header.definition_levels_byte_length);
|
@@ -418,7 +420,7 @@ void ColumnReader::PrepareDataPage(PageHeader &page_hdr) {
|
|
418
420
|
// where is it otherwise??
|
419
421
|
auto dict_width = block->read<uint8_t>();
|
420
422
|
// TODO somehow dict_width can be 0 ?
|
421
|
-
dict_decoder = make_uniq<RleBpDecoder>(
|
423
|
+
dict_decoder = make_uniq<RleBpDecoder>(block->ptr, block->len, dict_width);
|
422
424
|
block->inc(block->len);
|
423
425
|
break;
|
424
426
|
}
|
@@ -427,11 +429,11 @@ void ColumnReader::PrepareDataPage(PageHeader &page_hdr) {
|
|
427
429
|
throw std::runtime_error("RLE encoding is only supported for boolean data");
|
428
430
|
}
|
429
431
|
block->inc(sizeof(uint32_t));
|
430
|
-
rle_decoder = make_uniq<RleBpDecoder>(
|
432
|
+
rle_decoder = make_uniq<RleBpDecoder>(block->ptr, block->len, 1);
|
431
433
|
break;
|
432
434
|
}
|
433
435
|
case Encoding::DELTA_BINARY_PACKED: {
|
434
|
-
dbp_decoder = make_uniq<DbpDecoder>(
|
436
|
+
dbp_decoder = make_uniq<DbpDecoder>(block->ptr, block->len);
|
435
437
|
block->inc(block->len);
|
436
438
|
break;
|
437
439
|
}
|
@@ -452,10 +454,10 @@ void ColumnReader::PrepareDataPage(PageHeader &page_hdr) {
|
|
452
454
|
}
|
453
455
|
}
|
454
456
|
|
455
|
-
idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
|
457
|
+
idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr_t define_out, data_ptr_t repeat_out,
|
456
458
|
Vector &result) {
|
457
459
|
// we need to reset the location because multiple column readers share the same protocol
|
458
|
-
auto &trans =
|
460
|
+
auto &trans = reinterpret_cast<ThriftFileTransport &>(*protocol->getTransport());
|
459
461
|
trans.SetLocation(chunk_read_offset);
|
460
462
|
|
461
463
|
// Perform any skips that were not applied yet.
|
@@ -478,12 +480,12 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, uint8_t
|
|
478
480
|
|
479
481
|
if (HasRepeats()) {
|
480
482
|
D_ASSERT(repeated_decoder);
|
481
|
-
repeated_decoder->GetBatch<uint8_t>(
|
483
|
+
repeated_decoder->GetBatch<uint8_t>(repeat_out + result_offset, read_now);
|
482
484
|
}
|
483
485
|
|
484
486
|
if (HasDefines()) {
|
485
487
|
D_ASSERT(defined_decoder);
|
486
|
-
defined_decoder->GetBatch<uint8_t>(
|
488
|
+
defined_decoder->GetBatch<uint8_t>(define_out + result_offset, read_now);
|
487
489
|
}
|
488
490
|
|
489
491
|
idx_t null_count = 0;
|
@@ -501,7 +503,8 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, uint8_t
|
|
501
503
|
offset_buffer.resize(reader.allocator, sizeof(uint32_t) * (read_now - null_count));
|
502
504
|
dict_decoder->GetBatch<uint32_t>(offset_buffer.ptr, read_now - null_count);
|
503
505
|
DictReference(result);
|
504
|
-
Offsets(
|
506
|
+
Offsets(reinterpret_cast<uint32_t *>(offset_buffer.ptr), define_out, read_now, filter, result_offset,
|
507
|
+
result);
|
505
508
|
} else if (dbp_decoder) {
|
506
509
|
// TODO keep this in the state
|
507
510
|
auto read_buf = make_shared<ResizeableBuffer>();
|
@@ -566,7 +569,7 @@ void ColumnReader::ApplyPendingSkips(idx_t num_values) {
|
|
566
569
|
|
567
570
|
while (remaining) {
|
568
571
|
idx_t to_read = MinValue<idx_t>(remaining, STANDARD_VECTOR_SIZE);
|
569
|
-
read += Read(to_read, none_filter,
|
572
|
+
read += Read(to_read, none_filter, dummy_define.ptr, dummy_repeat.ptr, dummy_result);
|
570
573
|
remaining -= to_read;
|
571
574
|
}
|
572
575
|
|
@@ -623,14 +626,15 @@ void StringColumnReader::Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num
|
|
623
626
|
}
|
624
627
|
dict->available(str_len);
|
625
628
|
|
626
|
-
auto
|
627
|
-
|
629
|
+
auto dict_str = reinterpret_cast<const char *>(dict->ptr);
|
630
|
+
auto actual_str_len = VerifyString(dict_str, str_len);
|
631
|
+
dict_strings[dict_idx] = string_t(dict_str, actual_str_len);
|
628
632
|
dict->inc(str_len);
|
629
633
|
}
|
630
634
|
}
|
631
635
|
|
632
636
|
static shared_ptr<ResizeableBuffer> ReadDbpData(Allocator &allocator, ResizeableBuffer &buffer, idx_t &value_count) {
|
633
|
-
auto decoder = make_uniq<DbpDecoder>(
|
637
|
+
auto decoder = make_uniq<DbpDecoder>(buffer.ptr, buffer.len);
|
634
638
|
value_count = decoder->TotalValues();
|
635
639
|
auto result = make_shared<ResizeableBuffer>();
|
636
640
|
result->resize(allocator, sizeof(uint32_t) * value_count);
|
@@ -648,7 +652,7 @@ void StringColumnReader::PrepareDeltaLengthByteArray(ResizeableBuffer &buffer) {
|
|
648
652
|
byte_array_data = make_uniq<Vector>(LogicalType::VARCHAR, nullptr);
|
649
653
|
return;
|
650
654
|
}
|
651
|
-
auto length_data =
|
655
|
+
auto length_data = reinterpret_cast<uint32_t *>(length_buffer->ptr);
|
652
656
|
byte_array_data = make_uniq<Vector>(LogicalType::VARCHAR, value_count);
|
653
657
|
byte_array_count = value_count;
|
654
658
|
delta_offset = 0;
|
@@ -675,8 +679,8 @@ void StringColumnReader::PrepareDeltaByteArray(ResizeableBuffer &buffer) {
|
|
675
679
|
byte_array_data = make_uniq<Vector>(LogicalType::VARCHAR, nullptr);
|
676
680
|
return;
|
677
681
|
}
|
678
|
-
auto prefix_data =
|
679
|
-
auto suffix_data =
|
682
|
+
auto prefix_data = reinterpret_cast<uint32_t *>(prefix_buffer->ptr);
|
683
|
+
auto suffix_data = reinterpret_cast<uint32_t *>(suffix_buffer->ptr);
|
680
684
|
byte_array_data = make_uniq<Vector>(LogicalType::VARCHAR, prefix_count);
|
681
685
|
byte_array_count = prefix_count;
|
682
686
|
delta_offset = 0;
|
@@ -750,8 +754,9 @@ string_t StringParquetValueConversion::PlainRead(ByteBuffer &plain_data, ColumnR
|
|
750
754
|
auto &scr = reader.Cast<StringColumnReader>();
|
751
755
|
uint32_t str_len = scr.fixed_width_string_length == 0 ? plain_data.read<uint32_t>() : scr.fixed_width_string_length;
|
752
756
|
plain_data.available(str_len);
|
753
|
-
auto
|
754
|
-
auto
|
757
|
+
auto plain_str = char_ptr_cast(plain_data.ptr);
|
758
|
+
auto actual_str_len = reader.Cast<StringColumnReader>().VerifyString(plain_str, str_len);
|
759
|
+
auto ret_str = string_t(plain_str, actual_str_len);
|
755
760
|
plain_data.inc(str_len);
|
756
761
|
return ret_str;
|
757
762
|
}
|
@@ -765,8 +770,8 @@ void StringParquetValueConversion::PlainSkip(ByteBuffer &plain_data, ColumnReade
|
|
765
770
|
//===--------------------------------------------------------------------===//
|
766
771
|
// List Column Reader
|
767
772
|
//===--------------------------------------------------------------------===//
|
768
|
-
idx_t ListColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
|
769
|
-
Vector &result_out) {
|
773
|
+
idx_t ListColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr_t define_out,
|
774
|
+
data_ptr_t repeat_out, Vector &result_out) {
|
770
775
|
idx_t result_offset = 0;
|
771
776
|
auto result_ptr = FlatVector::GetData<list_entry_t>(result_out);
|
772
777
|
auto &result_mask = FlatVector::Validity(result_out);
|
@@ -935,8 +940,8 @@ void RowNumberColumnReader::InitializeRead(idx_t row_group_idx_p, const vector<C
|
|
935
940
|
}
|
936
941
|
}
|
937
942
|
|
938
|
-
idx_t RowNumberColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
|
939
|
-
|
943
|
+
idx_t RowNumberColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr_t define_out,
|
944
|
+
data_ptr_t repeat_out, Vector &result) {
|
940
945
|
|
941
946
|
auto data_ptr = FlatVector::GetData<int64_t>(result);
|
942
947
|
for (idx_t i = 0; i < num_values; i++) {
|
@@ -966,8 +971,8 @@ void CastColumnReader::InitializeRead(idx_t row_group_idx_p, const vector<Column
|
|
966
971
|
child_reader->InitializeRead(row_group_idx_p, columns, protocol_p);
|
967
972
|
}
|
968
973
|
|
969
|
-
idx_t CastColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
|
970
|
-
Vector &result) {
|
974
|
+
idx_t CastColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr_t define_out,
|
975
|
+
data_ptr_t repeat_out, Vector &result) {
|
971
976
|
intermediate_chunk.Reset();
|
972
977
|
auto &intermediate_vector = intermediate_chunk.data[0];
|
973
978
|
|
@@ -1018,8 +1023,8 @@ void StructColumnReader::InitializeRead(idx_t row_group_idx_p, const vector<Colu
|
|
1018
1023
|
}
|
1019
1024
|
}
|
1020
1025
|
|
1021
|
-
idx_t StructColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
|
1022
|
-
Vector &result) {
|
1026
|
+
idx_t StructColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr_t define_out,
|
1027
|
+
data_ptr_t repeat_out, Vector &result) {
|
1023
1028
|
auto &struct_entries = StructVector::GetEntries(result);
|
1024
1029
|
D_ASSERT(StructType::GetChildTypes(Type()).size() == struct_entries.size());
|
1025
1030
|
|
@@ -1099,7 +1104,7 @@ idx_t StructColumnReader::GroupRowsAvailable() {
|
|
1099
1104
|
template <class DUCKDB_PHYSICAL_TYPE, bool FIXED_LENGTH>
|
1100
1105
|
struct DecimalParquetValueConversion {
|
1101
1106
|
static DUCKDB_PHYSICAL_TYPE DictRead(ByteBuffer &dict, uint32_t &offset, ColumnReader &reader) {
|
1102
|
-
auto dict_ptr =
|
1107
|
+
auto dict_ptr = reinterpret_cast<DUCKDB_PHYSICAL_TYPE *>(dict.ptr);
|
1103
1108
|
return dict_ptr[offset];
|
1104
1109
|
}
|
1105
1110
|
|
@@ -1112,7 +1117,7 @@ struct DecimalParquetValueConversion {
|
|
1112
1117
|
}
|
1113
1118
|
plain_data.available(byte_len);
|
1114
1119
|
auto res =
|
1115
|
-
ParquetDecimalUtils::ReadDecimalValue<DUCKDB_PHYSICAL_TYPE>((
|
1120
|
+
ParquetDecimalUtils::ReadDecimalValue<DUCKDB_PHYSICAL_TYPE>(const_data_ptr_cast(plain_data.ptr), byte_len);
|
1116
1121
|
|
1117
1122
|
plain_data.inc(byte_len);
|
1118
1123
|
return res;
|
@@ -1186,7 +1191,7 @@ unique_ptr<ColumnReader> ParquetDecimalUtils::CreateReader(ParquetReader &reader
|
|
1186
1191
|
//===--------------------------------------------------------------------===//
|
1187
1192
|
struct UUIDValueConversion {
|
1188
1193
|
static hugeint_t DictRead(ByteBuffer &dict, uint32_t &offset, ColumnReader &reader) {
|
1189
|
-
auto dict_ptr =
|
1194
|
+
auto dict_ptr = reinterpret_cast<hugeint_t *>(dict.ptr);
|
1190
1195
|
return dict_ptr[offset];
|
1191
1196
|
}
|
1192
1197
|
|
@@ -1210,7 +1215,7 @@ struct UUIDValueConversion {
|
|
1210
1215
|
static hugeint_t PlainRead(ByteBuffer &plain_data, ColumnReader &reader) {
|
1211
1216
|
idx_t byte_len = sizeof(hugeint_t);
|
1212
1217
|
plain_data.available(byte_len);
|
1213
|
-
auto res = ReadParquetUUID((
|
1218
|
+
auto res = ReadParquetUUID(const_data_ptr_cast(plain_data.ptr));
|
1214
1219
|
|
1215
1220
|
plain_data.inc(byte_len);
|
1216
1221
|
return res;
|
@@ -1232,7 +1237,7 @@ public:
|
|
1232
1237
|
protected:
|
1233
1238
|
void Dictionary(shared_ptr<ResizeableBuffer> dictionary_data, idx_t num_entries) { // NOLINT
|
1234
1239
|
AllocateDict(num_entries * sizeof(hugeint_t));
|
1235
|
-
auto dict_ptr =
|
1240
|
+
auto dict_ptr = reinterpret_cast<hugeint_t *>(this->dict->ptr);
|
1236
1241
|
for (idx_t i = 0; i < num_entries; i++) {
|
1237
1242
|
dict_ptr[i] = UUIDValueConversion::PlainRead(*dictionary_data, *this);
|
1238
1243
|
}
|
@@ -1246,7 +1251,7 @@ struct IntervalValueConversion {
|
|
1246
1251
|
static constexpr const idx_t PARQUET_INTERVAL_SIZE = 12;
|
1247
1252
|
|
1248
1253
|
static interval_t DictRead(ByteBuffer &dict, uint32_t &offset, ColumnReader &reader) {
|
1249
|
-
auto dict_ptr =
|
1254
|
+
auto dict_ptr = reinterpret_cast<interval_t *>(dict.ptr);
|
1250
1255
|
return dict_ptr[offset];
|
1251
1256
|
}
|
1252
1257
|
|
@@ -1261,7 +1266,7 @@ struct IntervalValueConversion {
|
|
1261
1266
|
static interval_t PlainRead(ByteBuffer &plain_data, ColumnReader &reader) {
|
1262
1267
|
idx_t byte_len = PARQUET_INTERVAL_SIZE;
|
1263
1268
|
plain_data.available(byte_len);
|
1264
|
-
auto res = ReadParquetInterval((
|
1269
|
+
auto res = ReadParquetInterval(const_data_ptr_cast(plain_data.ptr));
|
1265
1270
|
|
1266
1271
|
plain_data.inc(byte_len);
|
1267
1272
|
return res;
|
@@ -1283,7 +1288,7 @@ public:
|
|
1283
1288
|
protected:
|
1284
1289
|
void Dictionary(shared_ptr<ResizeableBuffer> dictionary_data, idx_t num_entries) override { // NOLINT
|
1285
1290
|
AllocateDict(num_entries * sizeof(interval_t));
|
1286
|
-
auto dict_ptr =
|
1291
|
+
auto dict_ptr = reinterpret_cast<interval_t *>(this->dict->ptr);
|
1287
1292
|
for (idx_t i = 0; i < num_entries; i++) {
|
1288
1293
|
dict_ptr[i] = IntervalValueConversion::PlainRead(*dictionary_data, *this);
|
1289
1294
|
}
|