duckdb 0.8.2-dev157.0 → 0.8.2-dev1573.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +4619 -4446
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/extra_type_info.cpp +506 -0
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +14 -14
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/types.cpp +8 -655
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +4 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
- package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +79 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +44 -31
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_common.cpp +2 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
- package/src/statement.cpp +10 -3
- package/test/test_all_types.test.ts +233 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -375,7 +375,7 @@ int RemoveDirectoryRecursive(const char *path) {
|
|
375
375
|
continue;
|
376
376
|
}
|
377
377
|
len = path_len + (idx_t)strlen(p->d_name) + 2;
|
378
|
-
buf = new char[len];
|
378
|
+
buf = new (std::nothrow) char[len];
|
379
379
|
if (buf) {
|
380
380
|
struct stat statbuf;
|
381
381
|
snprintf(buf, len, "%s/%s", path, p->d_name);
|
@@ -559,7 +559,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
|
|
559
559
|
}
|
560
560
|
|
561
561
|
void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
|
562
|
-
auto &whandle = (
|
562
|
+
auto &whandle = handle.Cast<WindowsFileHandle>();
|
563
563
|
whandle.position = location;
|
564
564
|
LARGE_INTEGER wlocation;
|
565
565
|
wlocation.QuadPart = location;
|
@@ -567,7 +567,7 @@ void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
|
|
567
567
|
}
|
568
568
|
|
569
569
|
idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
|
570
|
-
return (
|
570
|
+
return handle.Cast<WindowsFileHandle>().position;
|
571
571
|
}
|
572
572
|
|
573
573
|
static DWORD FSInternalRead(FileHandle &handle, HANDLE hFile, void *buffer, int64_t nr_bytes, idx_t location) {
|
@@ -597,8 +597,8 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i
|
|
597
597
|
}
|
598
598
|
|
599
599
|
int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
|
600
|
-
HANDLE hFile = (
|
601
|
-
auto &pos = (
|
600
|
+
HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
|
601
|
+
auto &pos = handle.Cast<WindowsFileHandle>().position;
|
602
602
|
auto n = std::min<idx_t>(std::max<idx_t>(GetFileSize(handle), pos) - pos, nr_bytes);
|
603
603
|
auto bytes_read = FSInternalRead(handle, hFile, buffer, n, pos);
|
604
604
|
pos += bytes_read;
|
@@ -622,7 +622,7 @@ static DWORD FSInternalWrite(FileHandle &handle, HANDLE hFile, void *buffer, int
|
|
622
622
|
}
|
623
623
|
|
624
624
|
void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
|
625
|
-
HANDLE hFile = (
|
625
|
+
HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
|
626
626
|
auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, location);
|
627
627
|
if (bytes_written != nr_bytes) {
|
628
628
|
throw IOException("Could not write all bytes from file \"%s\": wanted=%lld wrote=%lld", handle.path, nr_bytes,
|
@@ -631,15 +631,15 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes,
|
|
631
631
|
}
|
632
632
|
|
633
633
|
int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
|
634
|
-
HANDLE hFile = (
|
635
|
-
auto &pos = (
|
634
|
+
HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
|
635
|
+
auto &pos = handle.Cast<WindowsFileHandle>().position;
|
636
636
|
auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, pos);
|
637
637
|
pos += bytes_written;
|
638
638
|
return bytes_written;
|
639
639
|
}
|
640
640
|
|
641
641
|
int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
|
642
|
-
HANDLE hFile = (
|
642
|
+
HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
|
643
643
|
LARGE_INTEGER result;
|
644
644
|
if (!GetFileSizeEx(hFile, &result)) {
|
645
645
|
return -1;
|
@@ -648,7 +648,7 @@ int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
|
|
648
648
|
}
|
649
649
|
|
650
650
|
time_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) {
|
651
|
-
HANDLE hFile = (
|
651
|
+
HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
|
652
652
|
|
653
653
|
// https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfiletime
|
654
654
|
FILETIME last_write;
|
@@ -674,7 +674,7 @@ time_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) {
|
|
674
674
|
}
|
675
675
|
|
676
676
|
void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
|
677
|
-
HANDLE hFile = (
|
677
|
+
HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
|
678
678
|
// seek to the location
|
679
679
|
SetFilePointer(handle, new_size);
|
680
680
|
// now set the end of file position
|
@@ -700,7 +700,7 @@ void LocalFileSystem::CreateDirectory(const string &directory) {
|
|
700
700
|
}
|
701
701
|
auto unicode_path = WindowsUtil::UTF8ToUnicode(directory.c_str());
|
702
702
|
if (directory.empty() || !CreateDirectoryW(unicode_path.c_str(), NULL) || !DirectoryExists(directory)) {
|
703
|
-
throw IOException("Could not create directory
|
703
|
+
throw IOException("Could not create directory: \'%s\'", directory.c_str());
|
704
704
|
}
|
705
705
|
}
|
706
706
|
|
@@ -767,7 +767,7 @@ bool LocalFileSystem::ListFiles(const string &directory, const std::function<voi
|
|
767
767
|
}
|
768
768
|
|
769
769
|
void LocalFileSystem::FileSync(FileHandle &handle) {
|
770
|
-
HANDLE hFile = (
|
770
|
+
HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
|
771
771
|
if (FlushFileBuffers(hFile) == 0) {
|
772
772
|
throw IOException("Could not flush file handle to disk!");
|
773
773
|
}
|
@@ -782,7 +782,7 @@ void LocalFileSystem::MoveFile(const string &source, const string &target) {
|
|
782
782
|
}
|
783
783
|
|
784
784
|
FileType LocalFileSystem::GetFileType(FileHandle &handle) {
|
785
|
-
auto path = (
|
785
|
+
auto path = handle.Cast<WindowsFileHandle>().path;
|
786
786
|
// pipes in windows are just files in '\\.\pipe\' folder
|
787
787
|
if (strncmp(path.c_str(), PIPE_PREFIX, strlen(PIPE_PREFIX)) == 0) {
|
788
788
|
return FileType::FILE_TYPE_FIFO;
|
@@ -6,6 +6,7 @@
|
|
6
6
|
#include "duckdb/common/exception.hpp"
|
7
7
|
#include "duckdb/function/function_set.hpp"
|
8
8
|
#include "duckdb/common/hive_partitioning.hpp"
|
9
|
+
#include "duckdb/common/types.hpp"
|
9
10
|
|
10
11
|
namespace duckdb {
|
11
12
|
|
@@ -13,6 +14,8 @@ void MultiFileReader::AddParameters(TableFunction &table_function) {
|
|
13
14
|
table_function.named_parameters["filename"] = LogicalType::BOOLEAN;
|
14
15
|
table_function.named_parameters["hive_partitioning"] = LogicalType::BOOLEAN;
|
15
16
|
table_function.named_parameters["union_by_name"] = LogicalType::BOOLEAN;
|
17
|
+
table_function.named_parameters["hive_types"] = LogicalType::ANY;
|
18
|
+
table_function.named_parameters["hive_types_autocast"] = LogicalType::BOOLEAN;
|
16
19
|
}
|
17
20
|
|
18
21
|
vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value &input, const string &name,
|
@@ -49,7 +52,8 @@ vector<string> MultiFileReader::GetFileList(ClientContext &context, const Value
|
|
49
52
|
return files;
|
50
53
|
}
|
51
54
|
|
52
|
-
bool MultiFileReader::ParseOption(const string &key, const Value &val, MultiFileReaderOptions &options
|
55
|
+
bool MultiFileReader::ParseOption(const string &key, const Value &val, MultiFileReaderOptions &options,
|
56
|
+
ClientContext &context) {
|
53
57
|
auto loption = StringUtil::Lower(key);
|
54
58
|
if (loption == "filename") {
|
55
59
|
options.filename = BooleanValue::Get(val);
|
@@ -58,6 +62,28 @@ bool MultiFileReader::ParseOption(const string &key, const Value &val, MultiFile
|
|
58
62
|
options.auto_detect_hive_partitioning = false;
|
59
63
|
} else if (loption == "union_by_name") {
|
60
64
|
options.union_by_name = BooleanValue::Get(val);
|
65
|
+
} else if (loption == "hive_types_autocast" || loption == "hive_type_autocast") {
|
66
|
+
options.hive_types_autocast = BooleanValue::Get(val);
|
67
|
+
} else if (loption == "hive_types" || loption == "hive_type") {
|
68
|
+
if (val.type().id() != LogicalTypeId::STRUCT) {
|
69
|
+
throw InvalidInputException(
|
70
|
+
"'hive_types' only accepts a STRUCT('name':VARCHAR, ...), but '%s' was provided",
|
71
|
+
val.type().ToString());
|
72
|
+
}
|
73
|
+
// verify that that all the children of the struct value are VARCHAR
|
74
|
+
auto &children = StructValue::GetChildren(val);
|
75
|
+
for (idx_t i = 0; i < children.size(); i++) {
|
76
|
+
const Value &child = children[i];
|
77
|
+
if (child.type().id() != LogicalType::VARCHAR) {
|
78
|
+
throw InvalidInputException("hive_types: '%s' must be a VARCHAR, instead: '%s' was provided",
|
79
|
+
StructType::GetChildName(val.type(), i), child.type().ToString());
|
80
|
+
}
|
81
|
+
// for every child of the struct, get the logical type
|
82
|
+
LogicalType transformed_type = TransformStringToLogicalType(child.ToString(), context);
|
83
|
+
const string &name = StructType::GetChildName(val.type(), i);
|
84
|
+
options.hive_types_schema[name] = transformed_type;
|
85
|
+
}
|
86
|
+
D_ASSERT(!options.hive_types_schema.empty());
|
61
87
|
} else {
|
62
88
|
return false;
|
63
89
|
}
|
@@ -80,8 +106,9 @@ bool MultiFileReader::ComplexFilterPushdown(ClientContext &context, vector<strin
|
|
80
106
|
}
|
81
107
|
|
82
108
|
auto start_files = files.size();
|
83
|
-
HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get.
|
84
|
-
options.
|
109
|
+
HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get, options.hive_partitioning,
|
110
|
+
options.filename);
|
111
|
+
|
85
112
|
if (files.size() != start_files) {
|
86
113
|
// we have pruned files
|
87
114
|
return true;
|
@@ -111,28 +138,28 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt
|
|
111
138
|
auto file_partitions = HivePartitioning::Parse(f);
|
112
139
|
for (auto &part_info : partitions) {
|
113
140
|
if (file_partitions.find(part_info.first) == file_partitions.end()) {
|
141
|
+
string error = "Hive partition mismatch between file \"%s\" and \"%s\": key \"%s\" not found";
|
114
142
|
if (options.auto_detect_hive_partitioning == true) {
|
115
|
-
throw
|
116
|
-
|
117
|
-
"mismatch between file \"%s\" and \"%s\": key \"%s\" not found\n\nTo switch off hive "
|
118
|
-
"partition, set: HIVE_PARTITIONING=0",
|
119
|
-
files[0], f, part_info.first);
|
143
|
+
throw InternalException(error + "(hive partitioning was autodetected)", files[0], f,
|
144
|
+
part_info.first);
|
120
145
|
}
|
121
|
-
throw BinderException(
|
122
|
-
"Hive partition mismatch between file \"%s\" and \"%s\": key \"%s\" not found", files[0], f,
|
123
|
-
part_info.first);
|
146
|
+
throw BinderException(error.c_str(), files[0], f, part_info.first);
|
124
147
|
}
|
125
148
|
}
|
126
149
|
if (partitions.size() != file_partitions.size()) {
|
150
|
+
string error_msg = "Hive partition mismatch between file \"%s\" and \"%s\"";
|
127
151
|
if (options.auto_detect_hive_partitioning == true) {
|
128
|
-
throw
|
129
|
-
"Hive partition mismatch between file \"%s\" and \"%s\"\n\nTo switch off "
|
130
|
-
"hive partition, set: HIVE_PARTITIONING=0",
|
131
|
-
files[0], f);
|
152
|
+
throw InternalException(error_msg + "(hive partitioning was autodetected)", files[0], f);
|
132
153
|
}
|
133
|
-
throw BinderException(
|
154
|
+
throw BinderException(error_msg.c_str(), files[0], f);
|
134
155
|
}
|
135
156
|
}
|
157
|
+
|
158
|
+
if (!options.hive_types_schema.empty()) {
|
159
|
+
// verify that all hive_types are existing partitions
|
160
|
+
options.VerifyHiveTypesArePartitions(partitions);
|
161
|
+
}
|
162
|
+
|
136
163
|
for (auto &part : partitions) {
|
137
164
|
idx_t hive_partitioning_index = DConstants::INVALID_INDEX;
|
138
165
|
auto lookup = std::find(names.begin(), names.end(), part.first);
|
@@ -140,11 +167,11 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt
|
|
140
167
|
// hive partitioning column also exists in file - override
|
141
168
|
auto idx = lookup - names.begin();
|
142
169
|
hive_partitioning_index = idx;
|
143
|
-
return_types[idx] =
|
170
|
+
return_types[idx] = options.GetHiveLogicalType(part.first);
|
144
171
|
} else {
|
145
172
|
// hive partitioning column does not exist in file - add a new column containing the key
|
146
173
|
hive_partitioning_index = names.size();
|
147
|
-
return_types.emplace_back(
|
174
|
+
return_types.emplace_back(options.GetHiveLogicalType(part.first));
|
148
175
|
names.emplace_back(part.first);
|
149
176
|
}
|
150
177
|
bind_data.hive_partitioning_indexes.emplace_back(part.first, hive_partitioning_index);
|
@@ -156,7 +183,9 @@ MultiFileReaderBindData MultiFileReader::BindOptions(MultiFileReaderOptions &opt
|
|
156
183
|
void MultiFileReader::FinalizeBind(const MultiFileReaderOptions &file_options, const MultiFileReaderBindData &options,
|
157
184
|
const string &filename, const vector<string> &local_names,
|
158
185
|
const vector<LogicalType> &global_types, const vector<string> &global_names,
|
159
|
-
const vector<column_t> &global_column_ids, MultiFileReaderData &reader_data
|
186
|
+
const vector<column_t> &global_column_ids, MultiFileReaderData &reader_data,
|
187
|
+
ClientContext &context) {
|
188
|
+
|
160
189
|
// create a map of name -> column index
|
161
190
|
case_insensitive_map_t<idx_t> name_map;
|
162
191
|
if (file_options.union_by_name) {
|
@@ -183,7 +212,8 @@ void MultiFileReader::FinalizeBind(const MultiFileReaderOptions &file_options, c
|
|
183
212
|
bool found_partition = false;
|
184
213
|
for (auto &entry : options.hive_partitioning_indexes) {
|
185
214
|
if (column_id == entry.index) {
|
186
|
-
|
215
|
+
Value value = file_options.GetHivePartitionValue(partitions[entry.value], entry.value, context);
|
216
|
+
reader_data.constant_map.emplace_back(i, value);
|
187
217
|
found_partition = true;
|
188
218
|
break;
|
189
219
|
}
|
@@ -313,7 +343,16 @@ void MultiFileReaderOptions::Serialize(Serializer &serializer) const {
|
|
313
343
|
FieldWriter writer(serializer);
|
314
344
|
writer.WriteField<bool>(filename);
|
315
345
|
writer.WriteField<bool>(hive_partitioning);
|
346
|
+
writer.WriteField<bool>(auto_detect_hive_partitioning);
|
316
347
|
writer.WriteField<bool>(union_by_name);
|
348
|
+
writer.WriteField<bool>(hive_types_autocast);
|
349
|
+
// serialize hive_types_schema
|
350
|
+
const uint32_t schema_size = hive_types_schema.size();
|
351
|
+
writer.WriteField<uint32_t>(schema_size);
|
352
|
+
for (auto &hive_type : hive_types_schema) {
|
353
|
+
writer.WriteString(hive_type.first);
|
354
|
+
writer.WriteString(hive_type.second.ToString());
|
355
|
+
}
|
317
356
|
writer.Finalize();
|
318
357
|
}
|
319
358
|
|
@@ -322,7 +361,16 @@ MultiFileReaderOptions MultiFileReaderOptions::Deserialize(Deserializer &source)
|
|
322
361
|
FieldReader reader(source);
|
323
362
|
result.filename = reader.ReadRequired<bool>();
|
324
363
|
result.hive_partitioning = reader.ReadRequired<bool>();
|
364
|
+
result.auto_detect_hive_partitioning = reader.ReadRequired<bool>();
|
325
365
|
result.union_by_name = reader.ReadRequired<bool>();
|
366
|
+
result.hive_types_autocast = reader.ReadRequired<bool>();
|
367
|
+
// deserialize hive_types_schema
|
368
|
+
const uint32_t schema_size = reader.ReadRequired<uint32_t>();
|
369
|
+
for (idx_t i = 0; i < schema_size; i++) {
|
370
|
+
const string name = reader.ReadRequired<string>();
|
371
|
+
const LogicalType type = TransformStringToLogicalType(reader.ReadRequired<string>());
|
372
|
+
result.hive_types_schema[name] = type;
|
373
|
+
}
|
326
374
|
reader.Finalize();
|
327
375
|
return result;
|
328
376
|
}
|
@@ -365,7 +413,9 @@ HivePartitioningIndex HivePartitioningIndex::Deserialize(Deserializer &source) {
|
|
365
413
|
void MultiFileReaderOptions::AddBatchInfo(BindInfo &bind_info) const {
|
366
414
|
bind_info.InsertOption("filename", Value::BOOLEAN(filename));
|
367
415
|
bind_info.InsertOption("hive_partitioning", Value::BOOLEAN(hive_partitioning));
|
416
|
+
bind_info.InsertOption("auto_detect_hive_partitioning", Value::BOOLEAN(auto_detect_hive_partitioning));
|
368
417
|
bind_info.InsertOption("union_by_name", Value::BOOLEAN(union_by_name));
|
418
|
+
bind_info.InsertOption("hive_types_autocast", Value::BOOLEAN(hive_types_autocast));
|
369
419
|
}
|
370
420
|
|
371
421
|
void UnionByName::CombineUnionTypes(const vector<string> &col_names, const vector<LogicalType> &sql_types,
|
@@ -390,4 +440,118 @@ void UnionByName::CombineUnionTypes(const vector<string> &col_names, const vecto
|
|
390
440
|
}
|
391
441
|
}
|
392
442
|
|
443
|
+
bool MultiFileReaderOptions::AutoDetectHivePartitioningInternal(const vector<string> &files) {
|
444
|
+
std::unordered_set<string> partitions;
|
445
|
+
|
446
|
+
auto splits_first_file = StringUtil::Split(files.front(), FileSystem::PathSeparator());
|
447
|
+
if (splits_first_file.size() < 2) {
|
448
|
+
return false;
|
449
|
+
}
|
450
|
+
for (auto it = splits_first_file.begin(); it != splits_first_file.end(); it++) {
|
451
|
+
auto partition = StringUtil::Split(*it, "=");
|
452
|
+
if (partition.size() == 2) {
|
453
|
+
partitions.insert(partition.front());
|
454
|
+
}
|
455
|
+
}
|
456
|
+
if (partitions.empty()) {
|
457
|
+
return false;
|
458
|
+
}
|
459
|
+
for (auto &file : files) {
|
460
|
+
auto splits = StringUtil::Split(file, FileSystem::PathSeparator());
|
461
|
+
if (splits.size() != splits_first_file.size()) {
|
462
|
+
return false;
|
463
|
+
}
|
464
|
+
for (auto it = splits.begin(); it != std::prev(splits.end()); it++) {
|
465
|
+
auto part = StringUtil::Split(*it, "=");
|
466
|
+
if (part.size() != 2) {
|
467
|
+
continue;
|
468
|
+
}
|
469
|
+
if (partitions.find(part.front()) == partitions.end()) {
|
470
|
+
return false;
|
471
|
+
}
|
472
|
+
}
|
473
|
+
}
|
474
|
+
return true;
|
475
|
+
}
|
476
|
+
void MultiFileReaderOptions::AutoDetectHiveTypesInternal(const string &file, ClientContext &context) {
|
477
|
+
std::map<string, string> partitions;
|
478
|
+
auto splits = StringUtil::Split(file, FileSystem::PathSeparator());
|
479
|
+
if (splits.size() < 2) {
|
480
|
+
return;
|
481
|
+
}
|
482
|
+
for (auto it = splits.begin(); it != std::prev(splits.end()); it++) {
|
483
|
+
auto part = StringUtil::Split(*it, "=");
|
484
|
+
if (part.size() == 2) {
|
485
|
+
partitions[part.front()] = part.back();
|
486
|
+
}
|
487
|
+
}
|
488
|
+
if (partitions.empty()) {
|
489
|
+
return;
|
490
|
+
}
|
491
|
+
|
492
|
+
const LogicalType candidates[] = {LogicalType::DATE, LogicalType::TIMESTAMP, LogicalType::BIGINT};
|
493
|
+
for (auto &part : partitions) {
|
494
|
+
const string &name = part.first;
|
495
|
+
if (hive_types_schema.find(name) != hive_types_schema.end()) {
|
496
|
+
continue;
|
497
|
+
}
|
498
|
+
Value value(part.second);
|
499
|
+
for (auto &candidate : candidates) {
|
500
|
+
const bool success = value.TryCastAs(context, candidate);
|
501
|
+
if (success) {
|
502
|
+
hive_types_schema[name] = candidate;
|
503
|
+
break;
|
504
|
+
}
|
505
|
+
}
|
506
|
+
}
|
507
|
+
}
|
508
|
+
void MultiFileReaderOptions::AutoDetectHivePartitioning(const vector<string> &files, ClientContext &context) {
|
509
|
+
D_ASSERT(!files.empty());
|
510
|
+
const bool hp_explicitly_disabled = !auto_detect_hive_partitioning && !hive_partitioning;
|
511
|
+
const bool ht_enabled = !hive_types_schema.empty();
|
512
|
+
if (hp_explicitly_disabled && ht_enabled) {
|
513
|
+
throw InvalidInputException("cannot disable hive_partitioning when hive_types is enabled");
|
514
|
+
}
|
515
|
+
if (ht_enabled && auto_detect_hive_partitioning && !hive_partitioning) {
|
516
|
+
// hive_types flag implies hive_partitioning
|
517
|
+
hive_partitioning = true;
|
518
|
+
auto_detect_hive_partitioning = false;
|
519
|
+
}
|
520
|
+
if (auto_detect_hive_partitioning) {
|
521
|
+
hive_partitioning = AutoDetectHivePartitioningInternal(files);
|
522
|
+
}
|
523
|
+
if (hive_partitioning && hive_types_autocast) {
|
524
|
+
AutoDetectHiveTypesInternal(files.front(), context);
|
525
|
+
}
|
526
|
+
}
|
527
|
+
void MultiFileReaderOptions::VerifyHiveTypesArePartitions(const std::map<string, string> &partitions) const {
|
528
|
+
for (auto &hive_type : hive_types_schema) {
|
529
|
+
if (partitions.find(hive_type.first) == partitions.end()) {
|
530
|
+
throw InvalidInputException("Unknown hive_type: \"%s\" does not appear to be a partition", hive_type.first);
|
531
|
+
}
|
532
|
+
}
|
533
|
+
}
|
534
|
+
LogicalType MultiFileReaderOptions::GetHiveLogicalType(const string &hive_partition_column) const {
|
535
|
+
if (!hive_types_schema.empty()) {
|
536
|
+
auto it = hive_types_schema.find(hive_partition_column);
|
537
|
+
if (it != hive_types_schema.end()) {
|
538
|
+
return it->second;
|
539
|
+
}
|
540
|
+
}
|
541
|
+
return LogicalType::VARCHAR;
|
542
|
+
}
|
543
|
+
Value MultiFileReaderOptions::GetHivePartitionValue(const string &base, const string &entry,
|
544
|
+
ClientContext &context) const {
|
545
|
+
Value value(base);
|
546
|
+
auto it = hive_types_schema.find(entry);
|
547
|
+
if (it == hive_types_schema.end()) {
|
548
|
+
return value;
|
549
|
+
}
|
550
|
+
if (!value.TryCastAs(context, it->second)) {
|
551
|
+
throw InvalidInputException("Unable to cast '%s' (from hive partition column '%s') to: '%s'", value.ToString(),
|
552
|
+
StringUtil::Upper(it->first), it->second.ToString());
|
553
|
+
}
|
554
|
+
return value;
|
555
|
+
}
|
556
|
+
|
393
557
|
} // namespace duckdb
|
@@ -1,4 +1,5 @@
|
|
1
1
|
#include "duckdb/common/operator/cast_operators.hpp"
|
2
|
+
#include "duckdb/common/hugeint.hpp"
|
2
3
|
#include "duckdb/common/operator/string_cast.hpp"
|
3
4
|
#include "duckdb/common/operator/numeric_cast.hpp"
|
4
5
|
#include "duckdb/common/operator/decimal_cast_operators.hpp"
|
@@ -1425,11 +1426,20 @@ string_t CastFromBlob::Operation(string_t input, Vector &vector) {
|
|
1425
1426
|
return result;
|
1426
1427
|
}
|
1427
1428
|
|
1429
|
+
template <>
|
1430
|
+
string_t CastFromBlobToBit::Operation(string_t input, Vector &vector) {
|
1431
|
+
idx_t result_size = input.GetSize() + 1;
|
1432
|
+
if (result_size <= 1) {
|
1433
|
+
throw ConversionException("Cannot cast empty BLOB to BIT");
|
1434
|
+
}
|
1435
|
+
return StringVector::AddStringOrBlob(vector, Bit::BlobToBit(input));
|
1436
|
+
}
|
1437
|
+
|
1428
1438
|
//===--------------------------------------------------------------------===//
|
1429
1439
|
// Cast From Bit
|
1430
1440
|
//===--------------------------------------------------------------------===//
|
1431
1441
|
template <>
|
1432
|
-
string_t
|
1442
|
+
string_t CastFromBitToString::Operation(string_t input, Vector &vector) {
|
1433
1443
|
|
1434
1444
|
idx_t result_size = Bit::BitLength(input);
|
1435
1445
|
string_t result = StringVector::EmptyString(vector, result_size);
|
@@ -1482,6 +1492,30 @@ bool TryCastToBit::Operation(string_t input, string_t &result, Vector &result_ve
|
|
1482
1492
|
return true;
|
1483
1493
|
}
|
1484
1494
|
|
1495
|
+
template <>
|
1496
|
+
bool CastFromBitToNumeric::Operation(string_t input, bool &result, bool strict) {
|
1497
|
+
D_ASSERT(input.GetSize() > 1);
|
1498
|
+
|
1499
|
+
uint8_t value;
|
1500
|
+
bool success = CastFromBitToNumeric::Operation(input, value, strict);
|
1501
|
+
result = (value > 0);
|
1502
|
+
return (success);
|
1503
|
+
}
|
1504
|
+
|
1505
|
+
template <>
|
1506
|
+
bool CastFromBitToNumeric::Operation(string_t input, hugeint_t &result, bool strict) {
|
1507
|
+
D_ASSERT(input.GetSize() > 1);
|
1508
|
+
|
1509
|
+
if (input.GetSize() - 1 > sizeof(hugeint_t)) {
|
1510
|
+
throw ConversionException("Bitstring doesn't fit inside of %s", GetTypeId<hugeint_t>());
|
1511
|
+
}
|
1512
|
+
Bit::BitToNumeric(input, result);
|
1513
|
+
if (result < NumericLimits<hugeint_t>::Minimum()) {
|
1514
|
+
throw ConversionException("Minimum limit for HUGEINT is %s", NumericLimits<hugeint_t>::Minimum().ToString());
|
1515
|
+
}
|
1516
|
+
return (true);
|
1517
|
+
}
|
1518
|
+
|
1485
1519
|
//===--------------------------------------------------------------------===//
|
1486
1520
|
// Cast From UUID
|
1487
1521
|
//===--------------------------------------------------------------------===//
|
@@ -8,9 +8,26 @@
|
|
8
8
|
|
9
9
|
namespace duckdb {
|
10
10
|
|
11
|
+
//! Templated radix partitioning constants, can be templated to the number of radix bits
|
12
|
+
template <idx_t radix_bits>
|
13
|
+
struct RadixPartitioningConstants {
|
14
|
+
public:
|
15
|
+
//! Bitmask of the upper bits starting at the 5th byte
|
16
|
+
static constexpr const idx_t NUM_PARTITIONS = RadixPartitioning::NumberOfPartitions(radix_bits);
|
17
|
+
static constexpr const idx_t SHIFT = RadixPartitioning::Shift(radix_bits);
|
18
|
+
static constexpr const hash_t MASK = RadixPartitioning::Mask(radix_bits);
|
19
|
+
|
20
|
+
public:
|
21
|
+
//! Apply bitmask and right shift to get a number between 0 and NUM_PARTITIONS
|
22
|
+
static inline hash_t ApplyMask(hash_t hash) {
|
23
|
+
D_ASSERT((hash & MASK) >> SHIFT < NUM_PARTITIONS);
|
24
|
+
return (hash & MASK) >> SHIFT;
|
25
|
+
}
|
26
|
+
};
|
27
|
+
|
11
28
|
template <class OP, class RETURN_TYPE, typename... ARGS>
|
12
29
|
RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
|
13
|
-
D_ASSERT(radix_bits <=
|
30
|
+
D_ASSERT(radix_bits <= RadixPartitioning::MAX_RADIX_BITS);
|
14
31
|
switch (radix_bits) {
|
15
32
|
case 1:
|
16
33
|
return OP::template Operation<1>(std::forward<ARGS>(args)...);
|
@@ -20,7 +37,7 @@ RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
|
|
20
37
|
return OP::template Operation<3>(std::forward<ARGS>(args)...);
|
21
38
|
case 4:
|
22
39
|
return OP::template Operation<4>(std::forward<ARGS>(args)...);
|
23
|
-
case 5:
|
40
|
+
case 5: // LCOV_EXCL_START
|
24
41
|
return OP::template Operation<5>(std::forward<ARGS>(args)...);
|
25
42
|
case 6:
|
26
43
|
return OP::template Operation<6>(std::forward<ARGS>(args)...);
|
@@ -32,9 +49,14 @@ RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
|
|
32
49
|
return OP::template Operation<9>(std::forward<ARGS>(args)...);
|
33
50
|
case 10:
|
34
51
|
return OP::template Operation<10>(std::forward<ARGS>(args)...);
|
52
|
+
case 11:
|
53
|
+
return OP::template Operation<10>(std::forward<ARGS>(args)...);
|
54
|
+
case 12:
|
55
|
+
return OP::template Operation<10>(std::forward<ARGS>(args)...);
|
35
56
|
default:
|
36
|
-
throw InternalException(
|
37
|
-
|
57
|
+
throw InternalException(
|
58
|
+
"radix_bits higher than RadixPartitioning::MAX_RADIX_BITS encountered in RadixBitsSwitch");
|
59
|
+
} // LCOV_EXCL_STOP
|
38
60
|
}
|
39
61
|
|
40
62
|
template <idx_t radix_bits>
|
@@ -69,10 +91,6 @@ struct HashsToBinsFunctor {
|
|
69
91
|
}
|
70
92
|
};
|
71
93
|
|
72
|
-
void RadixPartitioning::HashesToBins(Vector &hashes, idx_t radix_bits, Vector &bins, idx_t count) {
|
73
|
-
return RadixBitsSwitch<HashsToBinsFunctor, void>(radix_bits, hashes, bins, count);
|
74
|
-
}
|
75
|
-
|
76
94
|
//===--------------------------------------------------------------------===//
|
77
95
|
// Row Data Partitioning
|
78
96
|
//===--------------------------------------------------------------------===//
|
@@ -54,7 +54,7 @@ duckdb::vector<Match> RegexFindAll(const std::string &input, const Regex ®ex)
|
|
54
54
|
Match match;
|
55
55
|
while (RegexSearchInternal(input.c_str(), match, regex, RE2::UNANCHORED, position, input.size())) {
|
56
56
|
position += match.position(0) + match.length(0);
|
57
|
-
matches.emplace_back(
|
57
|
+
matches.emplace_back(match);
|
58
58
|
}
|
59
59
|
return matches;
|
60
60
|
}
|
@@ -99,7 +99,7 @@ void RowOperations::UnswizzleHeapPointer(const RowLayout &layout, const data_ptr
|
|
99
99
|
|
100
100
|
static inline void VerifyUnswizzledString(const RowLayout &layout, const idx_t &col_idx, const data_ptr_t &row_ptr) {
|
101
101
|
#ifdef DEBUG
|
102
|
-
if (layout.GetTypes()[col_idx]
|
102
|
+
if (layout.GetTypes()[col_idx].id() != LogicalTypeId::VARCHAR) {
|
103
103
|
return;
|
104
104
|
}
|
105
105
|
idx_t entry_idx;
|
@@ -273,16 +273,13 @@ void MergeSorter::ComputeMerge(const idx_t &count, bool left_smaller[]) {
|
|
273
273
|
break;
|
274
274
|
}
|
275
275
|
// Pin the radix sorting data
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
}
|
284
|
-
const idx_t &l_count = !l_done ? l_sorted_block.radix_sorting_data[l.block_idx]->count : 0;
|
285
|
-
const idx_t &r_count = !r_done ? r_sorted_block.radix_sorting_data[r.block_idx]->count : 0;
|
276
|
+
left->PinRadix(l.block_idx);
|
277
|
+
l_radix_ptr = left->RadixPtr();
|
278
|
+
right->PinRadix(r.block_idx);
|
279
|
+
r_radix_ptr = right->RadixPtr();
|
280
|
+
|
281
|
+
const idx_t l_count = l_sorted_block.radix_sorting_data[l.block_idx]->count;
|
282
|
+
const idx_t r_count = r_sorted_block.radix_sorting_data[r.block_idx]->count;
|
286
283
|
// Compute the merge
|
287
284
|
if (sort_layout.all_constant) {
|
288
285
|
// All sorting columns are constant size
|
@@ -298,12 +295,8 @@ void MergeSorter::ComputeMerge(const idx_t &count, bool left_smaller[]) {
|
|
298
295
|
}
|
299
296
|
} else {
|
300
297
|
// Pin the blob data
|
301
|
-
|
302
|
-
|
303
|
-
}
|
304
|
-
if (!r_done) {
|
305
|
-
right->PinData(*r_sorted_block.blob_sorting_data);
|
306
|
-
}
|
298
|
+
left->PinData(*l_sorted_block.blob_sorting_data);
|
299
|
+
right->PinData(*r_sorted_block.blob_sorting_data);
|
307
300
|
// Merge with variable size sorting columns
|
308
301
|
for (; compared < count && l.entry_idx < l_count && r.entry_idx < r_count; compared++) {
|
309
302
|
left_smaller[compared] =
|