duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +4619 -4446
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/extra_type_info.cpp +506 -0
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +14 -14
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/types.cpp +8 -655
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +13 -22
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +4 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
- package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +79 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +5 -0
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_common.cpp +2 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
- package/src/statement.cpp +10 -3
- package/test/test_all_types.test.ts +233 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -0,0 +1,132 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/optimizer/compressed_materialization.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/common/unordered_set.hpp"
|
12
|
+
#include "duckdb/function/scalar/compressed_materialization_functions.hpp"
|
13
|
+
#include "duckdb/planner/column_binding_map.hpp"
|
14
|
+
#include "duckdb/storage/statistics/base_statistics.hpp"
|
15
|
+
|
16
|
+
namespace duckdb {
|
17
|
+
|
18
|
+
class LogicalOperator;
|
19
|
+
struct JoinCondition;
|
20
|
+
|
21
|
+
struct CMChildInfo {
|
22
|
+
public:
|
23
|
+
CMChildInfo(LogicalOperator &op, const column_binding_set_t &referenced_bindings);
|
24
|
+
|
25
|
+
public:
|
26
|
+
//! Bindings and types before compressing
|
27
|
+
vector<ColumnBinding> bindings_before;
|
28
|
+
vector<LogicalType> &types;
|
29
|
+
//! Whether the input binding is eligible for compression
|
30
|
+
vector<bool> can_compress;
|
31
|
+
|
32
|
+
//! Bindings after compressing (projection on top)
|
33
|
+
vector<ColumnBinding> bindings_after;
|
34
|
+
};
|
35
|
+
|
36
|
+
struct CMBindingInfo {
|
37
|
+
public:
|
38
|
+
explicit CMBindingInfo(ColumnBinding binding, const LogicalType &type);
|
39
|
+
|
40
|
+
public:
|
41
|
+
ColumnBinding binding;
|
42
|
+
|
43
|
+
//! Type before compressing
|
44
|
+
LogicalType type;
|
45
|
+
bool needs_decompression;
|
46
|
+
unique_ptr<BaseStatistics> stats;
|
47
|
+
};
|
48
|
+
|
49
|
+
struct CompressedMaterializationInfo {
|
50
|
+
public:
|
51
|
+
CompressedMaterializationInfo(LogicalOperator &op, vector<idx_t> &&child_idxs,
|
52
|
+
const column_binding_set_t &referenced_bindings);
|
53
|
+
|
54
|
+
public:
|
55
|
+
//! Mapping from incoming bindings to outgoing bindings
|
56
|
+
column_binding_map_t<CMBindingInfo> binding_map;
|
57
|
+
|
58
|
+
//! Operator child info
|
59
|
+
vector<idx_t> child_idxs;
|
60
|
+
vector<CMChildInfo> child_info;
|
61
|
+
};
|
62
|
+
|
63
|
+
struct CompressExpression {
|
64
|
+
public:
|
65
|
+
CompressExpression(unique_ptr<Expression> expression, unique_ptr<BaseStatistics> stats);
|
66
|
+
|
67
|
+
public:
|
68
|
+
unique_ptr<Expression> expression;
|
69
|
+
unique_ptr<BaseStatistics> stats;
|
70
|
+
};
|
71
|
+
|
72
|
+
typedef column_binding_map_t<unique_ptr<BaseStatistics>> statistics_map_t;
|
73
|
+
|
74
|
+
//! The CompressedMaterialization optimizer compressed columns using projections, based on available statistics,
|
75
|
+
//! but only if the data enters a materializing operator
|
76
|
+
class CompressedMaterialization {
|
77
|
+
public:
|
78
|
+
explicit CompressedMaterialization(ClientContext &context, Binder &binder, statistics_map_t &&statistics_map);
|
79
|
+
|
80
|
+
void Compress(unique_ptr<LogicalOperator> &op);
|
81
|
+
|
82
|
+
private:
|
83
|
+
//! Depth-first traversal of the plan
|
84
|
+
void CompressInternal(unique_ptr<LogicalOperator> &op);
|
85
|
+
|
86
|
+
//! Compress materializing operators
|
87
|
+
void CompressAggregate(unique_ptr<LogicalOperator> &op);
|
88
|
+
void CompressDistinct(unique_ptr<LogicalOperator> &op);
|
89
|
+
void CompressOrder(unique_ptr<LogicalOperator> &op);
|
90
|
+
|
91
|
+
//! Update statistics after compressing
|
92
|
+
void UpdateAggregateStats(unique_ptr<LogicalOperator> &op);
|
93
|
+
void UpdateOrderStats(unique_ptr<LogicalOperator> &op);
|
94
|
+
|
95
|
+
//! Adds bindings referenced in expression to referenced_bindings
|
96
|
+
static void GetReferencedBindings(const Expression &expression, column_binding_set_t &referenced_bindings);
|
97
|
+
//! Updates CMBindingInfo in the binding_map in info
|
98
|
+
void UpdateBindingInfo(CompressedMaterializationInfo &info, const ColumnBinding &binding, bool needs_decompression);
|
99
|
+
|
100
|
+
//! Create (de)compress projections around the operator
|
101
|
+
void CreateProjections(unique_ptr<LogicalOperator> &op, CompressedMaterializationInfo &info);
|
102
|
+
bool TryCompressChild(CompressedMaterializationInfo &info, const CMChildInfo &child_info,
|
103
|
+
vector<unique_ptr<CompressExpression>> &compress_expressions);
|
104
|
+
void CreateCompressProjection(unique_ptr<LogicalOperator> &child_op,
|
105
|
+
vector<unique_ptr<CompressExpression>> &&compress_exprs,
|
106
|
+
CompressedMaterializationInfo &info, CMChildInfo &child_info);
|
107
|
+
void CreateDecompressProjection(unique_ptr<LogicalOperator> &op, CompressedMaterializationInfo &info);
|
108
|
+
|
109
|
+
//! Create expressions that apply a scalar compression function
|
110
|
+
unique_ptr<CompressExpression> GetCompressExpression(const ColumnBinding &binding, const LogicalType &type,
|
111
|
+
const bool &can_compress);
|
112
|
+
unique_ptr<CompressExpression> GetCompressExpression(unique_ptr<Expression> input, const BaseStatistics &stats);
|
113
|
+
unique_ptr<CompressExpression> GetIntegralCompress(unique_ptr<Expression> input, const BaseStatistics &stats);
|
114
|
+
unique_ptr<CompressExpression> GetStringCompress(unique_ptr<Expression> input, const BaseStatistics &stats);
|
115
|
+
|
116
|
+
//! Create an expression that applies a scalar decompression function
|
117
|
+
unique_ptr<Expression> GetDecompressExpression(unique_ptr<Expression> input, const LogicalType &result_type,
|
118
|
+
const BaseStatistics &stats);
|
119
|
+
unique_ptr<Expression> GetIntegralDecompress(unique_ptr<Expression> input, const LogicalType &result_type,
|
120
|
+
const BaseStatistics &stats);
|
121
|
+
unique_ptr<Expression> GetStringDecompress(unique_ptr<Expression> input, const BaseStatistics &stats);
|
122
|
+
|
123
|
+
private:
|
124
|
+
ClientContext &context;
|
125
|
+
Binder &binder;
|
126
|
+
statistics_map_t statistics_map;
|
127
|
+
unordered_set<idx_t> compression_table_indices;
|
128
|
+
unordered_set<idx_t> decompression_table_indices;
|
129
|
+
optional_ptr<LogicalOperator> root;
|
130
|
+
};
|
131
|
+
|
132
|
+
} // namespace duckdb
|
@@ -8,36 +8,33 @@
|
|
8
8
|
|
9
9
|
#pragma once
|
10
10
|
|
11
|
-
#include "duckdb/
|
12
|
-
#include "duckdb/planner/column_binding_map.hpp"
|
13
|
-
#include "duckdb/planner/logical_operator.hpp"
|
14
|
-
#include "duckdb/planner/logical_operator_visitor.hpp"
|
11
|
+
#include "duckdb/optimizer/column_binding_replacer.hpp"
|
15
12
|
|
16
13
|
namespace duckdb {
|
17
14
|
|
18
|
-
|
19
|
-
class DeliminatorPlanUpdater;
|
15
|
+
struct DelimCandidate;
|
20
16
|
|
21
17
|
//! The Deliminator optimizer traverses the logical operator tree and removes any redundant DelimGets/DelimJoins
|
22
18
|
class Deliminator {
|
23
19
|
public:
|
24
|
-
|
20
|
+
Deliminator() {
|
25
21
|
}
|
26
22
|
//! Perform DelimJoin elimination
|
27
23
|
unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
|
28
24
|
|
29
25
|
private:
|
30
|
-
//!
|
31
|
-
void FindCandidates(unique_ptr<LogicalOperator>
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
bool
|
37
|
-
|
26
|
+
//! Finds DelimJoins and their corresponding DelimGets
|
27
|
+
void FindCandidates(unique_ptr<LogicalOperator> &op, vector<DelimCandidate> &candidates);
|
28
|
+
void FindJoinWithDelimGet(unique_ptr<LogicalOperator> &op, DelimCandidate &candidate);
|
29
|
+
//! Remove joins with a DelimGet
|
30
|
+
bool RemoveJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
|
31
|
+
unique_ptr<LogicalOperator> &join, bool &all_equality_conditions);
|
32
|
+
bool RemoveInequalityJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
|
33
|
+
unique_ptr<LogicalOperator> &join,
|
34
|
+
const vector<ReplacementBinding> &replacement_bindings);
|
38
35
|
|
39
36
|
private:
|
40
|
-
|
37
|
+
optional_ptr<LogicalOperator> root;
|
41
38
|
};
|
42
39
|
|
43
40
|
} // namespace duckdb
|
@@ -19,8 +19,11 @@ class Optimizer;
|
|
19
19
|
class FilterPushdown {
|
20
20
|
public:
|
21
21
|
explicit FilterPushdown(Optimizer &optimizer);
|
22
|
+
|
22
23
|
//! Perform filter pushdown
|
23
24
|
unique_ptr<LogicalOperator> Rewrite(unique_ptr<LogicalOperator> op);
|
25
|
+
//! Return a reference to the client context (from the optimizer)
|
26
|
+
ClientContext &GetContext();
|
24
27
|
|
25
28
|
struct Filter {
|
26
29
|
unordered_set<idx_t> bindings;
|
@@ -34,7 +34,7 @@ struct RelationsToTDom {
|
|
34
34
|
bool has_tdom_hll;
|
35
35
|
vector<FilterInfo *> filters;
|
36
36
|
|
37
|
-
RelationsToTDom(column_binding_set_t column_binding_set)
|
37
|
+
RelationsToTDom(const column_binding_set_t &column_binding_set)
|
38
38
|
: equivalent_relations(column_binding_set), tdom_hll(0), tdom_no_hll(NumericLimits<idx_t>::Maximum()),
|
39
39
|
has_tdom_hll(false) {};
|
40
40
|
};
|
@@ -27,7 +27,10 @@ public:
|
|
27
27
|
T GetCardinality() const {
|
28
28
|
throw NotImplementedException("Unsupported type for GetCardinality");
|
29
29
|
}
|
30
|
-
|
30
|
+
template <class T>
|
31
|
+
T GetCost() const {
|
32
|
+
throw NotImplementedException("Unsupported type for GetCost");
|
33
|
+
}
|
31
34
|
void SetCost(double new_cost);
|
32
35
|
void SetCardinality(double cardinality);
|
33
36
|
|
@@ -45,4 +48,10 @@ double EstimatedProperties::GetCardinality() const;
|
|
45
48
|
template <>
|
46
49
|
idx_t EstimatedProperties::GetCardinality() const;
|
47
50
|
|
51
|
+
template <>
|
52
|
+
double EstimatedProperties::GetCost() const;
|
53
|
+
|
54
|
+
template <>
|
55
|
+
idx_t EstimatedProperties::GetCost() const;
|
56
|
+
|
48
57
|
} // namespace duckdb
|
@@ -90,7 +90,7 @@ private:
|
|
90
90
|
//! cancelling the dynamic programming step.
|
91
91
|
bool TryEmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector<reference<NeighborInfo>> &info);
|
92
92
|
|
93
|
-
bool EnumerateCmpRecursive(JoinRelationSet &left, JoinRelationSet &right, unordered_set<idx_t> exclusion_set);
|
93
|
+
bool EnumerateCmpRecursive(JoinRelationSet &left, JoinRelationSet &right, unordered_set<idx_t> &exclusion_set);
|
94
94
|
//! Emit a relation set node
|
95
95
|
bool EmitCSG(JoinRelationSet &node);
|
96
96
|
//! Enumerate the possible connected subgraphs that can be joined together in the join graph
|
@@ -53,7 +53,7 @@ public:
|
|
53
53
|
//! Create or get a JoinRelationSet from a single node with the given index
|
54
54
|
JoinRelationSet &GetJoinRelation(idx_t index);
|
55
55
|
//! Create or get a JoinRelationSet from a set of relation bindings
|
56
|
-
JoinRelationSet &GetJoinRelation(unordered_set<idx_t> &bindings);
|
56
|
+
JoinRelationSet &GetJoinRelation(const unordered_set<idx_t> &bindings);
|
57
57
|
//! Create or get a JoinRelationSet from a (sorted, duplicate-free!) list of relations
|
58
58
|
JoinRelationSet &GetJoinRelation(unsafe_unique_array<idx_t> relations, idx_t count);
|
59
59
|
//! Union two sets of relations together and create a new relation set
|
@@ -75,6 +75,9 @@ private:
|
|
75
75
|
//! Get the QueryEdge of a specific node
|
76
76
|
QueryEdge &GetQueryEdge(JoinRelationSet &left);
|
77
77
|
|
78
|
+
void EnumerateNeighborsDFS(JoinRelationSet &node, reference<QueryEdge> info, idx_t index,
|
79
|
+
const std::function<bool(NeighborInfo &)> &callback);
|
80
|
+
|
78
81
|
QueryEdge root;
|
79
82
|
};
|
80
83
|
|
@@ -23,6 +23,8 @@ public:
|
|
23
23
|
UNORDERED,
|
24
24
|
//! Only some entries have to be matched, the order of the matches does not matter
|
25
25
|
SOME,
|
26
|
+
//! Only some entries have to be matched. The order of the matches does matter.
|
27
|
+
SOME_ORDERED,
|
26
28
|
//! Not initialized
|
27
29
|
INVALID
|
28
30
|
};
|
@@ -79,6 +81,17 @@ public:
|
|
79
81
|
}
|
80
82
|
}
|
81
83
|
return true;
|
84
|
+
} else if (policy == Policy::SOME_ORDERED) {
|
85
|
+
if (entries.size() < matchers.size()) {
|
86
|
+
return false;
|
87
|
+
}
|
88
|
+
// now provided entries have to match in order
|
89
|
+
for (idx_t i = 0; i < matchers.size(); i++) {
|
90
|
+
if (!matchers[i]->Match(entries[i], bindings)) {
|
91
|
+
return false;
|
92
|
+
}
|
93
|
+
}
|
94
|
+
return true;
|
82
95
|
} else {
|
83
96
|
if (policy == Policy::UNORDERED && matchers.size() != entries.size()) {
|
84
97
|
// unordered policy, count does not match: no match
|
@@ -22,7 +22,10 @@ class Optimizer {
|
|
22
22
|
public:
|
23
23
|
Optimizer(Binder &binder, ClientContext &context);
|
24
24
|
|
25
|
+
//! Optimize a plan by running specialized optimizers
|
25
26
|
unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> plan);
|
27
|
+
//! Return a reference to the client context of this optimizer
|
28
|
+
ClientContext &GetContext();
|
26
29
|
|
27
30
|
ClientContext &context;
|
28
31
|
Binder &binder;
|
@@ -0,0 +1,40 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/optimizer/remove_duplicate_groups.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/planner/column_binding_map.hpp"
|
12
|
+
#include "duckdb/planner/logical_operator_visitor.hpp"
|
13
|
+
|
14
|
+
namespace duckdb {
|
15
|
+
|
16
|
+
class BoundColumnRefExpression;
|
17
|
+
|
18
|
+
//! The RemoveDuplicateGroups optimizer traverses the logical operator tree and removes any duplicate aggregate groups
|
19
|
+
//! Duplicate groups may be introduced when joins columns are removed, e.g., by Deliminator or RemoveUnusedColumns
|
20
|
+
class RemoveDuplicateGroups : public LogicalOperatorVisitor {
|
21
|
+
public:
|
22
|
+
RemoveDuplicateGroups() {
|
23
|
+
}
|
24
|
+
|
25
|
+
void VisitOperator(LogicalOperator &op) override;
|
26
|
+
|
27
|
+
private:
|
28
|
+
void VisitAggregate(LogicalAggregate &aggr);
|
29
|
+
|
30
|
+
protected:
|
31
|
+
unique_ptr<Expression> VisitReplace(BoundColumnRefExpression &expr, unique_ptr<Expression> *expr_ptr) override;
|
32
|
+
|
33
|
+
private:
|
34
|
+
//! The map of column references
|
35
|
+
column_binding_map_t<vector<reference<BoundColumnRefExpression>>> column_references;
|
36
|
+
//! Stored expressions (kept around so we don't have dangling pointers)
|
37
|
+
vector<unique_ptr<Expression>> stored_expressions;
|
38
|
+
};
|
39
|
+
|
40
|
+
} // namespace duckdb
|
@@ -18,6 +18,8 @@
|
|
18
18
|
#include "duckdb/storage/statistics/node_statistics.hpp"
|
19
19
|
|
20
20
|
namespace duckdb {
|
21
|
+
|
22
|
+
class Optimizer;
|
21
23
|
class ClientContext;
|
22
24
|
class LogicalOperator;
|
23
25
|
class TableFilter;
|
@@ -25,10 +27,14 @@ struct BoundOrderByNode;
|
|
25
27
|
|
26
28
|
class StatisticsPropagator {
|
27
29
|
public:
|
28
|
-
explicit StatisticsPropagator(
|
30
|
+
explicit StatisticsPropagator(Optimizer &optimizer);
|
29
31
|
|
30
32
|
unique_ptr<NodeStatistics> PropagateStatistics(unique_ptr<LogicalOperator> &node_ptr);
|
31
33
|
|
34
|
+
column_binding_map_t<unique_ptr<BaseStatistics>> GetStatisticsMap() {
|
35
|
+
return std::move(statistics_map);
|
36
|
+
}
|
37
|
+
|
32
38
|
private:
|
33
39
|
//! Propagate statistics through an operator
|
34
40
|
unique_ptr<NodeStatistics> PropagateStatistics(LogicalOperator &node, unique_ptr<LogicalOperator> *node_ptr);
|
@@ -75,6 +81,9 @@ private:
|
|
75
81
|
//! Multiply the cardinalities together (i.e. new max cardinality is stats.max * new_stats.max): used for
|
76
82
|
//! joins/cross products
|
77
83
|
void MultiplyCardinalities(unique_ptr<NodeStatistics> &stats, NodeStatistics &new_stats);
|
84
|
+
//! Creates and pushes down a filter based on join statistics
|
85
|
+
void CreateFilterFromJoinStats(unique_ptr<LogicalOperator> &child, unique_ptr<Expression> &expr,
|
86
|
+
const BaseStatistics &stats_before, const BaseStatistics &stats_after);
|
78
87
|
|
79
88
|
unique_ptr<BaseStatistics> PropagateExpression(unique_ptr<Expression> &expr);
|
80
89
|
unique_ptr<BaseStatistics> PropagateExpression(Expression &expr, unique_ptr<Expression> *expr_ptr);
|
@@ -90,14 +99,13 @@ private:
|
|
90
99
|
unique_ptr<BaseStatistics> PropagateExpression(BoundColumnRefExpression &expr, unique_ptr<Expression> *expr_ptr);
|
91
100
|
unique_ptr<BaseStatistics> PropagateExpression(BoundOperatorExpression &expr, unique_ptr<Expression> *expr_ptr);
|
92
101
|
|
93
|
-
void PropagateAndCompress(unique_ptr<Expression> &expr, unique_ptr<BaseStatistics> &stats);
|
94
|
-
|
95
102
|
void ReplaceWithEmptyResult(unique_ptr<LogicalOperator> &node);
|
96
103
|
|
97
104
|
bool ExpressionIsConstant(Expression &expr, const Value &val);
|
98
105
|
bool ExpressionIsConstantOrNull(Expression &expr, const Value &val);
|
99
106
|
|
100
107
|
private:
|
108
|
+
Optimizer &optimizer;
|
101
109
|
ClientContext &context;
|
102
110
|
//! The map of ColumnBinding -> statistics for the various nodes
|
103
111
|
column_binding_map_t<unique_ptr<BaseStatistics>> statistics_map;
|
@@ -18,6 +18,8 @@ class TopN {
|
|
18
18
|
public:
|
19
19
|
//! Optimize ORDER BY + LIMIT to TopN
|
20
20
|
unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
|
21
|
+
//! Whether we can perform the optimization on this operator
|
22
|
+
static bool CanOptimize(LogicalOperator &op);
|
21
23
|
};
|
22
24
|
|
23
25
|
} // namespace duckdb
|
@@ -30,6 +30,8 @@ public:
|
|
30
30
|
public:
|
31
31
|
//! Duplicate eliminated join scan dependencies
|
32
32
|
reference_map_t<const PhysicalOperator, reference<Pipeline>> delim_join_dependencies;
|
33
|
+
//! Materialized CTE scan dependencies
|
34
|
+
reference_map_t<const PhysicalOperator, reference<Pipeline>> cte_dependencies;
|
33
35
|
|
34
36
|
public:
|
35
37
|
void SetPipelineSource(Pipeline &pipeline, PhysicalOperator &op);
|
@@ -67,6 +67,9 @@ public:
|
|
67
67
|
//! Send signals to n threads, signalling for them to wake up and attempt to execute a task
|
68
68
|
void Signal(idx_t n);
|
69
69
|
|
70
|
+
//! Set the allocator flush threshold
|
71
|
+
void SetAllocatorFlushTreshold(idx_t threshold);
|
72
|
+
|
70
73
|
private:
|
71
74
|
void SetThreadsInternal(int32_t n);
|
72
75
|
|
@@ -80,6 +83,8 @@ private:
|
|
80
83
|
vector<unique_ptr<SchedulerThread>> threads;
|
81
84
|
//! Markers used by the various threads, if the markers are set to "false" the thread execution is stopped
|
82
85
|
vector<unique_ptr<atomic<bool>>> markers;
|
86
|
+
//! The threshold after which to flush the allocator after completing a task
|
87
|
+
atomic<idx_t> allocator_flush_threshold;
|
83
88
|
};
|
84
89
|
|
85
90
|
} // namespace duckdb
|
@@ -9,6 +9,7 @@
|
|
9
9
|
#pragma once
|
10
10
|
|
11
11
|
#include "duckdb/parser/statement/select_statement.hpp"
|
12
|
+
#include "duckdb/common/enums/cte_materialize.hpp"
|
12
13
|
|
13
14
|
namespace duckdb {
|
14
15
|
|
@@ -17,6 +18,7 @@ class SelectStatement;
|
|
17
18
|
struct CommonTableExpressionInfo {
|
18
19
|
vector<string> aliases;
|
19
20
|
unique_ptr<SelectStatement> query;
|
21
|
+
CTEMaterialize materialized = CTEMaterialize::CTE_MATERIALIZE_DEFAULT;
|
20
22
|
|
21
23
|
void FormatSerialize(FormatSerializer &serializer) const;
|
22
24
|
static unique_ptr<CommonTableExpressionInfo> FormatDeserialize(FormatDeserializer &deserializer);
|
@@ -36,5 +36,8 @@ public:
|
|
36
36
|
static unique_ptr<ParsedExpression> Deserialize(ExpressionType type, FieldReader &source);
|
37
37
|
void FormatSerialize(FormatSerializer &serializer) const override;
|
38
38
|
static unique_ptr<ParsedExpression> FormatDeserialize(ExpressionType type, FormatDeserializer &deserializer);
|
39
|
+
|
40
|
+
private:
|
41
|
+
CollateExpression();
|
39
42
|
};
|
40
43
|
} // namespace duckdb
|
@@ -50,5 +50,8 @@ public:
|
|
50
50
|
static unique_ptr<ParsedExpression> Deserialize(ExpressionType type, FieldReader &source);
|
51
51
|
void FormatSerialize(FormatSerializer &serializer) const override;
|
52
52
|
static unique_ptr<ParsedExpression> FormatDeserialize(ExpressionType type, FormatDeserializer &deserializer);
|
53
|
+
|
54
|
+
private:
|
55
|
+
ColumnRefExpression();
|
53
56
|
};
|
54
57
|
} // namespace duckdb
|
@@ -37,6 +37,9 @@ public:
|
|
37
37
|
|
38
38
|
void FormatSerialize(FormatSerializer &serializer) const override;
|
39
39
|
static unique_ptr<ParsedExpression> FormatDeserialize(ExpressionType type, FormatDeserializer &deserializer);
|
40
|
+
|
41
|
+
private:
|
42
|
+
ConstantExpression();
|
40
43
|
};
|
41
44
|
|
42
45
|
} // namespace duckdb
|
@@ -31,5 +31,6 @@ public:
|
|
31
31
|
void Serialize(FieldWriter &writer) const override;
|
32
32
|
static unique_ptr<ParsedExpression> Deserialize(ExpressionType type, FieldReader &source);
|
33
33
|
void FormatSerialize(FormatSerializer &serializer) const override;
|
34
|
+
static unique_ptr<ParsedExpression> FormatDeserialize(ExpressionType type, FormatDeserializer &deserializer);
|
34
35
|
};
|
35
36
|
} // namespace duckdb
|
@@ -42,6 +42,9 @@ public:
|
|
42
42
|
static unique_ptr<ParsedExpression> Deserialize(ExpressionType type, FieldReader &source);
|
43
43
|
void FormatSerialize(FormatSerializer &serializer) const override;
|
44
44
|
static unique_ptr<ParsedExpression> FormatDeserialize(ExpressionType type, FormatDeserializer &deserializer);
|
45
|
+
|
46
|
+
private:
|
47
|
+
LambdaExpression();
|
45
48
|
};
|
46
49
|
|
47
50
|
} // namespace duckdb
|
@@ -35,5 +35,8 @@ public:
|
|
35
35
|
static unique_ptr<ParsedExpression> Deserialize(ExpressionType type, FieldReader &source);
|
36
36
|
void FormatSerialize(FormatSerializer &serializer) const override;
|
37
37
|
static unique_ptr<ParsedExpression> FormatDeserialize(ExpressionType type, FormatDeserializer &deserializer);
|
38
|
+
|
39
|
+
private:
|
40
|
+
PositionalReferenceExpression();
|
38
41
|
};
|
39
42
|
} // namespace duckdb
|
@@ -0,0 +1,54 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/parser/query_node/cte_node.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/parser/parsed_expression.hpp"
|
12
|
+
#include "duckdb/parser/query_node.hpp"
|
13
|
+
#include "duckdb/parser/sql_statement.hpp"
|
14
|
+
|
15
|
+
namespace duckdb {
|
16
|
+
|
17
|
+
class CTENode : public QueryNode {
|
18
|
+
public:
|
19
|
+
static constexpr const QueryNodeType TYPE = QueryNodeType::CTE_NODE;
|
20
|
+
|
21
|
+
public:
|
22
|
+
CTENode() : QueryNode(QueryNodeType::CTE_NODE) {
|
23
|
+
}
|
24
|
+
|
25
|
+
string ctename;
|
26
|
+
//! The query of the CTE
|
27
|
+
unique_ptr<QueryNode> query;
|
28
|
+
//! Child
|
29
|
+
unique_ptr<QueryNode> child;
|
30
|
+
//! Aliases of the CTE node
|
31
|
+
vector<string> aliases;
|
32
|
+
|
33
|
+
const vector<unique_ptr<ParsedExpression>> &GetSelectList() const override {
|
34
|
+
return query->GetSelectList();
|
35
|
+
}
|
36
|
+
|
37
|
+
public:
|
38
|
+
//! Convert the query node to a string
|
39
|
+
string ToString() const override;
|
40
|
+
|
41
|
+
bool Equals(const QueryNode *other) const override;
|
42
|
+
//! Create a copy of this SelectNode
|
43
|
+
unique_ptr<QueryNode> Copy() const override;
|
44
|
+
|
45
|
+
//! Serializes a QueryNode to a stand-alone binary blob
|
46
|
+
void Serialize(FieldWriter &writer) const override;
|
47
|
+
//! Deserializes a blob back into a QueryNode
|
48
|
+
static unique_ptr<QueryNode> Deserialize(FieldReader &reader);
|
49
|
+
|
50
|
+
void FormatSerialize(FormatSerializer &serializer) const override;
|
51
|
+
static unique_ptr<QueryNode> FormatDeserialize(FormatDeserializer &source);
|
52
|
+
};
|
53
|
+
|
54
|
+
} // namespace duckdb
|
@@ -31,6 +31,7 @@ public:
|
|
31
31
|
//! Deserializes a blob back into a DummyTableRef
|
32
32
|
static unique_ptr<TableRef> Deserialize(FieldReader &source);
|
33
33
|
|
34
|
+
void FormatSerialize(FormatSerializer &serializer) const override;
|
34
35
|
static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
|
35
36
|
};
|
36
37
|
} // namespace duckdb
|
@@ -23,7 +23,7 @@ public:
|
|
23
23
|
static constexpr const TableReferenceType TYPE = TableReferenceType::JOIN;
|
24
24
|
|
25
25
|
public:
|
26
|
-
explicit JoinRef(JoinRefType ref_type)
|
26
|
+
explicit JoinRef(JoinRefType ref_type = JoinRefType::REGULAR)
|
27
27
|
: TableRef(TableReferenceType::JOIN), type(JoinType::INNER), ref_type(ref_type) {
|
28
28
|
}
|
29
29
|
|