duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +4619 -4446
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/extra_type_info.cpp +506 -0
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +14 -14
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +70 -50
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/types/value.cpp +11 -6
- package/src/duckdb/src/common/types.cpp +9 -656
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/arrow.cpp +19 -0
- package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +23 -8
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +5 -2
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +4 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
- package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +79 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +44 -31
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_common.cpp +2 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
- package/src/statement.cpp +10 -3
- package/test/columns.test.ts +24 -1
- package/test/test_all_types.test.ts +234 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -87,71 +87,76 @@ PartitionGlobalSinkState::PartitionGlobalSinkState(ClientContext &context,
|
|
87
87
|
const vector<unique_ptr<BaseStatistics>> &partition_stats,
|
88
88
|
idx_t estimated_cardinality)
|
89
89
|
: context(context), buffer_manager(BufferManager::GetBufferManager(context)), allocator(Allocator::Get(context)),
|
90
|
-
payload_types(payload_types), memory_per_thread(0), count(0) {
|
90
|
+
fixed_bits(0), payload_types(payload_types), memory_per_thread(0), max_bits(1), count(0) {
|
91
91
|
|
92
92
|
GenerateOrderings(partitions, orders, partition_bys, order_bys, partition_stats);
|
93
93
|
|
94
94
|
memory_per_thread = PhysicalOperator::GetMaxThreadMemory(context);
|
95
95
|
external = ClientConfig::GetConfig(context).force_external;
|
96
96
|
|
97
|
+
const auto thread_pages = PreviousPowerOfTwo(memory_per_thread / (4 * idx_t(Storage::BLOCK_ALLOC_SIZE)));
|
98
|
+
while (max_bits < 10 && (thread_pages >> max_bits) > 1) {
|
99
|
+
++max_bits;
|
100
|
+
}
|
101
|
+
|
97
102
|
if (!orders.empty()) {
|
98
|
-
|
99
|
-
|
103
|
+
auto types = payload_types;
|
104
|
+
types.push_back(LogicalType::HASH);
|
105
|
+
grouping_types.Initialize(types);
|
100
106
|
|
101
107
|
ResizeGroupingData(estimated_cardinality);
|
102
108
|
}
|
103
109
|
}
|
104
110
|
|
111
|
+
void PartitionGlobalSinkState::SyncPartitioning(const PartitionGlobalSinkState &other) {
|
112
|
+
fixed_bits = other.grouping_data ? other.grouping_data->GetRadixBits() : 0;
|
113
|
+
|
114
|
+
const auto old_bits = grouping_data ? grouping_data->GetRadixBits() : 0;
|
115
|
+
if (fixed_bits != old_bits) {
|
116
|
+
const auto hash_col_idx = payload_types.size();
|
117
|
+
grouping_data = make_uniq<RadixPartitionedTupleData>(buffer_manager, grouping_types, fixed_bits, hash_col_idx);
|
118
|
+
}
|
119
|
+
}
|
120
|
+
|
121
|
+
unique_ptr<RadixPartitionedTupleData> PartitionGlobalSinkState::CreatePartition(idx_t new_bits) const {
|
122
|
+
const auto hash_col_idx = payload_types.size();
|
123
|
+
return make_uniq<RadixPartitionedTupleData>(buffer_manager, grouping_types, new_bits, hash_col_idx);
|
124
|
+
}
|
125
|
+
|
105
126
|
void PartitionGlobalSinkState::ResizeGroupingData(idx_t cardinality) {
|
106
127
|
// Have we started to combine? Then just live with it.
|
107
|
-
if (grouping_data && !grouping_data->GetPartitions().empty()) {
|
128
|
+
if (fixed_bits || (grouping_data && !grouping_data->GetPartitions().empty())) {
|
108
129
|
return;
|
109
130
|
}
|
110
131
|
// Is the average partition size too large?
|
111
132
|
const idx_t partition_size = STANDARD_ROW_GROUPS_SIZE;
|
112
133
|
const auto bits = grouping_data ? grouping_data->GetRadixBits() : 0;
|
113
134
|
auto new_bits = bits ? bits : 4;
|
114
|
-
while (new_bits <
|
135
|
+
while (new_bits < max_bits && (cardinality / RadixPartitioning::NumberOfPartitions(new_bits)) > partition_size) {
|
115
136
|
++new_bits;
|
116
137
|
}
|
117
138
|
|
118
139
|
// Repartition the grouping data
|
119
140
|
if (new_bits != bits) {
|
120
|
-
|
121
|
-
grouping_data = make_uniq<RadixPartitionedColumnData>(context, grouping_types, new_bits, hash_col_idx);
|
141
|
+
grouping_data = CreatePartition(new_bits);
|
122
142
|
}
|
123
143
|
}
|
124
144
|
|
125
145
|
void PartitionGlobalSinkState::SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append) {
|
126
146
|
// We are done if the local_partition is right sized.
|
127
|
-
auto &local_radix = local_partition->Cast<
|
128
|
-
|
147
|
+
auto &local_radix = local_partition->Cast<RadixPartitionedTupleData>();
|
148
|
+
const auto new_bits = grouping_data->GetRadixBits();
|
149
|
+
if (local_radix.GetRadixBits() == new_bits) {
|
129
150
|
return;
|
130
151
|
}
|
131
152
|
|
132
153
|
// If the local partition is now too small, flush it and reallocate
|
133
|
-
auto new_partition =
|
134
|
-
auto new_append = make_uniq<PartitionedColumnDataAppendState>();
|
135
|
-
new_partition->InitializeAppendState(*new_append);
|
136
|
-
|
154
|
+
auto new_partition = CreatePartition(new_bits);
|
137
155
|
local_partition->FlushAppendState(*local_append);
|
138
|
-
|
139
|
-
for (auto &local_group : local_groups) {
|
140
|
-
ColumnDataScanState scanner;
|
141
|
-
local_group->InitializeScan(scanner);
|
142
|
-
|
143
|
-
DataChunk scan_chunk;
|
144
|
-
local_group->InitializeScanChunk(scan_chunk);
|
145
|
-
for (scan_chunk.Reset(); local_group->Scan(scanner, scan_chunk); scan_chunk.Reset()) {
|
146
|
-
new_partition->Append(*new_append, scan_chunk);
|
147
|
-
}
|
148
|
-
}
|
149
|
-
|
150
|
-
// The append state has stale pointers to the old local partition, so nuke it from orbit.
|
151
|
-
new_partition->FlushAppendState(*new_append);
|
156
|
+
local_partition->Repartition(*new_partition);
|
152
157
|
|
153
158
|
local_partition = std::move(new_partition);
|
154
|
-
local_append = make_uniq<
|
159
|
+
local_append = make_uniq<PartitionedTupleDataAppendState>();
|
155
160
|
local_partition->InitializeAppendState(*local_append);
|
156
161
|
}
|
157
162
|
|
@@ -160,8 +165,8 @@ void PartitionGlobalSinkState::UpdateLocalPartition(GroupingPartition &local_par
|
|
160
165
|
lock_guard<mutex> guard(lock);
|
161
166
|
|
162
167
|
if (!local_partition) {
|
163
|
-
local_partition = grouping_data->
|
164
|
-
local_append = make_uniq<
|
168
|
+
local_partition = CreatePartition(grouping_data->GetRadixBits());
|
169
|
+
local_append = make_uniq<PartitionedTupleDataAppendState>();
|
165
170
|
local_partition->InitializeAppendState(*local_append);
|
166
171
|
return;
|
167
172
|
}
|
@@ -186,9 +191,7 @@ void PartitionGlobalSinkState::CombineLocalPartition(GroupingPartition &local_pa
|
|
186
191
|
grouping_data->Combine(*local_partition);
|
187
192
|
}
|
188
193
|
|
189
|
-
void PartitionGlobalSinkState::BuildSortState(
|
190
|
-
auto &global_sort = *hash_group.global_sort;
|
191
|
-
|
194
|
+
void PartitionGlobalSinkState::BuildSortState(TupleDataCollection &group_data, GlobalSortState &global_sort) const {
|
192
195
|
// Set up the sort expression computation.
|
193
196
|
vector<LogicalType> sort_types;
|
194
197
|
ExpressionExecutor executor(context);
|
@@ -213,16 +216,9 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
|
|
213
216
|
for (column_t i = 0; i < payload_types.size(); ++i) {
|
214
217
|
column_ids.emplace_back(i);
|
215
218
|
}
|
216
|
-
|
217
|
-
|
218
|
-
chunk_state
|
219
|
-
scanner.InitializeScan();
|
220
|
-
for (auto chunk_idx = scanner.ChunkCount(); chunk_idx-- > 0;) {
|
221
|
-
if (!scanner.AssignChunk(chunk_state)) {
|
222
|
-
break;
|
223
|
-
}
|
224
|
-
scanner.ScanChunk(chunk_state, payload_chunk);
|
225
|
-
|
219
|
+
TupleDataScanState chunk_state;
|
220
|
+
group_data.InitializeScan(chunk_state, column_ids);
|
221
|
+
while (group_data.Scan(chunk_state, payload_chunk)) {
|
226
222
|
sort_chunk.Reset();
|
227
223
|
executor.Execute(payload_chunk, sort_chunk);
|
228
224
|
|
@@ -230,10 +226,13 @@ void PartitionGlobalSinkState::BuildSortState(ColumnDataCollection &group_data,
|
|
230
226
|
if (local_sort.SizeInBytes() > memory_per_thread) {
|
231
227
|
local_sort.Sort(global_sort, true);
|
232
228
|
}
|
233
|
-
scanner.FinishChunk(chunk_state);
|
234
229
|
}
|
235
230
|
|
236
231
|
global_sort.AddLocalState(local_sort);
|
232
|
+
}
|
233
|
+
|
234
|
+
void PartitionGlobalSinkState::BuildSortState(TupleDataCollection &group_data, PartitionGlobalHashGroup &hash_group) {
|
235
|
+
BuildSortState(group_data, *hash_group.global_sort);
|
237
236
|
|
238
237
|
hash_group.count += group_data.Count();
|
239
238
|
}
|
@@ -482,18 +481,29 @@ public:
|
|
482
481
|
TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override;
|
483
482
|
|
484
483
|
private:
|
484
|
+
struct ExecutorCallback : public PartitionGlobalMergeStates::Callback {
|
485
|
+
explicit ExecutorCallback(Executor &executor) : executor(executor) {
|
486
|
+
}
|
487
|
+
|
488
|
+
bool HasError() const override {
|
489
|
+
return executor.HasError();
|
490
|
+
}
|
491
|
+
|
492
|
+
Executor &executor;
|
493
|
+
};
|
494
|
+
|
485
495
|
shared_ptr<Event> event;
|
486
496
|
PartitionLocalMergeState local_state;
|
487
497
|
PartitionGlobalMergeStates &hash_groups;
|
488
498
|
};
|
489
499
|
|
490
|
-
|
500
|
+
bool PartitionGlobalMergeStates::ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback) {
|
491
501
|
// Loop until all hash groups are done
|
492
502
|
size_t sorted = 0;
|
493
|
-
while (sorted <
|
503
|
+
while (sorted < states.size()) {
|
494
504
|
// First check if there is an unfinished task for this thread
|
495
|
-
if (
|
496
|
-
return
|
505
|
+
if (callback.HasError()) {
|
506
|
+
return false;
|
497
507
|
}
|
498
508
|
if (!local_state.TaskFinished()) {
|
499
509
|
local_state.ExecuteTask();
|
@@ -501,8 +511,8 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
|
|
501
511
|
}
|
502
512
|
|
503
513
|
// Thread is done with its assigned task, try to fetch new work
|
504
|
-
for (auto group = sorted; group <
|
505
|
-
auto &global_state =
|
514
|
+
for (auto group = sorted; group < states.size(); ++group) {
|
515
|
+
auto &global_state = states[group];
|
506
516
|
if (global_state->IsSorted()) {
|
507
517
|
// This hash group is done
|
508
518
|
// Update the high water mark of densely completed groups
|
@@ -543,6 +553,16 @@ TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
|
|
543
553
|
}
|
544
554
|
}
|
545
555
|
|
556
|
+
return true;
|
557
|
+
}
|
558
|
+
|
559
|
+
TaskExecutionResult PartitionMergeTask::ExecuteTask(TaskExecutionMode mode) {
|
560
|
+
ExecutorCallback callback(executor);
|
561
|
+
|
562
|
+
if (!hash_groups.ExecuteTask(local_state, callback)) {
|
563
|
+
return TaskExecutionResult::TASK_ERROR;
|
564
|
+
}
|
565
|
+
|
546
566
|
event->FinishTask();
|
547
567
|
return TaskExecutionResult::TASK_FINISHED;
|
548
568
|
}
|
@@ -1,11 +1,14 @@
|
|
1
1
|
#include "duckdb/common/types/batched_data_collection.hpp"
|
2
|
+
|
3
|
+
#include "duckdb/common/optional_ptr.hpp"
|
2
4
|
#include "duckdb/common/printer.hpp"
|
3
5
|
#include "duckdb/storage/buffer_manager.hpp"
|
4
|
-
#include "duckdb/common/optional_ptr.hpp"
|
5
6
|
|
6
7
|
namespace duckdb {
|
7
8
|
|
8
|
-
BatchedDataCollection::BatchedDataCollection(vector<LogicalType> types_p
|
9
|
+
BatchedDataCollection::BatchedDataCollection(ClientContext &context_p, vector<LogicalType> types_p,
|
10
|
+
bool buffer_managed_p)
|
11
|
+
: context(context_p), types(std::move(types_p)), buffer_managed(buffer_managed_p) {
|
9
12
|
}
|
10
13
|
|
11
14
|
void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
|
@@ -20,6 +23,8 @@ void BatchedDataCollection::Append(DataChunk &input, idx_t batch_index) {
|
|
20
23
|
unique_ptr<ColumnDataCollection> new_collection;
|
21
24
|
if (last_collection.collection) {
|
22
25
|
new_collection = make_uniq<ColumnDataCollection>(*last_collection.collection);
|
26
|
+
} else if (buffer_managed) {
|
27
|
+
new_collection = make_uniq<ColumnDataCollection>(BufferManager::GetBufferManager(context), types);
|
23
28
|
} else {
|
24
29
|
new_collection = make_uniq<ColumnDataCollection>(Allocator::DefaultAllocator(), types);
|
25
30
|
}
|
@@ -1,4 +1,6 @@
|
|
1
|
+
#include "duckdb/common/assert.hpp"
|
1
2
|
#include "duckdb/common/operator/cast_operators.hpp"
|
3
|
+
#include "duckdb/common/typedefs.hpp"
|
2
4
|
#include "duckdb/common/types/bit.hpp"
|
3
5
|
#include "duckdb/common/types/string_type.hpp"
|
4
6
|
|
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
|
|
34
36
|
return str_len;
|
35
37
|
}
|
36
38
|
|
39
|
+
uint8_t Bit::GetFirstByte(const string_t &str) {
|
40
|
+
D_ASSERT(str.GetSize() > 1);
|
41
|
+
|
42
|
+
auto data = const_data_ptr_cast(str.GetData());
|
43
|
+
return data[1] & ((1 << (8 - data[0])) - 1);
|
44
|
+
}
|
45
|
+
|
37
46
|
void Bit::Finalize(string_t &str) {
|
38
47
|
// bit strings require all padding bits to be set to 1
|
39
48
|
// this method sets all padding bits to 1
|
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
|
|
146
155
|
return output_str.GetString();
|
147
156
|
}
|
148
157
|
|
158
|
+
void Bit::BlobToBit(string_t blob, string_t &output_str) {
|
159
|
+
auto data = const_data_ptr_cast(blob.GetData());
|
160
|
+
auto output = output_str.GetDataWriteable();
|
161
|
+
idx_t size = blob.GetSize();
|
162
|
+
|
163
|
+
*output = 0; // No padding
|
164
|
+
memcpy(output + 1, data, size);
|
165
|
+
}
|
166
|
+
|
167
|
+
string Bit::BlobToBit(string_t blob) {
|
168
|
+
auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
|
169
|
+
string_t output_str(buffer.get(), blob.GetSize() + 1);
|
170
|
+
Bit::BlobToBit(blob, output_str);
|
171
|
+
return output_str.GetString();
|
172
|
+
}
|
173
|
+
|
174
|
+
void Bit::BitToBlob(string_t bit, string_t &output_blob) {
|
175
|
+
D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
|
176
|
+
|
177
|
+
auto data = const_data_ptr_cast(bit.GetData());
|
178
|
+
auto output = output_blob.GetDataWriteable();
|
179
|
+
idx_t size = output_blob.GetSize();
|
180
|
+
|
181
|
+
output[0] = GetFirstByte(bit);
|
182
|
+
if (size > 2) {
|
183
|
+
++output;
|
184
|
+
// First byte in bitstring contains amount of padded bits,
|
185
|
+
// second byte in bitstring is the padded byte,
|
186
|
+
// therefore the rest of the data starts at data + 2 (third byte)
|
187
|
+
memcpy(output, data + 2, size - 1);
|
188
|
+
}
|
189
|
+
}
|
190
|
+
|
191
|
+
string Bit::BitToBlob(string_t bit) {
|
192
|
+
D_ASSERT(bit.GetSize() > 1);
|
193
|
+
|
194
|
+
auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
|
195
|
+
string_t output_str(buffer.get(), bit.GetSize() - 1);
|
196
|
+
Bit::BitToBlob(bit, output_str);
|
197
|
+
return output_str.GetString();
|
198
|
+
}
|
199
|
+
|
149
200
|
// **** scalar functions ****
|
150
201
|
void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
|
151
202
|
char *res_buf = result.GetDataWriteable();
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#include "duckdb/common/types/column/column_data_allocator.hpp"
|
2
2
|
|
3
3
|
#include "duckdb/common/types/column/column_data_collection_segment.hpp"
|
4
|
-
#include "duckdb/storage/buffer_manager.hpp"
|
5
4
|
#include "duckdb/storage/buffer/block_handle.hpp"
|
5
|
+
#include "duckdb/storage/buffer_manager.hpp"
|
6
6
|
|
7
7
|
namespace duckdb {
|
8
8
|
|
@@ -19,6 +19,7 @@ ColumnDataAllocator::ColumnDataAllocator(ClientContext &context, ColumnDataAlloc
|
|
19
19
|
: type(allocator_type) {
|
20
20
|
switch (type) {
|
21
21
|
case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
|
22
|
+
case ColumnDataAllocatorType::HYBRID:
|
22
23
|
alloc.buffer_manager = &BufferManager::GetBufferManager(context);
|
23
24
|
break;
|
24
25
|
case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
|
@@ -33,6 +34,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
|
|
33
34
|
type = other.GetType();
|
34
35
|
switch (type) {
|
35
36
|
case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
|
37
|
+
case ColumnDataAllocatorType::HYBRID:
|
36
38
|
alloc.allocator = other.alloc.allocator;
|
37
39
|
break;
|
38
40
|
case ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR:
|
@@ -44,7 +46,7 @@ ColumnDataAllocator::ColumnDataAllocator(ColumnDataAllocator &other) {
|
|
44
46
|
}
|
45
47
|
|
46
48
|
BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
|
47
|
-
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
|
49
|
+
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
|
48
50
|
shared_ptr<BlockHandle> handle;
|
49
51
|
if (shared) {
|
50
52
|
// we only need to grab the lock when accessing the vector, because vector access is not thread-safe:
|
@@ -58,7 +60,7 @@ BufferHandle ColumnDataAllocator::Pin(uint32_t block_id) {
|
|
58
60
|
}
|
59
61
|
|
60
62
|
BufferHandle ColumnDataAllocator::AllocateBlock(idx_t size) {
|
61
|
-
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR);
|
63
|
+
D_ASSERT(type == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR || type == ColumnDataAllocatorType::HYBRID);
|
62
64
|
auto block_size = MaxValue<idx_t>(size, Storage::BLOCK_SIZE);
|
63
65
|
BlockMetaData data;
|
64
66
|
data.size = 0;
|
@@ -136,6 +138,7 @@ void ColumnDataAllocator::AllocateData(idx_t size, uint32_t &block_id, uint32_t
|
|
136
138
|
ChunkManagementState *chunk_state) {
|
137
139
|
switch (type) {
|
138
140
|
case ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR:
|
141
|
+
case ColumnDataAllocatorType::HYBRID:
|
139
142
|
if (shared) {
|
140
143
|
lock_guard<mutex> guard(lock);
|
141
144
|
AllocateBuffer(size, block_id, offset, chunk_state);
|
@@ -174,8 +177,8 @@ data_ptr_t ColumnDataAllocator::GetDataPointer(ChunkManagementState &state, uint
|
|
174
177
|
return state.handles[block_id].Ptr() + offset;
|
175
178
|
}
|
176
179
|
|
177
|
-
void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result,
|
178
|
-
|
180
|
+
void ColumnDataAllocator::UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
|
181
|
+
uint32_t block_id, uint32_t offset) {
|
179
182
|
D_ASSERT(result.GetType().InternalType() == PhysicalType::VARCHAR);
|
180
183
|
lock_guard<mutex> guard(lock);
|
181
184
|
|
@@ -225,7 +228,7 @@ Allocator &ColumnDataAllocator::GetAllocator() {
|
|
225
228
|
}
|
226
229
|
|
227
230
|
void ColumnDataAllocator::InitializeChunkState(ChunkManagementState &state, ChunkMetaData &chunk) {
|
228
|
-
if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
|
231
|
+
if (type != ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR && type != ColumnDataAllocatorType::HYBRID) {
|
229
232
|
// nothing to pin
|
230
233
|
return;
|
231
234
|
}
|
@@ -333,7 +333,7 @@ struct StandardValueCopy : public BaseValueCopy<T> {
|
|
333
333
|
|
334
334
|
struct StringValueCopy : public BaseValueCopy<string_t> {
|
335
335
|
static string_t Operation(ColumnDataMetaData &meta_data, string_t input) {
|
336
|
-
return input.IsInlined() ? input : meta_data.segment.heap
|
336
|
+
return input.IsInlined() ? input : meta_data.segment.heap->AddBlob(input);
|
337
337
|
}
|
338
338
|
};
|
339
339
|
|
@@ -423,7 +423,8 @@ void ColumnDataCopy<string_t>(ColumnDataMetaData &meta_data, const UnifiedVector
|
|
423
423
|
idx_t offset, idx_t copy_count) {
|
424
424
|
|
425
425
|
const auto &allocator_type = meta_data.segment.allocator->GetType();
|
426
|
-
if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR
|
426
|
+
if (allocator_type == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
|
427
|
+
allocator_type == ColumnDataAllocatorType::HYBRID) {
|
427
428
|
// strings cannot be spilled to disk - use StringHeap
|
428
429
|
TemplatedColumnDataCopy<StringValueCopy>(meta_data, source_data, source, offset, copy_count);
|
429
430
|
return;
|
@@ -930,6 +931,7 @@ void ColumnDataCollection::Verify() {
|
|
930
931
|
#endif
|
931
932
|
}
|
932
933
|
|
934
|
+
// LCOV_EXCL_START
|
933
935
|
string ColumnDataCollection::ToString() const {
|
934
936
|
DataChunk chunk;
|
935
937
|
InitializeScanChunk(chunk);
|
@@ -950,6 +952,7 @@ string ColumnDataCollection::ToString() const {
|
|
950
952
|
|
951
953
|
return result;
|
952
954
|
}
|
955
|
+
// LCOV_EXCL_STOP
|
953
956
|
|
954
957
|
void ColumnDataCollection::Print() const {
|
955
958
|
Printer::Print(ToString());
|
@@ -1030,6 +1033,18 @@ bool ColumnDataCollection::ResultEquals(const ColumnDataCollection &left, const
|
|
1030
1033
|
return true;
|
1031
1034
|
}
|
1032
1035
|
|
1036
|
+
vector<shared_ptr<StringHeap>> ColumnDataCollection::GetHeapReferences() {
|
1037
|
+
vector<shared_ptr<StringHeap>> result(segments.size(), nullptr);
|
1038
|
+
for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
|
1039
|
+
result[segment_idx] = segments[segment_idx]->heap;
|
1040
|
+
}
|
1041
|
+
return result;
|
1042
|
+
}
|
1043
|
+
|
1044
|
+
ColumnDataAllocatorType ColumnDataCollection::GetAllocatorType() const {
|
1045
|
+
return allocator->GetType();
|
1046
|
+
}
|
1047
|
+
|
1033
1048
|
const vector<unique_ptr<ColumnDataCollectionSegment>> &ColumnDataCollection::GetSegments() const {
|
1034
1049
|
return segments;
|
1035
1050
|
}
|
@@ -6,7 +6,8 @@ namespace duckdb {
|
|
6
6
|
|
7
7
|
ColumnDataCollectionSegment::ColumnDataCollectionSegment(shared_ptr<ColumnDataAllocator> allocator_p,
|
8
8
|
vector<LogicalType> types_p)
|
9
|
-
: allocator(std::move(allocator_p)), types(std::move(types_p)), count(0),
|
9
|
+
: allocator(std::move(allocator_p)), types(std::move(types_p)), count(0),
|
10
|
+
heap(make_shared<StringHeap>(allocator->GetAllocator())) {
|
10
11
|
}
|
11
12
|
|
12
13
|
idx_t ColumnDataCollectionSegment::GetDataSize(idx_t type_size) {
|
@@ -26,7 +27,8 @@ VectorDataIndex ColumnDataCollectionSegment::AllocateVectorInternal(const Logica
|
|
26
27
|
auto type_size = internal_type == PhysicalType::STRUCT ? 0 : GetTypeIdSize(internal_type);
|
27
28
|
allocator->AllocateData(GetDataSize(type_size) + ValidityMask::STANDARD_MASK_SIZE, meta_data.block_id,
|
28
29
|
meta_data.offset, chunk_state);
|
29
|
-
if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR
|
30
|
+
if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR ||
|
31
|
+
allocator->GetType() == ColumnDataAllocatorType::HYBRID) {
|
30
32
|
chunk_meta.block_ids.insert(meta_data.block_id);
|
31
33
|
}
|
32
34
|
|
@@ -203,10 +205,17 @@ idx_t ColumnDataCollectionSegment::ReadVector(ChunkManagementState &state, Vecto
|
|
203
205
|
}
|
204
206
|
} else if (internal_type == PhysicalType::VARCHAR) {
|
205
207
|
if (allocator->GetType() == ColumnDataAllocatorType::BUFFER_MANAGER_ALLOCATOR) {
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
208
|
+
auto next_index = vector_index;
|
209
|
+
idx_t offset = 0;
|
210
|
+
while (next_index.IsValid()) {
|
211
|
+
auto ¤t_vdata = GetVectorData(next_index);
|
212
|
+
for (auto &swizzle_segment : current_vdata.swizzle_data) {
|
213
|
+
auto &string_heap_segment = GetVectorData(swizzle_segment.child_index);
|
214
|
+
allocator->UnswizzlePointers(state, result, offset + swizzle_segment.offset, swizzle_segment.count,
|
215
|
+
string_heap_segment.block_id, string_heap_segment.offset);
|
216
|
+
}
|
217
|
+
offset += current_vdata.count;
|
218
|
+
next_index = current_vdata.next_data;
|
210
219
|
}
|
211
220
|
}
|
212
221
|
if (state.properties == ColumnDataScanProperties::DISALLOW_ZERO_COPY) {
|
@@ -32,13 +32,13 @@ PartitionedColumnData::~PartitionedColumnData() {
|
|
32
32
|
|
33
33
|
void PartitionedColumnData::InitializeAppendState(PartitionedColumnDataAppendState &state) const {
|
34
34
|
state.partition_sel.Initialize();
|
35
|
-
state.slice_chunk.Initialize(context, types);
|
35
|
+
state.slice_chunk.Initialize(BufferAllocator::Get(context), types);
|
36
36
|
InitializeAppendStateInternal(state);
|
37
37
|
}
|
38
38
|
|
39
39
|
unique_ptr<DataChunk> PartitionedColumnData::CreatePartitionBuffer() const {
|
40
40
|
auto result = make_uniq<DataChunk>();
|
41
|
-
result->Initialize(
|
41
|
+
result->Initialize(BufferAllocator::Get(context), types, BufferSize());
|
42
42
|
return result;
|
43
43
|
}
|
44
44
|
|
@@ -309,7 +309,7 @@ void DataChunk::Hash(Vector &result) {
|
|
309
309
|
|
310
310
|
void DataChunk::Hash(vector<idx_t> &column_ids, Vector &result) {
|
311
311
|
D_ASSERT(result.GetType().id() == LogicalType::HASH);
|
312
|
-
D_ASSERT(column_ids.
|
312
|
+
D_ASSERT(!column_ids.empty());
|
313
313
|
|
314
314
|
VectorOperations::Hash(data[column_ids[0]], result, size());
|
315
315
|
for (idx_t i = 1; i < column_ids.size(); i++) {
|
@@ -327,7 +327,7 @@ void DataChunk::Verify() {
|
|
327
327
|
#endif
|
328
328
|
}
|
329
329
|
|
330
|
-
void DataChunk::Print() {
|
330
|
+
void DataChunk::Print() const {
|
331
331
|
Printer::Print(ToString());
|
332
332
|
}
|
333
333
|
|
@@ -441,6 +441,15 @@ int64_t Date::EpochMicroseconds(date_t date) {
|
|
441
441
|
return result;
|
442
442
|
}
|
443
443
|
|
444
|
+
int64_t Date::EpochMilliseconds(date_t date) {
|
445
|
+
int64_t result;
|
446
|
+
const auto MILLIS_PER_DAY = Interval::MICROS_PER_DAY / Interval::MICROS_PER_MSEC;
|
447
|
+
if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, MILLIS_PER_DAY, result)) {
|
448
|
+
throw ConversionException("Could not convert DATE (%s) to milliseconds", Date::ToString(date));
|
449
|
+
}
|
450
|
+
return result;
|
451
|
+
}
|
452
|
+
|
444
453
|
int32_t Date::ExtractYear(date_t d, int32_t *last_year) {
|
445
454
|
auto n = d.days;
|
446
455
|
// cached look up: check if year of this date is the same as the last one we looked up
|