duckdb 0.8.2-dev157.0 → 0.8.2-dev1573.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +4619 -4446
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/extra_type_info.cpp +506 -0
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +14 -14
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/types.cpp +8 -655
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +4 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
- package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +79 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +44 -31
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_common.cpp +2 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
- package/src/statement.cpp +10 -3
- package/test/test_all_types.test.ts +233 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -29,6 +29,30 @@ using duckdb_parquet::format::PageType;
|
|
29
29
|
using ParquetRowGroup = duckdb_parquet::format::RowGroup;
|
30
30
|
using duckdb_parquet::format::Type;
|
31
31
|
|
32
|
+
ChildFieldIDs::ChildFieldIDs() {
|
33
|
+
ids = make_uniq<case_insensitive_map_t<FieldID>>();
|
34
|
+
}
|
35
|
+
|
36
|
+
ChildFieldIDs ChildFieldIDs::Copy() const {
|
37
|
+
ChildFieldIDs result;
|
38
|
+
for (const auto &id : *ids) {
|
39
|
+
result.ids->emplace(id.first, id.second.Copy());
|
40
|
+
}
|
41
|
+
return result;
|
42
|
+
}
|
43
|
+
|
44
|
+
FieldID::FieldID() : set(false) {
|
45
|
+
}
|
46
|
+
|
47
|
+
FieldID::FieldID(int32_t field_id_p) : set(true), field_id(field_id_p) {
|
48
|
+
}
|
49
|
+
|
50
|
+
FieldID FieldID::Copy() const {
|
51
|
+
auto result = set ? FieldID(field_id) : FieldID();
|
52
|
+
result.child_field_ids = child_field_ids.Copy();
|
53
|
+
return result;
|
54
|
+
}
|
55
|
+
|
32
56
|
class MyTransport : public TTransport {
|
33
57
|
public:
|
34
58
|
explicit MyTransport(Serializer &serializer) : serializer(serializer) {
|
@@ -226,8 +250,9 @@ void VerifyUniqueNames(const vector<string> &names) {
|
|
226
250
|
}
|
227
251
|
|
228
252
|
ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalType> types_p, vector<string> names_p,
|
229
|
-
CompressionCodec::type codec)
|
230
|
-
: file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec)
|
253
|
+
CompressionCodec::type codec, ChildFieldIDs field_ids_p)
|
254
|
+
: file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
|
255
|
+
field_ids(std::move(field_ids_p)) {
|
231
256
|
// initialize the file writer
|
232
257
|
writer = make_uniq<BufferedFileWriter>(fs, file_name.c_str(),
|
233
258
|
FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
|
@@ -257,11 +282,18 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
|
|
257
282
|
vector<string> schema_path;
|
258
283
|
for (idx_t i = 0; i < sql_types.size(); i++) {
|
259
284
|
column_writers.push_back(ColumnWriter::CreateWriterRecursive(file_meta_data.schema, *this, sql_types[i],
|
260
|
-
unique_names[i], schema_path));
|
285
|
+
unique_names[i], schema_path, &field_ids));
|
261
286
|
}
|
262
287
|
}
|
263
288
|
|
264
289
|
void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result) {
|
290
|
+
// We write 8 columns at a time so that iterating over ColumnDataCollection is more efficient
|
291
|
+
static constexpr idx_t COLUMNS_PER_PASS = 8;
|
292
|
+
|
293
|
+
// We want these to be in-memory/hybrid so we don't have to copy over strings to the dictionary
|
294
|
+
D_ASSERT(buffer.GetAllocatorType() == ColumnDataAllocatorType::IN_MEMORY_ALLOCATOR ||
|
295
|
+
buffer.GetAllocatorType() == ColumnDataAllocatorType::HYBRID);
|
296
|
+
|
265
297
|
// set up a new row group for this chunk collection
|
266
298
|
auto &row_group = result.row_group;
|
267
299
|
row_group.num_rows = buffer.Count();
|
@@ -270,24 +302,52 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
|
|
270
302
|
auto &states = result.states;
|
271
303
|
// iterate over each of the columns of the chunk collection and write them
|
272
304
|
D_ASSERT(buffer.ColumnCount() == column_writers.size());
|
273
|
-
for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx
|
274
|
-
const auto
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
305
|
+
for (idx_t col_idx = 0; col_idx < buffer.ColumnCount(); col_idx += COLUMNS_PER_PASS) {
|
306
|
+
const auto next = MinValue<idx_t>(buffer.ColumnCount() - col_idx, COLUMNS_PER_PASS);
|
307
|
+
vector<column_t> column_ids;
|
308
|
+
vector<reference<ColumnWriter>> col_writers;
|
309
|
+
vector<unique_ptr<ColumnWriterState>> write_states;
|
310
|
+
for (idx_t i = 0; i < next; i++) {
|
311
|
+
column_ids.emplace_back(col_idx + i);
|
312
|
+
col_writers.emplace_back(*column_writers[column_ids.back()]);
|
313
|
+
write_states.emplace_back(col_writers.back().get().InitializeWriteState(row_group));
|
314
|
+
}
|
315
|
+
|
316
|
+
for (auto &chunk : buffer.Chunks({column_ids})) {
|
317
|
+
for (idx_t i = 0; i < next; i++) {
|
318
|
+
if (col_writers[i].get().HasAnalyze()) {
|
319
|
+
col_writers[i].get().Analyze(*write_states[i], nullptr, chunk.data[i], chunk.size());
|
320
|
+
}
|
321
|
+
}
|
322
|
+
}
|
323
|
+
|
324
|
+
for (idx_t i = 0; i < next; i++) {
|
325
|
+
if (col_writers[i].get().HasAnalyze()) {
|
326
|
+
col_writers[i].get().FinalizeAnalyze(*write_states[i]);
|
327
|
+
}
|
328
|
+
}
|
329
|
+
|
330
|
+
for (auto &chunk : buffer.Chunks({column_ids})) {
|
331
|
+
for (idx_t i = 0; i < next; i++) {
|
332
|
+
col_writers[i].get().Prepare(*write_states[i], nullptr, chunk.data[i], chunk.size());
|
279
333
|
}
|
280
|
-
col_writer->FinalizeAnalyze(*write_state);
|
281
334
|
}
|
282
|
-
|
283
|
-
|
335
|
+
|
336
|
+
for (idx_t i = 0; i < next; i++) {
|
337
|
+
col_writers[i].get().BeginWrite(*write_states[i]);
|
284
338
|
}
|
285
|
-
|
286
|
-
for (auto &chunk : buffer.Chunks()) {
|
287
|
-
|
339
|
+
|
340
|
+
for (auto &chunk : buffer.Chunks({column_ids})) {
|
341
|
+
for (idx_t i = 0; i < next; i++) {
|
342
|
+
col_writers[i].get().Write(*write_states[i], chunk.data[i], chunk.size());
|
343
|
+
}
|
344
|
+
}
|
345
|
+
|
346
|
+
for (auto &write_state : write_states) {
|
347
|
+
states.push_back(std::move(write_state));
|
288
348
|
}
|
289
|
-
states.push_back(std::move(write_state));
|
290
349
|
}
|
350
|
+
result.heaps = buffer.GetHeapReferences();
|
291
351
|
}
|
292
352
|
|
293
353
|
void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
|
@@ -307,6 +367,8 @@ void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
|
|
307
367
|
// append the row group to the file meta data
|
308
368
|
file_meta_data.row_groups.push_back(row_group);
|
309
369
|
file_meta_data.num_rows += row_group.num_rows;
|
370
|
+
|
371
|
+
prepared.heaps.clear();
|
310
372
|
}
|
311
373
|
|
312
374
|
void ParquetWriter::Flush(ColumnDataCollection &buffer) {
|
@@ -316,6 +378,7 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
|
|
316
378
|
|
317
379
|
PreparedRowGroup prepared_row_group;
|
318
380
|
PrepareRowGroup(buffer, prepared_row_group);
|
381
|
+
buffer.Reset();
|
319
382
|
|
320
383
|
FlushRowGroup(prepared_row_group);
|
321
384
|
}
|
@@ -155,7 +155,7 @@ void ZstdStreamWrapper::Close() {
|
|
155
155
|
|
156
156
|
class ZStdFile : public CompressedFile {
|
157
157
|
public:
|
158
|
-
ZStdFile(
|
158
|
+
ZStdFile(unique_ptr<FileHandle> child_handle_p, const string &path, bool write)
|
159
159
|
: CompressedFile(zstd_fs, std::move(child_handle_p), path) {
|
160
160
|
Initialize(write);
|
161
161
|
}
|
@@ -163,7 +163,7 @@ public:
|
|
163
163
|
ZStdFileSystem zstd_fs;
|
164
164
|
};
|
165
165
|
|
166
|
-
unique_ptr<FileHandle> ZStdFileSystem::OpenCompressedFile(
|
166
|
+
unique_ptr<FileHandle> ZStdFileSystem::OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) {
|
167
167
|
auto path = handle->path;
|
168
168
|
return make_uniq<ZStdFile>(std::move(handle), path, write);
|
169
169
|
}
|
@@ -298,7 +298,7 @@ unique_ptr<CatalogEntry> DuckTableEntry::AddColumn(ClientContext &context, AddCo
|
|
298
298
|
auto binder = Binder::CreateBinder(context);
|
299
299
|
auto bound_create_info = binder->BindCreateTableInfo(std::move(create_info));
|
300
300
|
auto new_storage =
|
301
|
-
make_shared<DataTable>(context, *storage, info.new_column, bound_create_info->bound_defaults.back()
|
301
|
+
make_shared<DataTable>(context, *storage, info.new_column, *bound_create_info->bound_defaults.back());
|
302
302
|
return make_uniq<DuckTableEntry>(catalog, schema, *bound_create_info, new_storage);
|
303
303
|
}
|
304
304
|
|
@@ -1,10 +1,10 @@
|
|
1
1
|
#include "duckdb/catalog/catalog_search_path.hpp"
|
2
2
|
|
3
|
+
#include "duckdb/catalog/catalog.hpp"
|
3
4
|
#include "duckdb/common/constants.hpp"
|
4
5
|
#include "duckdb/common/exception.hpp"
|
5
6
|
#include "duckdb/common/string_util.hpp"
|
6
7
|
#include "duckdb/main/client_context.hpp"
|
7
|
-
#include "duckdb/catalog/catalog.hpp"
|
8
8
|
#include "duckdb/main/database_manager.hpp"
|
9
9
|
|
10
10
|
namespace duckdb {
|
@@ -249,13 +249,14 @@ void CatalogSearchPath::SetPaths(vector<CatalogSearchEntry> new_paths) {
|
|
249
249
|
bool CatalogSearchPath::SchemaInSearchPath(ClientContext &context, const string &catalog_name,
|
250
250
|
const string &schema_name) {
|
251
251
|
for (auto &path : paths) {
|
252
|
-
if (path.schema
|
252
|
+
if (!StringUtil::CIEquals(path.schema, schema_name)) {
|
253
253
|
continue;
|
254
254
|
}
|
255
|
-
if (path.catalog
|
255
|
+
if (StringUtil::CIEquals(path.catalog, catalog_name)) {
|
256
256
|
return true;
|
257
257
|
}
|
258
|
-
if (IsInvalidCatalog(path.catalog) &&
|
258
|
+
if (IsInvalidCatalog(path.catalog) &&
|
259
|
+
StringUtil::CIEquals(catalog_name, DatabaseManager::GetDefaultDatabase(context))) {
|
259
260
|
return true;
|
260
261
|
}
|
261
262
|
}
|
@@ -99,6 +99,19 @@ static DefaultMacro internal_macros[] = {
|
|
99
99
|
{DEFAULT_SCHEMA, "count_if", {"l", nullptr}, "sum(if(l, 1, 0))"},
|
100
100
|
{DEFAULT_SCHEMA, "split_part", {"string", "delimiter", "position", nullptr}, "coalesce(string_split(string, delimiter)[position],'')"},
|
101
101
|
|
102
|
+
// FIXME implement as actual function if we encounter a lot of performance issues. Complexity now: n * m, with hashing possibly n + m
|
103
|
+
{DEFAULT_SCHEMA, "list_intersect", {"l1", "l2", nullptr}, "list_filter(l1, (x) -> list_contains(l2, x))"},
|
104
|
+
{DEFAULT_SCHEMA, "array_intersect", {"l1", "l2", nullptr}, "list_intersect(l1, l2)"},
|
105
|
+
|
106
|
+
{DEFAULT_SCHEMA, "list_has_any", {"l1", "l2", nullptr}, "CASE WHEN l1 IS NULL THEN NULL WHEN l2 IS NULL THEN NULL WHEN len(list_intersect(l1, l2)) > 0 THEN true ELSE false END"},
|
107
|
+
{DEFAULT_SCHEMA, "array_has_any", {"l1", "l2", nullptr}, "list_has_any(l1, l2)" },
|
108
|
+
{DEFAULT_SCHEMA, "&&", {"l1", "l2", nullptr}, "list_has_any(l1, l2)" }, // "&&" is the operator for "list_has_any
|
109
|
+
|
110
|
+
{DEFAULT_SCHEMA, "list_has_all", {"l1", "l2", nullptr}, "CASE WHEN l1 IS NULL THEN NULL WHEN l2 IS NULL THEN NULL WHEN len(list_intersect(l2, l1)) = len(list_filter(l2, x -> x IS NOT NULL)) THEN true ELSE false END"},
|
111
|
+
{DEFAULT_SCHEMA, "array_has_all", {"l1", "l2", nullptr}, "list_has_all(l1, l2)" },
|
112
|
+
{DEFAULT_SCHEMA, "@>", {"l1", "l2", nullptr}, "list_has_all(l1, l2)" }, // "@>" is the operator for "list_has_all
|
113
|
+
{DEFAULT_SCHEMA, "<@", {"l1", "l2", nullptr}, "list_has_all(l2, l1)" }, // "<@" is the operator for "list_has_all
|
114
|
+
|
102
115
|
// algebraic list aggregates
|
103
116
|
{DEFAULT_SCHEMA, "list_avg", {"l", nullptr}, "list_aggr(l, 'avg')"},
|
104
117
|
{DEFAULT_SCHEMA, "list_var_samp", {"l", nullptr}, "list_aggr(l, 'var_samp')"},
|
@@ -205,6 +218,9 @@ unique_ptr<CatalogEntry> DefaultFunctionGenerator::CreateDefaultEntry(ClientCont
|
|
205
218
|
vector<string> DefaultFunctionGenerator::GetDefaultEntries() {
|
206
219
|
vector<string> result;
|
207
220
|
for (idx_t index = 0; internal_macros[index].name != nullptr; index++) {
|
221
|
+
if (StringUtil::Lower(internal_macros[index].name) != internal_macros[index].name) {
|
222
|
+
throw InternalException("Default macro name %s should be lowercase", internal_macros[index].name);
|
223
|
+
}
|
208
224
|
if (internal_macros[index].schema == schema.name) {
|
209
225
|
result.emplace_back(internal_macros[index].name);
|
210
226
|
}
|
@@ -44,10 +44,23 @@ duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::A
|
|
44
44
|
driver->ConnectionRollback = duckdb_adbc::ConnectionRollback;
|
45
45
|
driver->ConnectionReadPartition = duckdb_adbc::ConnectionReadPartition;
|
46
46
|
driver->StatementExecutePartitions = duckdb_adbc::StatementExecutePartitions;
|
47
|
+
driver->ConnectionGetTableSchema = duckdb_adbc::ConnectionGetTableSchema;
|
48
|
+
driver->StatementSetSubstraitPlan = duckdb_adbc::StatementSetSubstraitPlan;
|
49
|
+
|
47
50
|
return ADBC_STATUS_OK;
|
48
51
|
}
|
49
52
|
|
50
53
|
namespace duckdb_adbc {
|
54
|
+
|
55
|
+
struct DuckDBAdbcStatementWrapper {
|
56
|
+
::duckdb_connection connection;
|
57
|
+
::duckdb_arrow result;
|
58
|
+
::duckdb_prepared_statement statement;
|
59
|
+
char *ingestion_table_name;
|
60
|
+
ArrowArrayStream *ingestion_stream;
|
61
|
+
};
|
62
|
+
static AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
|
63
|
+
struct AdbcError *error);
|
51
64
|
AdbcStatusCode SetErrorMaybe(const void *result, AdbcError *error, const std::string &error_message) {
|
52
65
|
if (!error) {
|
53
66
|
return ADBC_STATUS_INVALID_ARGUMENT;
|
@@ -109,6 +122,28 @@ AdbcStatusCode DatabaseNew(struct AdbcDatabase *database, struct AdbcError *erro
|
|
109
122
|
return CheckResult(res, error, "Failed to allocate");
|
110
123
|
}
|
111
124
|
|
125
|
+
AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *statement, const uint8_t *plan, size_t length,
|
126
|
+
struct AdbcError *error) {
|
127
|
+
if (!statement) {
|
128
|
+
SetError(error, "Statement is not set");
|
129
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
130
|
+
}
|
131
|
+
if (!plan) {
|
132
|
+
SetError(error, "Substrait Plan is not set");
|
133
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
134
|
+
}
|
135
|
+
if (length == 0) {
|
136
|
+
SetError(error, "Can't execute plan with size = 0");
|
137
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
138
|
+
}
|
139
|
+
auto wrapper = reinterpret_cast<DuckDBAdbcStatementWrapper *>(statement->private_data);
|
140
|
+
auto plan_str = std::string(reinterpret_cast<const char *>(plan), length);
|
141
|
+
auto query = "CALL from_substrait('" + plan_str + "'::BLOB)";
|
142
|
+
auto res = duckdb_prepare(wrapper->connection, query.c_str(), &wrapper->statement);
|
143
|
+
auto error_msg = duckdb_prepare_error(wrapper->statement);
|
144
|
+
return CheckResult(res, error, error_msg);
|
145
|
+
}
|
146
|
+
|
112
147
|
AdbcStatusCode DatabaseSetOption(struct AdbcDatabase *database, const char *key, const char *value,
|
113
148
|
struct AdbcError *error) {
|
114
149
|
auto status = SetErrorMaybe(database, error, "Missing database object");
|
@@ -159,6 +194,44 @@ AdbcStatusCode DatabaseRelease(struct AdbcDatabase *database, struct AdbcError *
|
|
159
194
|
return ADBC_STATUS_OK;
|
160
195
|
}
|
161
196
|
|
197
|
+
AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const char *catalog, const char *db_schema,
|
198
|
+
const char *table_name, struct ArrowSchema *schema, struct AdbcError *error) {
|
199
|
+
if (!connection) {
|
200
|
+
SetError(error, "Connection is not set");
|
201
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
202
|
+
}
|
203
|
+
if (catalog != nullptr && strlen(catalog) > 0) {
|
204
|
+
// In DuckDB this is the name of the database, not sure what's the expected functionality here, so for now,
|
205
|
+
// scream.
|
206
|
+
SetError(error, "Catalog Name is not used in DuckDB. It must be set to nullptr or an empty string");
|
207
|
+
return ADBC_STATUS_NOT_IMPLEMENTED;
|
208
|
+
} else if (db_schema == nullptr) {
|
209
|
+
SetError(error, "AdbcConnectionGetTableSchema: must provide db_schema");
|
210
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
211
|
+
} else if (table_name == nullptr) {
|
212
|
+
SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
|
213
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
214
|
+
} else if (strlen(table_name) == 0) {
|
215
|
+
SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
|
216
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
217
|
+
}
|
218
|
+
ArrowArrayStream arrow_stream;
|
219
|
+
|
220
|
+
std::string query = "SELECT * FROM ";
|
221
|
+
if (strlen(db_schema) > 0) {
|
222
|
+
query += std::string(db_schema) + ".";
|
223
|
+
}
|
224
|
+
query += std::string(table_name) + " LIMIT 0;";
|
225
|
+
|
226
|
+
auto success = QueryInternal(connection, &arrow_stream, query.c_str(), error);
|
227
|
+
if (success != ADBC_STATUS_OK) {
|
228
|
+
return success;
|
229
|
+
}
|
230
|
+
arrow_stream.get_schema(&arrow_stream, schema);
|
231
|
+
arrow_stream.release(&arrow_stream);
|
232
|
+
return ADBC_STATUS_OK;
|
233
|
+
}
|
234
|
+
|
162
235
|
AdbcStatusCode ConnectionNew(struct AdbcConnection *connection, struct AdbcError *error) {
|
163
236
|
auto status = SetErrorMaybe(connection, error, "Missing connection object");
|
164
237
|
if (status != ADBC_STATUS_OK) {
|
@@ -400,14 +473,6 @@ AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, stru
|
|
400
473
|
return ADBC_STATUS_OK;
|
401
474
|
}
|
402
475
|
|
403
|
-
struct DuckDBAdbcStatementWrapper {
|
404
|
-
::duckdb_connection connection;
|
405
|
-
::duckdb_arrow result;
|
406
|
-
::duckdb_prepared_statement statement;
|
407
|
-
char *ingestion_table_name;
|
408
|
-
ArrowArrayStream *ingestion_stream;
|
409
|
-
};
|
410
|
-
|
411
476
|
AdbcStatusCode StatementNew(struct AdbcConnection *connection, struct AdbcStatement *statement,
|
412
477
|
struct AdbcError *error) {
|
413
478
|
|
@@ -581,8 +646,8 @@ AdbcStatusCode StatementSetOption(struct AdbcStatement *statement, const char *k
|
|
581
646
|
return ADBC_STATUS_INVALID_ARGUMENT;
|
582
647
|
}
|
583
648
|
|
584
|
-
|
585
|
-
|
649
|
+
AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
|
650
|
+
struct AdbcError *error) {
|
586
651
|
AdbcStatement statement;
|
587
652
|
|
588
653
|
auto status = StatementNew(connection, &statement, error);
|
@@ -109,8 +109,9 @@ struct ManagerDriverState {
|
|
109
109
|
static AdbcStatusCode ReleaseDriver(struct AdbcDriver *driver, struct AdbcError *error) {
|
110
110
|
AdbcStatusCode status = ADBC_STATUS_OK;
|
111
111
|
|
112
|
-
if (!driver->private_manager)
|
112
|
+
if (!driver->private_manager) {
|
113
113
|
return status;
|
114
|
+
}
|
114
115
|
ManagerDriverState *state = reinterpret_cast<ManagerDriverState *>(driver->private_manager);
|
115
116
|
|
116
117
|
if (state->driver_release) {
|
@@ -138,11 +139,6 @@ AdbcStatusCode ConnectionGetInfo(struct AdbcConnection *connection, uint32_t *in
|
|
138
139
|
return ADBC_STATUS_NOT_IMPLEMENTED;
|
139
140
|
}
|
140
141
|
|
141
|
-
AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *, const char *, const char *, const char *,
|
142
|
-
struct ArrowSchema *, struct AdbcError *error) {
|
143
|
-
return ADBC_STATUS_NOT_IMPLEMENTED;
|
144
|
-
}
|
145
|
-
|
146
142
|
AdbcStatusCode StatementBind(struct AdbcStatement *, struct ArrowArray *, struct ArrowSchema *,
|
147
143
|
struct AdbcError *error) {
|
148
144
|
return ADBC_STATUS_NOT_IMPLEMENTED;
|
@@ -152,9 +148,6 @@ AdbcStatusCode StatementGetParameterSchema(struct AdbcStatement *statement, stru
|
|
152
148
|
struct AdbcError *error) {
|
153
149
|
return ADBC_STATUS_NOT_IMPLEMENTED;
|
154
150
|
}
|
155
|
-
AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *, const uint8_t *, size_t, struct AdbcError *error) {
|
156
|
-
return ADBC_STATUS_NOT_IMPLEMENTED;
|
157
|
-
}
|
158
151
|
|
159
152
|
/// Temporary state while the database is being configured.
|
160
153
|
struct TempDatabase {
|
@@ -351,15 +344,17 @@ AdbcStatusCode AdbcConnectionInit(struct AdbcConnection *connection, struct Adbc
|
|
351
344
|
delete args;
|
352
345
|
|
353
346
|
auto status = database->private_driver->ConnectionNew(connection, error);
|
354
|
-
if (status != ADBC_STATUS_OK)
|
347
|
+
if (status != ADBC_STATUS_OK) {
|
355
348
|
return status;
|
349
|
+
}
|
356
350
|
connection->private_driver = database->private_driver;
|
357
351
|
|
358
352
|
for (const auto &option : options) {
|
359
353
|
status = database->private_driver->ConnectionSetOption(connection, option.first.c_str(), option.second.c_str(),
|
360
354
|
error);
|
361
|
-
if (status != ADBC_STATUS_OK)
|
355
|
+
if (status != ADBC_STATUS_OK) {
|
362
356
|
return status;
|
357
|
+
}
|
363
358
|
}
|
364
359
|
return connection->private_driver->ConnectionInit(connection, database, error);
|
365
360
|
}
|
@@ -14,7 +14,13 @@
|
|
14
14
|
#include <execinfo.h>
|
15
15
|
#endif
|
16
16
|
|
17
|
-
#
|
17
|
+
#ifndef USE_JEMALLOC
|
18
|
+
#if defined(DUCKDB_EXTENSION_JEMALLOC_LINKED) && DUCKDB_EXTENSION_JEMALLOC_LINKED && !defined(WIN32)
|
19
|
+
#define USE_JEMALLOC
|
20
|
+
#endif
|
21
|
+
#endif
|
22
|
+
|
23
|
+
#ifdef USE_JEMALLOC
|
18
24
|
#include "jemalloc_extension.hpp"
|
19
25
|
#endif
|
20
26
|
|
@@ -89,7 +95,7 @@ PrivateAllocatorData::~PrivateAllocatorData() {
|
|
89
95
|
//===--------------------------------------------------------------------===//
|
90
96
|
// Allocator
|
91
97
|
//===--------------------------------------------------------------------===//
|
92
|
-
#
|
98
|
+
#ifdef USE_JEMALLOC
|
93
99
|
Allocator::Allocator()
|
94
100
|
: Allocator(JemallocExtension::Allocate, JemallocExtension::Free, JemallocExtension::Reallocate, nullptr) {
|
95
101
|
}
|
@@ -177,6 +183,12 @@ Allocator &Allocator::DefaultAllocator() {
|
|
177
183
|
return *DefaultAllocatorReference();
|
178
184
|
}
|
179
185
|
|
186
|
+
void Allocator::ThreadFlush(idx_t threshold) {
|
187
|
+
#ifdef USE_JEMALLOC
|
188
|
+
JemallocExtension::ThreadFlush(threshold);
|
189
|
+
#endif
|
190
|
+
}
|
191
|
+
|
180
192
|
//===--------------------------------------------------------------------===//
|
181
193
|
// Debug Info (extended)
|
182
194
|
//===--------------------------------------------------------------------===//
|
@@ -153,7 +153,9 @@ struct ArrowIntervalConverter {
|
|
153
153
|
template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
|
154
154
|
struct ArrowScalarBaseData {
|
155
155
|
static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
|
156
|
+
D_ASSERT(to >= from);
|
156
157
|
idx_t size = to - from;
|
158
|
+
D_ASSERT(size <= input_size);
|
157
159
|
UnifiedVectorFormat format;
|
158
160
|
input.ToUnifiedFormat(input_size, format);
|
159
161
|
|
@@ -489,16 +491,9 @@ struct ArrowListData {
|
|
489
491
|
SelectionVector child_sel(child_indices.data());
|
490
492
|
auto &child = ListVector::GetEntry(input);
|
491
493
|
auto child_size = child_indices.size();
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
child_copy.Slice(child, child_sel, child_size);
|
496
|
-
append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
|
497
|
-
} else {
|
498
|
-
// We don't care about the vector, slice it
|
499
|
-
child.Slice(child_sel, child_size);
|
500
|
-
append_data.child_data[0]->append_vector(*append_data.child_data[0], child, 0, child_size, child_size);
|
501
|
-
}
|
494
|
+
Vector child_copy(child.GetType());
|
495
|
+
child_copy.Slice(child, child_sel, child_size);
|
496
|
+
append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
|
502
497
|
append_data.row_count += size;
|
503
498
|
}
|
504
499
|
|
@@ -791,7 +786,6 @@ ArrowArray ArrowAppender::Finalize() {
|
|
791
786
|
|
792
787
|
// Configure root array
|
793
788
|
result.length = row_count;
|
794
|
-
result.n_children = types.size();
|
795
789
|
result.n_buffers = 1;
|
796
790
|
result.buffers = root_holder->buffers.data(); // there is no actual buffer there since we don't have NULLs
|
797
791
|
result.offset = 0;
|