duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +17 -37
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +192 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +5 -11
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +4619 -4446
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/extra_type_info.cpp +506 -0
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +14 -14
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/types.cpp +8 -655
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +414 -283
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +13 -22
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -10
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +8 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +4 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +96 -89
- package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +79 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +61 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +5 -0
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_common.cpp +2 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
- package/src/statement.cpp +10 -3
- package/test/test_all_types.test.ts +233 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -48,6 +48,10 @@ struct StringStats {
|
|
48
48
|
DUCKDB_API static uint32_t MaxStringLength(const BaseStatistics &stats);
|
49
49
|
//! Whether or not the strings can contain unicode
|
50
50
|
DUCKDB_API static bool CanContainUnicode(const BaseStatistics &stats);
|
51
|
+
//! Returns the min value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
|
52
|
+
DUCKDB_API static string Min(const BaseStatistics &stats);
|
53
|
+
//! Returns the max value (up to a length of StringStatsData::MAX_STRING_MINMAX_SIZE)
|
54
|
+
DUCKDB_API static string Max(const BaseStatistics &stats);
|
51
55
|
|
52
56
|
//! Resets the max string length so HasMaxStringLength() is false
|
53
57
|
DUCKDB_API static void ResetMaxStringLength(BaseStatistics &stats);
|
@@ -41,6 +41,7 @@ public:
|
|
41
41
|
//! Returns whether or not a single row in the ChunkInfo should be used or not for the given transaction
|
42
42
|
virtual bool Fetch(TransactionData transaction, row_t row) = 0;
|
43
43
|
virtual void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) = 0;
|
44
|
+
virtual idx_t GetCommittedDeletedCount(idx_t max_count) = 0;
|
44
45
|
|
45
46
|
virtual void Serialize(Serializer &serialize) = 0;
|
46
47
|
static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
|
@@ -79,6 +80,7 @@ public:
|
|
79
80
|
SelectionVector &sel_vector, idx_t max_count) override;
|
80
81
|
bool Fetch(TransactionData transaction, row_t row) override;
|
81
82
|
void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
|
83
|
+
idx_t GetCommittedDeletedCount(idx_t max_count) override;
|
82
84
|
|
83
85
|
void Serialize(Serializer &serialize) override;
|
84
86
|
static unique_ptr<ChunkInfo> Deserialize(Deserializer &source);
|
@@ -113,6 +115,7 @@ public:
|
|
113
115
|
SelectionVector &sel_vector, idx_t max_count) override;
|
114
116
|
bool Fetch(TransactionData transaction, row_t row) override;
|
115
117
|
void CommitAppend(transaction_t commit_id, idx_t start, idx_t end) override;
|
118
|
+
idx_t GetCommittedDeletedCount(idx_t max_count) override;
|
116
119
|
|
117
120
|
void Append(idx_t start, idx_t end, transaction_t commit_id);
|
118
121
|
|
@@ -74,7 +74,6 @@ public:
|
|
74
74
|
RowGroupCollection &GetCollection() {
|
75
75
|
return collection.get();
|
76
76
|
}
|
77
|
-
DatabaseInstance &GetDatabase();
|
78
77
|
BlockManager &GetBlockManager();
|
79
78
|
DataTableInfo &GetTableInfo();
|
80
79
|
|
@@ -82,7 +81,7 @@ public:
|
|
82
81
|
ExpressionExecutor &executor, CollectionScanState &scan_state,
|
83
82
|
DataChunk &scan_chunk);
|
84
83
|
unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
|
85
|
-
ExpressionExecutor &executor, Expression
|
84
|
+
ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
|
86
85
|
unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
|
87
86
|
|
88
87
|
void CommitDrop();
|
@@ -123,6 +122,7 @@ public:
|
|
123
122
|
idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
|
124
123
|
|
125
124
|
RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
|
125
|
+
bool AllDeleted();
|
126
126
|
RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
|
127
127
|
static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
|
128
128
|
static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
|
@@ -170,6 +170,7 @@ struct VersionNode {
|
|
170
170
|
unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
|
171
171
|
|
172
172
|
void SetStart(idx_t start);
|
173
|
+
idx_t GetCommittedDeletedCount(idx_t count);
|
173
174
|
};
|
174
175
|
|
175
176
|
} // namespace duckdb
|
@@ -45,7 +45,6 @@ public:
|
|
45
45
|
void AppendRowGroup(SegmentLock &l, idx_t start_row);
|
46
46
|
//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
|
47
47
|
RowGroup *GetRowGroup(int64_t index);
|
48
|
-
idx_t RowGroupCount();
|
49
48
|
void Verify();
|
50
49
|
|
51
50
|
void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
|
@@ -93,7 +92,7 @@ public:
|
|
93
92
|
const vector<LogicalType> &GetTypes() const;
|
94
93
|
|
95
94
|
shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
|
96
|
-
Expression
|
95
|
+
Expression &default_value);
|
97
96
|
shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
|
98
97
|
shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
|
99
98
|
vector<column_t> bound_columns, Expression &cast_expr);
|
@@ -104,7 +103,6 @@ public:
|
|
104
103
|
void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
|
105
104
|
|
106
105
|
AttachedDatabase &GetAttached();
|
107
|
-
DatabaseInstance &GetDatabase();
|
108
106
|
BlockManager &GetBlockManager() {
|
109
107
|
return block_manager;
|
110
108
|
}
|
@@ -32,7 +32,7 @@ public:
|
|
32
32
|
LocalTableStorage(DataTable &table, LocalTableStorage &parent, idx_t drop_idx);
|
33
33
|
// Create a LocalTableStorage from an ADD COLUMN
|
34
34
|
LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, ColumnDefinition &new_column,
|
35
|
-
|
35
|
+
Expression &default_value);
|
36
36
|
~LocalTableStorage();
|
37
37
|
|
38
38
|
reference<DataTable> table_ref;
|
@@ -143,8 +143,7 @@ public:
|
|
143
143
|
|
144
144
|
idx_t AddedRows(DataTable &table);
|
145
145
|
|
146
|
-
void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
|
147
|
-
optional_ptr<Expression> default_value);
|
146
|
+
void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column, Expression &default_value);
|
148
147
|
void DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t removed_column);
|
149
148
|
void ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t changed_idx, const LogicalType &target_type,
|
150
149
|
const vector<column_t> &bound_columns, Expression &cast_expr);
|
@@ -279,6 +279,9 @@ typedef struct _duckdb_appender {
|
|
279
279
|
typedef struct _duckdb_arrow {
|
280
280
|
void *__arrw;
|
281
281
|
} * duckdb_arrow;
|
282
|
+
typedef struct _duckdb_arrow_stream {
|
283
|
+
void *__arrwstr;
|
284
|
+
} * duckdb_arrow_stream;
|
282
285
|
typedef struct _duckdb_config {
|
283
286
|
void *__cnfg;
|
284
287
|
} * duckdb_config;
|
@@ -1078,6 +1081,31 @@ Executes the prepared statement with the given bound parameters, and returns an
|
|
1078
1081
|
DUCKDB_API duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_statement,
|
1079
1082
|
duckdb_arrow *out_result);
|
1080
1083
|
|
1084
|
+
/*!
|
1085
|
+
Scans the Arrow stream and creates a view with the given name.
|
1086
|
+
|
1087
|
+
* connection: The connection on which to execute the scan.
|
1088
|
+
* table_name: Name of the temporary view to create.
|
1089
|
+
* arrow: Arrow stream wrapper.
|
1090
|
+
* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
|
1091
|
+
*/
|
1092
|
+
DUCKDB_API duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name,
|
1093
|
+
duckdb_arrow_stream arrow);
|
1094
|
+
|
1095
|
+
/*!
|
1096
|
+
Scans the Arrow array and creates a view with the given name.
|
1097
|
+
|
1098
|
+
* connection: The connection on which to execute the scan.
|
1099
|
+
* table_name: Name of the temporary view to create.
|
1100
|
+
* arrow_schema: Arrow schema wrapper.
|
1101
|
+
* arrow_array: Arrow array wrapper.
|
1102
|
+
* out_stream: Output array stream that wraps around the passed schema, for releasing/deleting once done.
|
1103
|
+
* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
|
1104
|
+
*/
|
1105
|
+
DUCKDB_API duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
|
1106
|
+
duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
|
1107
|
+
duckdb_arrow_stream *out_stream);
|
1108
|
+
|
1081
1109
|
//===--------------------------------------------------------------------===//
|
1082
1110
|
// Extract Statements
|
1083
1111
|
//===--------------------------------------------------------------------===//
|
@@ -1,5 +1,6 @@
|
|
1
|
-
#include "duckdb/main/capi/capi_internal.hpp"
|
2
1
|
#include "duckdb/common/arrow/arrow_converter.hpp"
|
2
|
+
#include "duckdb/function/table/arrow.hpp"
|
3
|
+
#include "duckdb/main/capi/capi_internal.hpp"
|
3
4
|
|
4
5
|
using duckdb::ArrowConverter;
|
5
6
|
using duckdb::ArrowResultWrapper;
|
@@ -108,3 +109,156 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st
|
|
108
109
|
*out_result = reinterpret_cast<duckdb_arrow>(arrow_wrapper);
|
109
110
|
return !arrow_wrapper->result->HasError() ? DuckDBSuccess : DuckDBError;
|
110
111
|
}
|
112
|
+
|
113
|
+
namespace arrow_array_stream_wrapper {
|
114
|
+
namespace {
|
115
|
+
struct PrivateData {
|
116
|
+
ArrowSchema *schema;
|
117
|
+
ArrowArray *array;
|
118
|
+
bool done = false;
|
119
|
+
};
|
120
|
+
|
121
|
+
// LCOV_EXCL_START
|
122
|
+
// This function is never called, but used to set ArrowSchema's release functions to a non-null NOOP.
|
123
|
+
void EmptySchemaRelease(ArrowSchema *) {
|
124
|
+
}
|
125
|
+
// LCOV_EXCL_STOP
|
126
|
+
|
127
|
+
void EmptyArrayRelease(ArrowArray *) {
|
128
|
+
}
|
129
|
+
|
130
|
+
void EmptyStreamRelease(ArrowArrayStream *) {
|
131
|
+
}
|
132
|
+
|
133
|
+
void FactoryGetSchema(uintptr_t stream_factory_ptr, duckdb::ArrowSchemaWrapper &schema) {
|
134
|
+
auto private_data =
|
135
|
+
reinterpret_cast<PrivateData *>(reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr)->private_data);
|
136
|
+
schema.arrow_schema = *private_data->schema;
|
137
|
+
|
138
|
+
// Need to nullify the root schema's release function here, because streams don't allow us to set the release
|
139
|
+
// function. For the schema's children, we nullify the release functions in `duckdb_arrow_scan`, so we don't need to
|
140
|
+
// handle them again here. We set this to nullptr and not EmptySchemaRelease to prevent ArrowSchemaWrapper's
|
141
|
+
// destructor from destroying the schema (it's the caller's responsibility).
|
142
|
+
schema.arrow_schema.release = nullptr;
|
143
|
+
}
|
144
|
+
|
145
|
+
int GetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out) {
|
146
|
+
auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
|
147
|
+
if (private_data->schema == nullptr) {
|
148
|
+
return DuckDBError;
|
149
|
+
}
|
150
|
+
|
151
|
+
*out = *private_data->schema;
|
152
|
+
out->release = EmptySchemaRelease;
|
153
|
+
return DuckDBSuccess;
|
154
|
+
}
|
155
|
+
|
156
|
+
int GetNext(struct ArrowArrayStream *stream, struct ArrowArray *out) {
|
157
|
+
auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
|
158
|
+
*out = *private_data->array;
|
159
|
+
if (private_data->done) {
|
160
|
+
out->release = nullptr;
|
161
|
+
} else {
|
162
|
+
out->release = EmptyArrayRelease;
|
163
|
+
}
|
164
|
+
|
165
|
+
private_data->done = true;
|
166
|
+
return DuckDBSuccess;
|
167
|
+
}
|
168
|
+
|
169
|
+
duckdb::unique_ptr<duckdb::ArrowArrayStreamWrapper> FactoryGetNext(uintptr_t stream_factory_ptr,
|
170
|
+
duckdb::ArrowStreamParameters ¶meters) {
|
171
|
+
auto stream = reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr);
|
172
|
+
auto ret = duckdb::make_uniq<duckdb::ArrowArrayStreamWrapper>();
|
173
|
+
ret->arrow_array_stream = *stream;
|
174
|
+
ret->arrow_array_stream.release = EmptyStreamRelease;
|
175
|
+
return ret;
|
176
|
+
}
|
177
|
+
|
178
|
+
// LCOV_EXCL_START
|
179
|
+
// This function is never be called, because it's used to construct a stream wrapping around a caller-supplied
|
180
|
+
// ArrowArray. Thus, the stream itself cannot produce an error.
|
181
|
+
const char *GetLastError(struct ArrowArrayStream *stream) {
|
182
|
+
return nullptr;
|
183
|
+
}
|
184
|
+
// LCOV_EXCL_STOP
|
185
|
+
|
186
|
+
void Release(struct ArrowArrayStream *stream) {
|
187
|
+
if (stream->private_data != nullptr) {
|
188
|
+
delete reinterpret_cast<PrivateData *>(stream->private_data);
|
189
|
+
}
|
190
|
+
|
191
|
+
stream->private_data = nullptr;
|
192
|
+
stream->release = nullptr;
|
193
|
+
}
|
194
|
+
|
195
|
+
duckdb_state Ingest(duckdb_connection connection, const char *table_name, struct ArrowArrayStream *input) {
|
196
|
+
try {
|
197
|
+
auto cconn = reinterpret_cast<duckdb::Connection *>(connection);
|
198
|
+
cconn
|
199
|
+
->TableFunction("arrow_scan", {duckdb::Value::POINTER((uintptr_t)input),
|
200
|
+
duckdb::Value::POINTER((uintptr_t)FactoryGetNext),
|
201
|
+
duckdb::Value::POINTER((uintptr_t)FactoryGetSchema)})
|
202
|
+
->CreateView(table_name, true, false);
|
203
|
+
} catch (...) { // LCOV_EXCL_START
|
204
|
+
// Tried covering this in tests, but it proved harder than expected. At the time of writing:
|
205
|
+
// - Passing any name to `CreateView` worked without throwing an exception
|
206
|
+
// - Passing a null Arrow array worked without throwing an exception
|
207
|
+
// - Passing an invalid schema (without any columns) led to an InternalException with SIGABRT, which is meant to
|
208
|
+
// be un-catchable. This case likely needs to be handled gracefully within `arrow_scan`.
|
209
|
+
// Ref: https://discord.com/channels/909674491309850675/921100573732909107/1115230468699336785
|
210
|
+
return DuckDBError;
|
211
|
+
} // LCOV_EXCL_STOP
|
212
|
+
|
213
|
+
return DuckDBSuccess;
|
214
|
+
}
|
215
|
+
} // namespace
|
216
|
+
} // namespace arrow_array_stream_wrapper
|
217
|
+
|
218
|
+
duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name, duckdb_arrow_stream arrow) {
|
219
|
+
auto stream = reinterpret_cast<ArrowArrayStream *>(arrow);
|
220
|
+
|
221
|
+
// Backup release functions - we nullify children schema release functions because we don't want to release on
|
222
|
+
// behalf of the caller, downstream in our code. Note that Arrow releases target immediate children, but aren't
|
223
|
+
// recursive. So we only back up immediate children here and restore their functions.
|
224
|
+
ArrowSchema schema;
|
225
|
+
if (stream->get_schema(stream, &schema) == DuckDBError) {
|
226
|
+
return DuckDBError;
|
227
|
+
}
|
228
|
+
|
229
|
+
typedef void (*release_fn_t)(ArrowSchema *);
|
230
|
+
std::vector<release_fn_t> release_fns(schema.n_children);
|
231
|
+
for (int64_t i = 0; i < schema.n_children; i++) {
|
232
|
+
auto child = schema.children[i];
|
233
|
+
release_fns[i] = child->release;
|
234
|
+
child->release = arrow_array_stream_wrapper::EmptySchemaRelease;
|
235
|
+
}
|
236
|
+
|
237
|
+
auto ret = arrow_array_stream_wrapper::Ingest(connection, table_name, stream);
|
238
|
+
|
239
|
+
// Restore release functions.
|
240
|
+
for (int64_t i = 0; i < schema.n_children; i++) {
|
241
|
+
schema.children[i]->release = release_fns[i];
|
242
|
+
}
|
243
|
+
|
244
|
+
return ret;
|
245
|
+
}
|
246
|
+
|
247
|
+
duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
|
248
|
+
duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
|
249
|
+
duckdb_arrow_stream *out_stream) {
|
250
|
+
auto private_data = new arrow_array_stream_wrapper::PrivateData;
|
251
|
+
private_data->schema = reinterpret_cast<ArrowSchema *>(arrow_schema);
|
252
|
+
private_data->array = reinterpret_cast<ArrowArray *>(arrow_array);
|
253
|
+
private_data->done = false;
|
254
|
+
|
255
|
+
ArrowArrayStream *stream = new ArrowArrayStream;
|
256
|
+
*out_stream = reinterpret_cast<duckdb_arrow_stream>(stream);
|
257
|
+
stream->get_schema = arrow_array_stream_wrapper::GetSchema;
|
258
|
+
stream->get_next = arrow_array_stream_wrapper::GetNext;
|
259
|
+
stream->get_last_error = arrow_array_stream_wrapper::GetLastError;
|
260
|
+
stream->release = arrow_array_stream_wrapper::Release;
|
261
|
+
stream->private_data = private_data;
|
262
|
+
|
263
|
+
return duckdb_arrow_scan(connection, table_name, reinterpret_cast<duckdb_arrow_stream>(stream));
|
264
|
+
}
|
@@ -26,7 +26,7 @@ char *duckdb_get_varchar(duckdb_value value) {
|
|
26
26
|
auto str_val = val->DefaultCastAs(duckdb::LogicalType::VARCHAR);
|
27
27
|
auto &str = duckdb::StringValue::Get(str_val);
|
28
28
|
|
29
|
-
auto result = reinterpret_cast<char *>(malloc(sizeof(char
|
29
|
+
auto result = reinterpret_cast<char *>(malloc(sizeof(char) * (str.size() + 1)));
|
30
30
|
memcpy(result, str.c_str(), str.size());
|
31
31
|
result[str.size()] = '\0';
|
32
32
|
return result;
|
@@ -59,10 +59,12 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
|
|
59
59
|
DUCKDB_LOCAL(DebugForceExternal),
|
60
60
|
DUCKDB_LOCAL(DebugForceNoCrossProduct),
|
61
61
|
DUCKDB_LOCAL(DebugAsOfIEJoin),
|
62
|
+
DUCKDB_LOCAL(PreferRangeJoins),
|
62
63
|
DUCKDB_GLOBAL(DebugWindowMode),
|
63
64
|
DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
|
64
65
|
DUCKDB_GLOBAL(DefaultOrderSetting),
|
65
66
|
DUCKDB_GLOBAL(DefaultNullOrderSetting),
|
67
|
+
DUCKDB_GLOBAL(DisabledFileSystemsSetting),
|
66
68
|
DUCKDB_GLOBAL(DisabledOptimizersSetting),
|
67
69
|
DUCKDB_GLOBAL(EnableExternalAccessSetting),
|
68
70
|
DUCKDB_GLOBAL(EnableFSSTVectors),
|
@@ -92,6 +94,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
|
|
92
94
|
DUCKDB_LOCAL(OrderedAggregateThreshold),
|
93
95
|
DUCKDB_GLOBAL(PasswordSetting),
|
94
96
|
DUCKDB_LOCAL(PerfectHashThresholdSetting),
|
97
|
+
DUCKDB_LOCAL(PivotFilterThreshold),
|
95
98
|
DUCKDB_LOCAL(PivotLimitSetting),
|
96
99
|
DUCKDB_LOCAL(PreserveIdentifierCase),
|
97
100
|
DUCKDB_GLOBAL(PreserveInsertionOrder),
|
@@ -109,6 +112,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
|
|
109
112
|
DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
|
110
113
|
DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
|
111
114
|
DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),
|
115
|
+
DUCKDB_GLOBAL(FlushAllocatorSetting),
|
112
116
|
FINAL_SETTING};
|
113
117
|
|
114
118
|
vector<ConfigurationOption> DBConfig::GetOptions() {
|
@@ -178,7 +178,7 @@ void DatabaseInstance::CreateMainDatabase() {
|
|
178
178
|
void ThrowExtensionSetUnrecognizedOptions(const unordered_map<string, Value> &unrecognized_options) {
|
179
179
|
auto unrecognized_options_iter = unrecognized_options.begin();
|
180
180
|
string unrecognized_option_keys = unrecognized_options_iter->first;
|
181
|
-
|
181
|
+
while (++unrecognized_options_iter != unrecognized_options.end()) {
|
182
182
|
unrecognized_option_keys = "," + unrecognized_options_iter->first;
|
183
183
|
}
|
184
184
|
throw InvalidInputException("Unrecognized configuration property \"%s\"", unrecognized_option_keys);
|