duckdb 0.8.2-dev161.0 → 0.8.2-dev1724.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +4619 -4446
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/extra_type_info.cpp +506 -0
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +14 -14
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +44 -11
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/types/value.cpp +11 -6
- package/src/duckdb/src/common/types.cpp +9 -656
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/arrow.cpp +19 -0
- package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +13 -0
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +4 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
- package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +79 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +51 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +44 -31
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_common.cpp +2 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
- package/src/statement.cpp +10 -3
- package/test/columns.test.ts +24 -1
- package/test/test_all_types.test.ts +234 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -44,10 +44,23 @@ duckdb_adbc::AdbcStatusCode duckdb_adbc_init(size_t count, struct duckdb_adbc::A
|
|
44
44
|
driver->ConnectionRollback = duckdb_adbc::ConnectionRollback;
|
45
45
|
driver->ConnectionReadPartition = duckdb_adbc::ConnectionReadPartition;
|
46
46
|
driver->StatementExecutePartitions = duckdb_adbc::StatementExecutePartitions;
|
47
|
+
driver->ConnectionGetTableSchema = duckdb_adbc::ConnectionGetTableSchema;
|
48
|
+
driver->StatementSetSubstraitPlan = duckdb_adbc::StatementSetSubstraitPlan;
|
49
|
+
|
47
50
|
return ADBC_STATUS_OK;
|
48
51
|
}
|
49
52
|
|
50
53
|
namespace duckdb_adbc {
|
54
|
+
|
55
|
+
struct DuckDBAdbcStatementWrapper {
|
56
|
+
::duckdb_connection connection;
|
57
|
+
::duckdb_arrow result;
|
58
|
+
::duckdb_prepared_statement statement;
|
59
|
+
char *ingestion_table_name;
|
60
|
+
ArrowArrayStream *ingestion_stream;
|
61
|
+
};
|
62
|
+
static AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
|
63
|
+
struct AdbcError *error);
|
51
64
|
AdbcStatusCode SetErrorMaybe(const void *result, AdbcError *error, const std::string &error_message) {
|
52
65
|
if (!error) {
|
53
66
|
return ADBC_STATUS_INVALID_ARGUMENT;
|
@@ -109,6 +122,28 @@ AdbcStatusCode DatabaseNew(struct AdbcDatabase *database, struct AdbcError *erro
|
|
109
122
|
return CheckResult(res, error, "Failed to allocate");
|
110
123
|
}
|
111
124
|
|
125
|
+
AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *statement, const uint8_t *plan, size_t length,
|
126
|
+
struct AdbcError *error) {
|
127
|
+
if (!statement) {
|
128
|
+
SetError(error, "Statement is not set");
|
129
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
130
|
+
}
|
131
|
+
if (!plan) {
|
132
|
+
SetError(error, "Substrait Plan is not set");
|
133
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
134
|
+
}
|
135
|
+
if (length == 0) {
|
136
|
+
SetError(error, "Can't execute plan with size = 0");
|
137
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
138
|
+
}
|
139
|
+
auto wrapper = reinterpret_cast<DuckDBAdbcStatementWrapper *>(statement->private_data);
|
140
|
+
auto plan_str = std::string(reinterpret_cast<const char *>(plan), length);
|
141
|
+
auto query = "CALL from_substrait('" + plan_str + "'::BLOB)";
|
142
|
+
auto res = duckdb_prepare(wrapper->connection, query.c_str(), &wrapper->statement);
|
143
|
+
auto error_msg = duckdb_prepare_error(wrapper->statement);
|
144
|
+
return CheckResult(res, error, error_msg);
|
145
|
+
}
|
146
|
+
|
112
147
|
AdbcStatusCode DatabaseSetOption(struct AdbcDatabase *database, const char *key, const char *value,
|
113
148
|
struct AdbcError *error) {
|
114
149
|
auto status = SetErrorMaybe(database, error, "Missing database object");
|
@@ -159,6 +194,44 @@ AdbcStatusCode DatabaseRelease(struct AdbcDatabase *database, struct AdbcError *
|
|
159
194
|
return ADBC_STATUS_OK;
|
160
195
|
}
|
161
196
|
|
197
|
+
AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const char *catalog, const char *db_schema,
|
198
|
+
const char *table_name, struct ArrowSchema *schema, struct AdbcError *error) {
|
199
|
+
if (!connection) {
|
200
|
+
SetError(error, "Connection is not set");
|
201
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
202
|
+
}
|
203
|
+
if (catalog != nullptr && strlen(catalog) > 0) {
|
204
|
+
// In DuckDB this is the name of the database, not sure what's the expected functionality here, so for now,
|
205
|
+
// scream.
|
206
|
+
SetError(error, "Catalog Name is not used in DuckDB. It must be set to nullptr or an empty string");
|
207
|
+
return ADBC_STATUS_NOT_IMPLEMENTED;
|
208
|
+
} else if (db_schema == nullptr) {
|
209
|
+
SetError(error, "AdbcConnectionGetTableSchema: must provide db_schema");
|
210
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
211
|
+
} else if (table_name == nullptr) {
|
212
|
+
SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
|
213
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
214
|
+
} else if (strlen(table_name) == 0) {
|
215
|
+
SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
|
216
|
+
return ADBC_STATUS_INVALID_ARGUMENT;
|
217
|
+
}
|
218
|
+
ArrowArrayStream arrow_stream;
|
219
|
+
|
220
|
+
std::string query = "SELECT * FROM ";
|
221
|
+
if (strlen(db_schema) > 0) {
|
222
|
+
query += std::string(db_schema) + ".";
|
223
|
+
}
|
224
|
+
query += std::string(table_name) + " LIMIT 0;";
|
225
|
+
|
226
|
+
auto success = QueryInternal(connection, &arrow_stream, query.c_str(), error);
|
227
|
+
if (success != ADBC_STATUS_OK) {
|
228
|
+
return success;
|
229
|
+
}
|
230
|
+
arrow_stream.get_schema(&arrow_stream, schema);
|
231
|
+
arrow_stream.release(&arrow_stream);
|
232
|
+
return ADBC_STATUS_OK;
|
233
|
+
}
|
234
|
+
|
162
235
|
AdbcStatusCode ConnectionNew(struct AdbcConnection *connection, struct AdbcError *error) {
|
163
236
|
auto status = SetErrorMaybe(connection, error, "Missing connection object");
|
164
237
|
if (status != ADBC_STATUS_OK) {
|
@@ -400,14 +473,6 @@ AdbcStatusCode Ingest(duckdb_connection connection, const char *table_name, stru
|
|
400
473
|
return ADBC_STATUS_OK;
|
401
474
|
}
|
402
475
|
|
403
|
-
struct DuckDBAdbcStatementWrapper {
|
404
|
-
::duckdb_connection connection;
|
405
|
-
::duckdb_arrow result;
|
406
|
-
::duckdb_prepared_statement statement;
|
407
|
-
char *ingestion_table_name;
|
408
|
-
ArrowArrayStream *ingestion_stream;
|
409
|
-
};
|
410
|
-
|
411
476
|
AdbcStatusCode StatementNew(struct AdbcConnection *connection, struct AdbcStatement *statement,
|
412
477
|
struct AdbcError *error) {
|
413
478
|
|
@@ -581,8 +646,8 @@ AdbcStatusCode StatementSetOption(struct AdbcStatement *statement, const char *k
|
|
581
646
|
return ADBC_STATUS_INVALID_ARGUMENT;
|
582
647
|
}
|
583
648
|
|
584
|
-
|
585
|
-
|
649
|
+
AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArrayStream *out, const char *query,
|
650
|
+
struct AdbcError *error) {
|
586
651
|
AdbcStatement statement;
|
587
652
|
|
588
653
|
auto status = StatementNew(connection, &statement, error);
|
@@ -109,8 +109,9 @@ struct ManagerDriverState {
|
|
109
109
|
static AdbcStatusCode ReleaseDriver(struct AdbcDriver *driver, struct AdbcError *error) {
|
110
110
|
AdbcStatusCode status = ADBC_STATUS_OK;
|
111
111
|
|
112
|
-
if (!driver->private_manager)
|
112
|
+
if (!driver->private_manager) {
|
113
113
|
return status;
|
114
|
+
}
|
114
115
|
ManagerDriverState *state = reinterpret_cast<ManagerDriverState *>(driver->private_manager);
|
115
116
|
|
116
117
|
if (state->driver_release) {
|
@@ -138,11 +139,6 @@ AdbcStatusCode ConnectionGetInfo(struct AdbcConnection *connection, uint32_t *in
|
|
138
139
|
return ADBC_STATUS_NOT_IMPLEMENTED;
|
139
140
|
}
|
140
141
|
|
141
|
-
AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *, const char *, const char *, const char *,
|
142
|
-
struct ArrowSchema *, struct AdbcError *error) {
|
143
|
-
return ADBC_STATUS_NOT_IMPLEMENTED;
|
144
|
-
}
|
145
|
-
|
146
142
|
AdbcStatusCode StatementBind(struct AdbcStatement *, struct ArrowArray *, struct ArrowSchema *,
|
147
143
|
struct AdbcError *error) {
|
148
144
|
return ADBC_STATUS_NOT_IMPLEMENTED;
|
@@ -152,9 +148,6 @@ AdbcStatusCode StatementGetParameterSchema(struct AdbcStatement *statement, stru
|
|
152
148
|
struct AdbcError *error) {
|
153
149
|
return ADBC_STATUS_NOT_IMPLEMENTED;
|
154
150
|
}
|
155
|
-
AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *, const uint8_t *, size_t, struct AdbcError *error) {
|
156
|
-
return ADBC_STATUS_NOT_IMPLEMENTED;
|
157
|
-
}
|
158
151
|
|
159
152
|
/// Temporary state while the database is being configured.
|
160
153
|
struct TempDatabase {
|
@@ -351,15 +344,17 @@ AdbcStatusCode AdbcConnectionInit(struct AdbcConnection *connection, struct Adbc
|
|
351
344
|
delete args;
|
352
345
|
|
353
346
|
auto status = database->private_driver->ConnectionNew(connection, error);
|
354
|
-
if (status != ADBC_STATUS_OK)
|
347
|
+
if (status != ADBC_STATUS_OK) {
|
355
348
|
return status;
|
349
|
+
}
|
356
350
|
connection->private_driver = database->private_driver;
|
357
351
|
|
358
352
|
for (const auto &option : options) {
|
359
353
|
status = database->private_driver->ConnectionSetOption(connection, option.first.c_str(), option.second.c_str(),
|
360
354
|
error);
|
361
|
-
if (status != ADBC_STATUS_OK)
|
355
|
+
if (status != ADBC_STATUS_OK) {
|
362
356
|
return status;
|
357
|
+
}
|
363
358
|
}
|
364
359
|
return connection->private_driver->ConnectionInit(connection, database, error);
|
365
360
|
}
|
@@ -14,7 +14,13 @@
|
|
14
14
|
#include <execinfo.h>
|
15
15
|
#endif
|
16
16
|
|
17
|
-
#
|
17
|
+
#ifndef USE_JEMALLOC
|
18
|
+
#if defined(DUCKDB_EXTENSION_JEMALLOC_LINKED) && DUCKDB_EXTENSION_JEMALLOC_LINKED && !defined(WIN32)
|
19
|
+
#define USE_JEMALLOC
|
20
|
+
#endif
|
21
|
+
#endif
|
22
|
+
|
23
|
+
#ifdef USE_JEMALLOC
|
18
24
|
#include "jemalloc_extension.hpp"
|
19
25
|
#endif
|
20
26
|
|
@@ -89,7 +95,7 @@ PrivateAllocatorData::~PrivateAllocatorData() {
|
|
89
95
|
//===--------------------------------------------------------------------===//
|
90
96
|
// Allocator
|
91
97
|
//===--------------------------------------------------------------------===//
|
92
|
-
#
|
98
|
+
#ifdef USE_JEMALLOC
|
93
99
|
Allocator::Allocator()
|
94
100
|
: Allocator(JemallocExtension::Allocate, JemallocExtension::Free, JemallocExtension::Reallocate, nullptr) {
|
95
101
|
}
|
@@ -177,6 +183,12 @@ Allocator &Allocator::DefaultAllocator() {
|
|
177
183
|
return *DefaultAllocatorReference();
|
178
184
|
}
|
179
185
|
|
186
|
+
void Allocator::ThreadFlush(idx_t threshold) {
|
187
|
+
#ifdef USE_JEMALLOC
|
188
|
+
JemallocExtension::ThreadFlush(threshold);
|
189
|
+
#endif
|
190
|
+
}
|
191
|
+
|
180
192
|
//===--------------------------------------------------------------------===//
|
181
193
|
// Debug Info (extended)
|
182
194
|
//===--------------------------------------------------------------------===//
|
@@ -153,7 +153,9 @@ struct ArrowIntervalConverter {
|
|
153
153
|
template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
|
154
154
|
struct ArrowScalarBaseData {
|
155
155
|
static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
|
156
|
+
D_ASSERT(to >= from);
|
156
157
|
idx_t size = to - from;
|
158
|
+
D_ASSERT(size <= input_size);
|
157
159
|
UnifiedVectorFormat format;
|
158
160
|
input.ToUnifiedFormat(input_size, format);
|
159
161
|
|
@@ -392,6 +394,76 @@ struct ArrowVarcharData {
|
|
392
394
|
}
|
393
395
|
};
|
394
396
|
|
397
|
+
//===--------------------------------------------------------------------===//
|
398
|
+
// Unions
|
399
|
+
//===--------------------------------------------------------------------===//
|
400
|
+
/**
|
401
|
+
* Based on https://arrow.apache.org/docs/format/Columnar.html#union-layout &
|
402
|
+
* https://arrow.apache.org/docs/format/CDataInterface.html
|
403
|
+
*/
|
404
|
+
struct ArrowUnionData {
|
405
|
+
static void Initialize(ArrowAppendData &result, const LogicalType &type, idx_t capacity) {
|
406
|
+
result.main_buffer.reserve(capacity * sizeof(int8_t));
|
407
|
+
|
408
|
+
for (auto &child : UnionType::CopyMemberTypes(type)) {
|
409
|
+
auto child_buffer = InitializeArrowChild(child.second, capacity, result.options);
|
410
|
+
result.child_data.push_back(std::move(child_buffer));
|
411
|
+
}
|
412
|
+
}
|
413
|
+
|
414
|
+
static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
|
415
|
+
UnifiedVectorFormat format;
|
416
|
+
input.ToUnifiedFormat(input_size, format);
|
417
|
+
idx_t size = to - from;
|
418
|
+
|
419
|
+
auto &types_buffer = append_data.main_buffer;
|
420
|
+
|
421
|
+
duckdb::vector<Vector> child_vectors;
|
422
|
+
for (const auto &child : UnionType::CopyMemberTypes(input.GetType())) {
|
423
|
+
child_vectors.emplace_back(child.second);
|
424
|
+
}
|
425
|
+
|
426
|
+
for (idx_t input_idx = from; input_idx < to; input_idx++) {
|
427
|
+
const auto &val = input.GetValue(input_idx);
|
428
|
+
|
429
|
+
idx_t tag = 0;
|
430
|
+
Value resolved_value(nullptr);
|
431
|
+
if (!val.IsNull()) {
|
432
|
+
tag = UnionValue::GetTag(val);
|
433
|
+
|
434
|
+
resolved_value = UnionValue::GetValue(val);
|
435
|
+
}
|
436
|
+
|
437
|
+
for (idx_t child_idx = 0; child_idx < child_vectors.size(); child_idx++) {
|
438
|
+
child_vectors[child_idx].SetValue(input_idx, child_idx == tag ? resolved_value : Value(nullptr));
|
439
|
+
}
|
440
|
+
|
441
|
+
types_buffer.data()[input_idx] = tag;
|
442
|
+
}
|
443
|
+
|
444
|
+
for (idx_t child_idx = 0; child_idx < child_vectors.size(); child_idx++) {
|
445
|
+
auto &child_buffer = append_data.child_data[child_idx];
|
446
|
+
auto &child = child_vectors[child_idx];
|
447
|
+
child_buffer->append_vector(*child_buffer, child, from, to, size);
|
448
|
+
}
|
449
|
+
append_data.row_count += size;
|
450
|
+
}
|
451
|
+
|
452
|
+
static void Finalize(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result) {
|
453
|
+
result->n_buffers = 2;
|
454
|
+
result->buffers[1] = append_data.main_buffer.data();
|
455
|
+
|
456
|
+
auto &child_types = UnionType::CopyMemberTypes(type);
|
457
|
+
append_data.child_pointers.resize(child_types.size());
|
458
|
+
result->children = append_data.child_pointers.data();
|
459
|
+
result->n_children = child_types.size();
|
460
|
+
for (idx_t i = 0; i < child_types.size(); i++) {
|
461
|
+
auto &child_type = child_types[i].second;
|
462
|
+
append_data.child_pointers[i] = FinalizeArrowChild(child_type, *append_data.child_data[i]);
|
463
|
+
}
|
464
|
+
}
|
465
|
+
};
|
466
|
+
|
395
467
|
//===--------------------------------------------------------------------===//
|
396
468
|
// Structs
|
397
469
|
//===--------------------------------------------------------------------===//
|
@@ -489,16 +561,9 @@ struct ArrowListData {
|
|
489
561
|
SelectionVector child_sel(child_indices.data());
|
490
562
|
auto &child = ListVector::GetEntry(input);
|
491
563
|
auto child_size = child_indices.size();
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
child_copy.Slice(child, child_sel, child_size);
|
496
|
-
append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
|
497
|
-
} else {
|
498
|
-
// We don't care about the vector, slice it
|
499
|
-
child.Slice(child_sel, child_size);
|
500
|
-
append_data.child_data[0]->append_vector(*append_data.child_data[0], child, 0, child_size, child_size);
|
501
|
-
}
|
564
|
+
Vector child_copy(child.GetType());
|
565
|
+
child_copy.Slice(child, child_sel, child_size);
|
566
|
+
append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
|
502
567
|
append_data.row_count += size;
|
503
568
|
}
|
504
569
|
|
@@ -721,6 +786,9 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
|
|
721
786
|
case LogicalTypeId::INTERVAL:
|
722
787
|
InitializeFunctionPointers<ArrowScalarData<ArrowInterval, interval_t, ArrowIntervalConverter>>(append_data);
|
723
788
|
break;
|
789
|
+
case LogicalTypeId::UNION:
|
790
|
+
InitializeFunctionPointers<ArrowUnionData>(append_data);
|
791
|
+
break;
|
724
792
|
case LogicalTypeId::STRUCT:
|
725
793
|
InitializeFunctionPointers<ArrowStructData>(append_data);
|
726
794
|
break;
|
@@ -731,7 +799,7 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
|
|
731
799
|
InitializeFunctionPointers<ArrowMapData>(append_data);
|
732
800
|
break;
|
733
801
|
default:
|
734
|
-
throw
|
802
|
+
throw NotImplementedException("Unsupported type in DuckDB -> Arrow Conversion: %s\n", type.ToString());
|
735
803
|
}
|
736
804
|
}
|
737
805
|
|
@@ -791,7 +859,6 @@ ArrowArray ArrowAppender::Finalize() {
|
|
791
859
|
|
792
860
|
// Configure root array
|
793
861
|
result.length = row_count;
|
794
|
-
result.n_children = types.size();
|
795
862
|
result.n_buffers = 1;
|
796
863
|
result.buffers = root_holder->buffers.data(); // there is no actual buffer there since we don't have NULLs
|
797
864
|
result.offset = 0;
|
@@ -76,6 +76,15 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
|
|
76
76
|
SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
|
77
77
|
}
|
78
78
|
|
79
|
+
unsafe_unique_array<char> AddName(const string &name) {
|
80
|
+
auto name_ptr = make_unsafe_uniq_array<char>(name.size() + 1);
|
81
|
+
for (size_t i = 0; i < name.size(); i++) {
|
82
|
+
name_ptr[i] = name[i];
|
83
|
+
}
|
84
|
+
name_ptr[name.size()] = '\0';
|
85
|
+
return name_ptr;
|
86
|
+
}
|
87
|
+
|
79
88
|
void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
|
80
89
|
const ArrowOptions &options) {
|
81
90
|
switch (type.id()) {
|
@@ -135,12 +144,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
|
|
135
144
|
break;
|
136
145
|
case LogicalTypeId::TIMESTAMP_TZ: {
|
137
146
|
string format = "tsu:" + options.time_zone;
|
138
|
-
|
139
|
-
for (size_t i = 0; i < format.size(); i++) {
|
140
|
-
format_ptr[i] = format[i];
|
141
|
-
}
|
142
|
-
format_ptr[format.size()] = '\0';
|
143
|
-
root_holder.owned_type_names.push_back(std::move(format_ptr));
|
147
|
+
root_holder.owned_type_names.push_back(AddName(format));
|
144
148
|
child.format = root_holder.owned_type_names.back().get();
|
145
149
|
break;
|
146
150
|
}
|
@@ -160,12 +164,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
|
|
160
164
|
uint8_t width, scale;
|
161
165
|
type.GetDecimalProperties(width, scale);
|
162
166
|
string format = "d:" + to_string(width) + "," + to_string(scale);
|
163
|
-
|
164
|
-
for (size_t i = 0; i < format.size(); i++) {
|
165
|
-
format_ptr[i] = format[i];
|
166
|
-
}
|
167
|
-
format_ptr[format.size()] = '\0';
|
168
|
-
root_holder.owned_type_names.push_back(std::move(format_ptr));
|
167
|
+
root_holder.owned_type_names.push_back(AddName(format));
|
169
168
|
child.format = root_holder.owned_type_names.back().get();
|
170
169
|
break;
|
171
170
|
}
|
@@ -211,13 +210,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
|
|
211
210
|
|
212
211
|
InitializeChild(*child.children[type_idx]);
|
213
212
|
|
214
|
-
|
215
|
-
auto name_ptr = make_unsafe_uniq_array<char>(struct_col_name.size() + 1);
|
216
|
-
for (size_t i = 0; i < struct_col_name.size(); i++) {
|
217
|
-
name_ptr[i] = struct_col_name[i];
|
218
|
-
}
|
219
|
-
name_ptr[struct_col_name.size()] = '\0';
|
220
|
-
root_holder.owned_type_names.push_back(std::move(name_ptr));
|
213
|
+
root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
|
221
214
|
|
222
215
|
child.children[type_idx]->name = root_holder.owned_type_names.back().get();
|
223
216
|
SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, options);
|
@@ -228,6 +221,38 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
|
|
228
221
|
SetArrowMapFormat(root_holder, child, type, options);
|
229
222
|
break;
|
230
223
|
}
|
224
|
+
case LogicalTypeId::UNION: {
|
225
|
+
std::string format = "+us:";
|
226
|
+
|
227
|
+
auto &child_types = UnionType::CopyMemberTypes(type);
|
228
|
+
child.n_children = child_types.size();
|
229
|
+
root_holder.nested_children.emplace_back();
|
230
|
+
root_holder.nested_children.back().resize(child_types.size());
|
231
|
+
root_holder.nested_children_ptr.emplace_back();
|
232
|
+
root_holder.nested_children_ptr.back().resize(child_types.size());
|
233
|
+
for (idx_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
|
234
|
+
root_holder.nested_children_ptr.back()[type_idx] = &root_holder.nested_children.back()[type_idx];
|
235
|
+
}
|
236
|
+
child.children = &root_holder.nested_children_ptr.back()[0];
|
237
|
+
for (size_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
|
238
|
+
|
239
|
+
InitializeChild(*child.children[type_idx]);
|
240
|
+
|
241
|
+
root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
|
242
|
+
|
243
|
+
child.children[type_idx]->name = root_holder.owned_type_names.back().get();
|
244
|
+
SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, options);
|
245
|
+
|
246
|
+
format += to_string(type_idx) + ",";
|
247
|
+
}
|
248
|
+
|
249
|
+
format.pop_back();
|
250
|
+
|
251
|
+
root_holder.owned_type_names.push_back(AddName(format));
|
252
|
+
child.format = root_holder.owned_type_names.back().get();
|
253
|
+
|
254
|
+
break;
|
255
|
+
}
|
231
256
|
case LogicalTypeId::ENUM: {
|
232
257
|
// TODO what do we do with pointer enums here?
|
233
258
|
switch (EnumType::GetPhysicalType(type)) {
|