duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +15 -12
- package/binding.gyp.in +1 -1
- package/configure.py +1 -1
- package/duckdb_extension_config.cmake +10 -0
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-dateadd.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datefunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
- package/src/duckdb/extension/icu/icu-datetrunc.cpp +1 -1
- package/src/duckdb/extension/icu/icu-list-range.cpp +1 -1
- package/src/duckdb/extension/icu/icu-makedate.cpp +7 -0
- package/src/duckdb/extension/icu/icu-strptime.cpp +4 -4
- package/src/duckdb/extension/icu/icu-table-range.cpp +5 -5
- package/src/duckdb/extension/icu/icu-timebucket.cpp +16 -16
- package/src/duckdb/extension/icu/icu-timezone.cpp +8 -8
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +47 -231
- package/src/duckdb/extension/json/include/json_executors.hpp +49 -13
- package/src/duckdb/extension/json/include/json_functions.hpp +2 -1
- package/src/duckdb/extension/json/json_common.cpp +272 -40
- package/src/duckdb/extension/json/json_functions/json_create.cpp +21 -2
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +91 -38
- package/src/duckdb/extension/json/json_functions/json_type.cpp +1 -1
- package/src/duckdb/extension/json/json_functions.cpp +24 -24
- package/src/duckdb/extension/json/json_scan.cpp +3 -6
- package/src/duckdb/extension/parquet/column_reader.cpp +19 -21
- package/src/duckdb/extension/parquet/column_writer.cpp +77 -61
- package/src/duckdb/extension/parquet/include/cast_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/column_reader.hpp +14 -16
- package/src/duckdb/extension/parquet/include/column_writer.hpp +9 -7
- package/src/duckdb/extension/parquet/include/list_column_reader.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +3 -3
- package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_support.hpp +9 -11
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +24 -5
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -1
- package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +2 -3
- package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +2 -2
- package/src/duckdb/extension/parquet/parquet_extension.cpp +194 -20
- package/src/duckdb/extension/parquet/parquet_reader.cpp +6 -6
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +7 -6
- package/src/duckdb/extension/parquet/parquet_writer.cpp +79 -16
- package/src/duckdb/extension/parquet/zstd_file_system.cpp +2 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
- package/src/duckdb/src/catalog/catalog_search_path.cpp +5 -4
- package/src/duckdb/src/catalog/default/default_functions.cpp +16 -0
- package/src/duckdb/src/common/adbc/adbc.cpp +75 -10
- package/src/duckdb/src/common/adbc/driver_manager.cpp +6 -11
- package/src/duckdb/src/common/allocator.cpp +14 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +79 -12
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +44 -19
- package/src/duckdb/src/common/assert.cpp +3 -0
- package/src/duckdb/src/common/enum_util.cpp +4619 -4446
- package/src/duckdb/src/common/enums/logical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/enums/optimizer_type.cpp +2 -0
- package/src/duckdb/src/common/enums/physical_operator_type.cpp +4 -0
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/extra_type_info.cpp +506 -0
- package/src/duckdb/src/common/file_system.cpp +19 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +14 -14
- package/src/duckdb/src/common/multi_file_reader.cpp +184 -20
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/radix_partitioning.cpp +26 -8
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
- package/src/duckdb/src/common/sort/merge_sorter.cpp +9 -16
- package/src/duckdb/src/common/sort/partition_state.cpp +70 -50
- package/src/duckdb/src/common/types/batched_data_collection.cpp +7 -2
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/column/column_data_allocator.cpp +9 -6
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +17 -2
- package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +15 -6
- package/src/duckdb/src/common/types/column/partitioned_column_data.cpp +2 -2
- package/src/duckdb/src/common/types/data_chunk.cpp +2 -2
- package/src/duckdb/src/common/types/date.cpp +9 -0
- package/src/duckdb/src/common/types/list_segment.cpp +24 -74
- package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +3 -9
- package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +2 -0
- package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +2 -2
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/types/validity_mask.cpp +33 -0
- package/src/duckdb/src/common/types/value.cpp +11 -6
- package/src/duckdb/src/common/types.cpp +9 -656
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp +4 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp +5 -4
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp +8 -8
- package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp +4 -3
- package/src/duckdb/src/core_functions/function_list.cpp +4 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +208 -42
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -24
- package/src/duckdb/src/core_functions/scalar/date/make_date.cpp +19 -4
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +4 -2
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +34 -18
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/index/art/art.cpp +149 -139
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +1 -1
- package/src/duckdb/src/execution/index/art/iterator.cpp +129 -207
- package/src/duckdb/src/execution/index/art/leaf.cpp +8 -37
- package/src/duckdb/src/execution/index/art/node.cpp +113 -120
- package/src/duckdb/src/execution/index/art/node16.cpp +1 -10
- package/src/duckdb/src/execution/index/art/node256.cpp +1 -9
- package/src/duckdb/src/execution/index/art/node4.cpp +12 -13
- package/src/duckdb/src/execution/index/art/node48.cpp +1 -11
- package/src/duckdb/src/execution/index/art/prefix.cpp +228 -350
- package/src/duckdb/src/execution/join_hashtable.cpp +4 -4
- package/src/duckdb/src/execution/operator/aggregate/aggregate_object.cpp +1 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +8 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +32 -22
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +512 -300
- package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +4 -3
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -5
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +444 -284
- package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +21 -10
- package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +28 -12
- package/src/duckdb/src/execution/operator/join/physical_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +23 -4
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +41 -5
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +100 -13
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +20 -0
- package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +48 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +2 -3
- package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +6 -4
- package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +3 -3
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/projection/physical_pivot.cpp +2 -1
- package/src/duckdb/src/execution/operator/scan/physical_column_data_scan.cpp +19 -0
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/operator/set/physical_cte.cpp +160 -0
- package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp +15 -5
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +41 -6
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +30 -5
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +43 -10
- package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +56 -33
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +17 -13
- package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +33 -0
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_recursive_cte.cpp +25 -4
- package/src/duckdb/src/execution/physical_plan_generator.cpp +4 -0
- package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +290 -43
- package/src/duckdb/src/execution/window_segment_tree.cpp +286 -129
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +2 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +3 -1
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +5 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +212 -0
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +249 -0
- package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +29 -0
- package/src/duckdb/src/function/scalar/list/list_resize.cpp +162 -0
- package/src/duckdb/src/function/scalar/nested_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/string/like.cpp +12 -4
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +12 -5
- package/src/duckdb/src/function/table/arrow.cpp +19 -0
- package/src/duckdb/src/function/table/arrow_conversion.cpp +35 -1
- package/src/duckdb/src/function/table/copy_csv.cpp +8 -1
- package/src/duckdb/src/function/table/read_csv.cpp +100 -17
- package/src/duckdb/src/function/table/system/test_all_types.cpp +7 -0
- package/src/duckdb/src/function/table/system_functions.cpp +1 -0
- package/src/duckdb/src/function/table/table_scan.cpp +9 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +46 -2
- package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/bswap.hpp +42 -0
- package/src/duckdb/src/include/duckdb/common/dl.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +616 -584
- package/src/duckdb/src/include/duckdb/common/enums/cte_materialize.hpp +21 -0
- package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +219 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +6 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +10 -42
- package/src/duckdb/src/include/duckdb/common/mutex.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/radix.hpp +9 -20
- package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +6 -21
- package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +23 -8
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +11 -0
- package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +6 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp +3 -1
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/date.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +6 -8
- package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +5 -2
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -15
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp +4 -4
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/holistic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/nested_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_count.hpp +1 -0
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression/regr_slope.hpp +3 -3
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/regression_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/blob_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +31 -11
- package/src/duckdb/src/include/duckdb/core_functions/scalar/enum_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/generic_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/map_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/random_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/struct_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/union_functions.hpp +3 -1
- package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +21 -3
- package/src/duckdb/src/include/duckdb/execution/executor.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +4 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +31 -27
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -14
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +4 -10
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +5 -8
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +3 -6
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +63 -52
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +3 -10
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp +12 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +4 -3
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +8 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp +10 -0
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +62 -0
- package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp +8 -2
- package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +5 -1
- package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +10 -3
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +51 -40
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +1 -0
- package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +49 -0
- package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +5 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +2 -0
- package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/client_config.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/config.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/extension/generated_extension_loader.hpp +22 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +2 -0
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +39 -1
- package/src/duckdb/src/include/duckdb/optimizer/column_binding_replacer.hpp +47 -0
- package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp +132 -0
- package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +13 -16
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +10 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +13 -0
- package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +3 -0
- package/src/duckdb/src/include/duckdb/optimizer/remove_duplicate_groups.hpp +40 -0
- package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +11 -3
- package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +5 -0
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +54 -0
- package/src/duckdb/src/include/duckdb/parser/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +2 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
- package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +15 -8
- package/src/duckdb/src/include/duckdb/planner/binder.hpp +12 -5
- package/src/duckdb/src/include/duckdb/planner/bound_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +2 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +5 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +7 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +43 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +49 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +5 -4
- package/src/duckdb/src/include/duckdb/planner/query_node/bound_cte_node.hpp +44 -0
- package/src/duckdb/src/include/duckdb/planner/query_node/list.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
- package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +4 -1
- package/src/duckdb/src/include/duckdb/planner/subquery/recursive_dependent_join_planner.hpp +31 -0
- package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_correlated_expressions.hpp +8 -2
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +5 -2
- package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +22 -0
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +4 -0
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +1 -3
- package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -3
- package/src/duckdb/src/include/duckdb.h +28 -0
- package/src/duckdb/src/main/capi/arrow-c.cpp +155 -1
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +4 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_helper.cpp +93 -88
- package/src/duckdb/src/main/extension/extension_install.cpp +9 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +79 -18
- package/src/duckdb/src/optimizer/column_binding_replacer.cpp +43 -0
- package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
- package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +140 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +42 -0
- package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp +65 -0
- package/src/duckdb/src/optimizer/compressed_materialization.cpp +478 -0
- package/src/duckdb/src/optimizer/deliminator.cpp +176 -321
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -0
- package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +7 -0
- package/src/duckdb/src/optimizer/join_order/join_node.cpp +2 -2
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +113 -82
- package/src/duckdb/src/optimizer/join_order/join_relation_set.cpp +2 -6
- package/src/duckdb/src/optimizer/join_order/query_graph.cpp +22 -14
- package/src/duckdb/src/optimizer/optimizer.cpp +49 -14
- package/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +5 -5
- package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +0 -1
- package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +127 -0
- package/src/duckdb/src/optimizer/remove_unused_columns.cpp +4 -0
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +154 -15
- package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +65 -8
- package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp +1 -1
- package/src/duckdb/src/optimizer/statistics_propagator.cpp +7 -5
- package/src/duckdb/src/optimizer/topn_optimizer.cpp +20 -10
- package/src/duckdb/src/parallel/executor.cpp +15 -0
- package/src/duckdb/src/parallel/task_scheduler.cpp +11 -2
- package/src/duckdb/src/parser/common_table_expression_info.cpp +2 -0
- package/src/duckdb/src/parser/expression/between_expression.cpp +3 -15
- package/src/duckdb/src/parser/expression/case_expression.cpp +0 -13
- package/src/duckdb/src/parser/expression/cast_expression.cpp +3 -14
- package/src/duckdb/src/parser/expression/collate_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +3 -12
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +3 -13
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/constant_expression.cpp +3 -11
- package/src/duckdb/src/parser/expression/default_expression.cpp +0 -4
- package/src/duckdb/src/parser/expression/function_expression.cpp +3 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +4 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +0 -12
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +4 -11
- package/src/duckdb/src/parser/expression/star_expression.cpp +0 -19
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +0 -18
- package/src/duckdb/src/parser/expression/window_expression.cpp +3 -39
- package/src/duckdb/src/parser/parsed_expression.cpp +0 -70
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +7 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/parser/query_node/cte_node.cpp +58 -0
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +0 -19
- package/src/duckdb/src/parser/query_node/select_node.cpp +0 -29
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +0 -15
- package/src/duckdb/src/parser/query_node.cpp +15 -37
- package/src/duckdb/src/parser/result_modifier.cpp +0 -74
- package/src/duckdb/src/parser/tableref/basetableref.cpp +0 -19
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -4
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +0 -15
- package/src/duckdb/src/parser/tableref/joinref.cpp +3 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -23
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +3 -13
- package/src/duckdb/src/parser/tableref/table_function.cpp +0 -15
- package/src/duckdb/src/parser/tableref.cpp +0 -44
- package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +55 -3
- package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -0
- package/src/duckdb/src/parser/transform/expression/transform_multi_assign_reference.cpp +44 -0
- package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp +19 -1
- package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +13 -0
- package/src/duckdb/src/parser/transform/statement/transform_delete.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_insert.cpp +6 -1
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +7 -2
- package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +14 -11
- package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +11 -2
- package/src/duckdb/src/parser/transform/statement/transform_update.cpp +6 -1
- package/src/duckdb/src/parser/transformer.cpp +15 -0
- package/src/duckdb/src/planner/binder/query_node/bind_cte_node.cpp +64 -0
- package/src/duckdb/src/planner/binder/query_node/plan_cte_node.cpp +26 -0
- package/src/duckdb/src/planner/binder/query_node/plan_recursive_cte_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +4 -4
- package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +32 -29
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +11 -2
- package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +32 -5
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +116 -49
- package/src/duckdb/src/planner/binder/tableref/plan_cteref.cpp +1 -1
- package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +64 -26
- package/src/duckdb/src/planner/binder/tableref/plan_subqueryref.cpp +3 -3
- package/src/duckdb/src/planner/binder.cpp +44 -31
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +4 -31
- package/src/duckdb/src/planner/expression_binder.cpp +3 -0
- package/src/duckdb/src/planner/expression_iterator.cpp +6 -0
- package/src/duckdb/src/planner/logical_operator.cpp +5 -0
- package/src/duckdb/src/planner/logical_operator_visitor.cpp +2 -0
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +3 -1
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +26 -0
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +21 -0
- package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +90 -38
- package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +22 -7
- package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +65 -7
- package/src/duckdb/src/storage/arena_allocator.cpp +1 -2
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +3 -0
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/local_storage.cpp +3 -3
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +340 -0
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +122 -0
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +86 -0
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +166 -0
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +127 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +23 -0
- package/src/duckdb/src/storage/statistics/string_stats.cpp +21 -2
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +17 -0
- package/src/duckdb/src/storage/table/row_group.cpp +25 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +19 -18
- package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +2 -2
- package/src/duckdb/third_party/concurrentqueue/lightweightsemaphore.h +76 -0
- package/src/duckdb/third_party/fast_float/fast_float/fast_float.h +2 -0
- package/src/duckdb/third_party/httplib/httplib.hpp +10 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +9 -0
- package/src/duckdb/third_party/libpg_query/include/parser/gram.hpp +2 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12487 -12331
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/duckdb/ub_src_common.cpp +2 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/ub_src_execution_operator_persistent.cpp +2 -0
- package/src/duckdb/ub_src_execution_operator_set.cpp +2 -0
- package/src/duckdb/ub_src_execution_physical_plan.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar.cpp +2 -0
- package/src/duckdb/ub_src_function_scalar_compressed_materialization.cpp +4 -0
- package/src/duckdb/ub_src_function_scalar_list.cpp +2 -0
- package/src/duckdb/ub_src_optimizer.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_compressed_materialization.cpp +6 -0
- package/src/duckdb/ub_src_optimizer_statistics_expression.cpp +0 -2
- package/src/duckdb/ub_src_parser_query_node.cpp +2 -0
- package/src/duckdb/ub_src_parser_transform_expression.cpp +2 -0
- package/src/duckdb/ub_src_planner_binder_query_node.cpp +4 -0
- package/src/duckdb/ub_src_planner_operator.cpp +4 -0
- package/src/duckdb/ub_src_storage_serialization.cpp +10 -0
- package/src/statement.cpp +10 -3
- package/test/columns.test.ts +24 -1
- package/test/test_all_types.test.ts +234 -0
- package/tsconfig.json +1 -0
- package/src/duckdb/src/execution/index/art/prefix_segment.cpp +0 -42
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix_segment.hpp +0 -40
- package/src/duckdb/src/optimizer/statistics/expression/propagate_and_compress.cpp +0 -118
@@ -504,8 +504,7 @@ static void RegularLikeFunction(DataChunk &input, ExpressionState &state, Vector
|
|
504
504
|
}
|
505
505
|
void LikeFun::RegisterFunction(BuiltinFunctions &set) {
|
506
506
|
// like
|
507
|
-
set.AddFunction(
|
508
|
-
RegularLikeFunction<LikeOperator, false>, LikeBindFunction));
|
507
|
+
set.AddFunction(GetLikeFunction());
|
509
508
|
// not like
|
510
509
|
set.AddFunction(ScalarFunction("!~~", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::BOOLEAN,
|
511
510
|
RegularLikeFunction<NotLikeOperator, true>, LikeBindFunction));
|
@@ -522,9 +521,13 @@ void LikeFun::RegisterFunction(BuiltinFunctions &set) {
|
|
522
521
|
nullptr, ILikePropagateStats<NotILikeOperatorASCII>));
|
523
522
|
}
|
524
523
|
|
524
|
+
ScalarFunction LikeFun::GetLikeFunction() {
|
525
|
+
return ScalarFunction("~~", {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::BOOLEAN,
|
526
|
+
RegularLikeFunction<LikeOperator, false>, LikeBindFunction);
|
527
|
+
}
|
528
|
+
|
525
529
|
void LikeEscapeFun::RegisterFunction(BuiltinFunctions &set) {
|
526
|
-
set.AddFunction(
|
527
|
-
LogicalType::BOOLEAN, LikeEscapeFunction<LikeEscapeOperator>));
|
530
|
+
set.AddFunction(GetLikeEscapeFun());
|
528
531
|
set.AddFunction({"not_like_escape"},
|
529
532
|
ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR},
|
530
533
|
LogicalType::BOOLEAN, LikeEscapeFunction<NotLikeEscapeOperator>));
|
@@ -535,4 +538,9 @@ void LikeEscapeFun::RegisterFunction(BuiltinFunctions &set) {
|
|
535
538
|
ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR},
|
536
539
|
LogicalType::BOOLEAN, LikeEscapeFunction<NotILikeEscapeOperator>));
|
537
540
|
}
|
541
|
+
|
542
|
+
ScalarFunction LikeEscapeFun::GetLikeEscapeFun() {
|
543
|
+
return ScalarFunction("like_escape", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR},
|
544
|
+
LogicalType::BOOLEAN, LikeEscapeFunction<LikeEscapeOperator>);
|
545
|
+
}
|
538
546
|
} // namespace duckdb
|
@@ -1,11 +1,11 @@
|
|
1
1
|
#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
|
2
|
+
#include "duckdb/function/function_binder.hpp"
|
2
3
|
#include "duckdb/function/scalar/generic_functions.hpp"
|
3
4
|
#include "duckdb/main/client_context.hpp"
|
4
5
|
#include "duckdb/main/database.hpp"
|
5
6
|
#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
|
6
7
|
#include "duckdb/planner/expression/bound_constant_expression.hpp"
|
7
8
|
#include "duckdb/planner/expression/bound_function_expression.hpp"
|
8
|
-
#include "duckdb/function/function_binder.hpp"
|
9
9
|
|
10
10
|
namespace duckdb {
|
11
11
|
|
@@ -39,11 +39,14 @@ struct CombineState : public FunctionLocalState {
|
|
39
39
|
unsafe_unique_array<data_t> state_buffer0, state_buffer1;
|
40
40
|
Vector state_vector0, state_vector1;
|
41
41
|
|
42
|
+
ArenaAllocator allocator;
|
43
|
+
|
42
44
|
explicit CombineState(idx_t state_size_p)
|
43
45
|
: state_size(state_size_p), state_buffer0(make_unsafe_uniq_array<data_t>(state_size_p)),
|
44
46
|
state_buffer1(make_unsafe_uniq_array<data_t>(state_size_p)),
|
45
47
|
state_vector0(Value::POINTER(CastPointerToValue(state_buffer0.get()))),
|
46
|
-
state_vector1(Value::POINTER(CastPointerToValue(state_buffer1.get())))
|
48
|
+
state_vector1(Value::POINTER(CastPointerToValue(state_buffer1.get()))),
|
49
|
+
allocator(Allocator::DefaultAllocator()) {
|
47
50
|
}
|
48
51
|
};
|
49
52
|
|
@@ -58,10 +61,12 @@ struct FinalizeState : public FunctionLocalState {
|
|
58
61
|
unsafe_unique_array<data_t> state_buffer;
|
59
62
|
Vector addresses;
|
60
63
|
|
64
|
+
ArenaAllocator allocator;
|
65
|
+
|
61
66
|
explicit FinalizeState(idx_t state_size_p)
|
62
67
|
: state_size(state_size_p),
|
63
68
|
state_buffer(make_unsafe_uniq_array<data_t>(STANDARD_VECTOR_SIZE * AlignValue(state_size_p))),
|
64
|
-
addresses(LogicalType::POINTER) {
|
69
|
+
addresses(LogicalType::POINTER), allocator(Allocator::DefaultAllocator()) {
|
65
70
|
}
|
66
71
|
};
|
67
72
|
|
@@ -74,6 +79,7 @@ static unique_ptr<FunctionLocalState> InitFinalizeState(ExpressionState &state,
|
|
74
79
|
static void AggregateStateFinalize(DataChunk &input, ExpressionState &state_p, Vector &result) {
|
75
80
|
auto &bind_data = ExportAggregateBindData::GetFrom(state_p);
|
76
81
|
auto &local_state = ExecuteFunctionState::GetFunctionState(state_p)->Cast<FinalizeState>();
|
82
|
+
local_state.allocator.Reset();
|
77
83
|
|
78
84
|
D_ASSERT(bind_data.state_size == bind_data.aggr.state_size());
|
79
85
|
D_ASSERT(input.data.size() == 1);
|
@@ -100,7 +106,7 @@ static void AggregateStateFinalize(DataChunk &input, ExpressionState &state_p, V
|
|
100
106
|
state_vec_ptr[i] = data_ptr_cast(target_ptr);
|
101
107
|
}
|
102
108
|
|
103
|
-
AggregateInputData aggr_input_data(nullptr,
|
109
|
+
AggregateInputData aggr_input_data(nullptr, local_state.allocator);
|
104
110
|
bind_data.aggr.finalize(local_state.addresses, aggr_input_data, result, input.size(), 0);
|
105
111
|
|
106
112
|
for (idx_t i = 0; i < input.size(); i++) {
|
@@ -114,6 +120,7 @@ static void AggregateStateFinalize(DataChunk &input, ExpressionState &state_p, V
|
|
114
120
|
static void AggregateStateCombine(DataChunk &input, ExpressionState &state_p, Vector &result) {
|
115
121
|
auto &bind_data = ExportAggregateBindData::GetFrom(state_p);
|
116
122
|
auto &local_state = ExecuteFunctionState::GetFunctionState(state_p)->Cast<CombineState>();
|
123
|
+
local_state.allocator.Reset();
|
117
124
|
|
118
125
|
D_ASSERT(bind_data.state_size == bind_data.aggr.state_size());
|
119
126
|
|
@@ -164,7 +171,7 @@ static void AggregateStateCombine(DataChunk &input, ExpressionState &state_p, Ve
|
|
164
171
|
memcpy(local_state.state_buffer0.get(), state0.GetData(), bind_data.state_size);
|
165
172
|
memcpy(local_state.state_buffer1.get(), state1.GetData(), bind_data.state_size);
|
166
173
|
|
167
|
-
AggregateInputData aggr_input_data(nullptr,
|
174
|
+
AggregateInputData aggr_input_data(nullptr, local_state.allocator);
|
168
175
|
bind_data.aggr.combine(local_state.state_vector0, local_state.state_vector1, aggr_input_data, 1);
|
169
176
|
|
170
177
|
result_ptr[i] = StringVector::AddStringOrBlob(result, const_char_ptr_cast(local_state.state_buffer1.get()),
|
@@ -127,6 +127,25 @@ LogicalType ArrowTableFunction::GetArrowLogicalType(
|
|
127
127
|
}
|
128
128
|
return LogicalType::STRUCT(child_types);
|
129
129
|
|
130
|
+
} else if (format[0] == '+' && format[1] == 'u') {
|
131
|
+
if (format[2] != 's') {
|
132
|
+
throw NotImplementedException("Unsupported Internal Arrow Type: \"%c\" Union", format[2]);
|
133
|
+
}
|
134
|
+
D_ASSERT(format[3] == ':');
|
135
|
+
|
136
|
+
std::string prefix = "+us:";
|
137
|
+
// TODO: what are these type ids actually for?
|
138
|
+
auto type_ids = StringUtil::Split(format.substr(prefix.size()), ',');
|
139
|
+
|
140
|
+
child_list_t<LogicalType> members;
|
141
|
+
for (idx_t type_idx = 0; type_idx < (idx_t)schema.n_children; type_idx++) {
|
142
|
+
auto type = schema.children[type_idx];
|
143
|
+
|
144
|
+
members.emplace_back(type->name, GetArrowLogicalType(*type, arrow_convert_data, col_idx));
|
145
|
+
}
|
146
|
+
|
147
|
+
return LogicalType::UNION(members);
|
148
|
+
|
130
149
|
} else if (format == "+m") {
|
131
150
|
convert_data.variable_sz_type.emplace_back(ArrowVariableSizeType::NORMAL, 0);
|
132
151
|
|
@@ -644,8 +644,42 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
|
|
644
644
|
}
|
645
645
|
break;
|
646
646
|
}
|
647
|
+
case LogicalTypeId::UNION: {
|
648
|
+
auto type_ids = ArrowBufferData<int8_t>(array, array.n_buffers == 1 ? 0 : 1);
|
649
|
+
D_ASSERT(type_ids);
|
650
|
+
auto members = UnionType::CopyMemberTypes(vector.GetType());
|
651
|
+
|
652
|
+
auto &validity_mask = FlatVector::Validity(vector);
|
653
|
+
|
654
|
+
duckdb::vector<Vector> children;
|
655
|
+
for (idx_t type_idx = 0; type_idx < (::idx_t)array.n_children; type_idx++) {
|
656
|
+
Vector child(members[type_idx].second);
|
657
|
+
auto arrow_array = array.children[type_idx];
|
658
|
+
|
659
|
+
SetValidityMask(child, *arrow_array, scan_state, size, nested_offset);
|
660
|
+
|
661
|
+
ColumnArrowToDuckDB(child, *arrow_array, scan_state, size, arrow_convert_data, col_idx, arrow_convert_idx,
|
662
|
+
nested_offset, &validity_mask);
|
663
|
+
|
664
|
+
children.push_back(std::move(child));
|
665
|
+
}
|
666
|
+
|
667
|
+
for (idx_t row_idx = 0; row_idx < size; row_idx++) {
|
668
|
+
auto tag = type_ids[row_idx];
|
669
|
+
|
670
|
+
auto out_of_range = tag < 0 || tag >= array.n_children;
|
671
|
+
if (out_of_range) {
|
672
|
+
throw InvalidInputException("Arrow union tag out of range: %d", tag);
|
673
|
+
}
|
674
|
+
|
675
|
+
const Value &value = children[tag].GetValue(row_idx);
|
676
|
+
vector.SetValue(row_idx, value.IsNull() ? Value() : Value::UNION(members, tag, value));
|
677
|
+
}
|
678
|
+
|
679
|
+
break;
|
680
|
+
}
|
647
681
|
default:
|
648
|
-
throw NotImplementedException("Unsupported type %s", vector.GetType().ToString());
|
682
|
+
throw NotImplementedException("Unsupported type for arrow conversion: %s", vector.GetType().ToString());
|
649
683
|
}
|
650
684
|
}
|
651
685
|
|
@@ -116,13 +116,20 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, CopyInfo &in
|
|
116
116
|
for (auto &option : info.options) {
|
117
117
|
auto loption = StringUtil::Lower(option.first);
|
118
118
|
auto &set = option.second;
|
119
|
-
options.SetReadOption(loption, ConvertVectorToValue(
|
119
|
+
options.SetReadOption(loption, ConvertVectorToValue(set), expected_names);
|
120
120
|
}
|
121
121
|
// verify the parsed options
|
122
122
|
if (options.force_not_null.empty()) {
|
123
123
|
// no FORCE_QUOTE specified: initialize to false
|
124
124
|
options.force_not_null.resize(expected_types.size(), false);
|
125
125
|
}
|
126
|
+
|
127
|
+
// Look for rejects table options last
|
128
|
+
named_parameter_map_t options_map;
|
129
|
+
for (auto &option : info.options) {
|
130
|
+
options_map[option.first] = ConvertVectorToValue(std::move(option.second));
|
131
|
+
}
|
132
|
+
|
126
133
|
bind_data->FinalizeRead(context);
|
127
134
|
if (!bind_data->single_threaded && options.auto_detect) {
|
128
135
|
options.file_path = bind_data->files[0];
|
@@ -14,6 +14,8 @@
|
|
14
14
|
#include "duckdb/common/multi_file_reader.hpp"
|
15
15
|
#include "duckdb/main/client_data.hpp"
|
16
16
|
#include "duckdb/execution/operator/persistent/csv_line_info.hpp"
|
17
|
+
#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
|
18
|
+
|
17
19
|
#include <limits>
|
18
20
|
|
19
21
|
namespace duckdb {
|
@@ -42,6 +44,43 @@ void ReadCSVData::FinalizeRead(ClientContext &context) {
|
|
42
44
|
// not supported for parallel CSV reading
|
43
45
|
single_threaded = true;
|
44
46
|
}
|
47
|
+
|
48
|
+
// Validate rejects_table options
|
49
|
+
if (!options.rejects_table_name.empty()) {
|
50
|
+
if (!options.ignore_errors) {
|
51
|
+
throw BinderException("REJECTS_TABLE option is only supported when IGNORE_ERRORS is set to true");
|
52
|
+
}
|
53
|
+
if (options.file_options.union_by_name) {
|
54
|
+
throw BinderException("REJECTS_TABLE option is not supported when UNION_BY_NAME is set to true");
|
55
|
+
}
|
56
|
+
}
|
57
|
+
|
58
|
+
if (!options.rejects_recovery_columns.empty()) {
|
59
|
+
if (options.rejects_table_name.empty()) {
|
60
|
+
throw BinderException(
|
61
|
+
"REJECTS_RECOVERY_COLUMNS option is only supported when REJECTS_TABLE is set to a table name");
|
62
|
+
}
|
63
|
+
for (auto &recovery_col : options.rejects_recovery_columns) {
|
64
|
+
bool found = false;
|
65
|
+
for (idx_t col_idx = 0; col_idx < return_names.size(); col_idx++) {
|
66
|
+
if (StringUtil::CIEquals(return_names[col_idx], recovery_col)) {
|
67
|
+
options.rejects_recovery_column_ids.push_back(col_idx);
|
68
|
+
found = true;
|
69
|
+
break;
|
70
|
+
}
|
71
|
+
}
|
72
|
+
if (!found) {
|
73
|
+
throw BinderException("Unsupported parameter for REJECTS_RECOVERY_COLUMNS: column \"%s\" not found",
|
74
|
+
recovery_col);
|
75
|
+
}
|
76
|
+
}
|
77
|
+
}
|
78
|
+
|
79
|
+
if (options.rejects_limit != 0) {
|
80
|
+
if (options.rejects_table_name.empty()) {
|
81
|
+
throw BinderException("REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name");
|
82
|
+
}
|
83
|
+
}
|
45
84
|
}
|
46
85
|
|
47
86
|
uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
|
@@ -65,13 +104,14 @@ uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
|
|
65
104
|
|
66
105
|
static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctionBindInput &input,
|
67
106
|
vector<LogicalType> &return_types, vector<string> &names) {
|
107
|
+
|
68
108
|
auto result = make_uniq<ReadCSVData>();
|
69
109
|
auto &options = result->options;
|
70
110
|
result->files = MultiFileReader::GetFileList(context, input.inputs[0], "CSV");
|
71
111
|
|
72
112
|
bool explicitly_set_columns = false;
|
73
113
|
for (auto &kv : input.named_parameters) {
|
74
|
-
if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options)) {
|
114
|
+
if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options, context)) {
|
75
115
|
continue;
|
76
116
|
}
|
77
117
|
auto loption = StringUtil::Lower(kv.first);
|
@@ -180,9 +220,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
|
|
180
220
|
options.SetReadOption(loption, kv.second, names);
|
181
221
|
}
|
182
222
|
}
|
183
|
-
|
184
|
-
options.file_options.hive_partitioning = MultiFileReaderOptions::AutoDetectHivePartitioning(result->files);
|
185
|
-
}
|
223
|
+
options.file_options.AutoDetectHivePartitioning(result->files, context);
|
186
224
|
|
187
225
|
if (!options.auto_detect && return_types.empty()) {
|
188
226
|
throw BinderException("read_csv requires columns to be specified through the 'columns' option. Use "
|
@@ -235,9 +273,15 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
|
|
235
273
|
} else {
|
236
274
|
result->reader_bind = MultiFileReader::BindOptions(options.file_options, result->files, return_types, names);
|
237
275
|
}
|
276
|
+
|
238
277
|
result->return_types = return_types;
|
239
278
|
result->return_names = names;
|
240
279
|
result->FinalizeRead(context);
|
280
|
+
|
281
|
+
if (options.auto_detect) {
|
282
|
+
result->initial_reader->options = options;
|
283
|
+
}
|
284
|
+
|
241
285
|
return std::move(result);
|
242
286
|
}
|
243
287
|
|
@@ -261,10 +305,6 @@ public:
|
|
261
305
|
line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
|
262
306
|
file_handle->DisableReset();
|
263
307
|
current_file_path = files_path_p[0];
|
264
|
-
line_info.lines_read[0] = rows_to_skip;
|
265
|
-
if (has_header) {
|
266
|
-
line_info.lines_read[0]++;
|
267
|
-
}
|
268
308
|
file_size = file_handle->FileSize();
|
269
309
|
first_file_size = file_size;
|
270
310
|
on_disk_file = file_handle->OnDiskFile();
|
@@ -291,10 +331,17 @@ public:
|
|
291
331
|
// Initialize all the book-keeping variables
|
292
332
|
auto file_count = files_path_p.size();
|
293
333
|
line_info.current_batches.resize(file_count);
|
334
|
+
line_info.lines_read.resize(file_count);
|
294
335
|
tuple_start.resize(file_count);
|
295
336
|
tuple_end.resize(file_count);
|
296
337
|
tuple_end_to_batch.resize(file_count);
|
297
338
|
batch_to_tuple_end.resize(file_count);
|
339
|
+
|
340
|
+
// Initialize the lines read
|
341
|
+
line_info.lines_read[0][0] = rows_to_skip;
|
342
|
+
if (has_header) {
|
343
|
+
line_info.lines_read[0][0]++;
|
344
|
+
}
|
298
345
|
}
|
299
346
|
ParallelCSVGlobalState() : line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
|
300
347
|
running_threads = MaxThreads();
|
@@ -501,6 +548,9 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
|
|
501
548
|
current_csv_position = 0;
|
502
549
|
file_number++;
|
503
550
|
local_batch_index = 0;
|
551
|
+
|
552
|
+
line_info.lines_read[file_number][local_batch_index] = (bind_data.options.has_header ? 1 : 0);
|
553
|
+
|
504
554
|
current_buffer =
|
505
555
|
make_shared<CSVBuffer>(context, buffer_size, *file_handle, current_csv_position, file_number);
|
506
556
|
next_buffer = shared_ptr<CSVBuffer>(
|
@@ -512,7 +562,7 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
|
|
512
562
|
}
|
513
563
|
}
|
514
564
|
// set up the current buffer
|
515
|
-
line_info.current_batches.
|
565
|
+
line_info.current_batches[file_number].insert(local_batch_index);
|
516
566
|
auto result = make_uniq<CSVBufferRead>(current_buffer, next_buffer, next_byte, next_byte + bytes_per_local_state,
|
517
567
|
batch_index++, local_batch_index++, &line_info);
|
518
568
|
// move the byte index of the CSV reader to the next buffer
|
@@ -551,11 +601,12 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
|
|
551
601
|
reader->options.file_path = current_file_path;
|
552
602
|
MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
|
553
603
|
bind_data.return_types, bind_data.return_names, column_ids, nullptr,
|
554
|
-
bind_data.files.front());
|
604
|
+
bind_data.files.front(), context);
|
555
605
|
} else {
|
556
606
|
// update the current reader
|
557
607
|
reader->SetBufferRead(std::move(result));
|
558
608
|
}
|
609
|
+
|
559
610
|
return true;
|
560
611
|
}
|
561
612
|
void ParallelCSVGlobalState::UpdateVerification(VerificationPositions positions, idx_t file_number_p, idx_t batch_idx) {
|
@@ -574,7 +625,7 @@ void ParallelCSVGlobalState::UpdateLinesRead(CSVBufferRead &buffer_read, idx_t f
|
|
574
625
|
auto lines_read = buffer_read.lines_read;
|
575
626
|
lock_guard<mutex> parallel_lock(main_mutex);
|
576
627
|
line_info.current_batches[file_idx].erase(batch_idx);
|
577
|
-
line_info.lines_read[batch_idx] += lines_read;
|
628
|
+
line_info.lines_read[file_idx][batch_idx] += lines_read;
|
578
629
|
}
|
579
630
|
|
580
631
|
bool LineInfo::CanItGetLine(idx_t file_idx, idx_t batch_idx) {
|
@@ -593,21 +644,36 @@ bool LineInfo::CanItGetLine(idx_t file_idx, idx_t batch_idx) {
|
|
593
644
|
}
|
594
645
|
|
595
646
|
// Returns the 1-indexed line number
|
596
|
-
idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t cur_start, bool verify
|
647
|
+
idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t cur_start, bool verify,
|
648
|
+
bool stop_at_first) {
|
597
649
|
unique_ptr<lock_guard<mutex>> parallel_lock;
|
598
650
|
if (!verify) {
|
599
651
|
parallel_lock = duckdb::make_uniq<lock_guard<mutex>>(main_mutex);
|
600
652
|
}
|
601
653
|
idx_t line_count = 0;
|
654
|
+
|
655
|
+
if (!stop_at_first) {
|
656
|
+
// Figure out the amount of lines read in the current file
|
657
|
+
auto &file_batches = current_batches[file_idx];
|
658
|
+
for (auto &batch : file_batches) {
|
659
|
+
if (batch > batch_idx) {
|
660
|
+
break;
|
661
|
+
}
|
662
|
+
line_count += lines_read[file_idx][batch];
|
663
|
+
}
|
664
|
+
return line_count + line_error + 1;
|
665
|
+
}
|
666
|
+
|
667
|
+
// Otherwise, check if we already have an error on another thread
|
602
668
|
if (done) {
|
603
669
|
// line count is 0-indexed, but we want to return 1-indexed
|
604
670
|
return first_line + 1;
|
605
671
|
}
|
606
672
|
for (idx_t i = 0; i <= batch_idx; i++) {
|
607
|
-
if (lines_read.find(i) == lines_read.end() && i != batch_idx) {
|
673
|
+
if (lines_read[file_idx].find(i) == lines_read[file_idx].end() && i != batch_idx) {
|
608
674
|
throw InternalException("Missing batch index on Parallel CSV Reader GetLine");
|
609
675
|
}
|
610
|
-
line_count += lines_read[i];
|
676
|
+
line_count += lines_read[file_idx][i];
|
611
677
|
}
|
612
678
|
|
613
679
|
// before we are done, if this is not a call in Verify() we must check Verify up to this batch
|
@@ -782,7 +848,7 @@ private:
|
|
782
848
|
}
|
783
849
|
MultiFileReader::InitializeReader(*result, bind_data.options.file_options, bind_data.reader_bind,
|
784
850
|
bind_data.return_types, bind_data.return_names, column_ids, nullptr,
|
785
|
-
bind_data.files.front());
|
851
|
+
bind_data.files.front(), context);
|
786
852
|
}
|
787
853
|
total_size = result->file_handle->FileSize();
|
788
854
|
return result;
|
@@ -831,14 +897,14 @@ static unique_ptr<GlobalTableFunctionState> SingleThreadedCSVInit(ClientContext
|
|
831
897
|
}
|
832
898
|
MultiFileReader::InitializeReader(*result->initial_reader, bind_data.options.file_options, bind_data.reader_bind,
|
833
899
|
bind_data.return_types, bind_data.return_names, input.column_ids, input.filters,
|
834
|
-
bind_data.files.front());
|
900
|
+
bind_data.files.front(), context);
|
835
901
|
for (auto &reader : bind_data.union_readers) {
|
836
902
|
if (!reader) {
|
837
903
|
continue;
|
838
904
|
}
|
839
905
|
MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
|
840
906
|
bind_data.return_types, bind_data.return_names, input.column_ids,
|
841
|
-
input.filters, bind_data.files.front());
|
907
|
+
input.filters, bind_data.files.front(), context);
|
842
908
|
}
|
843
909
|
result->column_ids = input.column_ids;
|
844
910
|
|
@@ -914,6 +980,12 @@ static void SingleThreadedCSVFunction(ClientContext &context, TableFunctionInput
|
|
914
980
|
//===--------------------------------------------------------------------===//
|
915
981
|
static unique_ptr<GlobalTableFunctionState> ReadCSVInitGlobal(ClientContext &context, TableFunctionInitInput &input) {
|
916
982
|
auto &bind_data = input.bind_data->Cast<ReadCSVData>();
|
983
|
+
|
984
|
+
// Create the temporary rejects table
|
985
|
+
auto rejects_table = bind_data.options.rejects_table_name;
|
986
|
+
if (!rejects_table.empty()) {
|
987
|
+
CSVRejectsTable::GetOrCreate(context, rejects_table)->InitializeTable(context, bind_data);
|
988
|
+
}
|
917
989
|
if (bind_data.single_threaded) {
|
918
990
|
return SingleThreadedCSVInit(context, input);
|
919
991
|
} else {
|
@@ -974,6 +1046,9 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
|
|
974
1046
|
table_function.named_parameters["max_line_size"] = LogicalType::VARCHAR;
|
975
1047
|
table_function.named_parameters["maximum_line_size"] = LogicalType::VARCHAR;
|
976
1048
|
table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN;
|
1049
|
+
table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR;
|
1050
|
+
table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT;
|
1051
|
+
table_function.named_parameters["rejects_recovery_columns"] = LogicalType::LIST(LogicalType::VARCHAR);
|
977
1052
|
table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT;
|
978
1053
|
table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR;
|
979
1054
|
table_function.named_parameters["parallel"] = LogicalType::BOOLEAN;
|
@@ -1062,6 +1137,10 @@ void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
|
|
1062
1137
|
csv_formats.push_back(format.second.format_specifier);
|
1063
1138
|
}
|
1064
1139
|
writer.WriteList<string>(csv_formats);
|
1140
|
+
writer.WriteString(rejects_table_name);
|
1141
|
+
writer.WriteField<idx_t>(rejects_limit);
|
1142
|
+
writer.WriteList<string>(rejects_recovery_columns);
|
1143
|
+
writer.WriteList<idx_t>(rejects_recovery_column_ids);
|
1065
1144
|
}
|
1066
1145
|
|
1067
1146
|
void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
|
@@ -1109,6 +1188,10 @@ void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
|
|
1109
1188
|
has_format[type] = true;
|
1110
1189
|
StrTimeFormat::ParseFormatSpecifier(format, date_format[type]);
|
1111
1190
|
}
|
1191
|
+
rejects_table_name = reader.ReadRequired<string>();
|
1192
|
+
rejects_limit = reader.ReadRequired<idx_t>();
|
1193
|
+
rejects_recovery_columns = reader.ReadRequiredList<string>();
|
1194
|
+
rejects_recovery_column_ids = reader.ReadRequiredList<idx_t>();
|
1112
1195
|
}
|
1113
1196
|
|
1114
1197
|
static void CSVReaderSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {
|
@@ -204,6 +204,13 @@ vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
|
|
204
204
|
auto max_map_value = Value::MAP(ListType::GetChildType(map_type), map_values);
|
205
205
|
result.emplace_back(map_type, "map", std::move(min_map_value), std::move(max_map_value));
|
206
206
|
|
207
|
+
// union
|
208
|
+
child_list_t<LogicalType> members = {{"name", LogicalType::VARCHAR}, {"age", LogicalType::SMALLINT}};
|
209
|
+
auto union_type = LogicalType::UNION(members);
|
210
|
+
const Value &min = Value::UNION(members, 0, Value("Frank"));
|
211
|
+
const Value &max = Value::UNION(members, 1, Value::SMALLINT(5));
|
212
|
+
result.emplace_back(union_type, "union", min, max);
|
213
|
+
|
207
214
|
return result;
|
208
215
|
}
|
209
216
|
|
@@ -10,6 +10,7 @@ namespace duckdb {
|
|
10
10
|
|
11
11
|
void BuiltinFunctions::RegisterSQLiteFunctions() {
|
12
12
|
PragmaVersion::RegisterFunction(*this);
|
13
|
+
PragmaPlatform::RegisterFunction(*this);
|
13
14
|
PragmaCollations::RegisterFunction(*this);
|
14
15
|
PragmaTableInfo::RegisterFunction(*this);
|
15
16
|
PragmaStorageInfo::RegisterFunction(*this);
|
@@ -281,6 +281,15 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
|
|
281
281
|
if (bind_data.is_index_scan) {
|
282
282
|
return;
|
283
283
|
}
|
284
|
+
if (!get.table_filters.filters.empty()) {
|
285
|
+
// if there were filters before we can't convert this to an index scan
|
286
|
+
return;
|
287
|
+
}
|
288
|
+
if (!get.projection_ids.empty()) {
|
289
|
+
// if columns were pruned by RemoveUnusedColumns we can't convert this to an index scan,
|
290
|
+
// because index scan does not support filter_prune (yet)
|
291
|
+
return;
|
292
|
+
}
|
284
293
|
if (filters.empty()) {
|
285
294
|
// no indexes or no filters: skip the pushdown
|
286
295
|
return;
|
@@ -1,11 +1,12 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.8.2-
|
2
|
+
#define DUCKDB_VERSION "0.8.2-dev1764"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "07b0b0a2a4"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|
9
|
+
#include "duckdb/common/string_util.hpp"
|
9
10
|
|
10
11
|
#include <cstdint>
|
11
12
|
|
@@ -63,6 +64,13 @@ const char *DuckDB::LibraryVersion() {
|
|
63
64
|
}
|
64
65
|
|
65
66
|
string DuckDB::Platform() {
|
67
|
+
#if defined(DUCKDB_CUSTOM_PLATFORM)
|
68
|
+
return DUCKDB_QUOTE_DEFINE(DUCKDB_CUSTOM_PLATFORM);
|
69
|
+
#endif
|
70
|
+
#if defined(DUCKDB_WASM_VERSION)
|
71
|
+
// DuckDB-Wasm requires CUSTOM_PLATFORM to be defined
|
72
|
+
static_assert(0, "DUCKDB_WASM_VERSION should rely on CUSTOM_PLATFORM being provided");
|
73
|
+
#endif
|
66
74
|
string os = "linux";
|
67
75
|
#if INTPTR_MAX == INT64_MAX
|
68
76
|
string arch = "amd64";
|
@@ -97,4 +105,40 @@ string DuckDB::Platform() {
|
|
97
105
|
return os + "_" + arch + postfix;
|
98
106
|
}
|
99
107
|
|
108
|
+
struct PragmaPlatformData : public GlobalTableFunctionState {
|
109
|
+
PragmaPlatformData() : finished(false) {
|
110
|
+
}
|
111
|
+
|
112
|
+
bool finished;
|
113
|
+
};
|
114
|
+
|
115
|
+
static unique_ptr<FunctionData> PragmaPlatformBind(ClientContext &context, TableFunctionBindInput &input,
|
116
|
+
vector<LogicalType> &return_types, vector<string> &names) {
|
117
|
+
names.emplace_back("platform");
|
118
|
+
return_types.emplace_back(LogicalType::VARCHAR);
|
119
|
+
return nullptr;
|
120
|
+
}
|
121
|
+
|
122
|
+
static unique_ptr<GlobalTableFunctionState> PragmaPlatformInit(ClientContext &context, TableFunctionInitInput &input) {
|
123
|
+
return make_uniq<PragmaPlatformData>();
|
124
|
+
}
|
125
|
+
|
126
|
+
static void PragmaPlatformFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
|
127
|
+
auto &data = data_p.global_state->Cast<PragmaPlatformData>();
|
128
|
+
if (data.finished) {
|
129
|
+
// finished returning values
|
130
|
+
return;
|
131
|
+
}
|
132
|
+
output.SetCardinality(1);
|
133
|
+
output.SetValue(0, 0, DuckDB::Platform());
|
134
|
+
data.finished = true;
|
135
|
+
}
|
136
|
+
|
137
|
+
void PragmaPlatform::RegisterFunction(BuiltinFunctions &set) {
|
138
|
+
TableFunction pragma_platform("pragma_platform", {}, PragmaPlatformFunction);
|
139
|
+
pragma_platform.bind = PragmaPlatformBind;
|
140
|
+
pragma_platform.init_global = PragmaPlatformInit;
|
141
|
+
set.AddFunction(pragma_platform);
|
142
|
+
}
|
143
|
+
|
100
144
|
} // namespace duckdb
|
@@ -116,6 +116,8 @@ public:
|
|
116
116
|
DUCKDB_API static Allocator &DefaultAllocator();
|
117
117
|
DUCKDB_API static shared_ptr<Allocator> &DefaultAllocatorReference();
|
118
118
|
|
119
|
+
static void ThreadFlush(idx_t threshold);
|
120
|
+
|
119
121
|
private:
|
120
122
|
allocate_function_ptr_t allocate_function;
|
121
123
|
free_function_ptr_t free_function;
|
@@ -0,0 +1,42 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/common/bswap.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/common/common.hpp"
|
12
|
+
|
13
|
+
namespace duckdb {
|
14
|
+
|
15
|
+
#define BSWAP16(x) ((uint16_t)((((uint16_t)(x)&0xff00) >> 8) | (((uint16_t)(x)&0x00ff) << 8)))
|
16
|
+
|
17
|
+
#define BSWAP32(x) \
|
18
|
+
((uint32_t)((((uint32_t)(x)&0xff000000) >> 24) | (((uint32_t)(x)&0x00ff0000) >> 8) | \
|
19
|
+
(((uint32_t)(x)&0x0000ff00) << 8) | (((uint32_t)(x)&0x000000ff) << 24)))
|
20
|
+
|
21
|
+
#define BSWAP64(x) \
|
22
|
+
((uint64_t)((((uint64_t)(x)&0xff00000000000000ull) >> 56) | (((uint64_t)(x)&0x00ff000000000000ull) >> 40) | \
|
23
|
+
(((uint64_t)(x)&0x0000ff0000000000ull) >> 24) | (((uint64_t)(x)&0x000000ff00000000ull) >> 8) | \
|
24
|
+
(((uint64_t)(x)&0x00000000ff000000ull) << 8) | (((uint64_t)(x)&0x0000000000ff0000ull) << 24) | \
|
25
|
+
(((uint64_t)(x)&0x000000000000ff00ull) << 40) | (((uint64_t)(x)&0x00000000000000ffull) << 56)))
|
26
|
+
|
27
|
+
template <class T>
|
28
|
+
static inline T BSwap(const T &x) {
|
29
|
+
static_assert(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8,
|
30
|
+
"Size of type must be 1, 2, 4, or 8 for BSwap");
|
31
|
+
if (sizeof(T) == 1) {
|
32
|
+
return x;
|
33
|
+
} else if (sizeof(T) == 2) {
|
34
|
+
return BSWAP16(x);
|
35
|
+
} else if (sizeof(T) == 4) {
|
36
|
+
return BSWAP32(x);
|
37
|
+
} else {
|
38
|
+
return BSWAP64(x);
|
39
|
+
}
|
40
|
+
}
|
41
|
+
|
42
|
+
} // namespace duckdb
|
@@ -10,6 +10,7 @@
|
|
10
10
|
|
11
11
|
#include "duckdb/common/windows.hpp"
|
12
12
|
#include "duckdb/common/local_file_system.hpp"
|
13
|
+
#include "duckdb/common/windows_util.hpp"
|
13
14
|
|
14
15
|
#ifndef _WIN32
|
15
16
|
#include <dlfcn.h>
|
@@ -24,7 +25,8 @@ namespace duckdb {
|
|
24
25
|
|
25
26
|
inline void *dlopen(const char *file, int mode) {
|
26
27
|
D_ASSERT(file);
|
27
|
-
|
28
|
+
auto fpath = WindowsUtil::UTF8ToUnicode(file);
|
29
|
+
return (void *)LoadLibraryW(fpath.c_str());
|
28
30
|
}
|
29
31
|
|
30
32
|
inline void *dlsym(void *handle, const char *name) {
|