duckdb 1.1.2-dev2.0 → 1.1.2-dev6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +4 -2
- package/package.json +1 -1
- package/src/duckdb/extension/icu/third_party/icu/common/putil.cpp +0 -5
- package/src/duckdb/extension/icu/third_party/icu/common/rbbiscan.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/common/rbbitblb.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/common/ucurr.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/common/uresbund.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/common/uresimp.h +31 -31
- package/src/duckdb/extension/icu/third_party/icu/common/ustring.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/common/uvector.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/i18n/coleitr.cpp +12 -12
- package/src/duckdb/extension/icu/third_party/icu/i18n/format.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/i18n/listformatter.cpp +4 -4
- package/src/duckdb/extension/icu/third_party/icu/i18n/number_decimalquantity.h +1 -1
- package/src/duckdb/extension/icu/third_party/icu/i18n/tzgnames.cpp +1 -1
- package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/coleitr.h +28 -28
- package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/format.h +7 -7
- package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/ucol.h +1 -1
- package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/ucoleitr.h +41 -41
- package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/umsg.h +41 -41
- package/src/duckdb/extension/icu/third_party/icu/i18n/usrchimp.h +3 -3
- package/src/duckdb/extension/json/include/json_common.hpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_structure.cpp +13 -7
- package/src/duckdb/extension/parquet/column_writer.cpp +2 -1
- package/src/duckdb/extension/parquet/geo_parquet.cpp +24 -9
- package/src/duckdb/extension/parquet/include/geo_parquet.hpp +3 -1
- package/src/duckdb/extension/parquet/include/parquet_reader.hpp +1 -0
- package/src/duckdb/extension/parquet/include/parquet_rle_bp_decoder.hpp +1 -1
- package/src/duckdb/extension/parquet/include/templated_column_reader.hpp +0 -4
- package/src/duckdb/extension/parquet/parquet_extension.cpp +20 -6
- package/src/duckdb/extension/parquet/parquet_reader.cpp +1 -2
- package/src/duckdb/extension/parquet/parquet_writer.cpp +1 -1
- package/src/duckdb/extension/parquet/serialize_parquet.cpp +0 -2
- package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp +8 -1
- package/src/duckdb/src/catalog/default/default_functions.cpp +5 -5
- package/src/duckdb/src/common/allocator.cpp +3 -2
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +1 -0
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +11 -0
- package/src/duckdb/src/common/arrow/schema_metadata.cpp +6 -4
- package/src/duckdb/src/common/enum_util.cpp +33 -0
- package/src/duckdb/src/common/exception.cpp +3 -0
- package/src/duckdb/src/common/extra_type_info.cpp +1 -44
- package/src/duckdb/src/common/field_writer.cpp +97 -0
- package/src/duckdb/src/common/render_tree.cpp +7 -5
- package/src/duckdb/src/common/row_operations/row_match.cpp +359 -0
- package/src/duckdb/src/common/serializer/buffered_deserializer.cpp +27 -0
- package/src/duckdb/src/common/serializer/buffered_serializer.cpp +36 -0
- package/src/duckdb/src/common/serializer/format_serializer.cpp +15 -0
- package/src/duckdb/src/common/serializer.cpp +24 -0
- package/src/duckdb/src/common/sort/comparators.cpp +2 -2
- package/src/duckdb/src/common/types/bit.cpp +57 -34
- package/src/duckdb/src/common/types/data_chunk.cpp +32 -29
- package/src/duckdb/src/common/types/vector_cache.cpp +12 -6
- package/src/duckdb/src/common/vector_operations/comparison_operators.cpp +14 -0
- package/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp +20 -1
- package/src/duckdb/src/core_functions/aggregate/distributive/minmax.cpp +2 -2
- package/src/duckdb/src/core_functions/aggregate/holistic/approx_top_k.cpp +32 -7
- package/src/duckdb/src/core_functions/function_list.cpp +1 -2
- package/src/duckdb/src/core_functions/scalar/bit/bitstring.cpp +23 -5
- package/src/duckdb/src/core_functions/scalar/date/date_diff.cpp +12 -6
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +1 -1
- package/src/duckdb/src/execution/expression_executor/execute_between.cpp +4 -3
- package/src/duckdb/src/execution/expression_executor/execute_case.cpp +4 -3
- package/src/duckdb/src/execution/expression_executor/execute_cast.cpp +2 -1
- package/src/duckdb/src/execution/expression_executor/execute_comparison.cpp +3 -2
- package/src/duckdb/src/execution/expression_executor/execute_conjunction.cpp +2 -1
- package/src/duckdb/src/execution/expression_executor/execute_function.cpp +2 -1
- package/src/duckdb/src/execution/expression_executor/execute_operator.cpp +3 -2
- package/src/duckdb/src/execution/expression_executor/execute_reference.cpp +1 -1
- package/src/duckdb/src/execution/expression_executor.cpp +9 -3
- package/src/duckdb/src/execution/expression_executor_state.cpp +11 -9
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +238 -0
- package/src/duckdb/src/execution/index/art/plan_art.cpp +94 -0
- package/src/duckdb/src/execution/index/index_type_set.cpp +4 -1
- package/src/duckdb/src/execution/join_hashtable.cpp +7 -8
- package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +6 -4
- package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +4 -4
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/base_scanner.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/csv_schema.cpp +44 -5
- package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +28 -24
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +25 -26
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +5 -3
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +4 -4
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +2 -2
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_refinement.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_replacement.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp +2 -2
- package/src/duckdb/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +1 -1
- package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +73 -27
- package/src/duckdb/src/execution/operator/helper/physical_buffered_collector.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +695 -0
- package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +1487 -0
- package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp +72 -0
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +158 -0
- package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +280 -0
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +666 -0
- package/src/duckdb/src/execution/operator/persistent/physical_export.cpp +14 -4
- package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +207 -0
- package/src/duckdb/src/execution/partitionable_hashtable.cpp +207 -0
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +6 -1
- package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +0 -4
- package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +14 -87
- package/src/duckdb/src/execution/physical_plan/plan_export.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +1 -1
- package/src/duckdb/src/execution/reservoir_sample.cpp +1 -1
- package/src/duckdb/src/execution/window_executor.cpp +3 -3
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +1 -1
- package/src/duckdb/src/function/scalar/strftime_format.cpp +1 -2
- package/src/duckdb/src/function/scalar/string/concat.cpp +118 -151
- package/src/duckdb/src/function/table/arrow.cpp +13 -0
- package/src/duckdb/src/function/table/arrow_conversion.cpp +12 -7
- package/src/duckdb/src/function/table/copy_csv.cpp +1 -1
- package/src/duckdb/src/function/table/read_csv.cpp +2 -30
- package/src/duckdb/src/function/table/sniff_csv.cpp +2 -1
- package/src/duckdb/src/function/table/system/duckdb_secrets.cpp +15 -7
- package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry_retriever.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/atomic.hpp +13 -1
- package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +3 -4
- package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
- package/src/duckdb/src/include/duckdb/common/enums/metric_type.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/exception.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/extra_type_info/enum_type_info.hpp +53 -0
- package/src/duckdb/src/include/duckdb/common/insertion_order_preserving_map.hpp +5 -5
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +5 -0
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +36 -33
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +10 -13
- package/src/duckdb/src/include/duckdb/common/types/uhugeint.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/types/vector_cache.hpp +7 -5
- package/src/duckdb/src/include/duckdb/common/windows_undefs.hpp +2 -1
- package/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp +2 -0
- package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp +1 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +0 -6
- package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp +3 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +3 -0
- package/src/duckdb/src/include/duckdb/execution/index/index_type.hpp +16 -1
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +4 -4
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +4 -2
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp +3 -2
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +91 -36
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/sniff_result.hpp +36 -0
- package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp +0 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_export.hpp +2 -5
- package/src/duckdb/src/include/duckdb/function/table_function.hpp +1 -1
- package/src/duckdb/src/include/duckdb/main/database.hpp +5 -0
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/profiling_info.hpp +20 -22
- package/src/duckdb/src/include/duckdb/main/query_profiler.hpp +7 -9
- package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +8 -1
- package/src/duckdb/src/include/duckdb/main/table_description.hpp +14 -0
- package/src/duckdb/src/include/duckdb/optimizer/unnest_rewriter.hpp +5 -5
- package/src/duckdb/src/include/duckdb/parser/parsed_data/exported_table_data.hpp +15 -5
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +2 -0
- package/src/duckdb/src/include/duckdb/planner/expression_binder/order_binder.hpp +4 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_export.hpp +10 -13
- package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +1 -0
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp +0 -2
- package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +1 -0
- package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp +5 -1
- package/src/duckdb/src/include/duckdb.h +2 -2
- package/src/duckdb/src/main/appender.cpp +3 -0
- package/src/duckdb/src/main/capi/profiling_info-c.cpp +5 -2
- package/src/duckdb/src/main/client_context.cpp +8 -2
- package/src/duckdb/src/main/connection.cpp +1 -1
- package/src/duckdb/src/main/database.cpp +13 -0
- package/src/duckdb/src/main/extension/extension_helper.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_install.cpp +9 -1
- package/src/duckdb/src/main/extension/extension_load.cpp +3 -2
- package/src/duckdb/src/main/extension_install_info.cpp +1 -1
- package/src/duckdb/src/main/profiling_info.cpp +78 -58
- package/src/duckdb/src/main/query_profiler.cpp +79 -89
- package/src/duckdb/src/main/relation/read_csv_relation.cpp +1 -1
- package/src/duckdb/src/main/secret/secret.cpp +2 -1
- package/src/duckdb/src/main/secret/secret_manager.cpp +14 -0
- package/src/duckdb/src/optimizer/cte_filter_pusher.cpp +4 -2
- package/src/duckdb/src/optimizer/deliminator.cpp +0 -7
- package/src/duckdb/src/optimizer/in_clause_rewriter.cpp +7 -0
- package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp +4 -1
- package/src/duckdb/src/optimizer/unnest_rewriter.cpp +21 -21
- package/src/duckdb/src/parallel/task_scheduler.cpp +9 -0
- package/src/duckdb/src/parser/parsed_data/exported_table_data.cpp +22 -0
- package/src/duckdb/src/parser/parsed_expression_iterator.cpp +3 -0
- package/src/duckdb/src/parser/statement/insert_statement.cpp +7 -1
- package/src/duckdb/src/parser/transform/expression/transform_boolean_test.cpp +1 -1
- package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +89 -87
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +2 -2
- package/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp +4 -9
- package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +4 -0
- package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +2 -2
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +4 -1
- package/src/duckdb/src/planner/binder/statement/bind_export.cpp +4 -3
- package/src/duckdb/src/planner/expression_binder/order_binder.cpp +13 -3
- package/src/duckdb/src/planner/expression_binder.cpp +1 -1
- package/src/duckdb/src/planner/operator/logical_export.cpp +28 -0
- package/src/duckdb/src/planner/table_binding.cpp +1 -2
- package/src/duckdb/src/planner/table_filter.cpp +6 -2
- package/src/duckdb/src/storage/buffer/buffer_pool.cpp +2 -1
- package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -1
- package/src/duckdb/src/storage/compression/bitpacking.cpp +7 -3
- package/src/duckdb/src/storage/compression/dictionary_compression.cpp +1 -1
- package/src/duckdb/src/storage/metadata/metadata_manager.cpp +2 -2
- package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +16 -0
- package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +29 -0
- package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +15 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +2 -1
- package/src/duckdb/src/storage/statistics/distinct_statistics.cpp +3 -5
- package/src/duckdb/src/storage/storage_info.cpp +4 -4
- package/src/duckdb/src/storage/table/row_group_collection.cpp +1 -1
- package/src/duckdb/src/storage/table/row_version_manager.cpp +5 -1
- package/src/duckdb/src/storage/temporary_file_manager.cpp +1 -1
- package/src/duckdb/src/transaction/duck_transaction.cpp +15 -14
- package/src/duckdb/third_party/brotli/common/brotli_platform.h +1 -1
- package/src/duckdb/third_party/brotli/dec/decode.cpp +1 -1
- package/src/duckdb/third_party/brotli/enc/memory.cpp +4 -4
- package/src/duckdb/third_party/fsst/libfsst.cpp +1 -1
- package/src/duckdb/third_party/hyperloglog/sds.cpp +1 -1
- package/src/duckdb/third_party/hyperloglog/sds.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/common/keywords.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/datatype/timestamp.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/mb/pg_wchar.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/bitmapset.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/lockoptions.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/makefuncs.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/pg_list.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/nodes/value.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/parser/gramparse.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/parser/parser.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/parser/scanner.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/parser/scansup.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/include/pg_functions.hpp +1 -1
- package/src/duckdb/third_party/libpg_query/pg_functions.cpp +1 -1
- package/src/duckdb/third_party/libpg_query/src_backend_nodes_list.cpp +1 -1
- package/src/duckdb/third_party/libpg_query/src_backend_nodes_makefuncs.cpp +1 -1
- package/src/duckdb/third_party/libpg_query/src_backend_nodes_value.cpp +1 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +1964 -1964
- package/src/duckdb/third_party/libpg_query/src_backend_parser_parser.cpp +1 -1
- package/src/duckdb/third_party/libpg_query/src_backend_parser_scansup.cpp +1 -1
- package/src/duckdb/third_party/libpg_query/src_common_keywords.cpp +1 -1
- package/src/duckdb/third_party/lz4/lz4.cpp +1 -1
- package/src/duckdb/third_party/mbedtls/include/des_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/aes_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/aria_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/asn1write.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/camellia_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/ccm_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/chacha20.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/chachapoly.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/cmac.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/config_psa.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/ecdsa.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/ecp.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/gcm_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/md5.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/nist_kw.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/pkcs12.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/pkcs5.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/psa_util.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/ripemd160.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/threading.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls/timing.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/platform_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/psa/crypto.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/rsa_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/sha1_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/sha256_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/sha512_alt.h +1 -1
- package/src/duckdb/third_party/mbedtls/include/ssl_misc.h +1 -1
- package/src/duckdb/third_party/mbedtls/library/aesni.h +1 -1
- package/src/duckdb/third_party/mbedtls/library/padlock.h +1 -1
- package/src/duckdb/third_party/miniz/miniz.cpp +1 -1
- package/src/duckdb/third_party/parquet/parquet_types.cpp +1 -1
- package/src/duckdb/third_party/parquet/windows_compatibility.h +1 -1
- package/src/duckdb/third_party/pcg/pcg_extras.hpp +1 -1
- package/src/duckdb/third_party/pcg/pcg_uint128.hpp +1 -1
- package/src/duckdb/third_party/skiplist/Node.h +4 -4
- package/src/duckdb/third_party/snappy/snappy.cc +1 -1
- package/src/duckdb/third_party/snappy/snappy_version.hpp +1 -1
- package/src/duckdb/third_party/thrift/thrift/thrift-config.h +1 -1
- package/src/duckdb/third_party/zstd/decompress/zstd_decompress_block.cpp +1 -1
- package/src/duckdb/third_party/zstd/include/zstd_static.h +1 -1
- package/src/duckdb/ub_src_execution_index_art.cpp +2 -0
- package/src/duckdb/ub_src_parser_parsed_data.cpp +2 -0
- package/src/duckdb/ub_src_planner_operator.cpp +2 -0
@@ -28,6 +28,14 @@ struct DateDiff {
|
|
28
28
|
});
|
29
29
|
}
|
30
30
|
|
31
|
+
// We need to truncate down, not towards 0
|
32
|
+
static inline int64_t Truncate(int64_t value, int64_t units) {
|
33
|
+
return (value + (value < 0)) / units - (value < 0);
|
34
|
+
}
|
35
|
+
static inline int64_t Diff(int64_t start, int64_t end, int64_t units) {
|
36
|
+
return Truncate(end, units) - Truncate(start, units);
|
37
|
+
}
|
38
|
+
|
31
39
|
struct YearOperator {
|
32
40
|
template <class TA, class TB, class TR>
|
33
41
|
static inline TR Operation(TA startdate, TB enddate) {
|
@@ -204,30 +212,28 @@ template <>
|
|
204
212
|
int64_t DateDiff::MillisecondsOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
|
205
213
|
D_ASSERT(Timestamp::IsFinite(startdate));
|
206
214
|
D_ASSERT(Timestamp::IsFinite(enddate));
|
207
|
-
return
|
215
|
+
return Diff(startdate.value, enddate.value, Interval::MICROS_PER_MSEC);
|
208
216
|
}
|
209
217
|
|
210
218
|
template <>
|
211
219
|
int64_t DateDiff::SecondsOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
|
212
220
|
D_ASSERT(Timestamp::IsFinite(startdate));
|
213
221
|
D_ASSERT(Timestamp::IsFinite(enddate));
|
214
|
-
return
|
222
|
+
return Diff(startdate.value, enddate.value, Interval::MICROS_PER_SEC);
|
215
223
|
}
|
216
224
|
|
217
225
|
template <>
|
218
226
|
int64_t DateDiff::MinutesOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
|
219
227
|
D_ASSERT(Timestamp::IsFinite(startdate));
|
220
228
|
D_ASSERT(Timestamp::IsFinite(enddate));
|
221
|
-
return
|
222
|
-
Timestamp::GetEpochSeconds(startdate) / Interval::SECS_PER_MINUTE;
|
229
|
+
return Diff(startdate.value, enddate.value, Interval::MICROS_PER_MINUTE);
|
223
230
|
}
|
224
231
|
|
225
232
|
template <>
|
226
233
|
int64_t DateDiff::HoursOperator::Operation(timestamp_t startdate, timestamp_t enddate) {
|
227
234
|
D_ASSERT(Timestamp::IsFinite(startdate));
|
228
235
|
D_ASSERT(Timestamp::IsFinite(enddate));
|
229
|
-
return
|
230
|
-
Timestamp::GetEpochSeconds(startdate) / Interval::SECS_PER_HOUR;
|
236
|
+
return Diff(startdate.value, enddate.value, Interval::MICROS_PER_HOUR);
|
231
237
|
}
|
232
238
|
|
233
239
|
// TIME specialisations
|
@@ -412,7 +412,7 @@ struct DatePart {
|
|
412
412
|
D_ASSERT(input.ColumnCount() == 1);
|
413
413
|
|
414
414
|
UnaryExecutor::Execute<int64_t, timestamp_t>(input.data[0], result, input.size(), [&](int64_t input) {
|
415
|
-
//
|
415
|
+
// millisecond amounts provided to epoch_ms should never be considered infinite
|
416
416
|
// instead such values will just throw when converted to microseconds
|
417
417
|
return Timestamp::FromEpochMsPossiblyInfinite(input);
|
418
418
|
});
|
@@ -89,9 +89,10 @@ static idx_t BetweenLoopTypeSwitch(Vector &input, Vector &lower, Vector &upper,
|
|
89
89
|
unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundBetweenExpression &expr,
|
90
90
|
ExpressionExecutorState &root) {
|
91
91
|
auto result = make_uniq<ExpressionState>(expr, root);
|
92
|
-
result->AddChild(expr.input
|
93
|
-
result->AddChild(expr.lower
|
94
|
-
result->AddChild(expr.upper
|
92
|
+
result->AddChild(*expr.input);
|
93
|
+
result->AddChild(*expr.lower);
|
94
|
+
result->AddChild(*expr.upper);
|
95
|
+
|
95
96
|
result->Finalize();
|
96
97
|
return result;
|
97
98
|
}
|
@@ -18,10 +18,11 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundCaseE
|
|
18
18
|
ExpressionExecutorState &root) {
|
19
19
|
auto result = make_uniq<CaseExpressionState>(expr, root);
|
20
20
|
for (auto &case_check : expr.case_checks) {
|
21
|
-
result->AddChild(case_check.when_expr
|
22
|
-
result->AddChild(case_check.then_expr
|
21
|
+
result->AddChild(*case_check.when_expr);
|
22
|
+
result->AddChild(*case_check.then_expr);
|
23
23
|
}
|
24
|
-
result->AddChild(expr.else_expr
|
24
|
+
result->AddChild(*expr.else_expr);
|
25
|
+
|
25
26
|
result->Finalize();
|
26
27
|
return std::move(result);
|
27
28
|
}
|
@@ -8,8 +8,9 @@ namespace duckdb {
|
|
8
8
|
unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundCastExpression &expr,
|
9
9
|
ExpressionExecutorState &root) {
|
10
10
|
auto result = make_uniq<ExecuteFunctionState>(expr, root);
|
11
|
-
result->AddChild(expr.child
|
11
|
+
result->AddChild(*expr.child);
|
12
12
|
result->Finalize();
|
13
|
+
|
13
14
|
if (expr.bound_cast.init_local_state) {
|
14
15
|
CastLocalStateParameters parameters(root.executor->GetContext(), expr.bound_cast.cast_data);
|
15
16
|
result->local_state = expr.bound_cast.init_local_state(parameters);
|
@@ -12,8 +12,9 @@ namespace duckdb {
|
|
12
12
|
unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundComparisonExpression &expr,
|
13
13
|
ExpressionExecutorState &root) {
|
14
14
|
auto result = make_uniq<ExpressionState>(expr, root);
|
15
|
-
result->AddChild(expr.left
|
16
|
-
result->AddChild(expr.right
|
15
|
+
result->AddChild(*expr.left);
|
16
|
+
result->AddChild(*expr.right);
|
17
|
+
|
17
18
|
result->Finalize();
|
18
19
|
return result;
|
19
20
|
}
|
@@ -18,8 +18,9 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundConju
|
|
18
18
|
ExpressionExecutorState &root) {
|
19
19
|
auto result = make_uniq<ConjunctionState>(expr, root);
|
20
20
|
for (auto &child : expr.children) {
|
21
|
-
result->AddChild(child
|
21
|
+
result->AddChild(*child);
|
22
22
|
}
|
23
|
+
|
23
24
|
result->Finalize();
|
24
25
|
return std::move(result);
|
25
26
|
}
|
@@ -14,8 +14,9 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundFunct
|
|
14
14
|
ExpressionExecutorState &root) {
|
15
15
|
auto result = make_uniq<ExecuteFunctionState>(expr, root);
|
16
16
|
for (auto &child : expr.children) {
|
17
|
-
result->AddChild(child
|
17
|
+
result->AddChild(*child);
|
18
18
|
}
|
19
|
+
|
19
20
|
result->Finalize();
|
20
21
|
if (expr.function.init_local_state) {
|
21
22
|
result->local_state = expr.function.init_local_state(*result, expr, expr.bind_info.get());
|
@@ -8,8 +8,9 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundOpera
|
|
8
8
|
ExpressionExecutorState &root) {
|
9
9
|
auto result = make_uniq<ExpressionState>(expr, root);
|
10
10
|
for (auto &child : expr.children) {
|
11
|
-
result->AddChild(child
|
11
|
+
result->AddChild(*child);
|
12
12
|
}
|
13
|
+
|
13
14
|
result->Finalize();
|
14
15
|
return result;
|
15
16
|
}
|
@@ -33,7 +34,7 @@ void ExpressionExecutor::Execute(const BoundOperatorExpression &expr, Expression
|
|
33
34
|
intermediate.Reference(false_val);
|
34
35
|
|
35
36
|
// in rhs is a list of constants
|
36
|
-
// for every child, OR the result of the
|
37
|
+
// for every child, OR the result of the comparison with the left
|
37
38
|
// to get the overall result.
|
38
39
|
for (idx_t child = 1; child < expr.children.size(); child++) {
|
39
40
|
Vector vector_to_check(expr.children[child]->return_type);
|
@@ -6,7 +6,7 @@ namespace duckdb {
|
|
6
6
|
unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const BoundReferenceExpression &expr,
|
7
7
|
ExpressionExecutorState &root) {
|
8
8
|
auto result = make_uniq<ExpressionState>(expr, root);
|
9
|
-
result->Finalize(
|
9
|
+
result->Finalize();
|
10
10
|
return result;
|
11
11
|
}
|
12
12
|
|
@@ -170,10 +170,16 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const Expression
|
|
170
170
|
void ExpressionExecutor::Execute(const Expression &expr, ExpressionState *state, const SelectionVector *sel,
|
171
171
|
idx_t count, Vector &result) {
|
172
172
|
#ifdef DEBUG
|
173
|
-
//
|
174
|
-
//
|
173
|
+
// The result vector must be used for the first time, or must be reset.
|
174
|
+
// Otherwise, the validity mask can contain previous (now incorrect) data.
|
175
175
|
if (result.GetVectorType() == VectorType::FLAT_VECTOR) {
|
176
|
-
|
176
|
+
|
177
|
+
// We do not initialize vector caches for these expressions.
|
178
|
+
if (expr.GetExpressionClass() != ExpressionClass::BOUND_REF &&
|
179
|
+
expr.GetExpressionClass() != ExpressionClass::BOUND_CONSTANT &&
|
180
|
+
expr.GetExpressionClass() != ExpressionClass::BOUND_PARAMETER) {
|
181
|
+
D_ASSERT(FlatVector::Validity(result).CheckAllValid(count));
|
182
|
+
}
|
177
183
|
}
|
178
184
|
#endif
|
179
185
|
|
@@ -6,20 +6,22 @@
|
|
6
6
|
|
7
7
|
namespace duckdb {
|
8
8
|
|
9
|
-
void ExpressionState::AddChild(Expression
|
10
|
-
types.push_back(
|
11
|
-
|
9
|
+
void ExpressionState::AddChild(Expression &child_expr) {
|
10
|
+
types.push_back(child_expr.return_type);
|
11
|
+
auto child_state = ExpressionExecutor::InitializeState(child_expr, root);
|
12
|
+
child_states.push_back(std::move(child_state));
|
13
|
+
|
14
|
+
auto expr_class = child_expr.GetExpressionClass();
|
15
|
+
auto initialize_child = expr_class != ExpressionClass::BOUND_REF && expr_class != ExpressionClass::BOUND_CONSTANT &&
|
16
|
+
expr_class != ExpressionClass::BOUND_PARAMETER;
|
17
|
+
initialize.push_back(initialize_child);
|
12
18
|
}
|
13
19
|
|
14
|
-
void ExpressionState::Finalize(
|
20
|
+
void ExpressionState::Finalize() {
|
15
21
|
if (types.empty()) {
|
16
22
|
return;
|
17
23
|
}
|
18
|
-
|
19
|
-
intermediate_chunk.InitializeEmpty(types);
|
20
|
-
} else {
|
21
|
-
intermediate_chunk.Initialize(GetAllocator(), types);
|
22
|
-
}
|
24
|
+
intermediate_chunk.Initialize(GetAllocator(), types, initialize);
|
23
25
|
}
|
24
26
|
|
25
27
|
Allocator &ExpressionState::GetAllocator() {
|
@@ -0,0 +1,238 @@
|
|
1
|
+
#include "duckdb/execution/index/art/fixed_size_allocator.hpp"
|
2
|
+
|
3
|
+
namespace duckdb {
|
4
|
+
|
5
|
+
constexpr idx_t FixedSizeAllocator::BASE[];
|
6
|
+
constexpr uint8_t FixedSizeAllocator::SHIFT[];
|
7
|
+
|
8
|
+
FixedSizeAllocator::FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator)
|
9
|
+
: allocation_size(allocation_size), total_allocations(0), allocator(allocator) {
|
10
|
+
|
11
|
+
// calculate how many allocations fit into one buffer
|
12
|
+
|
13
|
+
idx_t bits_per_value = sizeof(validity_t) * 8;
|
14
|
+
idx_t curr_alloc_size = 0;
|
15
|
+
|
16
|
+
bitmask_count = 0;
|
17
|
+
allocations_per_buffer = 0;
|
18
|
+
|
19
|
+
while (curr_alloc_size < BUFFER_ALLOC_SIZE) {
|
20
|
+
if (!bitmask_count || (bitmask_count * bits_per_value) % allocations_per_buffer == 0) {
|
21
|
+
bitmask_count++;
|
22
|
+
curr_alloc_size += sizeof(validity_t);
|
23
|
+
}
|
24
|
+
|
25
|
+
auto remaining_alloc_size = BUFFER_ALLOC_SIZE - curr_alloc_size;
|
26
|
+
auto remaining_allocations = MinValue(remaining_alloc_size / allocation_size, bits_per_value);
|
27
|
+
|
28
|
+
if (remaining_allocations == 0) {
|
29
|
+
break;
|
30
|
+
}
|
31
|
+
|
32
|
+
allocations_per_buffer += remaining_allocations;
|
33
|
+
curr_alloc_size += remaining_allocations * allocation_size;
|
34
|
+
}
|
35
|
+
|
36
|
+
allocation_offset = bitmask_count * sizeof(validity_t);
|
37
|
+
}
|
38
|
+
|
39
|
+
FixedSizeAllocator::~FixedSizeAllocator() {
|
40
|
+
for (auto &buffer : buffers) {
|
41
|
+
allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
|
42
|
+
}
|
43
|
+
}
|
44
|
+
|
45
|
+
Node FixedSizeAllocator::New() {
|
46
|
+
|
47
|
+
// no more free pointers
|
48
|
+
if (buffers_with_free_space.empty()) {
|
49
|
+
|
50
|
+
// add a new buffer
|
51
|
+
idx_t buffer_id = buffers.size();
|
52
|
+
D_ASSERT(buffer_id <= (uint32_t)DConstants::INVALID_INDEX);
|
53
|
+
auto buffer = allocator.AllocateData(BUFFER_ALLOC_SIZE);
|
54
|
+
buffers.emplace_back(buffer, 0);
|
55
|
+
buffers_with_free_space.insert(buffer_id);
|
56
|
+
|
57
|
+
// set the bitmask
|
58
|
+
ValidityMask mask(reinterpret_cast<validity_t *>(buffer));
|
59
|
+
mask.SetAllValid(allocations_per_buffer);
|
60
|
+
}
|
61
|
+
|
62
|
+
// return a pointer
|
63
|
+
D_ASSERT(!buffers_with_free_space.empty());
|
64
|
+
auto buffer_id = (uint32_t)*buffers_with_free_space.begin();
|
65
|
+
|
66
|
+
auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[buffer_id].ptr);
|
67
|
+
ValidityMask mask(bitmask_ptr);
|
68
|
+
auto offset = GetOffset(mask, buffers[buffer_id].allocation_count);
|
69
|
+
|
70
|
+
buffers[buffer_id].allocation_count++;
|
71
|
+
total_allocations++;
|
72
|
+
if (buffers[buffer_id].allocation_count == allocations_per_buffer) {
|
73
|
+
buffers_with_free_space.erase(buffer_id);
|
74
|
+
}
|
75
|
+
|
76
|
+
return Node(buffer_id, offset);
|
77
|
+
}
|
78
|
+
|
79
|
+
void FixedSizeAllocator::Free(const Node ptr) {
|
80
|
+
auto bitmask_ptr = reinterpret_cast<validity_t *>(buffers[ptr.GetBufferId()].ptr);
|
81
|
+
ValidityMask mask(bitmask_ptr);
|
82
|
+
D_ASSERT(!mask.RowIsValid(ptr.GetOffset()));
|
83
|
+
mask.SetValid(ptr.GetOffset());
|
84
|
+
buffers_with_free_space.insert(ptr.GetBufferId());
|
85
|
+
|
86
|
+
D_ASSERT(total_allocations > 0);
|
87
|
+
D_ASSERT(buffers[ptr.GetBufferId()].allocation_count > 0);
|
88
|
+
buffers[ptr.GetBufferId()].allocation_count--;
|
89
|
+
total_allocations--;
|
90
|
+
}
|
91
|
+
|
92
|
+
void FixedSizeAllocator::Reset() {
|
93
|
+
|
94
|
+
for (auto &buffer : buffers) {
|
95
|
+
allocator.FreeData(buffer.ptr, BUFFER_ALLOC_SIZE);
|
96
|
+
}
|
97
|
+
buffers.clear();
|
98
|
+
buffers_with_free_space.clear();
|
99
|
+
total_allocations = 0;
|
100
|
+
}
|
101
|
+
|
102
|
+
void FixedSizeAllocator::Merge(FixedSizeAllocator &other) {
|
103
|
+
|
104
|
+
D_ASSERT(allocation_size == other.allocation_size);
|
105
|
+
|
106
|
+
// remember the buffer count and merge the buffers
|
107
|
+
idx_t buffer_count = buffers.size();
|
108
|
+
for (auto &buffer : other.buffers) {
|
109
|
+
buffers.push_back(buffer);
|
110
|
+
}
|
111
|
+
other.buffers.clear();
|
112
|
+
|
113
|
+
// merge the buffers with free spaces
|
114
|
+
for (auto &buffer_id : other.buffers_with_free_space) {
|
115
|
+
buffers_with_free_space.insert(buffer_id + buffer_count);
|
116
|
+
}
|
117
|
+
other.buffers_with_free_space.clear();
|
118
|
+
|
119
|
+
// add the total allocations
|
120
|
+
total_allocations += other.total_allocations;
|
121
|
+
}
|
122
|
+
|
123
|
+
bool FixedSizeAllocator::InitializeVacuum() {
|
124
|
+
|
125
|
+
if (total_allocations == 0) {
|
126
|
+
Reset();
|
127
|
+
return false;
|
128
|
+
}
|
129
|
+
|
130
|
+
auto total_available_allocations = allocations_per_buffer * buffers.size();
|
131
|
+
D_ASSERT(total_available_allocations >= total_allocations);
|
132
|
+
auto total_free_positions = total_available_allocations - total_allocations;
|
133
|
+
|
134
|
+
// vacuum_count buffers can be freed
|
135
|
+
auto vacuum_count = total_free_positions / allocations_per_buffer;
|
136
|
+
|
137
|
+
// calculate the vacuum threshold adaptively
|
138
|
+
D_ASSERT(vacuum_count < buffers.size());
|
139
|
+
idx_t memory_usage = GetMemoryUsage();
|
140
|
+
idx_t excess_memory_usage = vacuum_count * BUFFER_ALLOC_SIZE;
|
141
|
+
auto excess_percentage = (double)excess_memory_usage / (double)memory_usage;
|
142
|
+
auto threshold = (double)VACUUM_THRESHOLD / 100.0;
|
143
|
+
if (excess_percentage < threshold) {
|
144
|
+
return false;
|
145
|
+
}
|
146
|
+
|
147
|
+
min_vacuum_buffer_id = buffers.size() - vacuum_count;
|
148
|
+
|
149
|
+
// remove all invalid buffers from the available buffer list to ensure that we do not reuse them
|
150
|
+
auto it = buffers_with_free_space.begin();
|
151
|
+
while (it != buffers_with_free_space.end()) {
|
152
|
+
if (*it >= min_vacuum_buffer_id) {
|
153
|
+
it = buffers_with_free_space.erase(it);
|
154
|
+
} else {
|
155
|
+
it++;
|
156
|
+
}
|
157
|
+
}
|
158
|
+
|
159
|
+
return true;
|
160
|
+
}
|
161
|
+
|
162
|
+
void FixedSizeAllocator::FinalizeVacuum() {
|
163
|
+
|
164
|
+
// free all (now unused) buffers
|
165
|
+
while (min_vacuum_buffer_id < buffers.size()) {
|
166
|
+
allocator.FreeData(buffers.back().ptr, BUFFER_ALLOC_SIZE);
|
167
|
+
buffers.pop_back();
|
168
|
+
}
|
169
|
+
}
|
170
|
+
|
171
|
+
Node FixedSizeAllocator::VacuumPointer(const Node ptr) {
|
172
|
+
|
173
|
+
// we do not need to adjust the bitmask of the old buffer, because we will free the entire
|
174
|
+
// buffer after the vacuum operation
|
175
|
+
|
176
|
+
auto new_ptr = New();
|
177
|
+
|
178
|
+
// new increases the allocation count
|
179
|
+
total_allocations--;
|
180
|
+
|
181
|
+
memcpy(Get(new_ptr), Get(ptr), allocation_size);
|
182
|
+
return new_ptr;
|
183
|
+
}
|
184
|
+
|
185
|
+
void FixedSizeAllocator::Verify() const {
|
186
|
+
#ifdef DEBUG
|
187
|
+
auto total_available_allocations = allocations_per_buffer * buffers.size();
|
188
|
+
D_ASSERT(total_available_allocations >= total_allocations);
|
189
|
+
D_ASSERT(buffers.size() >= buffers_with_free_space.size());
|
190
|
+
#endif
|
191
|
+
}
|
192
|
+
|
193
|
+
uint32_t FixedSizeAllocator::GetOffset(ValidityMask &mask, const idx_t allocation_count) {
|
194
|
+
|
195
|
+
auto data = mask.GetData();
|
196
|
+
|
197
|
+
// fills up a buffer sequentially before searching for free bits
|
198
|
+
if (mask.RowIsValid(allocation_count)) {
|
199
|
+
mask.SetInvalid(allocation_count);
|
200
|
+
return allocation_count;
|
201
|
+
}
|
202
|
+
|
203
|
+
// get an entry with free bits
|
204
|
+
for (idx_t entry_idx = 0; entry_idx < bitmask_count; entry_idx++) {
|
205
|
+
if (data[entry_idx] != 0) {
|
206
|
+
|
207
|
+
// find the position of the free bit
|
208
|
+
auto entry = data[entry_idx];
|
209
|
+
idx_t first_valid_bit = 0;
|
210
|
+
|
211
|
+
// this loop finds the position of the rightmost set bit in entry and stores it
|
212
|
+
// in first_valid_bit
|
213
|
+
for (idx_t i = 0; i < 6; i++) {
|
214
|
+
// set the left half of the bits of this level to zero and test if the entry is still not zero
|
215
|
+
if (entry & BASE[i]) {
|
216
|
+
// first valid bit is in the rightmost s[i] bits
|
217
|
+
// permanently set the left half of the bits to zero
|
218
|
+
entry &= BASE[i];
|
219
|
+
} else {
|
220
|
+
// first valid bit is in the leftmost s[i] bits
|
221
|
+
// shift by s[i] for the next iteration and add s[i] to the position of the rightmost set bit
|
222
|
+
entry >>= SHIFT[i];
|
223
|
+
first_valid_bit += SHIFT[i];
|
224
|
+
}
|
225
|
+
}
|
226
|
+
D_ASSERT(entry);
|
227
|
+
|
228
|
+
auto prev_bits = entry_idx * sizeof(validity_t) * 8;
|
229
|
+
D_ASSERT(mask.RowIsValid(prev_bits + first_valid_bit));
|
230
|
+
mask.SetInvalid(prev_bits + first_valid_bit);
|
231
|
+
return (prev_bits + first_valid_bit);
|
232
|
+
}
|
233
|
+
}
|
234
|
+
|
235
|
+
throw InternalException("Invalid bitmask of FixedSizeAllocator");
|
236
|
+
}
|
237
|
+
|
238
|
+
} // namespace duckdb
|
@@ -0,0 +1,94 @@
|
|
1
|
+
|
2
|
+
#include "duckdb/execution/operator/order/physical_order.hpp"
|
3
|
+
#include "duckdb/execution/operator/projection/physical_projection.hpp"
|
4
|
+
#include "duckdb/execution/operator/filter/physical_filter.hpp"
|
5
|
+
#include "duckdb/execution/operator/schema/physical_create_art_index.hpp"
|
6
|
+
|
7
|
+
#include "duckdb/planner/expression/bound_operator_expression.hpp"
|
8
|
+
#include "duckdb/planner/expression/bound_reference_expression.hpp"
|
9
|
+
#include "duckdb/planner/operator/logical_create_index.hpp"
|
10
|
+
|
11
|
+
#include "duckdb/execution/index/art/art.hpp"
|
12
|
+
|
13
|
+
namespace duckdb {
|
14
|
+
|
15
|
+
unique_ptr<PhysicalOperator> ART::CreatePlan(PlanIndexInput &input) {
|
16
|
+
// generate a physical plan for the parallel index creation which consists of the following operators
|
17
|
+
// table scan - projection (for expression execution) - filter (NOT NULL) - order (if applicable) - create index
|
18
|
+
|
19
|
+
auto &op = input.op;
|
20
|
+
auto &table_scan = input.table_scan;
|
21
|
+
|
22
|
+
vector<LogicalType> new_column_types;
|
23
|
+
vector<unique_ptr<Expression>> select_list;
|
24
|
+
for (idx_t i = 0; i < op.expressions.size(); i++) {
|
25
|
+
new_column_types.push_back(op.expressions[i]->return_type);
|
26
|
+
select_list.push_back(std::move(op.expressions[i]));
|
27
|
+
}
|
28
|
+
new_column_types.emplace_back(LogicalType::ROW_TYPE);
|
29
|
+
select_list.push_back(make_uniq<BoundReferenceExpression>(LogicalType::ROW_TYPE, op.info->scan_types.size() - 1));
|
30
|
+
|
31
|
+
auto projection = make_uniq<PhysicalProjection>(new_column_types, std::move(select_list), op.estimated_cardinality);
|
32
|
+
projection->children.push_back(std::move(table_scan));
|
33
|
+
|
34
|
+
// filter operator for IS_NOT_NULL on each key column
|
35
|
+
|
36
|
+
vector<LogicalType> filter_types;
|
37
|
+
vector<unique_ptr<Expression>> filter_select_list;
|
38
|
+
|
39
|
+
for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
|
40
|
+
filter_types.push_back(new_column_types[i]);
|
41
|
+
auto is_not_null_expr =
|
42
|
+
make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
|
43
|
+
auto bound_ref = make_uniq<BoundReferenceExpression>(new_column_types[i], i);
|
44
|
+
is_not_null_expr->children.push_back(std::move(bound_ref));
|
45
|
+
filter_select_list.push_back(std::move(is_not_null_expr));
|
46
|
+
}
|
47
|
+
|
48
|
+
auto null_filter =
|
49
|
+
make_uniq<PhysicalFilter>(std::move(filter_types), std::move(filter_select_list), op.estimated_cardinality);
|
50
|
+
null_filter->types.emplace_back(LogicalType::ROW_TYPE);
|
51
|
+
null_filter->children.push_back(std::move(projection));
|
52
|
+
|
53
|
+
// determine if we sort the data prior to index creation
|
54
|
+
// we don't sort, if either VARCHAR or compound key
|
55
|
+
auto perform_sorting = true;
|
56
|
+
if (op.unbound_expressions.size() > 1) {
|
57
|
+
perform_sorting = false;
|
58
|
+
} else if (op.unbound_expressions[0]->return_type.InternalType() == PhysicalType::VARCHAR) {
|
59
|
+
perform_sorting = false;
|
60
|
+
}
|
61
|
+
|
62
|
+
// actual physical create index operator
|
63
|
+
|
64
|
+
auto physical_create_index =
|
65
|
+
make_uniq<PhysicalCreateARTIndex>(op, op.table, op.info->column_ids, std::move(op.info),
|
66
|
+
std::move(op.unbound_expressions), op.estimated_cardinality, perform_sorting);
|
67
|
+
|
68
|
+
if (perform_sorting) {
|
69
|
+
|
70
|
+
// optional order operator
|
71
|
+
vector<BoundOrderByNode> orders;
|
72
|
+
vector<idx_t> projections;
|
73
|
+
for (idx_t i = 0; i < new_column_types.size() - 1; i++) {
|
74
|
+
auto col_expr = make_uniq_base<Expression, BoundReferenceExpression>(new_column_types[i], i);
|
75
|
+
orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_FIRST, std::move(col_expr));
|
76
|
+
projections.emplace_back(i);
|
77
|
+
}
|
78
|
+
projections.emplace_back(new_column_types.size() - 1);
|
79
|
+
|
80
|
+
auto physical_order = make_uniq<PhysicalOrder>(new_column_types, std::move(orders), std::move(projections),
|
81
|
+
op.estimated_cardinality);
|
82
|
+
physical_order->children.push_back(std::move(null_filter));
|
83
|
+
|
84
|
+
physical_create_index->children.push_back(std::move(physical_order));
|
85
|
+
} else {
|
86
|
+
|
87
|
+
// no ordering
|
88
|
+
physical_create_index->children.push_back(std::move(null_filter));
|
89
|
+
}
|
90
|
+
|
91
|
+
return std::move(physical_create_index);
|
92
|
+
}
|
93
|
+
|
94
|
+
} // namespace duckdb
|
@@ -5,10 +5,13 @@
|
|
5
5
|
namespace duckdb {
|
6
6
|
|
7
7
|
IndexTypeSet::IndexTypeSet() {
|
8
|
-
|
8
|
+
|
9
|
+
// Register the ART index type by default
|
9
10
|
IndexType art_index_type;
|
10
11
|
art_index_type.name = ART::TYPE_NAME;
|
11
12
|
art_index_type.create_instance = ART::Create;
|
13
|
+
art_index_type.create_plan = ART::CreatePlan;
|
14
|
+
|
12
15
|
RegisterIndexType(art_index_type);
|
13
16
|
}
|
14
17
|
|
@@ -453,23 +453,21 @@ static inline data_ptr_t InsertRowToEntry(atomic<ht_entry_t> &entry, const data_
|
|
453
453
|
// if we expect the entry to be empty, if the operation fails we need to cancel the whole operation as another
|
454
454
|
// key might have been inserted in the meantime that does not match the current key
|
455
455
|
if (EXPECT_EMPTY) {
|
456
|
-
|
457
456
|
// add nullptr to the end of the list to mark the end
|
458
457
|
StorePointer(nullptr, row_ptr_to_insert + pointer_offset);
|
459
458
|
|
460
459
|
ht_entry_t new_empty_entry = ht_entry_t::GetDesiredEntry(row_ptr_to_insert, salt);
|
461
460
|
ht_entry_t expected_empty_entry = ht_entry_t::GetEmptyEntry();
|
462
|
-
|
461
|
+
entry.compare_exchange_strong(expected_empty_entry, new_empty_entry, std::memory_order_acquire,
|
462
|
+
std::memory_order_relaxed);
|
463
463
|
|
464
464
|
// if the expected empty entry actually was null, we can just return the pointer, and it will be a nullptr
|
465
465
|
// if the expected entry was filled in the meantime, we need to cancel the operation and will return the
|
466
466
|
// pointer to the next entry
|
467
467
|
return expected_empty_entry.GetPointerOrNull();
|
468
|
-
}
|
469
|
-
|
470
|
-
|
471
|
-
// just keep trying until we succeed
|
472
|
-
else {
|
468
|
+
} else {
|
469
|
+
// if we expect the entry to be full, we know that even if the insert fails the keys still match so we can
|
470
|
+
// just keep trying until we succeed
|
473
471
|
ht_entry_t expected_current_entry = entry.load(std::memory_order_relaxed);
|
474
472
|
ht_entry_t desired_new_entry = ht_entry_t::GetDesiredEntry(row_ptr_to_insert, salt);
|
475
473
|
D_ASSERT(expected_current_entry.IsOccupied());
|
@@ -477,7 +475,8 @@ static inline data_ptr_t InsertRowToEntry(atomic<ht_entry_t> &entry, const data_
|
|
477
475
|
do {
|
478
476
|
data_ptr_t current_row_pointer = expected_current_entry.GetPointer();
|
479
477
|
StorePointer(current_row_pointer, row_ptr_to_insert + pointer_offset);
|
480
|
-
} while (!
|
478
|
+
} while (!entry.compare_exchange_weak(expected_current_entry, desired_new_entry, std::memory_order_release,
|
479
|
+
std::memory_order_relaxed));
|
481
480
|
|
482
481
|
return nullptr;
|
483
482
|
}
|
@@ -319,15 +319,17 @@ void PhysicalHashAggregate::SinkDistinctGrouping(ExecutionContext &context, Data
|
|
319
319
|
for (idx_t group_idx = 0; group_idx < grouped_aggregate_data.groups.size(); group_idx++) {
|
320
320
|
auto &group = grouped_aggregate_data.groups[group_idx];
|
321
321
|
auto &bound_ref = group->Cast<BoundReferenceExpression>();
|
322
|
-
filtered_input.data[bound_ref.index]
|
322
|
+
auto &col = filtered_input.data[bound_ref.index];
|
323
|
+
col.Reference(chunk.data[bound_ref.index]);
|
324
|
+
col.Slice(sel_vec, count);
|
323
325
|
}
|
324
326
|
for (idx_t child_idx = 0; child_idx < aggregate.children.size(); child_idx++) {
|
325
327
|
auto &child = aggregate.children[child_idx];
|
326
328
|
auto &bound_ref = child->Cast<BoundReferenceExpression>();
|
327
|
-
|
328
|
-
|
329
|
+
auto &col = filtered_input.data[bound_ref.index];
|
330
|
+
col.Reference(chunk.data[bound_ref.index]);
|
331
|
+
col.Slice(sel_vec, count);
|
329
332
|
}
|
330
|
-
filtered_input.Slice(sel_vec, count);
|
331
333
|
filtered_input.SetCardinality(count);
|
332
334
|
|
333
335
|
radix_table.Sink(context, filtered_input, sink_input, empty_chunk, empty_filter);
|
@@ -119,15 +119,15 @@ void CSVBufferManager::ResetBuffer(const idx_t buffer_idx) {
|
|
119
119
|
}
|
120
120
|
}
|
121
121
|
|
122
|
-
idx_t CSVBufferManager::GetBufferSize() {
|
122
|
+
idx_t CSVBufferManager::GetBufferSize() const {
|
123
123
|
return buffer_size;
|
124
124
|
}
|
125
125
|
|
126
|
-
idx_t CSVBufferManager::BufferCount() {
|
126
|
+
idx_t CSVBufferManager::BufferCount() const {
|
127
127
|
return cached_buffers.size();
|
128
128
|
}
|
129
129
|
|
130
|
-
bool CSVBufferManager::Done() {
|
130
|
+
bool CSVBufferManager::Done() const {
|
131
131
|
return done;
|
132
132
|
}
|
133
133
|
|
@@ -144,7 +144,7 @@ void CSVBufferManager::ResetBufferManager() {
|
|
144
144
|
}
|
145
145
|
}
|
146
146
|
|
147
|
-
string CSVBufferManager::GetFilePath() {
|
147
|
+
string CSVBufferManager::GetFilePath() const {
|
148
148
|
return file_path;
|
149
149
|
}
|
150
150
|
|