duckdb 0.8.2-dev4126.0 → 0.8.2-dev4203.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +0 -2
- package/lib/duckdb.js +3 -3
- package/package.json +1 -1
- package/src/connection.cpp +9 -8
- package/src/database.cpp +5 -8
- package/src/duckdb/extension/icu/icu-datepart.cpp +2 -2
- package/src/duckdb/extension/icu/icu-strptime.cpp +0 -9
- package/src/duckdb/extension/icu/icu_extension.cpp +3 -3
- package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/gregocal.h +1 -1
- package/src/duckdb/extension/json/buffered_json_reader.cpp +3 -17
- package/src/duckdb/extension/json/include/buffered_json_reader.hpp +2 -5
- package/src/duckdb/extension/json/include/json_deserializer.hpp +2 -2
- package/src/duckdb/extension/json/include/json_scan.hpp +5 -12
- package/src/duckdb/extension/json/include/json_serializer.hpp +3 -3
- package/src/duckdb/extension/json/include/json_transform.hpp +2 -5
- package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +1 -1
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +2 -18
- package/src/duckdb/extension/json/json_scan.cpp +7 -76
- package/src/duckdb/extension/json/serialize_json.cpp +8 -8
- package/src/duckdb/extension/parquet/column_writer.cpp +46 -45
- package/src/duckdb/extension/parquet/include/column_writer.hpp +2 -2
- package/src/duckdb/extension/parquet/include/parquet_reader.hpp +2 -5
- package/src/duckdb/extension/parquet/include/parquet_rle_bp_encoder.hpp +4 -4
- package/src/duckdb/extension/parquet/parquet_extension.cpp +74 -55
- package/src/duckdb/extension/parquet/parquet_writer.cpp +3 -2
- package/src/duckdb/extension/parquet/serialize_parquet.cpp +4 -4
- package/src/duckdb/src/catalog/catalog.cpp +20 -0
- package/src/duckdb/src/catalog/catalog_entry/index_catalog_entry.cpp +0 -1
- package/src/duckdb/src/catalog/catalog_entry/macro_catalog_entry.cpp +0 -1
- package/src/duckdb/src/catalog/catalog_entry/schema_catalog_entry.cpp +0 -1
- package/src/duckdb/src/catalog/catalog_entry/sequence_catalog_entry.cpp +0 -1
- package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +0 -2
- package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp +1 -3
- package/src/duckdb/src/catalog/catalog_entry/view_catalog_entry.cpp +0 -1
- package/src/duckdb/src/catalog/catalog_entry.cpp +2 -11
- package/src/duckdb/src/catalog/catalog_set.cpp +16 -14
- package/src/duckdb/src/catalog/default/default_functions.cpp +2 -2
- package/src/duckdb/src/common/arrow/arrow_converter.cpp +3 -1
- package/src/duckdb/src/common/constants.cpp +0 -1
- package/src/duckdb/src/common/enum_util.cpp +0 -10
- package/src/duckdb/src/common/exception.cpp +1 -1
- package/src/duckdb/src/common/extra_type_info.cpp +20 -192
- package/src/duckdb/src/common/multi_file_reader.cpp +0 -68
- package/src/duckdb/src/common/serializer/binary_serializer.cpp +9 -4
- package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +3 -11
- package/src/duckdb/src/common/serializer/buffered_file_writer.cpp +1 -1
- package/src/duckdb/src/common/serializer/memory_stream.cpp +61 -0
- package/src/duckdb/src/common/serializer/{format_serializer.cpp → serializer.cpp} +2 -2
- package/src/duckdb/src/common/types/column/column_data_collection.cpp +4 -4
- package/src/duckdb/src/common/types/data_chunk.cpp +11 -43
- package/src/duckdb/src/common/types/hyperloglog.cpp +4 -23
- package/src/duckdb/src/common/types/interval.cpp +2 -2
- package/src/duckdb/src/common/types/value.cpp +10 -135
- package/src/duckdb/src/common/types/vector.cpp +17 -149
- package/src/duckdb/src/common/types.cpp +2 -20
- package/src/duckdb/src/core_functions/aggregate/algebraic/avg.cpp +0 -1
- package/src/duckdb/src/core_functions/aggregate/distributive/string_agg.cpp +6 -7
- package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +5 -28
- package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +30 -78
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +5 -30
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +6 -9
- package/src/duckdb/src/core_functions/scalar/list/list_aggregates.cpp +21 -16
- package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +15 -23
- package/src/duckdb/src/function/macro_function.cpp +0 -42
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +10 -8
- package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +13 -12
- package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +9 -11
- package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +7 -33
- package/src/duckdb/src/function/scalar_function.cpp +1 -2
- package/src/duckdb/src/function/scalar_macro_function.cpp +0 -10
- package/src/duckdb/src/function/table/copy_csv.cpp +38 -38
- package/src/duckdb/src/function/table/read_csv.cpp +5 -174
- package/src/duckdb/src/function/table/table_scan.cpp +5 -42
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/function/table_function.cpp +3 -4
- package/src/duckdb/src/function/table_macro_function.cpp +0 -10
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +1 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/schema_catalog_entry.hpp +0 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp +0 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp +0 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry.hpp +3 -8
- package/src/duckdb/src/include/duckdb/common/constants.hpp +4 -2
- package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +16 -46
- package/src/duckdb/src/include/duckdb/common/index_vector.hpp +4 -4
- package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +2 -6
- package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +0 -4
- package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp +44 -23
- package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp +31 -23
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +4 -6
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp +3 -3
- package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp +17 -0
- package/src/duckdb/src/include/duckdb/common/serializer/{format_deserializer.hpp → deserializer.hpp} +13 -13
- package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp +62 -0
- package/src/duckdb/src/include/duckdb/common/serializer/read_stream.hpp +38 -0
- package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +15 -13
- package/src/duckdb/src/include/duckdb/common/serializer/{format_serializer.hpp → serializer.hpp} +13 -12
- package/src/duckdb/src/include/duckdb/common/serializer/write_stream.hpp +36 -0
- package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +2 -2
- package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +3 -8
- package/src/duckdb/src/include/duckdb/common/types/hyperloglog.hpp +4 -10
- package/src/duckdb/src/include/duckdb/common/types/interval.hpp +4 -4
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +1 -5
- package/src/duckdb/src/include/duckdb/common/types/vector.hpp +2 -7
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -10
- package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_reader_options.hpp +2 -11
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +4 -14
- package/src/duckdb/src/include/duckdb/function/copy_function.hpp +3 -3
- package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +15 -98
- package/src/duckdb/src/include/duckdb/function/macro_function.hpp +1 -7
- package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +9 -10
- package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +2 -12
- package/src/duckdb/src/include/duckdb/function/scalar_macro_function.hpp +2 -7
- package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp +4 -16
- package/src/duckdb/src/include/duckdb/function/table_function.hpp +2 -11
- package/src/duckdb/src/include/duckdb/function/table_macro_function.hpp +2 -7
- package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +1 -0
- package/src/duckdb/src/include/duckdb/parser/column_definition.hpp +1 -4
- package/src/duckdb/src/include/duckdb/parser/column_list.hpp +2 -6
- package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/constraint.hpp +2 -12
- package/src/duckdb/src/include/duckdb/parser/constraints/check_constraint.hpp +2 -5
- package/src/duckdb/src/include/duckdb/parser/constraints/foreign_key_constraint.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/constraints/not_null_constraint.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/constraints/unique_constraint.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/parser/expression/bound_expression.hpp +1 -4
- package/src/duckdb/src/include/duckdb/parser/expression/case_expression.hpp +4 -6
- package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/conjunction_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/parameter_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/subquery_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_info.hpp +2 -5
- package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_scalar_function_info.hpp +0 -2
- package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_table_function_info.hpp +0 -2
- package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_table_info.hpp +24 -51
- package/src/duckdb/src/include/duckdb/parser/parsed_data/attach_info.hpp +1 -4
- package/src/duckdb/src/include/duckdb/parser/parsed_data/copy_info.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_collation_info.hpp +0 -3
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_copy_function_info.hpp +0 -3
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_function_info.hpp +0 -5
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_index_info.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_info.hpp +1 -13
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_macro_info.hpp +2 -8
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_schema_info.hpp +2 -12
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_sequence_info.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_table_info.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_type_info.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/parsed_data/create_view_info.hpp +2 -7
- package/src/duckdb/src/include/duckdb/parser/parsed_data/detach_info.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/parsed_data/drop_info.hpp +2 -5
- package/src/duckdb/src/include/duckdb/parser/parsed_data/load_info.hpp +2 -21
- package/src/duckdb/src/include/duckdb/parser/parsed_data/parse_info.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/parsed_data/pragma_info.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/parsed_data/sample_options.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/parsed_data/transaction_info.hpp +1 -4
- package/src/duckdb/src/include/duckdb/parser/parsed_data/vacuum_info.hpp +5 -7
- package/src/duckdb/src/include/duckdb/parser/parsed_expression.hpp +3 -16
- package/src/duckdb/src/include/duckdb/parser/query_node/cte_node.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/query_node/recursive_cte_node.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/query_node/select_node.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp +2 -4
- package/src/duckdb/src/include/duckdb/parser/query_node.hpp +7 -14
- package/src/duckdb/src/include/duckdb/parser/result_modifier.hpp +14 -32
- package/src/duckdb/src/include/duckdb/parser/statement/select_statement.hpp +4 -9
- package/src/duckdb/src/include/duckdb/parser/tableref/basetableref.hpp +2 -6
- package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +2 -6
- package/src/duckdb/src/include/duckdb/parser/tableref/expressionlistref.hpp +2 -6
- package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +2 -6
- package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp +6 -14
- package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +2 -6
- package/src/duckdb/src/include/duckdb/parser/tableref/table_function_ref.hpp +2 -6
- package/src/duckdb/src/include/duckdb/parser/tableref.hpp +2 -10
- package/src/duckdb/src/include/duckdb/parser/transformer.hpp +1 -0
- package/src/duckdb/src/include/duckdb/planner/bound_constraint.hpp +0 -1
- package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp +2 -8
- package/src/duckdb/src/include/duckdb/planner/column_binding.hpp +4 -4
- package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/expression/bound_between_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/expression/bound_case_expression.hpp +3 -9
- package/src/duckdb/src/include/duckdb/planner/expression/bound_cast_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_columnref_expression.hpp +2 -8
- package/src/duckdb/src/include/duckdb/planner/expression/bound_comparison_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/expression/bound_conjunction_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_constant_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_default_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_function_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_lambda_expression.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/expression/bound_lambdaref_expression.hpp +2 -8
- package/src/duckdb/src/include/duckdb/planner/expression/bound_operator_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp +2 -19
- package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_reference_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp +0 -3
- package/src/duckdb/src/include/duckdb/planner/expression/bound_unnest_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression/bound_window_expression.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/expression.hpp +2 -14
- package/src/duckdb/src/include/duckdb/planner/filter/conjunction_filter.hpp +4 -8
- package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/filter/null_filter.hpp +4 -8
- package/src/duckdb/src/include/duckdb/planner/joinside.hpp +1 -6
- package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +2 -17
- package/src/duckdb/src/include/duckdb/planner/operator/logical_aggregate.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_any_join.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_column_data_get.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +2 -6
- package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_create.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_create_index.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_create_table.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cross_product.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_cteref.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_delete.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_get.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +0 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_distinct.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_dummy_scan.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_empty_result.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_execute.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_explain.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_export.hpp +0 -3
- package/src/duckdb/src/include/duckdb/planner/operator/logical_expression_get.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_extension_operator.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_filter.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_join.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_limit.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_limit_percent.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_materialized_cte.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_order.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_pivot.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_positional_join.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_pragma.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_prepare.hpp +0 -2
- package/src/duckdb/src/include/duckdb/planner/operator/logical_projection.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_recursive_cte.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_reset.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_sample.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_set.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_set_operation.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_show.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_simple.hpp +2 -5
- package/src/duckdb/src/include/duckdb/planner/operator/logical_top_n.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_unnest.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_update.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator/logical_window.hpp +2 -4
- package/src/duckdb/src/include/duckdb/planner/operator_extension.hpp +1 -3
- package/src/duckdb/src/include/duckdb/planner/parsed_data/bound_create_table_info.hpp +0 -5
- package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +3 -12
- package/src/duckdb/src/include/duckdb/planner/tableref/bound_pivotref.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/block.hpp +7 -4
- package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +6 -6
- package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +18 -18
- package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp +6 -0
- package/src/duckdb/src/include/duckdb/storage/data_table.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +4 -5
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp +2 -1
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp +2 -11
- package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp +3 -5
- package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp +3 -11
- package/src/duckdb/src/include/duckdb/storage/statistics/list_stats.hpp +2 -7
- package/src/duckdb/src/include/duckdb/storage/statistics/node_statistics.hpp +0 -26
- package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +2 -7
- package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +2 -7
- package/src/duckdb/src/include/duckdb/storage/statistics/struct_stats.hpp +2 -7
- package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +4 -6
- package/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +8 -17
- package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +2 -7
- package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp +4 -7
- package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +27 -29
- package/src/duckdb/src/include/duckdb/verification/statement_verifier.hpp +0 -2
- package/src/duckdb/src/main/client_context.cpp +0 -6
- package/src/duckdb/src/main/client_verify.cpp +0 -2
- package/src/duckdb/src/main/extension/extension_helper.cpp +18 -0
- package/src/duckdb/src/main/extension/extension_install.cpp +5 -3
- package/src/duckdb/src/main/extension/extension_load.cpp +3 -3
- package/src/duckdb/src/parser/column_definition.cpp +0 -25
- package/src/duckdb/src/parser/column_list.cpp +0 -13
- package/src/duckdb/src/parser/constraint.cpp +0 -33
- package/src/duckdb/src/parser/constraints/check_constraint.cpp +0 -11
- package/src/duckdb/src/parser/constraints/foreign_key_constraint.cpp +0 -27
- package/src/duckdb/src/parser/constraints/not_null_constraint.cpp +0 -11
- package/src/duckdb/src/parser/constraints/unique_constraint.cpp +0 -24
- package/src/duckdb/src/parser/expression/between_expression.cpp +2 -16
- package/src/duckdb/src/parser/expression/case_expression.cpp +2 -29
- package/src/duckdb/src/parser/expression/cast_expression.cpp +2 -16
- package/src/duckdb/src/parser/expression/collate_expression.cpp +2 -14
- package/src/duckdb/src/parser/expression/columnref_expression.cpp +2 -13
- package/src/duckdb/src/parser/expression/comparison_expression.cpp +2 -14
- package/src/duckdb/src/parser/expression/conjunction_expression.cpp +2 -13
- package/src/duckdb/src/parser/expression/constant_expression.cpp +2 -12
- package/src/duckdb/src/parser/expression/default_expression.cpp +2 -9
- package/src/duckdb/src/parser/expression/function_expression.cpp +2 -32
- package/src/duckdb/src/parser/expression/lambda_expression.cpp +2 -14
- package/src/duckdb/src/parser/expression/operator_expression.cpp +2 -13
- package/src/duckdb/src/parser/expression/parameter_expression.cpp +2 -13
- package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +2 -12
- package/src/duckdb/src/parser/expression/star_expression.cpp +2 -43
- package/src/duckdb/src/parser/expression/subquery_expression.cpp +2 -28
- package/src/duckdb/src/parser/expression/window_expression.cpp +2 -53
- package/src/duckdb/src/parser/parsed_data/alter_info.cpp +0 -35
- package/src/duckdb/src/parser/parsed_data/alter_scalar_function_info.cpp +0 -18
- package/src/duckdb/src/parser/parsed_data/alter_table_function_info.cpp +0 -13
- package/src/duckdb/src/parser/parsed_data/alter_table_info.cpp +1 -189
- package/src/duckdb/src/parser/parsed_data/attach_info.cpp +0 -29
- package/src/duckdb/src/parser/parsed_data/create_collation_info.cpp +0 -4
- package/src/duckdb/src/parser/parsed_data/create_copy_function_info.cpp +0 -4
- package/src/duckdb/src/parser/parsed_data/create_index_info.cpp +0 -48
- package/src/duckdb/src/parser/parsed_data/create_info.cpp +0 -47
- package/src/duckdb/src/parser/parsed_data/create_macro_info.cpp +1 -28
- package/src/duckdb/src/parser/parsed_data/create_sequence_info.cpp +0 -31
- package/src/duckdb/src/parser/parsed_data/create_table_info.cpp +0 -23
- package/src/duckdb/src/parser/parsed_data/create_type_info.cpp +0 -22
- package/src/duckdb/src/parser/parsed_data/create_view_info.cpp +0 -23
- package/src/duckdb/src/parser/parsed_data/detach_info.cpp +0 -19
- package/src/duckdb/src/parser/parsed_data/drop_info.cpp +0 -26
- package/src/duckdb/src/parser/parsed_data/sample_options.cpp +2 -25
- package/src/duckdb/src/parser/parsed_data/transaction_info.cpp +0 -16
- package/src/duckdb/src/parser/parsed_data/vacuum_info.cpp +0 -20
- package/src/duckdb/src/parser/parsed_expression.cpp +2 -79
- package/src/duckdb/src/parser/query_node/cte_node.cpp +2 -19
- package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +2 -21
- package/src/duckdb/src/parser/query_node/select_node.cpp +2 -47
- package/src/duckdb/src/parser/query_node/set_operation_node.cpp +2 -17
- package/src/duckdb/src/parser/query_node.cpp +2 -62
- package/src/duckdb/src/parser/result_modifier.cpp +2 -96
- package/src/duckdb/src/parser/statement/select_statement.cpp +2 -13
- package/src/duckdb/src/parser/tableref/basetableref.cpp +2 -21
- package/src/duckdb/src/parser/tableref/emptytableref.cpp +0 -9
- package/src/duckdb/src/parser/tableref/expressionlistref.cpp +2 -28
- package/src/duckdb/src/parser/tableref/joinref.cpp +2 -23
- package/src/duckdb/src/parser/tableref/pivotref.cpp +0 -61
- package/src/duckdb/src/parser/tableref/subqueryref.cpp +2 -15
- package/src/duckdb/src/parser/tableref/table_function.cpp +2 -17
- package/src/duckdb/src/parser/tableref.cpp +2 -52
- package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +1 -4
- package/src/duckdb/src/planner/bound_result_modifier.cpp +0 -33
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +6 -30
- package/src/duckdb/src/planner/expression/bound_between_expression.cpp +0 -19
- package/src/duckdb/src/planner/expression/bound_case_expression.cpp +0 -36
- package/src/duckdb/src/planner/expression/bound_cast_expression.cpp +0 -15
- package/src/duckdb/src/planner/expression/bound_columnref_expression.cpp +0 -20
- package/src/duckdb/src/planner/expression/bound_comparison_expression.cpp +0 -13
- package/src/duckdb/src/planner/expression/bound_conjunction_expression.cpp +0 -13
- package/src/duckdb/src/planner/expression/bound_constant_expression.cpp +0 -11
- package/src/duckdb/src/planner/expression/bound_expression.cpp +1 -5
- package/src/duckdb/src/planner/expression/bound_function_expression.cpp +8 -28
- package/src/duckdb/src/planner/expression/bound_lambda_expression.cpp +0 -4
- package/src/duckdb/src/planner/expression/bound_lambdaref_expression.cpp +0 -23
- package/src/duckdb/src/planner/expression/bound_operator_expression.cpp +0 -16
- package/src/duckdb/src/planner/expression/bound_parameter_expression.cpp +0 -18
- package/src/duckdb/src/planner/expression/bound_reference_expression.cpp +0 -16
- package/src/duckdb/src/planner/expression/bound_subquery_expression.cpp +0 -4
- package/src/duckdb/src/planner/expression/bound_unnest_expression.cpp +0 -15
- package/src/duckdb/src/planner/expression/bound_window_expression.cpp +7 -60
- package/src/duckdb/src/planner/expression.cpp +0 -74
- package/src/duckdb/src/planner/filter/conjunction_filter.cpp +0 -21
- package/src/duckdb/src/planner/filter/constant_filter.cpp +0 -12
- package/src/duckdb/src/planner/filter/null_filter.cpp +0 -14
- package/src/duckdb/src/planner/joinside.cpp +0 -24
- package/src/duckdb/src/planner/logical_operator.cpp +15 -215
- package/src/duckdb/src/planner/operator/logical_aggregate.cpp +0 -46
- package/src/duckdb/src/planner/operator/logical_any_join.cpp +0 -14
- package/src/duckdb/src/planner/operator/logical_column_data_get.cpp +0 -23
- package/src/duckdb/src/planner/operator/logical_comparison_join.cpp +0 -23
- package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp +2 -56
- package/src/duckdb/src/planner/operator/logical_create.cpp +0 -12
- package/src/duckdb/src/planner/operator/logical_create_index.cpp +0 -25
- package/src/duckdb/src/planner/operator/logical_create_table.cpp +0 -10
- package/src/duckdb/src/planner/operator/logical_cross_product.cpp +0 -9
- package/src/duckdb/src/planner/operator/logical_cteref.cpp +0 -18
- package/src/duckdb/src/planner/operator/logical_delete.cpp +0 -22
- package/src/duckdb/src/planner/operator/logical_delim_get.cpp +0 -12
- package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +0 -4
- package/src/duckdb/src/planner/operator/logical_distinct.cpp +0 -16
- package/src/duckdb/src/planner/operator/logical_dummy_scan.cpp +0 -10
- package/src/duckdb/src/planner/operator/logical_empty_result.cpp +0 -15
- package/src/duckdb/src/planner/operator/logical_expression_get.cpp +0 -24
- package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +6 -19
- package/src/duckdb/src/planner/operator/logical_filter.cpp +0 -15
- package/src/duckdb/src/planner/operator/logical_get.cpp +12 -97
- package/src/duckdb/src/planner/operator/logical_insert.cpp +0 -77
- package/src/duckdb/src/planner/operator/logical_join.cpp +0 -17
- package/src/duckdb/src/planner/operator/logical_limit.cpp +0 -16
- package/src/duckdb/src/planner/operator/logical_limit_percent.cpp +0 -16
- package/src/duckdb/src/planner/operator/logical_materialized_cte.cpp +0 -13
- package/src/duckdb/src/planner/operator/logical_order.cpp +0 -15
- package/src/duckdb/src/planner/operator/logical_pivot.cpp +0 -20
- package/src/duckdb/src/planner/operator/logical_positional_join.cpp +0 -10
- package/src/duckdb/src/planner/operator/logical_pragma.cpp +0 -8
- package/src/duckdb/src/planner/operator/logical_prepare.cpp +0 -8
- package/src/duckdb/src/planner/operator/logical_projection.cpp +0 -12
- package/src/duckdb/src/planner/operator/logical_recursive_cte.cpp +0 -15
- package/src/duckdb/src/planner/operator/logical_reset.cpp +0 -12
- package/src/duckdb/src/planner/operator/logical_sample.cpp +0 -11
- package/src/duckdb/src/planner/operator/logical_set.cpp +0 -14
- package/src/duckdb/src/planner/operator/logical_set_operation.cpp +0 -13
- package/src/duckdb/src/planner/operator/logical_simple.cpp +0 -60
- package/src/duckdb/src/planner/operator/logical_top_n.cpp +0 -14
- package/src/duckdb/src/planner/operator/logical_unnest.cpp +0 -14
- package/src/duckdb/src/planner/operator/logical_update.cpp +0 -28
- package/src/duckdb/src/planner/operator/logical_window.cpp +0 -13
- package/src/duckdb/src/planner/planner.cpp +13 -22
- package/src/duckdb/src/planner/table_filter.cpp +0 -59
- package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp +7 -17
- package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +7 -4
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +24 -22
- package/src/duckdb/src/storage/checkpoint_manager.cpp +203 -180
- package/src/duckdb/src/storage/data_table.cpp +2 -2
- package/src/duckdb/src/storage/metadata/metadata_manager.cpp +11 -9
- package/src/duckdb/src/storage/serialization/serialize_constraint.cpp +20 -20
- package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +46 -42
- package/src/duckdb/src/storage/serialization/serialize_expression.cpp +63 -63
- package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +167 -165
- package/src/duckdb/src/storage/serialization/serialize_macro_function.cpp +12 -12
- package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +54 -54
- package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +88 -88
- package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +72 -72
- package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +20 -20
- package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +22 -22
- package/src/duckdb/src/storage/serialization/serialize_statement.cpp +4 -4
- package/src/duckdb/src/storage/serialization/serialize_storage.cpp +41 -6
- package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +24 -24
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +32 -32
- package/src/duckdb/src/storage/serialization/serialize_types.cpp +29 -29
- package/src/duckdb/src/storage/single_file_block_manager.cpp +17 -22
- package/src/duckdb/src/storage/statistics/base_statistics.cpp +31 -85
- package/src/duckdb/src/storage/statistics/column_statistics.cpp +6 -19
- package/src/duckdb/src/storage/statistics/distinct_statistics.cpp +0 -26
- package/src/duckdb/src/storage/statistics/list_stats.cpp +6 -22
- package/src/duckdb/src/storage/statistics/numeric_stats.cpp +20 -135
- package/src/duckdb/src/storage/statistics/string_stats.cpp +5 -28
- package/src/duckdb/src/storage/statistics/struct_stats.cpp +8 -30
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/chunk_info.cpp +9 -88
- package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +2 -2
- package/src/duckdb/src/storage/table/column_data.cpp +20 -25
- package/src/duckdb/src/storage/table/list_column_data.cpp +16 -9
- package/src/duckdb/src/storage/table/row_group.cpp +16 -92
- package/src/duckdb/src/storage/table/row_group_collection.cpp +5 -1
- package/src/duckdb/src/storage/table/standard_column_data.cpp +10 -7
- package/src/duckdb/src/storage/table/struct_column_data.cpp +17 -11
- package/src/duckdb/src/storage/table/table_statistics.cpp +13 -22
- package/src/duckdb/src/storage/wal_replay.cpp +108 -106
- package/src/duckdb/src/storage/write_ahead_log.cpp +135 -64
- package/src/duckdb/src/transaction/commit_state.cpp +22 -8
- package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +11 -6
- package/src/duckdb/src/verification/statement_verifier.cpp +0 -5
- package/src/duckdb/third_party/parquet/parquet_types.h +1 -1
- package/src/duckdb/ub_src_common.cpp +0 -4
- package/src/duckdb/ub_src_common_serializer.cpp +2 -4
- package/src/duckdb/ub_src_planner.cpp +0 -2
- package/src/duckdb/ub_src_planner_expression.cpp +0 -2
- package/src/duckdb/ub_src_planner_operator.cpp +0 -8
- package/src/duckdb_node.cpp +6 -12
- package/src/duckdb_node.hpp +23 -13
- package/src/statement.cpp +15 -16
- package/test/database_fail.test.ts +20 -0
- package/test/worker.js +7 -0
- package/src/duckdb/src/common/field_writer.cpp +0 -97
- package/src/duckdb/src/common/serializer/buffered_deserializer.cpp +0 -27
- package/src/duckdb/src/common/serializer/buffered_serializer.cpp +0 -36
- package/src/duckdb/src/common/serializer.cpp +0 -24
- package/src/duckdb/src/include/duckdb/common/field_writer.hpp +0 -379
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_deserializer.hpp +0 -41
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_serializer.hpp +0 -50
- package/src/duckdb/src/include/duckdb/common/serializer.hpp +0 -151
- package/src/duckdb/src/include/duckdb/planner/plan_serialization.hpp +0 -44
- package/src/duckdb/src/include/duckdb/verification/deserialized_statement_verifier_v2.hpp +0 -32
- package/src/duckdb/src/planner/expression/bound_default_expression.cpp +0 -16
- package/src/duckdb/src/planner/operator/logical_execute.cpp +0 -12
- package/src/duckdb/src/planner/operator/logical_explain.cpp +0 -22
- package/src/duckdb/src/planner/operator/logical_export.cpp +0 -13
- package/src/duckdb/src/planner/operator/logical_show.cpp +0 -21
- package/src/duckdb/src/planner/parsed_data/bound_create_table_info.cpp +0 -22
- package/src/duckdb/src/planner/plan_serialization.cpp +0 -20
- package/src/duckdb/src/verification/deserialized_statement_verifier_v2.cpp +0 -34
- package/src/duckdb/ub_src_planner_parsed_data.cpp +0 -2
@@ -10,7 +10,6 @@
|
|
10
10
|
#include "duckdb/common/mutex.hpp"
|
11
11
|
#include "duckdb/common/operator/comparison_operators.hpp"
|
12
12
|
#include "duckdb/common/serializer/buffered_file_writer.hpp"
|
13
|
-
#include "duckdb/common/serializer/buffered_serializer.hpp"
|
14
13
|
#include "duckdb/common/string_map_set.hpp"
|
15
14
|
#include "duckdb/common/types/chunk_collection.hpp"
|
16
15
|
#include "duckdb/common/types/date.hpp"
|
@@ -18,6 +17,8 @@
|
|
18
17
|
#include "duckdb/common/types/string_heap.hpp"
|
19
18
|
#include "duckdb/common/types/time.hpp"
|
20
19
|
#include "duckdb/common/types/timestamp.hpp"
|
20
|
+
#include "duckdb/common/serializer/write_stream.hpp"
|
21
|
+
#include "duckdb/common/serializer/memory_stream.hpp"
|
21
22
|
#endif
|
22
23
|
|
23
24
|
#include "miniz_wrapper.hpp"
|
@@ -41,7 +42,7 @@ using duckdb_parquet::format::Type;
|
|
41
42
|
|
42
43
|
#define PARQUET_DEFINE_VALID 65535
|
43
44
|
|
44
|
-
static void VarintEncode(uint32_t val,
|
45
|
+
static void VarintEncode(uint32_t val, WriteStream &ser) {
|
45
46
|
do {
|
46
47
|
uint8_t byte = val & 127;
|
47
48
|
val >>= 7;
|
@@ -124,13 +125,13 @@ idx_t RleBpEncoder::GetByteCount() {
|
|
124
125
|
return byte_count;
|
125
126
|
}
|
126
127
|
|
127
|
-
void RleBpEncoder::BeginWrite(
|
128
|
+
void RleBpEncoder::BeginWrite(WriteStream &writer, uint32_t first_value) {
|
128
129
|
// start the RLE runs
|
129
130
|
last_value = first_value;
|
130
131
|
current_run_count = 1;
|
131
132
|
}
|
132
133
|
|
133
|
-
void RleBpEncoder::WriteRun(
|
134
|
+
void RleBpEncoder::WriteRun(WriteStream &writer) {
|
134
135
|
// write the header of the run
|
135
136
|
VarintEncode(current_run_count << 1, writer);
|
136
137
|
// now write the value
|
@@ -156,7 +157,7 @@ void RleBpEncoder::WriteRun(Serializer &writer) {
|
|
156
157
|
current_run_count = 1;
|
157
158
|
}
|
158
159
|
|
159
|
-
void RleBpEncoder::WriteValue(
|
160
|
+
void RleBpEncoder::WriteValue(WriteStream &writer, uint32_t value) {
|
160
161
|
if (value != last_value) {
|
161
162
|
WriteRun(writer);
|
162
163
|
last_value = value;
|
@@ -165,7 +166,7 @@ void RleBpEncoder::WriteValue(Serializer &writer, uint32_t value) {
|
|
165
166
|
}
|
166
167
|
}
|
167
168
|
|
168
|
-
void RleBpEncoder::FinishWrite(
|
169
|
+
void RleBpEncoder::FinishWrite(WriteStream &writer) {
|
169
170
|
WriteRun(writer);
|
170
171
|
}
|
171
172
|
|
@@ -183,36 +184,36 @@ ColumnWriter::~ColumnWriter() {
|
|
183
184
|
ColumnWriterState::~ColumnWriterState() {
|
184
185
|
}
|
185
186
|
|
186
|
-
void ColumnWriter::CompressPage(
|
187
|
+
void ColumnWriter::CompressPage(MemoryStream &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
|
187
188
|
unique_ptr<data_t[]> &compressed_buf) {
|
188
189
|
switch (writer.GetCodec()) {
|
189
190
|
case CompressionCodec::UNCOMPRESSED:
|
190
|
-
compressed_size = temp_writer.
|
191
|
-
compressed_data = temp_writer.
|
191
|
+
compressed_size = temp_writer.GetPosition();
|
192
|
+
compressed_data = temp_writer.GetData();
|
192
193
|
break;
|
193
194
|
case CompressionCodec::SNAPPY: {
|
194
|
-
compressed_size = duckdb_snappy::MaxCompressedLength(temp_writer.
|
195
|
+
compressed_size = duckdb_snappy::MaxCompressedLength(temp_writer.GetPosition());
|
195
196
|
compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
|
196
|
-
duckdb_snappy::RawCompress(const_char_ptr_cast(temp_writer.
|
197
|
+
duckdb_snappy::RawCompress(const_char_ptr_cast(temp_writer.GetData()), temp_writer.GetPosition(),
|
197
198
|
char_ptr_cast(compressed_buf.get()), &compressed_size);
|
198
199
|
compressed_data = compressed_buf.get();
|
199
|
-
D_ASSERT(compressed_size <= duckdb_snappy::MaxCompressedLength(temp_writer.
|
200
|
+
D_ASSERT(compressed_size <= duckdb_snappy::MaxCompressedLength(temp_writer.GetPosition()));
|
200
201
|
break;
|
201
202
|
}
|
202
203
|
case CompressionCodec::GZIP: {
|
203
204
|
MiniZStream s;
|
204
|
-
compressed_size = s.MaxCompressedLength(temp_writer.
|
205
|
+
compressed_size = s.MaxCompressedLength(temp_writer.GetPosition());
|
205
206
|
compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
|
206
|
-
s.Compress(const_char_ptr_cast(temp_writer.
|
207
|
+
s.Compress(const_char_ptr_cast(temp_writer.GetData()), temp_writer.GetPosition(),
|
207
208
|
char_ptr_cast(compressed_buf.get()), &compressed_size);
|
208
209
|
compressed_data = compressed_buf.get();
|
209
210
|
break;
|
210
211
|
}
|
211
212
|
case CompressionCodec::ZSTD: {
|
212
|
-
compressed_size = duckdb_zstd::ZSTD_compressBound(temp_writer.
|
213
|
+
compressed_size = duckdb_zstd::ZSTD_compressBound(temp_writer.GetPosition());
|
213
214
|
compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
|
214
215
|
compressed_size = duckdb_zstd::ZSTD_compress((void *)compressed_buf.get(), compressed_size,
|
215
|
-
(const void *)temp_writer.
|
216
|
+
(const void *)temp_writer.GetData(), temp_writer.GetPosition(),
|
216
217
|
ZSTD_CLEVEL_DEFAULT);
|
217
218
|
compressed_data = compressed_buf.get();
|
218
219
|
break;
|
@@ -223,7 +224,7 @@ void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compres
|
|
223
224
|
|
224
225
|
if (compressed_size > idx_t(NumericLimits<int32_t>::Maximum())) {
|
225
226
|
throw InternalException("Parquet writer: %d compressed page size out of range for type integer",
|
226
|
-
temp_writer.
|
227
|
+
temp_writer.GetPosition());
|
227
228
|
}
|
228
229
|
}
|
229
230
|
|
@@ -303,7 +304,7 @@ struct PageInformation {
|
|
303
304
|
|
304
305
|
struct PageWriteInformation {
|
305
306
|
PageHeader page_header;
|
306
|
-
unique_ptr<
|
307
|
+
unique_ptr<MemoryStream> temp_writer;
|
307
308
|
unique_ptr<ColumnWriterPageState> page_state;
|
308
309
|
idx_t write_page_idx = 0;
|
309
310
|
idx_t write_count = 0;
|
@@ -362,7 +363,7 @@ public:
|
|
362
363
|
void FinalizeWrite(ColumnWriterState &state) override;
|
363
364
|
|
364
365
|
protected:
|
365
|
-
void WriteLevels(
|
366
|
+
void WriteLevels(WriteStream &temp_writer, const vector<uint16_t> &levels, idx_t max_value, idx_t start_offset,
|
366
367
|
idx_t count);
|
367
368
|
|
368
369
|
virtual duckdb_parquet::format::Encoding::type GetEncoding(BasicColumnWriterState &state);
|
@@ -377,12 +378,12 @@ protected:
|
|
377
378
|
virtual unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state);
|
378
379
|
|
379
380
|
//! Flushes the writer for a specific page. Only used for scalar types.
|
380
|
-
virtual void FlushPageState(
|
381
|
+
virtual void FlushPageState(WriteStream &temp_writer, ColumnWriterPageState *state);
|
381
382
|
|
382
383
|
//! Retrieves the row size of a vector at the specified location. Only used for scalar types.
|
383
384
|
virtual idx_t GetRowSize(Vector &vector, idx_t index, BasicColumnWriterState &state);
|
384
385
|
//! Writes a (subset of a) vector to the specified serializer. Only used for scalar types.
|
385
|
-
virtual void WriteVector(
|
386
|
+
virtual void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats, ColumnWriterPageState *page_state,
|
386
387
|
Vector &vector, idx_t chunk_start, idx_t chunk_end) = 0;
|
387
388
|
|
388
389
|
virtual bool HasDictionary(BasicColumnWriterState &state_p) {
|
@@ -390,7 +391,7 @@ protected:
|
|
390
391
|
}
|
391
392
|
//! The number of elements in the dictionary
|
392
393
|
virtual idx_t DictionarySize(BasicColumnWriterState &state_p);
|
393
|
-
void WriteDictionary(BasicColumnWriterState &state, unique_ptr<
|
394
|
+
void WriteDictionary(BasicColumnWriterState &state, unique_ptr<MemoryStream> temp_writer, idx_t row_count);
|
394
395
|
virtual void FlushDictionary(BasicColumnWriterState &state, ColumnWriterStatistics *stats);
|
395
396
|
|
396
397
|
void SetParquetStatistics(BasicColumnWriterState &state, duckdb_parquet::format::ColumnChunk &column);
|
@@ -417,7 +418,7 @@ unique_ptr<ColumnWriterPageState> BasicColumnWriter::InitializePageState(BasicCo
|
|
417
418
|
return nullptr;
|
418
419
|
}
|
419
420
|
|
420
|
-
void BasicColumnWriter::FlushPageState(
|
421
|
+
void BasicColumnWriter::FlushPageState(WriteStream &temp_writer, ColumnWriterPageState *state) {
|
421
422
|
}
|
422
423
|
|
423
424
|
void BasicColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count) {
|
@@ -481,7 +482,7 @@ void BasicColumnWriter::BeginWrite(ColumnWriterState &state_p) {
|
|
481
482
|
hdr.data_page_header.definition_level_encoding = Encoding::RLE;
|
482
483
|
hdr.data_page_header.repetition_level_encoding = Encoding::RLE;
|
483
484
|
|
484
|
-
write_info.temp_writer = make_uniq<
|
485
|
+
write_info.temp_writer = make_uniq<MemoryStream>();
|
485
486
|
write_info.write_count = page_info.empty_count;
|
486
487
|
write_info.max_write_count = page_info.row_count;
|
487
488
|
write_info.page_state = InitializePageState(state);
|
@@ -496,7 +497,7 @@ void BasicColumnWriter::BeginWrite(ColumnWriterState &state_p) {
|
|
496
497
|
NextPage(state);
|
497
498
|
}
|
498
499
|
|
499
|
-
void BasicColumnWriter::WriteLevels(
|
500
|
+
void BasicColumnWriter::WriteLevels(WriteStream &temp_writer, const vector<uint16_t> &levels, idx_t max_value,
|
500
501
|
idx_t offset, idx_t count) {
|
501
502
|
if (levels.empty() || count == 0) {
|
502
503
|
return;
|
@@ -557,11 +558,11 @@ void BasicColumnWriter::FlushPage(BasicColumnWriterState &state) {
|
|
557
558
|
FlushPageState(temp_writer, write_info.page_state.get());
|
558
559
|
|
559
560
|
// now that we have finished writing the data we know the uncompressed size
|
560
|
-
if (temp_writer.
|
561
|
+
if (temp_writer.GetPosition() > idx_t(NumericLimits<int32_t>::Maximum())) {
|
561
562
|
throw InternalException("Parquet writer: %d uncompressed page size out of range for type integer",
|
562
|
-
temp_writer.
|
563
|
+
temp_writer.GetPosition());
|
563
564
|
}
|
564
|
-
hdr.uncompressed_page_size = temp_writer.
|
565
|
+
hdr.uncompressed_page_size = temp_writer.GetPosition();
|
565
566
|
|
566
567
|
// compress the data
|
567
568
|
CompressPage(temp_writer, write_info.compressed_size, write_info.compressed_data, write_info.compressed_buf);
|
@@ -695,16 +696,16 @@ idx_t BasicColumnWriter::DictionarySize(BasicColumnWriterState &state) {
|
|
695
696
|
throw InternalException("This page does not have a dictionary");
|
696
697
|
}
|
697
698
|
|
698
|
-
void BasicColumnWriter::WriteDictionary(BasicColumnWriterState &state, unique_ptr<
|
699
|
+
void BasicColumnWriter::WriteDictionary(BasicColumnWriterState &state, unique_ptr<MemoryStream> temp_writer,
|
699
700
|
idx_t row_count) {
|
700
701
|
D_ASSERT(temp_writer);
|
701
|
-
D_ASSERT(temp_writer->
|
702
|
+
D_ASSERT(temp_writer->GetPosition() > 0);
|
702
703
|
|
703
704
|
// write the dictionary page header
|
704
705
|
PageWriteInformation write_info;
|
705
706
|
// set up the header
|
706
707
|
auto &hdr = write_info.page_header;
|
707
|
-
hdr.uncompressed_page_size = temp_writer->
|
708
|
+
hdr.uncompressed_page_size = temp_writer->GetPosition();
|
708
709
|
hdr.type = PageType::DICTIONARY_PAGE;
|
709
710
|
hdr.__isset.dictionary_page_header = true;
|
710
711
|
|
@@ -813,7 +814,7 @@ struct ParquetHugeintOperator {
|
|
813
814
|
|
814
815
|
template <class SRC, class TGT, class OP = ParquetCastOperator>
|
815
816
|
static void TemplatedWritePlain(Vector &col, ColumnWriterStatistics *stats, idx_t chunk_start, idx_t chunk_end,
|
816
|
-
ValidityMask &mask,
|
817
|
+
ValidityMask &mask, WriteStream &ser) {
|
817
818
|
auto *ptr = FlatVector::GetData<SRC>(col);
|
818
819
|
for (idx_t r = chunk_start; r < chunk_end; r++) {
|
819
820
|
if (mask.RowIsValid(r)) {
|
@@ -838,7 +839,7 @@ public:
|
|
838
839
|
return OP::template InitializeStats<SRC, TGT>();
|
839
840
|
}
|
840
841
|
|
841
|
-
void WriteVector(
|
842
|
+
void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats, ColumnWriterPageState *page_state,
|
842
843
|
Vector &input_column, idx_t chunk_start, idx_t chunk_end) override {
|
843
844
|
auto &mask = FlatVector::Validity(input_column);
|
844
845
|
TemplatedWritePlain<SRC, TGT, OP>(input_column, stats, chunk_start, chunk_end, mask, temp_writer);
|
@@ -898,7 +899,7 @@ public:
|
|
898
899
|
return make_uniq<BooleanStatisticsState>();
|
899
900
|
}
|
900
901
|
|
901
|
-
void WriteVector(
|
902
|
+
void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats_p, ColumnWriterPageState *state_p,
|
902
903
|
Vector &input_column, idx_t chunk_start, idx_t chunk_end) override {
|
903
904
|
auto &stats = stats_p->Cast<BooleanStatisticsState>();
|
904
905
|
auto &state = state_p->Cast<BooleanWriterPageState>();
|
@@ -929,7 +930,7 @@ public:
|
|
929
930
|
return make_uniq<BooleanWriterPageState>();
|
930
931
|
}
|
931
932
|
|
932
|
-
void FlushPageState(
|
933
|
+
void FlushPageState(WriteStream &temp_writer, ColumnWriterPageState *state_p) override {
|
933
934
|
auto &state = state_p->Cast<BooleanWriterPageState>();
|
934
935
|
if (state.byte_pos > 0) {
|
935
936
|
temp_writer.Write<uint8_t>(state.byte);
|
@@ -1023,7 +1024,7 @@ public:
|
|
1023
1024
|
return make_uniq<FixedDecimalStatistics>();
|
1024
1025
|
}
|
1025
1026
|
|
1026
|
-
void WriteVector(
|
1027
|
+
void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats_p, ColumnWriterPageState *page_state,
|
1027
1028
|
Vector &input_column, idx_t chunk_start, idx_t chunk_end) override {
|
1028
1029
|
auto &mask = FlatVector::Validity(input_column);
|
1029
1030
|
auto *ptr = FlatVector::GetData<hugeint_t>(input_column);
|
@@ -1072,7 +1073,7 @@ public:
|
|
1072
1073
|
}
|
1073
1074
|
}
|
1074
1075
|
|
1075
|
-
void WriteVector(
|
1076
|
+
void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats_p, ColumnWriterPageState *page_state,
|
1076
1077
|
Vector &input_column, idx_t chunk_start, idx_t chunk_end) override {
|
1077
1078
|
auto &mask = FlatVector::Validity(input_column);
|
1078
1079
|
auto *ptr = FlatVector::GetData<hugeint_t>(input_column);
|
@@ -1114,7 +1115,7 @@ public:
|
|
1114
1115
|
Store<uint32_t>(input.micros / 1000, result + sizeof(uint32_t) * 2);
|
1115
1116
|
}
|
1116
1117
|
|
1117
|
-
void WriteVector(
|
1118
|
+
void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats_p, ColumnWriterPageState *page_state,
|
1118
1119
|
Vector &input_column, idx_t chunk_start, idx_t chunk_end) override {
|
1119
1120
|
auto &mask = FlatVector::Validity(input_column);
|
1120
1121
|
auto *ptr = FlatVector::GetData<interval_t>(input_column);
|
@@ -1313,7 +1314,7 @@ public:
|
|
1313
1314
|
}
|
1314
1315
|
}
|
1315
1316
|
|
1316
|
-
void WriteVector(
|
1317
|
+
void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats_p, ColumnWriterPageState *page_state_p,
|
1317
1318
|
Vector &input_column, idx_t chunk_start, idx_t chunk_end) override {
|
1318
1319
|
auto &page_state = page_state_p->Cast<StringWriterPageState>();
|
1319
1320
|
auto &mask = FlatVector::Validity(input_column);
|
@@ -1356,7 +1357,7 @@ public:
|
|
1356
1357
|
return make_uniq<StringWriterPageState>(state.key_bit_width, state.dictionary);
|
1357
1358
|
}
|
1358
1359
|
|
1359
|
-
void FlushPageState(
|
1360
|
+
void FlushPageState(WriteStream &temp_writer, ColumnWriterPageState *state_p) override {
|
1360
1361
|
auto &page_state = state_p->Cast<StringWriterPageState>();
|
1361
1362
|
if (page_state.bit_width != 0) {
|
1362
1363
|
if (!page_state.written_value) {
|
@@ -1398,7 +1399,7 @@ public:
|
|
1398
1399
|
values[entry.second] = entry.first;
|
1399
1400
|
}
|
1400
1401
|
// first write the contents of the dictionary page to a temporary buffer
|
1401
|
-
auto temp_writer = make_uniq<
|
1402
|
+
auto temp_writer = make_uniq<MemoryStream>();
|
1402
1403
|
for (idx_t r = 0; r < values.size(); r++) {
|
1403
1404
|
auto &value = values[r];
|
1404
1405
|
// update the statistics
|
@@ -1453,7 +1454,7 @@ public:
|
|
1453
1454
|
}
|
1454
1455
|
|
1455
1456
|
template <class T>
|
1456
|
-
void WriteEnumInternal(
|
1457
|
+
void WriteEnumInternal(WriteStream &temp_writer, Vector &input_column, idx_t chunk_start, idx_t chunk_end,
|
1457
1458
|
EnumWriterPageState &page_state) {
|
1458
1459
|
auto &mask = FlatVector::Validity(input_column);
|
1459
1460
|
auto *ptr = FlatVector::GetData<T>(input_column);
|
@@ -1473,7 +1474,7 @@ public:
|
|
1473
1474
|
}
|
1474
1475
|
}
|
1475
1476
|
|
1476
|
-
void WriteVector(
|
1477
|
+
void WriteVector(WriteStream &temp_writer, ColumnWriterStatistics *stats_p, ColumnWriterPageState *page_state_p,
|
1477
1478
|
Vector &input_column, idx_t chunk_start, idx_t chunk_end) override {
|
1478
1479
|
auto &page_state = page_state_p->Cast<EnumWriterPageState>();
|
1479
1480
|
switch (enum_type.InternalType()) {
|
@@ -1495,7 +1496,7 @@ public:
|
|
1495
1496
|
return make_uniq<EnumWriterPageState>(bit_width);
|
1496
1497
|
}
|
1497
1498
|
|
1498
|
-
void FlushPageState(
|
1499
|
+
void FlushPageState(WriteStream &temp_writer, ColumnWriterPageState *state_p) override {
|
1499
1500
|
auto &page_state = state_p->Cast<EnumWriterPageState>();
|
1500
1501
|
if (!page_state.written_value) {
|
1501
1502
|
// all values are null
|
@@ -1525,7 +1526,7 @@ public:
|
|
1525
1526
|
auto enum_count = EnumType::GetSize(enum_type);
|
1526
1527
|
auto string_values = FlatVector::GetData<string_t>(enum_values);
|
1527
1528
|
// first write the contents of the dictionary page to a temporary buffer
|
1528
|
-
auto temp_writer = make_uniq<
|
1529
|
+
auto temp_writer = make_uniq<MemoryStream>();
|
1529
1530
|
for (idx_t r = 0; r < enum_count; r++) {
|
1530
1531
|
D_ASSERT(!FlatVector::IsNull(enum_values, r));
|
1531
1532
|
// update the statistics
|
@@ -12,7 +12,7 @@
|
|
12
12
|
#include "parquet_types.h"
|
13
13
|
|
14
14
|
namespace duckdb {
|
15
|
-
class
|
15
|
+
class MemoryStream;
|
16
16
|
class ParquetWriter;
|
17
17
|
class ColumnWriterPageState;
|
18
18
|
class BasicColumnWriterState;
|
@@ -113,7 +113,7 @@ protected:
|
|
113
113
|
uint16_t define_value, uint16_t null_value);
|
114
114
|
void HandleRepeatLevels(ColumnWriterState &state_p, ColumnWriterState *parent, idx_t count, idx_t max_repeat);
|
115
115
|
|
116
|
-
void CompressPage(
|
116
|
+
void CompressPage(MemoryStream &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
|
117
117
|
unique_ptr<data_t[]> &compressed_buf);
|
118
118
|
};
|
119
119
|
|
@@ -74,11 +74,8 @@ struct ParquetOptions {
|
|
74
74
|
MultiFileReaderOptions file_options;
|
75
75
|
|
76
76
|
public:
|
77
|
-
void Serialize(
|
78
|
-
|
79
|
-
|
80
|
-
void FormatSerialize(FormatSerializer &serializer) const;
|
81
|
-
static ParquetOptions FormatDeserialize(FormatDeserializer &deserializer);
|
77
|
+
void Serialize(Serializer &serializer) const;
|
78
|
+
static ParquetOptions Deserialize(Deserializer &deserializer);
|
82
79
|
};
|
83
80
|
|
84
81
|
class ParquetReader {
|
@@ -26,9 +26,9 @@ public:
|
|
26
26
|
void PrepareValue(uint32_t value);
|
27
27
|
void FinishPrepare();
|
28
28
|
|
29
|
-
void BeginWrite(
|
30
|
-
void WriteValue(
|
31
|
-
void FinishWrite(
|
29
|
+
void BeginWrite(WriteStream &writer, uint32_t first_value);
|
30
|
+
void WriteValue(WriteStream &writer, uint32_t value);
|
31
|
+
void FinishWrite(WriteStream &writer);
|
32
32
|
|
33
33
|
idx_t GetByteCount();
|
34
34
|
|
@@ -43,7 +43,7 @@ private:
|
|
43
43
|
|
44
44
|
private:
|
45
45
|
void FinishRun();
|
46
|
-
void WriteRun(
|
46
|
+
void WriteRun(WriteStream &writer);
|
47
47
|
};
|
48
48
|
|
49
49
|
} // namespace duckdb
|
@@ -18,11 +18,8 @@
|
|
18
18
|
#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
|
19
19
|
#include "duckdb/common/constants.hpp"
|
20
20
|
#include "duckdb/common/enums/file_compression_type.hpp"
|
21
|
-
#include "duckdb/common/field_writer.hpp"
|
22
21
|
#include "duckdb/common/file_system.hpp"
|
23
22
|
#include "duckdb/common/multi_file_reader.hpp"
|
24
|
-
#include "duckdb/common/serializer/format_deserializer.hpp"
|
25
|
-
#include "duckdb/common/serializer/format_serializer.hpp"
|
26
23
|
#include "duckdb/common/types/chunk_collection.hpp"
|
27
24
|
#include "duckdb/function/copy_function.hpp"
|
28
25
|
#include "duckdb/function/table_function.hpp"
|
@@ -37,7 +34,8 @@
|
|
37
34
|
#include "duckdb/planner/operator/logical_get.hpp"
|
38
35
|
#include "duckdb/storage/statistics/base_statistics.hpp"
|
39
36
|
#include "duckdb/storage/table/row_group.hpp"
|
40
|
-
|
37
|
+
#include "duckdb/common/serializer/serializer.hpp"
|
38
|
+
#include "duckdb/common/serializer/deserializer.hpp"
|
41
39
|
#endif
|
42
40
|
|
43
41
|
namespace duckdb {
|
@@ -143,18 +141,6 @@ struct ParquetWriteLocalState : public LocalFunctionData {
|
|
143
141
|
ColumnDataAppendState append_state;
|
144
142
|
};
|
145
143
|
|
146
|
-
void ParquetOptions::Serialize(FieldWriter &writer) const {
|
147
|
-
writer.WriteField<bool>(binary_as_string);
|
148
|
-
writer.WriteField<bool>(file_row_number);
|
149
|
-
writer.WriteSerializable(file_options);
|
150
|
-
}
|
151
|
-
|
152
|
-
void ParquetOptions::Deserialize(FieldReader &reader) {
|
153
|
-
binary_as_string = reader.ReadRequired<bool>();
|
154
|
-
file_row_number = reader.ReadRequired<bool>();
|
155
|
-
file_options = reader.ReadRequiredSerializable<MultiFileReaderOptions, MultiFileReaderOptions>();
|
156
|
-
}
|
157
|
-
|
158
144
|
BindInfo ParquetGetBatchInfo(const FunctionData *bind_data) {
|
159
145
|
auto bind_info = BindInfo(ScanType::PARQUET);
|
160
146
|
auto &parquet_bind = bind_data->Cast<ParquetReadBindData>();
|
@@ -186,8 +172,6 @@ public:
|
|
186
172
|
table_function.get_batch_index = ParquetScanGetBatchIndex;
|
187
173
|
table_function.serialize = ParquetScanSerialize;
|
188
174
|
table_function.deserialize = ParquetScanDeserialize;
|
189
|
-
table_function.format_serialize = ParquetScanFormatSerialize;
|
190
|
-
table_function.format_deserialize = ParquetScanFormatDeserialize;
|
191
175
|
table_function.get_batch_info = ParquetGetBatchInfo;
|
192
176
|
table_function.projection_pushdown = true;
|
193
177
|
table_function.filter_pushdown = true;
|
@@ -420,38 +404,16 @@ public:
|
|
420
404
|
return data.batch_index;
|
421
405
|
}
|
422
406
|
|
423
|
-
static void ParquetScanSerialize(
|
407
|
+
static void ParquetScanSerialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data_p,
|
424
408
|
const TableFunction &function) {
|
425
409
|
auto &bind_data = bind_data_p->Cast<ParquetReadBindData>();
|
426
|
-
writer.WriteList<string>(bind_data.files);
|
427
|
-
writer.WriteRegularSerializableList(bind_data.types);
|
428
|
-
writer.WriteList<string>(bind_data.names);
|
429
|
-
bind_data.parquet_options.Serialize(writer);
|
430
|
-
}
|
431
|
-
|
432
|
-
static unique_ptr<FunctionData> ParquetScanDeserialize(PlanDeserializationState &state, FieldReader &reader,
|
433
|
-
TableFunction &function) {
|
434
|
-
auto &context = state.context;
|
435
|
-
auto files = reader.ReadRequiredList<string>();
|
436
|
-
auto types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
|
437
|
-
auto names = reader.ReadRequiredList<string>();
|
438
|
-
ParquetOptions options(context);
|
439
|
-
options.Deserialize(reader);
|
440
|
-
|
441
|
-
return ParquetScanBindInternal(context, files, types, names, options);
|
442
|
-
}
|
443
|
-
|
444
|
-
static void ParquetScanFormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
|
445
|
-
const TableFunction &function) {
|
446
|
-
auto &bind_data = bind_data_p->Cast<ParquetReadBindData>();
|
447
410
|
serializer.WriteProperty(100, "files", bind_data.files);
|
448
411
|
serializer.WriteProperty(101, "types", bind_data.types);
|
449
412
|
serializer.WriteProperty(102, "names", bind_data.names);
|
450
413
|
serializer.WriteProperty(103, "parquet_options", bind_data.parquet_options);
|
451
414
|
}
|
452
415
|
|
453
|
-
static unique_ptr<FunctionData>
|
454
|
-
TableFunction &function) {
|
416
|
+
static unique_ptr<FunctionData> ParquetScanDeserialize(Deserializer &deserializer, TableFunction &function) {
|
455
417
|
auto &context = deserializer.Get<ClientContext &>();
|
456
418
|
auto files = deserializer.ReadProperty<vector<string>>(100, "files");
|
457
419
|
auto types = deserializer.ReadProperty<vector<LogicalType>>(101, "types");
|
@@ -892,23 +854,80 @@ unique_ptr<LocalFunctionData> ParquetWriteInitializeLocal(ExecutionContext &cont
|
|
892
854
|
}
|
893
855
|
|
894
856
|
// LCOV_EXCL_START
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
899
|
-
|
900
|
-
|
857
|
+
|
858
|
+
// FIXME: Have these be generated instead
|
859
|
+
template <>
|
860
|
+
const char *EnumUtil::ToChars<duckdb_parquet::format::CompressionCodec::type>(
|
861
|
+
duckdb_parquet::format::CompressionCodec::type value) {
|
862
|
+
switch (value) {
|
863
|
+
case CompressionCodec::UNCOMPRESSED:
|
864
|
+
return "UNCOMPRESSED";
|
865
|
+
break;
|
866
|
+
case CompressionCodec::SNAPPY:
|
867
|
+
return "SNAPPY";
|
868
|
+
break;
|
869
|
+
case CompressionCodec::GZIP:
|
870
|
+
return "GZIP";
|
871
|
+
break;
|
872
|
+
case CompressionCodec::LZO:
|
873
|
+
return "LZO";
|
874
|
+
break;
|
875
|
+
case CompressionCodec::BROTLI:
|
876
|
+
return "BROTLI";
|
877
|
+
break;
|
878
|
+
case CompressionCodec::LZ4:
|
879
|
+
return "LZ4";
|
880
|
+
break;
|
881
|
+
case CompressionCodec::ZSTD:
|
882
|
+
return "ZSTD";
|
883
|
+
break;
|
884
|
+
default:
|
885
|
+
throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
|
886
|
+
}
|
901
887
|
}
|
902
888
|
|
903
|
-
|
904
|
-
|
905
|
-
|
889
|
+
template <>
|
890
|
+
duckdb_parquet::format::CompressionCodec::type
|
891
|
+
EnumUtil::FromString<duckdb_parquet::format::CompressionCodec::type>(const char *value) {
|
892
|
+
if (StringUtil::Equals(value, "UNCOMPRESSED")) {
|
893
|
+
return CompressionCodec::UNCOMPRESSED;
|
894
|
+
}
|
895
|
+
if (StringUtil::Equals(value, "SNAPPY")) {
|
896
|
+
return CompressionCodec::SNAPPY;
|
897
|
+
}
|
898
|
+
if (StringUtil::Equals(value, "GZIP")) {
|
899
|
+
return CompressionCodec::GZIP;
|
900
|
+
}
|
901
|
+
if (StringUtil::Equals(value, "LZO")) {
|
902
|
+
return CompressionCodec::LZO;
|
903
|
+
}
|
904
|
+
if (StringUtil::Equals(value, "BROTLI")) {
|
905
|
+
return CompressionCodec::BROTLI;
|
906
|
+
}
|
907
|
+
if (StringUtil::Equals(value, "LZ4")) {
|
908
|
+
return CompressionCodec::LZ4;
|
909
|
+
}
|
910
|
+
if (StringUtil::Equals(value, "ZSTD")) {
|
911
|
+
return CompressionCodec::ZSTD;
|
912
|
+
}
|
913
|
+
throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
|
914
|
+
}
|
906
915
|
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
916
|
+
static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bind_data_p,
|
917
|
+
const CopyFunction &function) {
|
918
|
+
auto &bind_data = bind_data_p.Cast<ParquetWriteBindData>();
|
919
|
+
serializer.WriteProperty(100, "sql_types", bind_data.sql_types);
|
920
|
+
serializer.WriteProperty(101, "column_names", bind_data.column_names);
|
921
|
+
serializer.WriteProperty(102, "codec", bind_data.codec);
|
922
|
+
serializer.WriteProperty(103, "row_group_size", bind_data.row_group_size);
|
923
|
+
}
|
911
924
|
|
925
|
+
static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserializer, CopyFunction &function) {
|
926
|
+
auto data = make_uniq<ParquetWriteBindData>();
|
927
|
+
data->sql_types = deserializer.ReadProperty<vector<LogicalType>>(100, "sql_types");
|
928
|
+
data->column_names = deserializer.ReadProperty<vector<string>>(101, "column_names");
|
929
|
+
data->codec = deserializer.ReadProperty<duckdb_parquet::format::CompressionCodec::type>(102, "codec");
|
930
|
+
data->row_group_size = deserializer.ReadProperty<idx_t>(103, "row_group_size");
|
912
931
|
return std::move(data);
|
913
932
|
}
|
914
933
|
// LCOV_EXCL_STOP
|
@@ -9,6 +9,7 @@
|
|
9
9
|
#include "duckdb/function/table_function.hpp"
|
10
10
|
#include "duckdb/main/client_context.hpp"
|
11
11
|
#include "duckdb/main/connection.hpp"
|
12
|
+
#include "duckdb/common/serializer/write_stream.hpp"
|
12
13
|
#include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
|
13
14
|
#include "duckdb/parser/parsed_data/create_table_function_info.hpp"
|
14
15
|
#endif
|
@@ -55,7 +56,7 @@ FieldID FieldID::Copy() const {
|
|
55
56
|
|
56
57
|
class MyTransport : public TTransport {
|
57
58
|
public:
|
58
|
-
explicit MyTransport(
|
59
|
+
explicit MyTransport(WriteStream &serializer) : serializer(serializer) {
|
59
60
|
}
|
60
61
|
|
61
62
|
bool isOpen() const override {
|
@@ -73,7 +74,7 @@ public:
|
|
73
74
|
}
|
74
75
|
|
75
76
|
private:
|
76
|
-
|
77
|
+
WriteStream &serializer;
|
77
78
|
};
|
78
79
|
|
79
80
|
bool ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type, Type::type &parquet_type) {
|
@@ -3,19 +3,19 @@
|
|
3
3
|
// Do not edit this file manually, your changes will be overwritten
|
4
4
|
//===----------------------------------------------------------------------===//
|
5
5
|
|
6
|
-
#include "duckdb/common/serializer/
|
7
|
-
#include "duckdb/common/serializer/
|
6
|
+
#include "duckdb/common/serializer/serializer.hpp"
|
7
|
+
#include "duckdb/common/serializer/deserializer.hpp"
|
8
8
|
#include "parquet_reader.hpp"
|
9
9
|
|
10
10
|
namespace duckdb {
|
11
11
|
|
12
|
-
void ParquetOptions::
|
12
|
+
void ParquetOptions::Serialize(Serializer &serializer) const {
|
13
13
|
serializer.WriteProperty(100, "binary_as_string", binary_as_string);
|
14
14
|
serializer.WriteProperty(101, "file_row_number", file_row_number);
|
15
15
|
serializer.WriteProperty(102, "file_options", file_options);
|
16
16
|
}
|
17
17
|
|
18
|
-
ParquetOptions ParquetOptions::
|
18
|
+
ParquetOptions ParquetOptions::Deserialize(Deserializer &deserializer) {
|
19
19
|
ParquetOptions result;
|
20
20
|
deserializer.ReadProperty(100, "binary_as_string", result.binary_as_string);
|
21
21
|
deserializer.ReadProperty(101, "file_row_number", result.file_row_number);
|
@@ -431,6 +431,26 @@ void FindMinimalQualification(ClientContext &context, const string &catalog_name
|
|
431
431
|
qualify_schema = true;
|
432
432
|
}
|
433
433
|
|
434
|
+
bool Catalog::TryAutoLoad(ClientContext &context, const string &extension_name) noexcept {
|
435
|
+
if (context.db->ExtensionIsLoaded(extension_name)) {
|
436
|
+
return true;
|
437
|
+
}
|
438
|
+
#ifndef DUCKDB_DISABLE_EXTENSION_LOAD
|
439
|
+
auto &dbconfig = DBConfig::GetConfig(context);
|
440
|
+
if (!dbconfig.options.autoload_known_extensions) {
|
441
|
+
return false;
|
442
|
+
}
|
443
|
+
try {
|
444
|
+
if (ExtensionHelper::CanAutoloadExtension(extension_name)) {
|
445
|
+
return ExtensionHelper::TryAutoLoadExtension(context, extension_name);
|
446
|
+
}
|
447
|
+
} catch (...) {
|
448
|
+
return false;
|
449
|
+
}
|
450
|
+
#endif
|
451
|
+
return false;
|
452
|
+
}
|
453
|
+
|
434
454
|
void Catalog::AutoloadExtensionByConfigName(ClientContext &context, const string &configuration_name) {
|
435
455
|
#ifndef DUCKDB_DISABLE_EXTENSION_LOAD
|
436
456
|
auto &dbconfig = DBConfig::GetConfig(context);
|