duckdb 0.7.2-dev1034.0 → 0.7.2-dev1138.0

package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp

@@ -18,8 +18,6 @@ class SegmentBase {
 public:
 	SegmentBase(idx_t start, idx_t count) : start(start), count(count), next(nullptr) {
 	}
-	virtual ~SegmentBase() {
-	}
 	T *Next() {
 #ifndef DUCKDB_R_BUILD
 		return next.load();
@@ -33,12 +31,13 @@ public:
 	//! The amount of entries in this storage chunk
 	atomic<idx_t> count;
 	//! The next segment after this one
-
 #ifndef DUCKDB_R_BUILD
 	atomic<T *> next;
 #else
 	T *next;
 #endif
+	//! The index within the segment tree
+	idx_t index;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -56,7 +56,7 @@ public:
 		if (nodes.empty()) {
 			LoadNextSegment(l);
 		}
-		return nodes.empty() ? nullptr : nodes[0].node.get();
+		return GetRootSegmentInternal();
 	}
 	//! Obtains ownership of the data of the segment tree
 	vector<SegmentNode<T>> MoveSegments(SegmentLock &l) {
@@ -258,6 +258,10 @@ private:
 	mutex node_lock;
 
 private:
+	T *GetRootSegmentInternal() {
+		return nodes.empty() ? nullptr : nodes[0].node.get();
+	}
+
 	class SegmentIterationHelper {
 	public:
 		explicit SegmentIterationHelper(SegmentTree &tree) : tree(tree) {
@@ -304,6 +308,9 @@ private:
 
 	//! Load the next segment, if there are any left to load
 	bool LoadNextSegment(SegmentLock &l) {
+		if (!SUPPORTS_LAZY_LOADING) {
+			return false;
+		}
 		if (finished_loading) {
 			return false;
 		}
@@ -317,6 +324,9 @@ private:
 
 	//! Load all segments, if there are any left to load
 	void LoadAllSegments(SegmentLock &l) {
+		if (!SUPPORTS_LAZY_LOADING) {
+			return;
+		}
 		while (LoadNextSegment(l))
 			;
 	}

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -59,10 +59,6 @@ public:
 	void GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, TableStorageInfo &result) override;
 
 	void Verify(RowGroup &parent) override;
-
-private:
-	template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
-	void TemplatedScan(Transaction *transaction, ColumnScanState &state, Vector &result);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "duckdb/storage/table/row_group_collection.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table/table_index_list.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
 
@@ -18,6 +17,7 @@ class AttachedDatabase;
 class DataTable;
 class Transaction;
 class WriteAheadLog;
+struct LocalAppendState;
 struct TableAppendState;
 
 class OptimisticDataWriter {
@@ -118,6 +118,9 @@ public:
 
 public:
 	struct CommitState {
+		CommitState();
+		~CommitState();
+
 		unordered_map<DataTable *, unique_ptr<TableAppendState>> append_states;
 	};
 

package/src/duckdb/src/include/duckdb.h

@@ -1710,6 +1710,16 @@ Adds a parameter to the table function.
 */
 DUCKDB_API void duckdb_table_function_add_parameter(duckdb_table_function table_function, duckdb_logical_type type);
 
+/*!
+Adds a named parameter to the table function.
+
+* table_function: The table function
+* name: The name of the parameter
+* type: The type of the parameter to add.
+*/
+DUCKDB_API void duckdb_table_function_add_named_parameter(duckdb_table_function table_function, const char *name,
+                                                          duckdb_logical_type type);
+
 /*!
 Assigns extra information to the table function that can be fetched during binding, etc.
 
@@ -1818,6 +1828,17 @@ The result must be destroyed with `duckdb_destroy_value`.
 */
 DUCKDB_API duckdb_value duckdb_bind_get_parameter(duckdb_bind_info info, idx_t index);
 
+/*!
+Retrieves a named parameter with the given name.
+
+The result must be destroyed with `duckdb_destroy_value`.
+
+* info: The info object
+* name: The name of the parameter
+* returns: The value of the parameter. Must be destroyed with `duckdb_destroy_value`.
+*/
+DUCKDB_API duckdb_value duckdb_bind_get_named_parameter(duckdb_bind_info info, const char *name);
+
 /*!
 Sets the user-provided bind data in the bind object. This object can be retrieved again during execution.
 

package/src/duckdb/src/main/capi/table_function-c.cpp

@@ -208,6 +208,16 @@ void duckdb_table_function_add_parameter(duckdb_table_function function, duckdb_
 	tf->arguments.push_back(*logical_type);
 }
 
+void duckdb_table_function_add_named_parameter(duckdb_table_function function, const char *name,
+                                               duckdb_logical_type type) {
+	if (!function || !type) {
+		return;
+	}
+	auto tf = (duckdb::TableFunction *)function;
+	auto logical_type = (duckdb::LogicalType *)type;
+	tf->named_parameters.insert({name, *logical_type});
+}
+
 void duckdb_table_function_set_extra_info(duckdb_table_function function, void *extra_info,
                                           duckdb_delete_callback_t destroy) {
 	if (!function) {
@@ -319,6 +329,19 @@ duckdb_value duckdb_bind_get_parameter(duckdb_bind_info info, idx_t index) {
 	return reinterpret_cast<duckdb_value>(new duckdb::Value(bind_info->input.inputs[index]));
 }
 
+duckdb_value duckdb_bind_get_named_parameter(duckdb_bind_info info, const char *name) {
+	if (!info || !name) {
+		return nullptr;
+	}
+	auto bind_info = (duckdb::CTableInternalBindInfo *)info;
+	auto t = bind_info->input.named_parameters.find(name);
+	if (t == bind_info->input.named_parameters.end()) {
+		return nullptr;
+	} else {
+		return reinterpret_cast<duckdb_value>(new duckdb::Value(t->second));
+	}
+}
+
 void duckdb_bind_set_bind_data(duckdb_bind_info info, void *bind_data, duckdb_delete_callback_t destroy) {
 	if (!info) {
 		return;

package/src/duckdb/src/main/settings/settings.cpp

@@ -454,11 +454,15 @@ Value CustomExtensionRepository::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 
 void EnableProgressBarSetting::ResetLocal(ClientContext &context) {
-	ClientConfig::GetConfig(context).enable_progress_bar = ClientConfig().enable_progress_bar;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.enable_progress_bar = ClientConfig().enable_progress_bar;
 }
 
 void EnableProgressBarSetting::SetLocal(ClientContext &context, const Value &input) {
-	ClientConfig::GetConfig(context).enable_progress_bar = input.GetValue<bool>();
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.enable_progress_bar = input.GetValue<bool>();
 }
 
 Value EnableProgressBarSetting::GetSetting(ClientContext &context) {
@@ -469,11 +473,15 @@ Value EnableProgressBarSetting::GetSetting(ClientContext &context) {
 // Enable Progress Bar Print
 //===--------------------------------------------------------------------===//
 void EnableProgressBarPrintSetting::SetLocal(ClientContext &context, const Value &input) {
-	ClientConfig::GetConfig(context).print_progress_bar = input.GetValue<bool>();
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.print_progress_bar = input.GetValue<bool>();
 }
 
 void EnableProgressBarPrintSetting::ResetLocal(ClientContext &context) {
-	ClientConfig::GetConfig(context).print_progress_bar = ClientConfig().print_progress_bar;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.print_progress_bar = ClientConfig().print_progress_bar;
 }
 
 Value EnableProgressBarPrintSetting::GetSetting(ClientContext &context) {
@@ -878,13 +886,17 @@ Value ProfilingModeSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 
 void ProgressBarTimeSetting::ResetLocal(ClientContext &context) {
-	ClientConfig::GetConfig(context).wait_time = ClientConfig().wait_time;
-	ClientConfig::GetConfig(context).enable_progress_bar = ClientConfig().enable_progress_bar;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.wait_time = ClientConfig().wait_time;
+	config.enable_progress_bar = ClientConfig().enable_progress_bar;
 }
 
 void ProgressBarTimeSetting::SetLocal(ClientContext &context, const Value &input) {
-	ClientConfig::GetConfig(context).wait_time = input.GetValue<int32_t>();
-	ClientConfig::GetConfig(context).enable_progress_bar = true;
+	auto &config = ClientConfig::GetConfig(context);
+	ProgressBar::SystemOverrideCheck(config);
+	config.wait_time = input.GetValue<int32_t>();
+	config.enable_progress_bar = true;
 }
 
 Value ProgressBarTimeSetting::GetSetting(ClientContext &context) {

package/src/duckdb/src/optimizer/filter_combiner.cpp

@@ -2,21 +2,18 @@
 
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/expression/bound_between_expression.hpp"
+#include "duckdb/planner/expression/bound_cast_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
 #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
-#include "duckdb/planner/operator/logical_empty_result.hpp"
-#include "duckdb/planner/operator/logical_filter.hpp"
+#include "duckdb/planner/expression/bound_operator_expression.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/planner/filter/constant_filter.hpp"
 #include "duckdb/planner/filter/null_filter.hpp"
 #include "duckdb/optimizer/optimizer.hpp"
-#include "duckdb/common/operator/cast_operators.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -8,6 +8,8 @@
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
+#include <cmath>
+
 namespace duckdb {
 
 static TableCatalogEntry *GetCatalogTableEntry(LogicalOperator *op) {

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -7,6 +7,7 @@
 #include "duckdb/planner/operator/list.hpp"
 
 #include <algorithm>
+#include <cmath>
 
 namespace std {
 

package/src/duckdb/src/parallel/meta_pipeline.cpp

@@ -156,9 +156,6 @@ Pipeline *MetaPipeline::CreateUnionPipeline(Pipeline &current, bool order_matter
 void MetaPipeline::CreateChildPipeline(Pipeline &current, PhysicalOperator *op, Pipeline *last_pipeline) {
 	// rule 2: 'current' must be fully built (down to the source) before creating the child pipeline
 	D_ASSERT(current.source);
-	if (HasRecursiveCTE()) {
-		throw NotImplementedException("Child pipelines are not supported in recursive CTEs yet");
-	}
 
 	// create the child pipeline (same batch index)
 	pipelines.emplace_back(state.CreateChildPipeline(executor, current, op));

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -1,7 +1,9 @@
+#include "duckdb/common/serializer/enum_serializer.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/to_string.hpp"
 #include "duckdb/parser/expression/case_expression.hpp"
 #include "duckdb/parser/expression/cast_expression.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 
 #include "duckdb/parser/expression/operator_expression.hpp"
@@ -310,6 +312,26 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		coalesce_op->children.push_back(std::move(children[0]));
 		coalesce_op->children.push_back(std::move(children[1]));
 		return std::move(coalesce_op);
+	} else if (lowercase_name == "list" && order_bys->orders.size() == 1) {
+		// list(expr ORDER BY expr <sense> <nulls>) => list_sort(list(expr), <sense>, <nulls>)
+		if (children.size() != 1) {
+			throw ParserException("Wrong number of arguments to LIST.");
+		}
+		auto arg_expr = children[0].get();
+		auto &order_by = order_bys->orders[0];
+		if (arg_expr->Equals(order_by.expression.get())) {
+			auto sense = make_unique<ConstantExpression>(EnumSerializer::EnumToString(order_by.type));
+			auto nulls = make_unique<ConstantExpression>(EnumSerializer::EnumToString(order_by.null_order));
+			order_bys = nullptr;
+			auto unordered = make_unique<FunctionExpression>(
+			    catalog, schema, lowercase_name.c_str(), std::move(children), std::move(filter_expr),
+			    std::move(order_bys), root->agg_distinct, false, root->export_state);
+			lowercase_name = "list_sort";
+			children.clear();
+			children.emplace_back(std::move(unordered));
+			children.emplace_back(std::move(sense));
+			children.emplace_back(std::move(nulls));
+		}
 	}
 
 	auto function = make_unique<FunctionExpression>(std::move(catalog), std::move(schema), lowercase_name.c_str(),

package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/planner/binder.hpp"
 #include "duckdb/planner/constraints/list.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/planner/expression_binder/check_binder.hpp"
 #include "duckdb/planner/expression_binder/constant_binder.hpp"
 #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -1,7 +1,6 @@
 #include "duckdb/common/bitpacking.hpp"
 
 #include "duckdb/common/limits.hpp"
-#include "duckdb/common/types/null_value.hpp"
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
@@ -11,6 +10,7 @@
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
 #include "duckdb/storage/compression/bitpacking.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 #include <functional>
 

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -64,7 +65,7 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 	auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 	if (type.InternalType() == PhysicalType::VARCHAR) {
 		auto &state = (UncompressedStringSegmentState &)*compressed_segment->GetSegmentState();
-		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().block_manager);
+		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().GetBlockManager());
 	}
 	current_segment = std::move(compressed_segment);
 	current_segment->InitializeAppend(append_state);

package/src/duckdb/src/storage/compression/numeric_constant.cpp

@@ -1,10 +1,10 @@
 #include "duckdb/function/compression/compression.hpp"
-#include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/vector.hpp"
 
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/segment/uncompressed.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/storage/compression/rle.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/common/types/null_value.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include <functional>
 
 namespace duckdb {

package/src/duckdb/src/storage/compression/validity_uncompressed.cpp

@@ -6,7 +6,7 @@
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/function/compression_function.hpp"
-#include "duckdb/main/config.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/storage/data_table.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/sort/sort.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/parser/constraints/list.hpp"
@@ -23,6 +22,8 @@
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/common/types/conflict_manager.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -728,8 +729,7 @@ void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::functi
 	CreateIndexScanState state;
 
 	InitializeScanWithOffset(state, column_ids, row_start, row_start + count);
-	auto row_start_aligned = state.table_state.row_group_state.row_group->start +
-	                         state.table_state.row_group_state.vector_index * STANDARD_VECTOR_SIZE;
+	auto row_start_aligned = state.table_state.row_group->start + state.table_state.vector_index * STANDARD_VECTOR_SIZE;
 
 	idx_t current_row = row_start_aligned;
 	while (current_row < end) {

package/src/duckdb/src/storage/local_storage.cpp

@@ -11,6 +11,7 @@
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -355,6 +356,12 @@ LocalStorage::LocalStorage(ClientContext &context, DuckTransaction &transaction)
     : context(context), transaction(transaction) {
 }
 
+LocalStorage::CommitState::CommitState() {
+}
+
+LocalStorage::CommitState::~CommitState() {
+}
+
 LocalStorage &LocalStorage::Get(DuckTransaction &transaction) {
 	return transaction.GetLocalStorage();
 }

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 48;
+const uint64_t VERSION_NUMBER = 49;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table/column_data.cpp

@@ -10,27 +10,35 @@
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/list_column_data.hpp"
 #include "duckdb/storage/table/standard_column_data.hpp"
-#include "duckdb/transaction/transaction.hpp"
 
 #include "duckdb/storage/table/struct_column_data.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/main/attached_database.hpp"
 
 namespace duckdb {
 
 ColumnData::ColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t column_index, idx_t start_row,
-                       LogicalType type, ColumnData *parent)
-    : block_manager(block_manager), info(info), column_index(column_index), start(start_row), type(std::move(type)),
-      parent(parent), version(0) {
+                       LogicalType type_p, ColumnData *parent)
+    : start(start_row), count(0), block_manager(block_manager), info(info), column_index(column_index),
+      type(std::move(type_p)), parent(parent), version(0) {
+	if (!parent) {
+		stats = make_unique<SegmentStatistics>(type);
+	}
 }
 
 ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
-    : block_manager(other.block_manager), info(other.info), column_index(other.column_index), start(start),
-      type(std::move(other.type)), parent(parent), version(parent ? parent->version + 1 : 0) {
+    : start(start), count(other.count), block_manager(other.block_manager), info(other.info),
+      column_index(other.column_index), type(std::move(other.type)), parent(parent),
+      version(parent ? parent->version + 1 : 0) {
 	if (other.updates) {
 		updates = make_unique<UpdateSegment>(*other.updates, *this);
 	}
+	if (other.stats) {
+		stats = make_unique<SegmentStatistics>(other.stats->statistics.Copy());
+	}
 	idx_t offset = 0;
 	for (auto &segment : other.data.Segments()) {
 		this->data.AppendSegment(ColumnSegment::CreateSegment(segment, start + offset));
@@ -61,16 +69,7 @@ void ColumnData::IncrementVersion() {
 }
 
 idx_t ColumnData::GetMaxEntry() {
-	auto l = data.Lock();
-	auto first_segment = data.GetRootSegment(l);
-	auto last_segment = data.GetLastSegment(l);
-	if (!first_segment) {
-		D_ASSERT(!last_segment);
-		return 0;
-	} else {
-		D_ASSERT(last_segment->start >= first_segment->start);
-		return last_segment->start + last_segment->count - first_segment->start;
-	}
+	return count;
 }
 
 void ColumnData::InitializeScan(ColumnScanState &state) {
@@ -81,6 +80,7 @@ void ColumnData::InitializeScan(ColumnScanState &state) {
 	state.initialized = false;
 	state.version = version;
 	state.scan_state.reset();
+	state.last_offset = 0;
 }
 
 void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
@@ -91,6 +91,7 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.initialized = false;
 	state.version = version;
 	state.scan_state.reset();
+	state.last_offset = 0;
 }
 
 idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining) {
@@ -230,6 +231,46 @@ void ColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector
 	AppendData(stats, state, vdata, count);
 }
 
+void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t count) {
+	if (parent || !stats) {
+		throw InternalException("ColumnData::Append called on a column with a parent or without stats");
+	}
+	Append(stats->statistics, state, vector, count);
+}
+
+bool ColumnData::CheckZonemap(TableFilter &filter) {
+	if (!stats) {
+		throw InternalException("ColumnData::CheckZonemap called on a column without stats");
+	}
+	auto propagate_result = filter.CheckStatistics(stats->statistics);
+	if (propagate_result == FilterPropagateResult::FILTER_ALWAYS_FALSE ||
+	    propagate_result == FilterPropagateResult::FILTER_FALSE_OR_NULL) {
+		return false;
+	}
+	return true;
+}
+
+unique_ptr<BaseStatistics> ColumnData::GetStatistics() {
+	if (!stats) {
+		throw InternalException("ColumnData::GetStatistics called on a column without stats");
+	}
+	return stats->statistics.ToUnique();
+}
+
+void ColumnData::MergeStatistics(const BaseStatistics &other) {
+	if (!stats) {
+		throw InternalException("ColumnData::MergeStatistics called on a column without stats");
+	}
+	return stats->statistics.Merge(other);
+}
+
+void ColumnData::MergeIntoStatistics(BaseStatistics &other) {
+	if (!stats) {
+		throw InternalException("ColumnData::MergeIntoStatistics called on a column without stats");
+	}
+	return other.Merge(stats->statistics);
+}
+
 void ColumnData::InitializeAppend(ColumnAppendState &state) {
 	auto l = data.Lock();
 	if (data.IsEmpty(l)) {
@@ -253,6 +294,7 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 
 void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) {
 	idx_t offset = 0;
+	this->count += count;
 	while (true) {
 		// append the data from the vector
 		idx_t copied_elements = state.current->Append(state, vdata, offset, count);
@@ -292,6 +334,7 @@ void ColumnData::RevertAppend(row_t start_row) {
 	// remove any segments AFTER this segment: they should be deleted entirely
 	data.EraseSegments(l, segment_index);
 
+	this->count = start_row - this->start;
 	segment->next = nullptr;
 	transient.RevertAppend(start_row);
 }
@@ -412,6 +455,7 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 
 void ColumnData::DeserializeColumn(Deserializer &source) {
 	// load the data pointers for the column
+	this->count = 0;
 	idx_t data_pointer_count = source.Read<idx_t>();
 	for (idx_t data_ptr = 0; data_ptr < data_pointer_count; data_ptr++) {
 		// read the data pointer
@@ -420,15 +464,20 @@ void ColumnData::DeserializeColumn(Deserializer &source) {
 		auto block_pointer_block_id = source.Read<block_id_t>();
 		auto block_pointer_offset = source.Read<uint32_t>();
 		auto compression_type = source.Read<CompressionType>();
-		auto stats = BaseStatistics::Deserialize(source, type);
+		auto segment_stats = BaseStatistics::Deserialize(source, type);
+		if (stats) {
+			stats->statistics.Merge(segment_stats);
+		}
 
-		DataPointer data_pointer(std::move(stats));
+		DataPointer data_pointer(std::move(segment_stats));
 		data_pointer.row_start = row_start;
 		data_pointer.tuple_count = tuple_count;
 		data_pointer.block_pointer.block_id = block_pointer_block_id;
 		data_pointer.block_pointer.offset = block_pointer_offset;
 		data_pointer.compression_type = compression_type;
 
+		this->count += tuple_count;
+
 		// create a persistent segment
 		auto segment = ColumnSegment::CreatePersistentSegment(
 		    GetDatabase(), block_manager, data_pointer.block_pointer.block_id, data_pointer.block_pointer.offset, type,
@@ -495,14 +544,22 @@ void ColumnData::Verify(RowGroup &parent) {
 #ifdef DEBUG
 	D_ASSERT(this->start == parent.start);
 	data.Verify();
+	if (type.InternalType() == PhysicalType::STRUCT) {
+		// structs don't have segments
+		D_ASSERT(!data.GetRootSegment());
+		return;
+	}
 	idx_t current_index = 0;
 	idx_t current_start = this->start;
+	idx_t total_count = 0;
 	for (auto &segment : data.Segments()) {
 		D_ASSERT(segment.index == current_index);
 		D_ASSERT(segment.start == current_start);
 		current_start += segment.count;
+		total_count += segment.count;
 		current_index++;
 	}
+	D_ASSERT(this->count == total_count);
 #endif
 }