duckdb 0.7.2-dev16.0 → 0.7.2-dev314.0

package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp

@@ -23,23 +23,75 @@ class DatabaseInstance;
 class TemporaryDirectoryHandle;
 struct EvictionQueue;
 
+//! The BufferPool is in charge of handling memory management for one or more databases. It defines memory limits
+//! and implements priority eviction among all users of the pool.
+class BufferPool {
+	friend class BlockHandle;
+	friend class BlockManager;
+	friend class BufferManager;
+
+public:
+	explicit BufferPool(idx_t maximum_memory);
+	virtual ~BufferPool();
+
+	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
+	//! blocks can be evicted
+	void SetLimit(idx_t limit, const char *exception_postscript);
+
+	idx_t GetUsedMemory() {
+		return current_memory;
+	}
+	idx_t GetMaxMemory() {
+		return maximum_memory;
+	}
+
+protected:
+	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
+	//! (i.e. not enough blocks could be evicted)
+	//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
+	//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
+	//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
+	//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
+	struct EvictionResult {
+		bool success;
+		TempBufferPoolReservation reservation;
+	};
+	virtual EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
+	                                   unique_ptr<FileBuffer> *buffer = nullptr);
+
+	//! Garbage collect eviction queue
+	void PurgeQueue();
+	void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
+
+private:
+	//! The lock for changing the memory limit
+	mutex limit_lock;
+	//! The current amount of memory that is occupied by the buffer manager (in bytes)
+	atomic<idx_t> current_memory;
+	//! The maximum amount of memory that the buffer manager can keep (in bytes)
+	atomic<idx_t> maximum_memory;
+	//! Eviction queue
+	unique_ptr<EvictionQueue> queue;
+	//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
+	atomic<uint32_t> queue_insertions;
+};
+
 struct TemporaryFileInformation {
 	string path;
 	idx_t size;
 };
 
-//! The buffer manager is in charge of handling memory management for the database. It hands out memory buffers that can
-//! be used by the database internally.
-//
-//! BlockIds are NOT unique within the context of a BufferManager. A buffer manager
-//! can be shared by many BlockManagers.
+//! The BufferManager is in charge of handling memory management for a singke database. It cooperatively shares a
+//! BufferPool with other BufferManagers, belonging to different databases. It hands out memory buffers that can
+//! be used by the database internally, and offers configuration options specific to a database, which need not be
+//! shared by the BufferPool, including whether to support swapping temp buffers to disk, and where to swap them to.
 class BufferManager {
 	friend class BufferHandle;
 	friend class BlockHandle;
 	friend class BlockManager;
 
 public:
-	BufferManager(DatabaseInstance &db, string temp_directory, idx_t maximum_memory);
+	BufferManager(DatabaseInstance &db, string temp_directory);
 	virtual ~BufferManager();
 
 	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
@@ -58,21 +110,17 @@ public:
 	BufferHandle Pin(shared_ptr<BlockHandle> &handle);
 	void Unpin(shared_ptr<BlockHandle> &handle);
 
-	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
-	//! blocks can be evicted
-	void SetLimit(idx_t limit = (idx_t)-1);
-
 	DUCKDB_API static BufferManager &GetBufferManager(ClientContext &context);
 	DUCKDB_API static BufferManager &GetBufferManager(DatabaseInstance &db);
 	DUCKDB_API static BufferManager &GetBufferManager(AttachedDatabase &db);
 
 	//! Returns the currently allocated memory
 	idx_t GetUsedMemory() {
-		return current_memory;
+		return buffer_pool.current_memory;
 	}
 	//! Returns the maximum available memory
 	idx_t GetMaxMemory() {
-		return maximum_memory;
+		return buffer_pool.maximum_memory;
 	}
 
 	//! Increases the currently allocated memory, but the actual allocation does not go through the buffer manager
@@ -92,6 +140,10 @@ public:
 		return db;
 	}
 
+	BufferPool &GetBufferPool() {
+		return buffer_pool;
+	}
+
 	static idx_t GetAllocSize(idx_t block_size) {
 		return AlignValue<idx_t, Storage::SECTOR_SIZE>(block_size + Storage::BLOCK_HEADER_SIZE);
 	}
@@ -103,6 +155,11 @@ public:
 	DUCKDB_API void ReserveMemory(idx_t size);
 	DUCKDB_API void FreeReservedMemory(idx_t size);
 
+	//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
+	//! blocks can be evicted. (Sugar for calling method directly on the BufferPool.)
+	void SetLimit(idx_t limit = (idx_t)-1) {
+		buffer_pool.SetLimit(limit, InMemoryWarning());
+	}
 	//! Returns a list of all temporary files
 	vector<TemporaryFileInformation> GetTemporaryFiles();
 
@@ -113,25 +170,6 @@ private:
 	//! This needs to be private to prevent creating blocks without ever pinning them:
 	//! blocks that are never pinned are never added to the eviction queue
 	shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
-	//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
-	//! (i.e. not enough blocks could be evicted)
-	//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
-	//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
-	//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
-	//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
-	struct EvictionResult {
-		bool success;
-		TempBufferPoolReservation reservation;
-	};
-	EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit, unique_ptr<FileBuffer> *buffer = nullptr);
-
-	//! Helper
-	template <typename... ARGS>
-	TempBufferPoolReservation EvictBlocksOrThrow(idx_t extra_memory, idx_t limit, unique_ptr<FileBuffer> *buffer,
-	                                             ARGS...);
-
-	//! Garbage collect eviction queue
-	void PurgeQueue();
 
 	//! Write a temporary buffer to disk
 	void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer);
@@ -144,9 +182,7 @@ private:
 
 	void RequireTemporaryDirectory();
 
-	void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
-
-	string InMemoryWarning();
+	const char *InMemoryWarning();
 
 	static data_ptr_t BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size);
 	static void BufferAllocatorFree(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t size);
@@ -157,27 +193,23 @@ private:
 	//! overwrites the data within with garbage. Any readers that do not hold the pin will notice
 	void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
 
+	//! Helper
+	template <typename... ARGS>
+	TempBufferPoolReservation EvictBlocksOrThrow(idx_t extra_memory, unique_ptr<FileBuffer> *buffer, ARGS...);
+
 private:
 	//! The database instance
 	DatabaseInstance &db;
-	//! The lock for changing the memory limit
-	mutex limit_lock;
-	//! The current amount of memory that is occupied by the buffer manager (in bytes)
-	atomic<idx_t> current_memory;
-	//! The maximum amount of memory that the buffer manager can keep (in bytes)
-	atomic<idx_t> maximum_memory;
+	//! The buffer pool
+	BufferPool &buffer_pool;
 	//! The directory name where temporary files are stored
 	string temp_directory;
 	//! Lock for creating the temp handle
 	mutex temp_handle_lock;
 	//! Handle for the temporary directory
 	unique_ptr<TemporaryDirectoryHandle> temp_directory_handle;
-	//! Eviction queue
-	unique_ptr<EvictionQueue> queue;
 	//! The temporary id used for managed buffers
 	atomic<block_id_t> temporary_id;
-	//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
-	atomic<uint32_t> queue_insertions;
 	//! Allocator associated with the buffer manager, that passes all allocations through this buffer manager
 	Allocator buffer_allocator;
 	//! Block manager for temp data

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -12,6 +12,7 @@
 
 namespace duckdb {
 class DuckTableEntry;
+class TableStatistics;
 
 //! The table data writer is responsible for writing the data of a table to
 //! storage.
@@ -30,7 +31,7 @@ public:
 
 	CompressionType GetColumnCompressionType(idx_t i);
 
-	virtual void FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) = 0;
+	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) = 0;
 	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) = 0;
 
 	virtual void AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> &&writer);
@@ -47,7 +48,7 @@ public:
 	                          MetaBlockWriter &table_data_writer, MetaBlockWriter &meta_data_writer);
 
 public:
-	virtual void FinalizeTable(vector<unique_ptr<BaseStatistics>> &&global_stats, DataTableInfo *info) override;
+	virtual void FinalizeTable(TableStatistics &&global_stats, DataTableInfo *info) override;
 	virtual unique_ptr<RowGroupWriter> GetRowGroupWriter(RowGroup &row_group) override;
 
 private:

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_compress.hpp

@@ -18,7 +18,7 @@
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
@@ -150,7 +150,7 @@ public:
 
 		if (is_valid) {
 			T floating_point_value = Load<T>((const_data_ptr_t)&value);
-			NumericStatistics::Update<T>(current_segment->stats, floating_point_value);
+			NumericStats::Update<T>(current_segment->stats.statistics, floating_point_value);
 		} else {
 			//! FIXME: find a cheaper alternative to storing a NULL
 			// store this as "value_identical", only using 9 bits for a NULL

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_fetch.hpp

@@ -17,7 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_scan.hpp

@@ -17,7 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_compress.hpp

@@ -18,7 +18,7 @@
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"
@@ -49,7 +49,7 @@ public:
 			}
 
 			if (is_valid) {
-				NumericStatistics::Update<VALUE_TYPE>(state_wrapper->current_segment->stats, value);
+				NumericStats::Update<VALUE_TYPE>(state_wrapper->current_segment->stats.statistics, value);
 			}
 
 			state_wrapper->WriteValue(Load<EXACT_TYPE>((const_data_ptr_t)&value));

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_fetch.hpp

@@ -17,7 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_scan.hpp

@@ -17,7 +17,7 @@
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
+
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
 #include "duckdb/common/operator/subtract.hpp"

package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp

@@ -18,12 +18,15 @@
 namespace duckdb {
 
 struct DataPointer {
+	DataPointer(BaseStatistics stats) : statistics(std::move(stats)) {
+	}
+
 	uint64_t row_start;
 	uint64_t tuple_count;
 	BlockPointer block_pointer;
 	CompressionType compression_type;
 	//! Type-specific statistics of the segment
-	unique_ptr<BaseStatistics> statistics;
+	BaseStatistics statistics;
 };
 
 struct RowGroupPointer {
@@ -32,7 +35,7 @@ struct RowGroupPointer {
 	//! The data pointers of the column segments stored in the row group
 	vector<BlockPointer> data_pointers;
 	//! The per-column statistics of the row group
-	vector<unique_ptr<BaseStatistics>> statistics;
+	vector<BaseStatistics> statistics;
 	//! The versions information of the row group (if any)
 	shared_ptr<VersionNode> versions;
 };

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -169,7 +169,7 @@ public:
 	//! Get statistics of a physical column within the table
 	unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, column_t column_id);
 	//! Sets statistics of a physical column within the table
-	void SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun);
+	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	//! Checkpoint the table to the specified table data writer
 	void Checkpoint(TableDataWriter &writer);

package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp

@@ -13,6 +13,8 @@
 #include "duckdb/common/operator/comparison_operators.hpp"
 #include "duckdb/common/enums/expression_type.hpp"
 #include "duckdb/common/types/value.hpp"
+#include "duckdb/storage/statistics/numeric_stats.hpp"
+#include "duckdb/storage/statistics/string_stats.hpp"
 
 namespace duckdb {
 struct SelectionVector;
@@ -22,59 +24,121 @@ class Deserializer;
 class FieldWriter;
 class FieldReader;
 class Vector;
-class ValidityStatistics;
-class DistinctStatistics;
 struct UnifiedVectorFormat;
 
-enum StatisticsType { LOCAL_STATS = 0, GLOBAL_STATS = 1 };
+enum class StatsInfo : uint8_t {
+	CAN_HAVE_NULL_VALUES = 0,
+	CANNOT_HAVE_NULL_VALUES = 1,
+	CAN_HAVE_VALID_VALUES = 2,
+	CANNOT_HAVE_VALID_VALUES = 3,
+	CAN_HAVE_NULL_AND_VALID_VALUES = 4
+};
+
+enum class StatisticsType : uint8_t { NUMERIC_STATS, STRING_STATS, LIST_STATS, STRUCT_STATS, BASE_STATS };
 
 class BaseStatistics {
-public:
-	BaseStatistics(LogicalType type, StatisticsType stats_type);
-	virtual ~BaseStatistics();
+	friend struct NumericStats;
+	friend struct StringStats;
+	friend struct StructStats;
+	friend struct ListStats;
 
-	//! The type of the logical segment
-	LogicalType type;
-	//! The validity stats of the column (if any)
-	unique_ptr<BaseStatistics> validity_stats;
-	//! The approximate count distinct stats of the column (if any)
-	unique_ptr<BaseStatistics> distinct_stats;
-	//! Whether these are 'global' stats, i.e., over a whole table, or just over a segment
-	//! Some statistics are more expensive to keep, therefore we only keep them globally
-	StatisticsType stats_type;
+public:
+	DUCKDB_API ~BaseStatistics();
+	// disable copy constructors
+	BaseStatistics(const BaseStatistics &other) = delete;
+	BaseStatistics &operator=(const BaseStatistics &) = delete;
+	//! enable move constructors
+	DUCKDB_API BaseStatistics(BaseStatistics &&other) noexcept;
+	DUCKDB_API BaseStatistics &operator=(BaseStatistics &&) noexcept;
 
 public:
-	static unique_ptr<BaseStatistics> CreateEmpty(LogicalType type, StatisticsType stats_type);
+	//! Creates a set of statistics for data that is unknown, i.e. "has_null" is true, "has_no_null" is true, etc
+	//! This can be used in case nothing is known about the data - or can be used as a baseline when only a few things
+	//! are known
+	static BaseStatistics CreateUnknown(LogicalType type);
+	//! Creates statistics for an empty database, i.e. "has_null" is false, "has_no_null" is false, etc
+	//! This is used when incrementally constructing statistics by constantly adding new values
+	static BaseStatistics CreateEmpty(LogicalType type);
+
+	DUCKDB_API StatisticsType GetStatsType() const;
+	DUCKDB_API static StatisticsType GetStatsType(const LogicalType &type);
 
 	DUCKDB_API bool CanHaveNull() const;
 	DUCKDB_API bool CanHaveNoNull() const;
 
-	void UpdateDistinctStatistics(Vector &v, idx_t count);
+	void SetDistinctCount(idx_t distinct_count);
+
+	bool IsConstant() const;
 
-	virtual bool IsConstant() const {
-		return false;
+	const LogicalType &GetType() const {
+		return type;
 	}
 
-	virtual void Merge(const BaseStatistics &other);
+	void Set(StatsInfo info);
+	void CombineValidity(BaseStatistics &left, BaseStatistics &right);
+	void CopyValidity(BaseStatistics &stats);
+	inline void SetHasNull() {
+		has_null = true;
+	}
+	inline void SetHasNoNull() {
+		has_no_null = true;
+	}
 
-	virtual unique_ptr<BaseStatistics> Copy() const;
+	void Merge(const BaseStatistics &other);
+
+	void Copy(const BaseStatistics &other);
+
+	BaseStatistics Copy() const;
+	unique_ptr<BaseStatistics> ToUnique() const;
 	void CopyBase(const BaseStatistics &orig);
 
-	virtual void Serialize(Serializer &serializer) const;
-	virtual void Serialize(FieldWriter &writer) const;
+	void Serialize(Serializer &serializer) const;
+	void Serialize(FieldWriter &writer) const;
 
-	virtual idx_t GetDistinctCount();
+	idx_t GetDistinctCount();
 
-	static unique_ptr<BaseStatistics> Deserialize(Deserializer &source, LogicalType type);
+	static BaseStatistics Deserialize(Deserializer &source, LogicalType type);
 
 	//! Verify that a vector does not violate the statistics
-	virtual void Verify(Vector &vector, const SelectionVector &sel, idx_t count) const;
+	void Verify(Vector &vector, const SelectionVector &sel, idx_t count) const;
 	void Verify(Vector &vector, idx_t count) const;
 
-	virtual string ToString() const;
+	string ToString() const;
+
+	static BaseStatistics FromConstant(const Value &input);
+
+private:
+	BaseStatistics();
+	explicit BaseStatistics(LogicalType type);
+
+	static void Construct(BaseStatistics &stats, LogicalType type);
 
-protected:
-	void InitializeBase();
+	void InitializeUnknown();
+	void InitializeEmpty();
+
+	static BaseStatistics CreateUnknownType(LogicalType type);
+	static BaseStatistics CreateEmptyType(LogicalType type);
+	static BaseStatistics DeserializeType(FieldReader &reader, LogicalType type);
+	static BaseStatistics FromConstantType(const Value &input);
+
+private:
+	//! The type of the logical segment
+	LogicalType type;
+	//! Whether or not the segment can contain NULL values
+	bool has_null;
+	//! Whether or not the segment can contain values that are not null
+	bool has_no_null;
+	// estimate that one may have even if distinct_stats==nullptr
+	idx_t distinct_count;
+	//! Numeric and String stats
+	union {
+		//! Numeric stats data, for numeric stats
+		NumericStatsData numeric_data;
+		//! String stats data, for string stats
+		StringStatsData string_data;
+	} stats_union;
+	//! Child stats (for LIST and STRUCT)
+	unique_ptr<BaseStatistics[]> child_stats;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp

@@ -9,17 +9,36 @@
 #pragma once
 
 #include "duckdb/storage/statistics/base_statistics.hpp"
+#include "duckdb/storage/statistics/distinct_statistics.hpp"
 
 namespace duckdb {
 
 class ColumnStatistics {
 public:
-	explicit ColumnStatistics(unique_ptr<BaseStatistics> stats_p);
-
-	unique_ptr<BaseStatistics> stats;
+	explicit ColumnStatistics(BaseStatistics stats_p);
+	ColumnStatistics(BaseStatistics stats_p, unique_ptr<DistinctStatistics> distinct_stats_p);
 
 public:
 	static shared_ptr<ColumnStatistics> CreateEmptyStats(const LogicalType &type);
+
+	void Merge(ColumnStatistics &other);
+
+	void UpdateDistinctStatistics(Vector &v, idx_t count);
+
+	BaseStatistics &Statistics();
+
+	bool HasDistinctStats();
+	DistinctStatistics &DistinctStats();
+	void SetDistinct(unique_ptr<DistinctStatistics> distinct_stats);
+
+	shared_ptr<ColumnStatistics> Copy() const;
+	void Serialize(Serializer &serializer) const;
+	static shared_ptr<ColumnStatistics> Deserialize(Deserializer &source, const LogicalType &type);
+
+private:
+	BaseStatistics stats;
+	//! The approximate count distinct stats of the column
+	unique_ptr<DistinctStatistics> distinct_stats;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp

@@ -17,7 +17,7 @@ class Serializer;
 class Deserializer;
 class Vector;
 
-class DistinctStatistics : public BaseStatistics {
+class DistinctStatistics {
 public:
 	DistinctStatistics();
 	explicit DistinctStatistics(unique_ptr<HyperLogLog> log, idx_t sample_count, idx_t total_count);
@@ -30,12 +30,12 @@ public:
 	atomic<idx_t> total_count;
 
 public:
-	void Merge(const BaseStatistics &other) override;
+	void Merge(const DistinctStatistics &other);
 
-	unique_ptr<BaseStatistics> Copy() const override;
+	unique_ptr<DistinctStatistics> Copy() const;
 
-	void Serialize(Serializer &serializer) const override;
-	void Serialize(FieldWriter &writer) const override;
+	void Serialize(Serializer &serializer) const;
+	void Serialize(FieldWriter &writer) const;
 
 	static unique_ptr<DistinctStatistics> Deserialize(Deserializer &source);
 	static unique_ptr<DistinctStatistics> Deserialize(FieldReader &reader);
@@ -43,7 +43,7 @@ public:
 	void Update(Vector &update, idx_t count, bool sample = true);
 	void Update(UnifiedVectorFormat &update_data, const LogicalType &ptype, idx_t count, bool sample = true);
 
-	string ToString() const override;
+	string ToString() const;
 	idx_t GetCount() const;
 
 private:

package/src/duckdb/src/include/duckdb/storage/statistics/list_stats.hpp

@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/statistics/list_stats.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/types/hugeint.hpp"
+
+namespace duckdb {
+class BaseStatistics;
+class FieldWriter;
+class FieldReader;
+struct SelectionVector;
+class Vector;
+
+struct ListStats {
+	DUCKDB_API static void Construct(BaseStatistics &stats);
+	DUCKDB_API static BaseStatistics CreateUnknown(LogicalType type);
+	DUCKDB_API static BaseStatistics CreateEmpty(LogicalType type);
+
+	DUCKDB_API static const BaseStatistics &GetChildStats(const BaseStatistics &stats);
+	DUCKDB_API static BaseStatistics &GetChildStats(BaseStatistics &stats);
+	DUCKDB_API static void SetChildStats(BaseStatistics &stats, unique_ptr<BaseStatistics> new_stats);
+
+	DUCKDB_API static void Serialize(const BaseStatistics &stats, FieldWriter &writer);
+	DUCKDB_API static BaseStatistics Deserialize(FieldReader &reader, LogicalType type);
+
+	DUCKDB_API static string ToString(const BaseStatistics &stats);
+
+	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);
+	DUCKDB_API static void Copy(BaseStatistics &stats, const BaseStatistics &other);
+	DUCKDB_API static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/node_statistics.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/serializer.hpp"
 
 namespace duckdb {
 
@@ -23,6 +24,31 @@ public:
 	    : has_estimated_cardinality(true), estimated_cardinality(estimated_cardinality), has_max_cardinality(true),
 	      max_cardinality(max_cardinality) {
 	}
+	void Serialize(Serializer &serializer) const {
+		serializer.Write(has_estimated_cardinality);
+		if (has_estimated_cardinality) {
+			serializer.Write(estimated_cardinality);
+			serializer.Write(has_max_cardinality);
+			if (has_max_cardinality) {
+				serializer.Write(max_cardinality);
+			}
+		} else {
+			D_ASSERT(!has_max_cardinality);
+		}
+	}
+	static unique_ptr<NodeStatistics> Deserialize(Deserializer &source) {
+		bool has_estimated_cardinality = source.Read<bool>();
+		if (!has_estimated_cardinality) {
+			return make_unique<NodeStatistics>();
+		}
+		idx_t estimated_cardinality = source.Read<idx_t>();
+		bool has_max_cardinality = source.Read<bool>();
+		if (!has_max_cardinality) {
+			return make_unique<NodeStatistics>(estimated_cardinality);
+		}
+		idx_t max_cardinality = source.Read<idx_t>();
+		return make_unique<NodeStatistics>(estimated_cardinality, max_cardinality);
+	}
 
 	//! Whether or not the node has an estimated cardinality specified
 	bool has_estimated_cardinality;