duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -3,13 +3,12 @@
 #include "duckdb/common/allocator.hpp"
 #include "duckdb/common/checksum.hpp"
 #include "duckdb/common/exception.hpp"
-#include "duckdb/common/serializer/buffered_deserializer.hpp"
-#include "duckdb/common/serializer/buffered_serializer.hpp"
-#include "duckdb/common/field_writer.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/database.hpp"
 
 #include <algorithm>
 #include <cstring>
@@ -18,14 +17,22 @@ namespace duckdb {
 
 const char MainHeader::MAGIC_BYTES[] = "DUCK";
 
-void MainHeader::Serialize(Serializer &ser) {
+void SerializeVersionNumber(WriteStream &ser, const string &version_str) {
+	constexpr const idx_t MAX_VERSION_SIZE = 32;
+	data_t version[MAX_VERSION_SIZE];
+	memset(version, 0, MAX_VERSION_SIZE);
+	memcpy(version, version_str.c_str(), MinValue<idx_t>(version_str.size(), MAX_VERSION_SIZE));
+	ser.WriteData(version, MAX_VERSION_SIZE);
+}
+
+void MainHeader::Write(WriteStream &ser) {
 	ser.WriteData(const_data_ptr_cast(MAGIC_BYTES), MAGIC_BYTE_SIZE);
 	ser.Write<uint64_t>(version_number);
-	FieldWriter writer(ser);
 	for (idx_t i = 0; i < FLAG_COUNT; i++) {
-		writer.WriteField<uint64_t>(flags[i]);
+		ser.Write<uint64_t>(flags[i]);
 	}
-	writer.Finalize();
+	SerializeVersionNumber(ser, DuckDB::LibraryVersion());
+	SerializeVersionNumber(ser, DuckDB::SourceID());
 }
 
 void MainHeader::CheckMagicBytes(FileHandle &handle) {
@@ -39,7 +46,7 @@ void MainHeader::CheckMagicBytes(FileHandle &handle) {
 	}
 }
 
-MainHeader MainHeader::Deserialize(Deserializer &source) {
+MainHeader MainHeader::Read(ReadStream &source) {
 	data_t magic_bytes[MAGIC_BYTE_SIZE];
 	MainHeader header;
 	source.ReadData(magic_bytes, MainHeader::MAGIC_BYTE_SIZE);
@@ -71,40 +78,38 @@ MainHeader MainHeader::Deserialize(Deserializer &source) {
 		    header.version_number, VERSION_NUMBER, version_text);
 	}
 	// read the flags
-	FieldReader reader(source);
 	for (idx_t i = 0; i < FLAG_COUNT; i++) {
-		header.flags[i] = reader.ReadRequired<uint64_t>();
+		header.flags[i] = source.Read<uint64_t>();
 	}
-	reader.Finalize();
 	return header;
 }
 
-void DatabaseHeader::Serialize(Serializer &ser) {
+void DatabaseHeader::Write(WriteStream &ser) {
 	ser.Write<uint64_t>(iteration);
-	ser.Write<block_id_t>(meta_block);
-	ser.Write<block_id_t>(free_list);
+	ser.Write<idx_t>(meta_block);
+	ser.Write<idx_t>(free_list);
 	ser.Write<uint64_t>(block_count);
 }
 
-DatabaseHeader DatabaseHeader::Deserialize(Deserializer &source) {
+DatabaseHeader DatabaseHeader::Read(ReadStream &source) {
 	DatabaseHeader header;
 	header.iteration = source.Read<uint64_t>();
-	header.meta_block = source.Read<block_id_t>();
-	header.free_list = source.Read<block_id_t>();
+	header.meta_block = source.Read<idx_t>();
+	header.free_list = source.Read<idx_t>();
 	header.block_count = source.Read<uint64_t>();
 	return header;
 }
 
 template <class T>
 void SerializeHeaderStructure(T header, data_ptr_t ptr) {
-	BufferedSerializer ser(ptr, Storage::FILE_HEADER_SIZE);
-	header.Serialize(ser);
+	MemoryStream ser(ptr, Storage::FILE_HEADER_SIZE);
+	header.Write(ser);
 }
 
 template <class T>
 T DeserializeHeaderStructure(data_ptr_t ptr) {
-	BufferedDeserializer source(ptr, Storage::FILE_HEADER_SIZE);
-	return T::Deserialize(source);
+	MemoryStream source(ptr, Storage::FILE_HEADER_SIZE);
+	return T::Read(source);
 }
 
 SingleFileBlockManager::SingleFileBlockManager(AttachedDatabase &db, string path_p, StorageManagerOptions options)
@@ -241,11 +246,12 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 }
 
 void SingleFileBlockManager::LoadFreeList() {
-	if (free_list_id == INVALID_BLOCK) {
+	MetaBlockPointer free_pointer(free_list_id, 0);
+	if (!free_pointer.IsValid()) {
 		// no free list
 		return;
 	}
-	MetaBlockReader reader(*this, free_list_id);
+	MetadataReader reader(GetMetadataManager(), free_pointer, nullptr, BlockReaderType::REGISTER_BLOCKS);
 	auto free_list_count = reader.Read<uint64_t>();
 	free_list.clear();
 	for (idx_t i = 0; i < free_list_count; i++) {
@@ -258,10 +264,12 @@ void SingleFileBlockManager::LoadFreeList() {
 		auto usage_count = reader.Read<uint32_t>();
 		multi_use_blocks[block_id] = usage_count;
 	}
+	GetMetadataManager().Read(reader);
+	GetMetadataManager().MarkBlocksAsModified();
 }
 
-bool SingleFileBlockManager::IsRootBlock(block_id_t root) {
-	return root == meta_block;
+bool SingleFileBlockManager::IsRootBlock(MetaBlockPointer root) {
+	return root.block_pointer == meta_block;
 }
 
 block_id_t SingleFileBlockManager::GetFreeBlockId() {
@@ -327,7 +335,7 @@ void SingleFileBlockManager::IncreaseBlockReferenceCount(block_id_t block_id) {
 	}
 }
 
-block_id_t SingleFileBlockManager::GetMetaBlock() {
+idx_t SingleFileBlockManager::GetMetaBlock() {
 	return meta_block;
 }
 
@@ -368,50 +376,70 @@ void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
 	ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
 }
 
-vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
-	vector<block_id_t> free_list_blocks;
+void SingleFileBlockManager::Truncate() {
+	BlockManager::Truncate();
+	idx_t blocks_to_truncate = 0;
+	// reverse iterate over the free-list
+	for (auto entry = free_list.rbegin(); entry != free_list.rend(); entry++) {
+		auto block_id = *entry;
+		if (block_id + 1 != max_block) {
+			break;
+		}
+		blocks_to_truncate++;
+		max_block--;
+	}
+	if (blocks_to_truncate == 0) {
+		// nothing to truncate
+		return;
+	}
+	// truncate the file
+	for (idx_t i = 0; i < blocks_to_truncate; i++) {
+		free_list.erase(max_block + i);
+	}
+	handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
+}
+
+vector<MetadataHandle> SingleFileBlockManager::GetFreeListBlocks() {
+	vector<MetadataHandle> free_list_blocks;
 
-	if (!free_list.empty() || !multi_use_blocks.empty() || !modified_blocks.empty()) {
-		// there are blocks in the free list or multi_use_blocks
-		// figure out how many blocks we need to write these to the file
+	// reserve all blocks that we are going to write the free list to
+	// since these blocks are no longer free we cannot just include them in the free list!
+	auto block_size = MetadataManager::METADATA_BLOCK_SIZE - sizeof(idx_t);
+	idx_t allocated_size = 0;
+	while (true) {
 		auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
 		auto multi_use_blocks_size =
 		    sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
-		auto total_size = free_list_size + multi_use_blocks_size;
-		// because of potential alignment issues and needing to store a next pointer in a block we subtract
-		// a bit from the max block size
-		auto space_in_block = Storage::BLOCK_SIZE - 4 * sizeof(block_id_t);
-		auto total_blocks = (total_size + space_in_block - 1) / space_in_block;
-		D_ASSERT(total_size > 0);
-		D_ASSERT(total_blocks > 0);
-
-		// reserve the blocks that we are going to write
-		// since these blocks are no longer free we cannot just include them in the free list!
-		for (idx_t i = 0; i < total_blocks; i++) {
-			auto block_id = GetFreeBlockId();
-			free_list_blocks.push_back(block_id);
+		auto metadata_blocks =
+		    sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(idx_t)) * GetMetadataManager().BlockCount();
+		auto total_size = free_list_size + multi_use_blocks_size + metadata_blocks;
+		if (total_size < allocated_size) {
+			break;
 		}
+		auto free_list_handle = GetMetadataManager().AllocateHandle();
+		free_list_blocks.push_back(std::move(free_list_handle));
+		allocated_size += block_size;
 	}
 
 	return free_list_blocks;
 }
 
-class FreeListBlockWriter : public MetaBlockWriter {
+class FreeListBlockWriter : public MetadataWriter {
 public:
-	FreeListBlockWriter(BlockManager &block_manager, vector<block_id_t> &free_list_blocks_p)
-	    : MetaBlockWriter(block_manager, free_list_blocks_p[0]), free_list_blocks(free_list_blocks_p), index(1) {
+	FreeListBlockWriter(MetadataManager &manager, vector<MetadataHandle> free_list_blocks_p)
+	    : MetadataWriter(manager), free_list_blocks(std::move(free_list_blocks_p)), index(0) {
 	}
 
-	vector<block_id_t> &free_list_blocks;
+	vector<MetadataHandle> free_list_blocks;
 	idx_t index;
 
 protected:
-	block_id_t GetNextBlockId() override {
+	MetadataHandle NextHandle() override {
 		if (index >= free_list_blocks.size()) {
 			throw InternalException(
 			    "Free List Block Writer ran out of blocks, this means not enough blocks were allocated up front");
 		}
-		return free_list_blocks[index++];
+		return std::move(free_list_blocks[index++]);
 	}
 };
 
@@ -419,10 +447,12 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	// set the iteration count
 	header.iteration = ++iteration_count;
 
-	vector<block_id_t> free_list_blocks = GetFreeListBlocks();
+	auto free_list_blocks = GetFreeListBlocks();
 
 	// now handle the free list
+	auto &metadata_manager = GetMetadataManager();
 	// add all modified blocks to the free list: they can now be written to again
+	metadata_manager.MarkBlocksAsModified();
 	for (auto &block : modified_blocks) {
 		free_list.insert(block);
 	}
@@ -431,17 +461,12 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	if (!free_list_blocks.empty()) {
 		// there are blocks to write, either in the free_list or in the modified_blocks
 		// we write these blocks specifically to the free_list_blocks
-		// a normal MetaBlockWriter will fetch blocks to use from the free_list
+		// a normal MetadataWriter will fetch blocks to use from the free_list
 		// but since we are WRITING the free_list, this behavior is sub-optimal
+		FreeListBlockWriter writer(metadata_manager, std::move(free_list_blocks));
 
-		FreeListBlockWriter writer(*this, free_list_blocks);
-
-		auto ptr = writer.GetBlockPointer();
-		D_ASSERT(ptr.block_id == free_list_blocks[0]);
-		header.free_list = ptr.block_id;
-		for (auto &block_id : free_list_blocks) {
-			modified_blocks.insert(block_id);
-		}
+		auto ptr = writer.GetMetaBlockPointer();
+		header.free_list = ptr.block_pointer;
 
 		writer.Write<uint64_t>(free_list.size());
 		for (auto &block_id : free_list) {
@@ -452,11 +477,13 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 			writer.Write<block_id_t>(entry.first);
 			writer.Write<uint32_t>(entry.second);
 		}
+		GetMetadataManager().Write(writer);
 		writer.Flush();
 	} else {
 		// no blocks in the free list
-		header.free_list = INVALID_BLOCK;
+		header.free_list = DConstants::INVALID_INDEX;
 	}
+	metadata_manager.Flush();
 	header.block_count = max_block;
 
 	auto &config = DBConfig::Get(db);
@@ -471,7 +498,9 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	}
 	// set the header inside the buffer
 	header_buffer.Clear();
-	Store<DatabaseHeader>(header, header_buffer.buffer);
+	MemoryStream serializer;
+	header.Write(serializer);
+	memcpy(header_buffer.buffer, serializer.GetData(), serializer.GetPosition());
 	// now write the header to the file, active_header determines whether we write to h1 or h2
 	// note that if active_header is h1 we write to h2, and vice versa
 	ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -3,11 +3,11 @@
 #include "duckdb/common/allocator.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/set.hpp"
-#include "duckdb/storage/in_memory_block_manager.hpp"
-#include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/storage/buffer/buffer_pool.hpp"
+#include "duckdb/storage/in_memory_block_manager.hpp"
+#include "duckdb/storage/storage_manager.hpp"
 
 namespace duckdb {
 
@@ -77,14 +77,6 @@ idx_t StandardBufferManager::GetMaxMemory() const {
 	return buffer_pool.GetMaxMemory();
 }
 
-// POTENTIALLY PROBLEMATIC
-// void StandardBufferManager::IncreaseUsedMemory(idx_t size, bool unsafe) {
-//	if (!unsafe && buffer_pool.GetUsedMemory() + size > buffer_pool.GetMaxMemory()) {
-//		throw OutOfMemoryException("Failed to allocate data of size %lld%s", size, InMemoryWarning());
-//	}
-//	buffer_pool.IncreaseUsedMemory(size);
-//}
-
 template <typename... ARGS>
 TempBufferPoolReservation StandardBufferManager::EvictBlocksOrThrow(idx_t memory_delta, unique_ptr<FileBuffer> *buffer,
                                                                     ARGS... args) {
@@ -329,12 +321,13 @@ private:
 };
 
 class TemporaryFileHandle {
-	constexpr static idx_t MAX_ALLOWED_INDEX = 4000;
+	constexpr static idx_t MAX_ALLOWED_INDEX_BASE = 4000;
 
 public:
-	TemporaryFileHandle(DatabaseInstance &db, const string &temp_directory, idx_t index)
-	    : db(db), file_index(index), path(FileSystem::GetFileSystem(db).JoinPath(
-	                                     temp_directory, "duckdb_temp_storage-" + to_string(index) + ".tmp")) {
+	TemporaryFileHandle(idx_t temp_file_count, DatabaseInstance &db, const string &temp_directory, idx_t index)
+	    : max_allowed_index((1 << temp_file_count) * MAX_ALLOWED_INDEX_BASE), db(db), file_index(index),
+	      path(FileSystem::GetFileSystem(db).JoinPath(temp_directory,
+	                                                  "duckdb_temp_storage-" + to_string(index) + ".tmp")) {
 	}
 
 public:
@@ -348,7 +341,7 @@ public:
 public:
 	TemporaryFileIndex TryGetBlockIndex() {
 		TemporaryFileLock lock(file_lock);
-		if (index_manager.GetMaxIndex() >= MAX_ALLOWED_INDEX && index_manager.HasFreeBlocks()) {
+		if (index_manager.GetMaxIndex() >= max_allowed_index && index_manager.HasFreeBlocks()) {
 			// file is at capacity
 			return TemporaryFileIndex();
 		}
@@ -426,6 +419,7 @@ private:
 	}
 
 private:
+	const idx_t max_allowed_index;
 	DatabaseInstance &db;
 	unique_ptr<FileHandle> handle;
 	idx_t file_index;
@@ -467,7 +461,7 @@ public:
 			if (!handle) {
 				// no existing handle to write to; we need to create & open a new file
 				auto new_file_index = index_manager.GetNewBlockIndex();
-				auto new_file = make_uniq<TemporaryFileHandle>(db, temp_directory, new_file_index);
+				auto new_file = make_uniq<TemporaryFileHandle>(files.size(), db, temp_directory, new_file_index);
 				handle = new_file.get();
 				files[new_file_index] = std::move(new_file);
 

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -1,11 +1,13 @@
 #include "duckdb/common/exception.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/storage/statistics/struct_stats.hpp"
 
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+
 namespace duckdb {
 
 BaseStatistics::BaseStatistics() : type(LogicalType::INVALID) {
@@ -267,63 +269,70 @@ void BaseStatistics::CopyValidity(BaseStatistics &stats) {
 	has_no_null = stats.has_no_null;
 }
 
-void BaseStatistics::Serialize(Serializer &serializer) const {
-	FieldWriter writer(serializer);
-	writer.WriteField<bool>(has_null);
-	writer.WriteField<bool>(has_no_null);
-	writer.WriteField<idx_t>(distinct_count);
-	Serialize(writer);
-	writer.Finalize();
-}
-
 void BaseStatistics::SetDistinctCount(idx_t count) {
 	this->distinct_count = count;
 }
 
-void BaseStatistics::Serialize(FieldWriter &writer) const {
-	switch (GetStatsType()) {
-	case StatisticsType::NUMERIC_STATS:
-		NumericStats::Serialize(*this, writer);
-		break;
-	case StatisticsType::STRING_STATS:
-		StringStats::Serialize(*this, writer);
-		break;
-	case StatisticsType::LIST_STATS:
-		ListStats::Serialize(*this, writer);
-		break;
-	case StatisticsType::STRUCT_STATS:
-		StructStats::Serialize(*this, writer);
-		break;
-	default:
-		break;
-	}
-}
-BaseStatistics BaseStatistics::DeserializeType(FieldReader &reader, LogicalType type) {
-	switch (GetStatsType(type)) {
-	case StatisticsType::NUMERIC_STATS:
-		return NumericStats::Deserialize(reader, std::move(type));
-	case StatisticsType::STRING_STATS:
-		return StringStats::Deserialize(reader, std::move(type));
-	case StatisticsType::LIST_STATS:
-		return ListStats::Deserialize(reader, std::move(type));
-	case StatisticsType::STRUCT_STATS:
-		return StructStats::Deserialize(reader, std::move(type));
-	default:
-		return BaseStatistics(std::move(type));
-	}
-}
+void BaseStatistics::Serialize(Serializer &serializer) const {
+	serializer.WriteProperty(100, "has_null", has_null);
+	serializer.WriteProperty(101, "has_no_null", has_no_null);
+	serializer.WriteProperty(102, "distinct_count", distinct_count);
+	serializer.WriteObject(103, "type_stats", [&](Serializer &serializer) {
+		switch (GetStatsType()) {
+		case StatisticsType::NUMERIC_STATS:
+			NumericStats::Serialize(*this, serializer);
+			break;
+		case StatisticsType::STRING_STATS:
+			StringStats::Serialize(*this, serializer);
+			break;
+		case StatisticsType::LIST_STATS:
+			ListStats::Serialize(*this, serializer);
+			break;
+		case StatisticsType::STRUCT_STATS:
+			StructStats::Serialize(*this, serializer);
+			break;
+		default:
+			break;
+		}
+	});
+}
+
+BaseStatistics BaseStatistics::Deserialize(Deserializer &deserializer) {
+	auto has_null = deserializer.ReadProperty<bool>(100, "has_null");
+	auto has_no_null = deserializer.ReadProperty<bool>(101, "has_no_null");
+	auto distinct_count = deserializer.ReadProperty<idx_t>(102, "distinct_count");
+
+	// Get the logical type from the deserializer context.
+	auto type = deserializer.Get<LogicalType &>();
+
+	auto stats_type = GetStatsType(type);
+
+	BaseStatistics stats(std::move(type));
+
+	stats.has_null = has_null;
+	stats.has_no_null = has_no_null;
+	stats.distinct_count = distinct_count;
+
+	deserializer.ReadObject(103, "type_stats", [&](Deserializer &obj) {
+		switch (stats_type) {
+		case StatisticsType::NUMERIC_STATS:
+			NumericStats::Deserialize(obj, stats);
+			break;
+		case StatisticsType::STRING_STATS:
+			StringStats::Deserialize(obj, stats);
+			break;
+		case StatisticsType::LIST_STATS:
+			ListStats::Deserialize(obj, stats);
+			break;
+		case StatisticsType::STRUCT_STATS:
+			StructStats::Deserialize(obj, stats);
+			break;
+		default:
+			break;
+		}
+	});
 
-BaseStatistics BaseStatistics::Deserialize(Deserializer &source, LogicalType type) {
-	FieldReader reader(source);
-	bool has_null = reader.ReadRequired<bool>();
-	bool has_no_null = reader.ReadRequired<bool>();
-	idx_t distinct_count = reader.ReadRequired<idx_t>();
-	auto result = DeserializeType(reader, std::move(type));
-	result.has_null = has_null;
-	result.has_no_null = has_no_null;
-	result.distinct_count = distinct_count;
-	reader.Finalize();
-	return result;
+	return stats;
 }
 
 string BaseStatistics::ToString() const {

package/src/duckdb/src/storage/statistics/column_statistics.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/storage/statistics/column_statistics.hpp"
-#include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
 
 namespace duckdb {
 
@@ -53,15 +54,17 @@ void ColumnStatistics::UpdateDistinctStatistics(Vector &v, idx_t count) {
 shared_ptr<ColumnStatistics> ColumnStatistics::Copy() const {
 	return make_shared<ColumnStatistics>(stats.Copy(), distinct_stats ? distinct_stats->Copy() : nullptr);
 }
+
 void ColumnStatistics::Serialize(Serializer &serializer) const {
-	stats.Serialize(serializer);
-	serializer.WriteOptional(distinct_stats);
+	serializer.WriteProperty(100, "statistics", stats);
+	serializer.WritePropertyWithDefault(101, "distinct", distinct_stats, unique_ptr<DistinctStatistics>());
 }
 
-shared_ptr<ColumnStatistics> ColumnStatistics::Deserialize(Deserializer &source, const LogicalType &type) {
-	auto stats = BaseStatistics::Deserialize(source, type);
-	auto distinct_stats = source.ReadOptional<DistinctStatistics>();
-	return make_shared<ColumnStatistics>(stats.Copy(), std::move(distinct_stats));
+shared_ptr<ColumnStatistics> ColumnStatistics::Deserialize(Deserializer &deserializer) {
+	auto stats = deserializer.ReadProperty<BaseStatistics>(100, "statistics");
+	auto distinct_stats = deserializer.ReadPropertyWithDefault<unique_ptr<DistinctStatistics>>(
+	    101, "distinct", unique_ptr<DistinctStatistics>());
+	return make_shared<ColumnStatistics>(std::move(stats), std::move(distinct_stats));
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/statistics/distinct_statistics.cpp

@@ -1,6 +1,5 @@
 #include "duckdb/storage/statistics/distinct_statistics.hpp"
 
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/string_util.hpp"
 
 #include <math.h>
@@ -24,31 +23,6 @@ void DistinctStatistics::Merge(const DistinctStatistics &other) {
 	total_count += other.total_count;
 }
 
-void DistinctStatistics::Serialize(Serializer &serializer) const {
-	FieldWriter writer(serializer);
-	Serialize(writer);
-	writer.Finalize();
-}
-
-void DistinctStatistics::Serialize(FieldWriter &writer) const {
-	writer.WriteField<idx_t>(sample_count);
-	writer.WriteField<idx_t>(total_count);
-	log->Serialize(writer);
-}
-
-unique_ptr<DistinctStatistics> DistinctStatistics::Deserialize(Deserializer &source) {
-	FieldReader reader(source);
-	auto result = Deserialize(reader);
-	reader.Finalize();
-	return result;
-}
-
-unique_ptr<DistinctStatistics> DistinctStatistics::Deserialize(FieldReader &reader) {
-	auto sample_count = reader.ReadRequired<idx_t>();
-	auto total_count = reader.ReadRequired<idx_t>();
-	return make_uniq<DistinctStatistics>(HyperLogLog::Deserialize(reader), sample_count, total_count);
-}
-
 void DistinctStatistics::Update(Vector &v, idx_t count, bool sample) {
 	UnifiedVectorFormat vdata;
 	v.ToUnifiedFormat(count, vdata);

package/src/duckdb/src/storage/statistics/list_stats.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+
 namespace duckdb {
 
 void ListStats::Construct(BaseStatistics &stats) {
@@ -66,17 +68,20 @@ void ListStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
 	child_stats.Merge(other_child_stats);
 }
 
-void ListStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
+void ListStats::Serialize(const BaseStatistics &stats, Serializer &serializer) {
 	auto &child_stats = ListStats::GetChildStats(stats);
-	writer.WriteSerializable(child_stats);
+	serializer.WriteProperty(200, "child_stats", child_stats);
 }
 
-BaseStatistics ListStats::Deserialize(FieldReader &reader, LogicalType type) {
+void ListStats::Deserialize(Deserializer &deserializer, BaseStatistics &base) {
+	auto &type = base.GetType();
 	D_ASSERT(type.InternalType() == PhysicalType::LIST);
 	auto &child_type = ListType::GetChildType(type);
-	BaseStatistics result(std::move(type));
-	result.child_stats[0].Copy(reader.ReadRequiredSerializable<BaseStatistics, BaseStatistics>(child_type));
-	return result;
+
+	// Push the logical type of the child type to the deserialization context
+	deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_type));
+	base.child_stats[0].Copy(deserializer.ReadProperty<BaseStatistics>(200, "child_stats"));
+	deserializer.Unset<LogicalType>();
 }
 
 string ListStats::ToString(const BaseStatistics &stats) {