duckdb 0.8.2-dev87.0 → 0.9.0

package/src/duckdb/src/storage/statistics/numeric_stats.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/storage/statistics/numeric_stats.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
 
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+
 namespace duckdb {
 
 template <>
@@ -403,115 +405,123 @@ Value NumericStats::MaxOrNull(const BaseStatistics &stats) {
 	return NumericStats::Max(stats);
 }
 
-void SerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value, FieldWriter &writer) {
-	writer.WriteField<bool>(!has_value);
+static void SerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value,
+                                       Serializer &serializer) {
+	serializer.WriteProperty(100, "has_value", has_value);
 	if (!has_value) {
 		return;
 	}
 	switch (type.InternalType()) {
 	case PhysicalType::BOOL:
-		writer.WriteField<bool>(val.value_.boolean);
+		serializer.WriteProperty(101, "value", val.value_.boolean);
 		break;
 	case PhysicalType::INT8:
-		writer.WriteField<int8_t>(val.value_.tinyint);
+		serializer.WriteProperty(101, "value", val.value_.tinyint);
 		break;
 	case PhysicalType::INT16:
-		writer.WriteField<int16_t>(val.value_.smallint);
+		serializer.WriteProperty(101, "value", val.value_.smallint);
 		break;
 	case PhysicalType::INT32:
-		writer.WriteField<int32_t>(val.value_.integer);
+		serializer.WriteProperty(101, "value", val.value_.integer);
 		break;
 	case PhysicalType::INT64:
-		writer.WriteField<int64_t>(val.value_.bigint);
+		serializer.WriteProperty(101, "value", val.value_.bigint);
 		break;
 	case PhysicalType::UINT8:
-		writer.WriteField<int8_t>(val.value_.utinyint);
+		serializer.WriteProperty(101, "value", val.value_.utinyint);
 		break;
 	case PhysicalType::UINT16:
-		writer.WriteField<int16_t>(val.value_.usmallint);
+		serializer.WriteProperty(101, "value", val.value_.usmallint);
 		break;
 	case PhysicalType::UINT32:
-		writer.WriteField<int32_t>(val.value_.uinteger);
+		serializer.WriteProperty(101, "value", val.value_.uinteger);
 		break;
 	case PhysicalType::UINT64:
-		writer.WriteField<int64_t>(val.value_.ubigint);
+		serializer.WriteProperty(101, "value", val.value_.ubigint);
 		break;
 	case PhysicalType::INT128:
-		writer.WriteField<hugeint_t>(val.value_.hugeint);
+		serializer.WriteProperty(101, "value", val.value_.hugeint);
 		break;
 	case PhysicalType::FLOAT:
-		writer.WriteField<float>(val.value_.float_);
+		serializer.WriteProperty(101, "value", val.value_.float_);
 		break;
 	case PhysicalType::DOUBLE:
-		writer.WriteField<double>(val.value_.double_);
+		serializer.WriteProperty(101, "value", val.value_.double_);
 		break;
 	default:
 		throw InternalException("Unsupported type for serializing numeric statistics");
 	}
 }
 
-void NumericStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
-	auto &numeric_stats = NumericStats::GetDataUnsafe(stats);
-	SerializeNumericStatsValue(stats.GetType(), numeric_stats.min, numeric_stats.has_min, writer);
-	SerializeNumericStatsValue(stats.GetType(), numeric_stats.max, numeric_stats.has_max, writer);
-}
-
-void DeserializeNumericStatsValue(const LogicalType &type, FieldReader &reader, NumericValueUnion &result,
-                                  bool &has_stats) {
-	auto is_null = reader.ReadRequired<bool>();
-	if (is_null) {
+static void DeserializeNumericStatsValue(const LogicalType &type, NumericValueUnion &result, bool &has_stats,
+                                         Deserializer &deserializer) {
+	auto has_value = deserializer.ReadProperty<bool>(100, "has_value");
+	if (!has_value) {
 		has_stats = false;
 		return;
 	}
 	has_stats = true;
 	switch (type.InternalType()) {
 	case PhysicalType::BOOL:
-		result.value_.boolean = reader.ReadRequired<bool>();
+		result.value_.boolean = deserializer.ReadProperty<bool>(101, "value");
 		break;
 	case PhysicalType::INT8:
-		result.value_.tinyint = reader.ReadRequired<int8_t>();
+		result.value_.tinyint = deserializer.ReadProperty<int8_t>(101, "value");
 		break;
 	case PhysicalType::INT16:
-		result.value_.smallint = reader.ReadRequired<int16_t>();
+		result.value_.smallint = deserializer.ReadProperty<int16_t>(101, "value");
 		break;
 	case PhysicalType::INT32:
-		result.value_.integer = reader.ReadRequired<int32_t>();
+		result.value_.integer = deserializer.ReadProperty<int32_t>(101, "value");
 		break;
 	case PhysicalType::INT64:
-		result.value_.bigint = reader.ReadRequired<int64_t>();
+		result.value_.bigint = deserializer.ReadProperty<int64_t>(101, "value");
 		break;
 	case PhysicalType::UINT8:
-		result.value_.utinyint = reader.ReadRequired<uint8_t>();
+		result.value_.utinyint = deserializer.ReadProperty<uint8_t>(101, "value");
 		break;
 	case PhysicalType::UINT16:
-		result.value_.usmallint = reader.ReadRequired<uint16_t>();
+		result.value_.usmallint = deserializer.ReadProperty<uint16_t>(101, "value");
 		break;
 	case PhysicalType::UINT32:
-		result.value_.uinteger = reader.ReadRequired<uint32_t>();
+		result.value_.uinteger = deserializer.ReadProperty<uint32_t>(101, "value");
 		break;
 	case PhysicalType::UINT64:
-		result.value_.ubigint = reader.ReadRequired<uint64_t>();
+		result.value_.ubigint = deserializer.ReadProperty<uint64_t>(101, "value");
 		break;
 	case PhysicalType::INT128:
-		result.value_.hugeint = reader.ReadRequired<hugeint_t>();
+		result.value_.hugeint = deserializer.ReadProperty<hugeint_t>(101, "value");
 		break;
 	case PhysicalType::FLOAT:
-		result.value_.float_ = reader.ReadRequired<float>();
+		result.value_.float_ = deserializer.ReadProperty<float>(101, "value");
 		break;
 	case PhysicalType::DOUBLE:
-		result.value_.double_ = reader.ReadRequired<double>();
+		result.value_.double_ = deserializer.ReadProperty<double>(101, "value");
 		break;
 	default:
-		throw InternalException("Unsupported type for deserializing numeric statistics");
+		throw InternalException("Unsupported type for serializing numeric statistics");
 	}
 }
 
-BaseStatistics NumericStats::Deserialize(FieldReader &reader, LogicalType type) {
-	BaseStatistics result(std::move(type));
+void NumericStats::Serialize(const BaseStatistics &stats, Serializer &serializer) {
+	auto &numeric_stats = NumericStats::GetDataUnsafe(stats);
+	serializer.WriteObject(200, "max", [&](Serializer &object) {
+		SerializeNumericStatsValue(stats.GetType(), numeric_stats.min, numeric_stats.has_min, object);
+	});
+	serializer.WriteObject(201, "min", [&](Serializer &object) {
+		SerializeNumericStatsValue(stats.GetType(), numeric_stats.max, numeric_stats.has_max, object);
+	});
+}
+
+void NumericStats::Deserialize(Deserializer &deserializer, BaseStatistics &result) {
 	auto &numeric_stats = NumericStats::GetDataUnsafe(result);
-	DeserializeNumericStatsValue(result.GetType(), reader, numeric_stats.min, numeric_stats.has_min);
-	DeserializeNumericStatsValue(result.GetType(), reader, numeric_stats.max, numeric_stats.has_max);
-	return result;
+
+	deserializer.ReadObject(200, "max", [&](Deserializer &object) {
+		DeserializeNumericStatsValue(result.GetType(), numeric_stats.min, numeric_stats.has_min, object);
+	});
+	deserializer.ReadObject(201, "min", [&](Deserializer &object) {
+		DeserializeNumericStatsValue(result.GetType(), numeric_stats.max, numeric_stats.has_max, object);
+	});
 }
 
 string NumericStats::ToString(const BaseStatistics &stats) {

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -1,10 +1,13 @@
 #include "duckdb/storage/statistics/string_stats.hpp"
-#include "duckdb/storage/statistics/base_statistics.hpp"
-#include "duckdb/common/field_writer.hpp"
-#include "utf8proc_wrapper.hpp"
+
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/main/error_manager.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "utf8proc_wrapper.hpp"
+
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -67,6 +70,24 @@ bool StringStats::CanContainUnicode(const BaseStatistics &stats) {
 	return StringStats::GetDataUnsafe(stats).has_unicode;
 }
 
+string GetStringMinMaxValue(const data_t data[]) {
+	idx_t len;
+	for (len = 0; len < StringStatsData::MAX_STRING_MINMAX_SIZE; len++) {
+		if (!data[len]) {
+			break;
+		}
+	}
+	return string(const_char_ptr_cast(data), len);
+}
+
+string StringStats::Min(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).min);
+}
+
+string StringStats::Max(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).max);
+}
+
 void StringStats::ResetMaxStringLength(BaseStatistics &stats) {
 	StringStats::GetDataUnsafe(stats).has_max_string_length = false;
 }
@@ -75,24 +96,22 @@ void StringStats::SetContainsUnicode(BaseStatistics &stats) {
 	StringStats::GetDataUnsafe(stats).has_unicode = true;
 }
 
-void StringStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
+void StringStats::Serialize(const BaseStatistics &stats, Serializer &serializer) {
 	auto &string_data = StringStats::GetDataUnsafe(stats);
-	writer.WriteBlob(string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
-	writer.WriteBlob(string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
-	writer.WriteField<bool>(string_data.has_unicode);
-	writer.WriteField<bool>(string_data.has_max_string_length);
-	writer.WriteField<uint32_t>(string_data.max_string_length);
+	serializer.WriteProperty(200, "min", string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	serializer.WriteProperty(201, "max", string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	serializer.WriteProperty(202, "has_unicode", string_data.has_unicode);
+	serializer.WriteProperty(203, "has_max_string_length", string_data.has_max_string_length);
+	serializer.WriteProperty(204, "max_string_length", string_data.max_string_length);
 }
 
-BaseStatistics StringStats::Deserialize(FieldReader &reader, LogicalType type) {
-	BaseStatistics result(std::move(type));
-	auto &string_data = StringStats::GetDataUnsafe(result);
-	reader.ReadBlob(string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
-	reader.ReadBlob(string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
-	string_data.has_unicode = reader.ReadRequired<bool>();
-	string_data.has_max_string_length = reader.ReadRequired<bool>();
-	string_data.max_string_length = reader.ReadRequired<uint32_t>();
-	return result;
+void StringStats::Deserialize(Deserializer &deserializer, BaseStatistics &base) {
+	auto &string_data = StringStats::GetDataUnsafe(base);
+	deserializer.ReadProperty(200, "min", string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	deserializer.ReadProperty(201, "max", string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	deserializer.ReadProperty(202, "has_unicode", string_data.has_unicode);
+	deserializer.ReadProperty(203, "has_max_string_length", string_data.has_max_string_length);
+	deserializer.ReadProperty(204, "max_string_length", string_data.max_string_length);
 }
 
 static int StringValueComparison(const_data_ptr_t data, idx_t len, const_data_ptr_t comparison) {

package/src/duckdb/src/storage/statistics/struct_stats.cpp

@@ -1,8 +1,10 @@
 #include "duckdb/storage/statistics/struct_stats.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/types/vector.hpp"
 
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+
 namespace duckdb {
 
 void StructStats::Construct(BaseStatistics &stats) {
@@ -90,23 +92,26 @@ void StructStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
 	}
 }
 
-void StructStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
+void StructStats::Serialize(const BaseStatistics &stats, Serializer &serializer) {
 	auto child_stats = StructStats::GetChildStats(stats);
 	auto child_count = StructType::GetChildCount(stats.GetType());
-	for (idx_t i = 0; i < child_count; i++) {
-		writer.WriteSerializable(child_stats[i]);
-	}
+
+	serializer.WriteList(200, "child_stats", child_count,
+	                     [&](Serializer::List &list, idx_t i) { list.WriteElement(child_stats[i]); });
 }
 
-BaseStatistics StructStats::Deserialize(FieldReader &reader, LogicalType type) {
+void StructStats::Deserialize(Deserializer &deserializer, BaseStatistics &base) {
+	auto &type = base.GetType();
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
+
 	auto &child_types = StructType::GetChildTypes(type);
-	BaseStatistics result(std::move(type));
-	for (idx_t i = 0; i < child_types.size(); i++) {
-		result.child_stats[i].Copy(
-		    reader.ReadRequiredSerializable<BaseStatistics, BaseStatistics>(child_types[i].second));
-	}
-	return result;
+
+	deserializer.ReadList(200, "child_stats", [&](Deserializer::List &list, idx_t i) {
+		deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_types[i].second));
+		auto stat = list.ReadElement<BaseStatistics>();
+		base.child_stats[i].Copy(stat);
+		deserializer.Unset<LogicalType>();
+	});
 }
 
 string StructStats::ToString(const BaseStatistics &stats) {

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,14 +2,15 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 51;
+const uint64_t VERSION_NUMBER = 64;
 
 struct StorageVersionInfo {
 	const char *version_name;
 	idx_t storage_version;
 };
 
-static StorageVersionInfo storage_version_info[] = {{"v0.7.0 or v0.7.1", 43},
+static StorageVersionInfo storage_version_info[] = {{"v0.8.0 or v0.8.1", 51},
+                                                    {"v0.7.0 or v0.7.1", 43},
                                                     {"v0.6.0 or v0.6.1", 39},
                                                     {"v0.5.0 or v0.5.1", 38},
                                                     {"v0.3.3, v0.3.4 or v0.4.0", 33},

package/src/duckdb/src/storage/storage_manager.cpp

@@ -82,6 +82,9 @@ public:
 	BlockManager &GetBlockManagerForRowData() override {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager() override {
+		return block_manager.GetMetadataManager();
+	}
 };
 
 SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only)
@@ -94,8 +97,13 @@ void SingleFileStorageManager::LoadDatabase() {
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 		return;
 	}
-
-	string wal_path = path + ".wal";
+	std::size_t question_mark_pos = path.find('?');
+	auto wal_path = path;
+	if (question_mark_pos != std::string::npos) {
+		wal_path.insert(question_mark_pos, ".wal");
+	} else {
+		wal_path += ".wal";
+	}
 	auto &fs = FileSystem::Get(db);
 	auto &config = DBConfig::Get(db);
 	bool truncate_wal = false;
@@ -135,8 +143,6 @@ void SingleFileStorageManager::LoadDatabase() {
 		//! Load from storage
 		auto checkpointer = SingleFileCheckpointReader(*this);
 		checkpointer.LoadFromStorage();
-		// finish load checkpoint, clear the cached handles of meta blocks
-		block_manager->ClearMetaBlockHandles();
 		// check if the WAL file exists
 		if (fs.FileExists(wal_path)) {
 			// replay the WAL
@@ -218,7 +224,7 @@ unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(T
 	return make_uniq<SingleFileStorageCommitState>(*this, checkpoint);
 }
 
-bool SingleFileStorageManager::IsCheckpointClean(block_id_t checkpoint_id) {
+bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id) {
 	return block_manager->IsRootBlock(checkpoint_id);
 }
 
@@ -229,8 +235,12 @@ void SingleFileStorageManager::CreateCheckpoint(bool delete_wal, bool force_chec
 	auto &config = DBConfig::Get(db);
 	if (wal->GetWALSize() > 0 || config.options.force_checkpoint || force_checkpoint) {
 		// we only need to checkpoint if there is anything in the WAL
-		SingleFileCheckpointWriter checkpointer(db, *block_manager);
-		checkpointer.CreateCheckpoint();
+		try {
+			SingleFileCheckpointWriter checkpointer(db, *block_manager);
+			checkpointer.CreateCheckpoint();
+		} catch (std::exception &ex) {
+			throw FatalException("Failed to create checkpoint because of error: %s", ex.what());
+		}
 	}
 	if (delete_wal) {
 		wal->Delete();
@@ -254,6 +264,11 @@ DatabaseSize SingleFileStorageManager::GetDatabaseSize() {
 	return ds;
 }
 
+vector<MetadataBlockInfo> SingleFileStorageManager::GetMetadataInfo() {
+	auto &metadata_manager = block_manager->GetMetadataManager();
+	return metadata_manager.GetMetadataInfo();
+}
+
 bool SingleFileStorageManager::AutomaticCheckpoint(idx_t estimated_wal_bytes) {
 	auto log = GetWriteAheadLog();
 	if (!log) {

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/storage/table/chunk_info.hpp"
 #include "duckdb/transaction/transaction.hpp"
-#include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
 
 namespace duckdb {
 
@@ -28,15 +30,19 @@ static bool UseVersion(TransactionData transaction, transaction_t id) {
 	return TransactionVersionOperator::UseInsertedVersion(transaction.start_time, transaction.transaction_id, id);
 }
 
-unique_ptr<ChunkInfo> ChunkInfo::Deserialize(Deserializer &source) {
-	auto type = source.Read<ChunkInfoType>();
+void ChunkInfo::Write(WriteStream &writer) const {
+	writer.Write<ChunkInfoType>(type);
+}
+
+unique_ptr<ChunkInfo> ChunkInfo::Read(ReadStream &reader) {
+	auto type = reader.Read<ChunkInfoType>();
 	switch (type) {
 	case ChunkInfoType::EMPTY_INFO:
 		return nullptr;
 	case ChunkInfoType::CONSTANT_INFO:
-		return ChunkConstantInfo::Deserialize(source);
+		return ChunkConstantInfo::Read(reader);
 	case ChunkInfoType::VECTOR_INFO:
-		return ChunkVectorInfo::Deserialize(source);
+		return ChunkVectorInfo::Read(reader);
 	default:
 		throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type");
 	}
@@ -51,7 +57,7 @@ ChunkConstantInfo::ChunkConstantInfo(idx_t start)
 
 template <class OP>
 idx_t ChunkConstantInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id,
-                                               SelectionVector &sel_vector, idx_t max_count) {
+                                               SelectionVector &sel_vector, idx_t max_count) const {
 	if (OP::UseInsertedVersion(start_time, transaction_id, insert_id) &&
 	    OP::UseDeletedVersion(start_time, transaction_id, delete_id)) {
 		return max_count;
@@ -78,20 +84,23 @@ void ChunkConstantInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t
 	insert_id = commit_id;
 }
 
-void ChunkConstantInfo::Serialize(Serializer &serializer) {
-	// we only need to write this node if any tuple deletions have been committed
+bool ChunkConstantInfo::HasDeletes() const {
 	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
-	if (!is_deleted) {
-		serializer.Write<ChunkInfoType>(ChunkInfoType::EMPTY_INFO);
-		return;
-	}
-	serializer.Write<ChunkInfoType>(type);
-	serializer.Write<idx_t>(start);
+	return is_deleted;
 }
 
-unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
-	auto start = source.Read<idx_t>();
+idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
+	return delete_id < TRANSACTION_ID_START ? max_count : 0;
+}
+
+void ChunkConstantInfo::Write(WriteStream &writer) const {
+	D_ASSERT(HasDeletes());
+	ChunkInfo::Write(writer);
+	writer.Write<idx_t>(start);
+}
 
+unique_ptr<ChunkInfo> ChunkConstantInfo::Read(ReadStream &reader) {
+	auto start = reader.Read<idx_t>();
 	auto info = make_uniq<ChunkConstantInfo>(start);
 	info->insert_id = 0;
 	info->delete_id = 0;
@@ -111,7 +120,7 @@ ChunkVectorInfo::ChunkVectorInfo(idx_t start)
 
 template <class OP>
 idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id,
-                                             SelectionVector &sel_vector, idx_t max_count) {
+                                             SelectionVector &sel_vector, idx_t max_count) const {
 	idx_t count = 0;
 	if (same_inserted_id && !any_deleted) {
 		// all tuples have the same inserted id: and no tuples were deleted
@@ -150,7 +159,7 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti
 }
 
 idx_t ChunkVectorInfo::GetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-                                    idx_t max_count) {
+                                    idx_t max_count) const {
 	return TemplatedGetSelVector<TransactionVersionOperator>(start_time, transaction_id, sel_vector, max_count);
 }
 
@@ -215,44 +224,58 @@ void ChunkVectorInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t e
 	}
 }
 
-void ChunkVectorInfo::Serialize(Serializer &serializer) {
+bool ChunkVectorInfo::HasDeletes() const {
+	return any_deleted;
+}
+
+idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
+	if (!any_deleted) {
+		return 0;
+	}
+	idx_t delete_count = 0;
+	for (idx_t i = 0; i < max_count; i++) {
+		if (deleted[i] < TRANSACTION_ID_START) {
+			delete_count++;
+		}
+	}
+	return delete_count;
+}
+
+void ChunkVectorInfo::Write(WriteStream &writer) const {
 	SelectionVector sel(STANDARD_VECTOR_SIZE);
 	transaction_t start_time = TRANSACTION_ID_START - 1;
 	transaction_t transaction_id = DConstants::INVALID_INDEX;
 	idx_t count = GetSelVector(start_time, transaction_id, sel, STANDARD_VECTOR_SIZE);
 	if (count == STANDARD_VECTOR_SIZE) {
 		// nothing is deleted: skip writing anything
-		serializer.Write<ChunkInfoType>(ChunkInfoType::EMPTY_INFO);
+		writer.Write<ChunkInfoType>(ChunkInfoType::EMPTY_INFO);
 		return;
 	}
 	if (count == 0) {
 		// everything is deleted: write a constant vector
-		serializer.Write<ChunkInfoType>(ChunkInfoType::CONSTANT_INFO);
-		serializer.Write<idx_t>(start);
+		writer.Write<ChunkInfoType>(ChunkInfoType::CONSTANT_INFO);
+		writer.Write<idx_t>(start);
 		return;
 	}
 	// write a boolean vector
-	serializer.Write<ChunkInfoType>(ChunkInfoType::VECTOR_INFO);
-	serializer.Write<idx_t>(start);
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		deleted_tuples[i] = true;
-	}
+	ChunkInfo::Write(writer);
+	writer.Write<idx_t>(start);
+	ValidityMask mask(STANDARD_VECTOR_SIZE);
+	mask.Initialize(STANDARD_VECTOR_SIZE);
 	for (idx_t i = 0; i < count; i++) {
-		deleted_tuples[sel.get_index(i)] = false;
+		mask.SetInvalid(sel.get_index(i));
 	}
-	serializer.WriteData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
+	mask.Write(writer, STANDARD_VECTOR_SIZE);
 }
 
-unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
-	auto start = source.Read<idx_t>();
-
+unique_ptr<ChunkInfo> ChunkVectorInfo::Read(ReadStream &reader) {
+	auto start = reader.Read<idx_t>();
 	auto result = make_uniq<ChunkVectorInfo>(start);
 	result->any_deleted = true;
-	bool deleted_tuples[STANDARD_VECTOR_SIZE];
-	source.ReadData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
+	ValidityMask mask;
+	mask.Read(reader, STANDARD_VECTOR_SIZE);
 	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
-		if (deleted_tuples[i]) {
+		if (mask.RowIsValid(i)) {
 			result->deleted[i] = 0;
 		}
 	}