duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -6,6 +6,9 @@
 #include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/storage/statistics/struct_stats.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 BaseStatistics::BaseStatistics() : type(LogicalType::INVALID) {
@@ -267,6 +270,10 @@ void BaseStatistics::CopyValidity(BaseStatistics &stats) {
 	has_no_null = stats.has_no_null;
 }
 
+void BaseStatistics::SetDistinctCount(idx_t count) {
+	this->distinct_count = count;
+}
+
 void BaseStatistics::Serialize(Serializer &serializer) const {
 	FieldWriter writer(serializer);
 	writer.WriteField<bool>(has_null);
@@ -276,10 +283,6 @@ void BaseStatistics::Serialize(Serializer &serializer) const {
 	writer.Finalize();
 }
 
-void BaseStatistics::SetDistinctCount(idx_t count) {
-	this->distinct_count = count;
-}
-
 void BaseStatistics::Serialize(FieldWriter &writer) const {
 	switch (GetStatsType()) {
 	case StatisticsType::NUMERIC_STATS:
@@ -298,6 +301,66 @@ void BaseStatistics::Serialize(FieldWriter &writer) const {
 		break;
 	}
 }
+
+void BaseStatistics::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "has_null", has_null);
+	serializer.WriteProperty(101, "has_no_null", has_no_null);
+	serializer.WriteProperty(102, "distinct_count", distinct_count);
+	serializer.WriteProperty(103, "stats_type", GetStatsType());
+	serializer.WriteObject(104, "stats", [this](FormatSerializer &serializer) {
+		switch (GetStatsType()) {
+		case StatisticsType::NUMERIC_STATS:
+			NumericStats::FormatSerialize(*this, serializer);
+			break;
+		case StatisticsType::STRING_STATS:
+			StringStats::FormatSerialize(*this, serializer);
+			break;
+		case StatisticsType::LIST_STATS:
+			ListStats::FormatSerialize(*this, serializer);
+			break;
+		case StatisticsType::STRUCT_STATS:
+			StructStats::FormatSerialize(*this, serializer);
+			break;
+		default:
+			throw NotImplementedException("Unrecognized StatisticsType for BaseStatistics::FormatSerialize");
+		}
+	});
+}
+
+BaseStatistics BaseStatistics::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto has_null = deserializer.ReadProperty<bool>(100, "has_null");
+	auto has_no_null = deserializer.ReadProperty<bool>(101, "has_no_null");
+	auto distinct_count = deserializer.ReadProperty<idx_t>(102, "distinct_count");
+
+	// Get the logical type from the deserializer context.
+	auto type = deserializer.Get<LogicalType &>();
+
+	BaseStatistics stats;
+	switch (GetStatsType(type)) {
+	case StatisticsType::NUMERIC_STATS:
+		stats = NumericStats::FormatDeserialize(deserializer, type);
+		break;
+	case StatisticsType::STRING_STATS:
+		stats = StringStats::FormatDeserialize(deserializer, type);
+		break;
+	case StatisticsType::LIST_STATS:
+		stats = ListStats::FormatDeserialize(deserializer, type);
+		break;
+	case StatisticsType::STRUCT_STATS:
+		stats = StructStats::FormatDeserialize(deserializer, type);
+		break;
+	default:
+		stats = BaseStatistics(std::move(type));
+		break;
+	}
+
+	stats.has_null = has_null;
+	stats.has_no_null = has_no_null;
+	stats.distinct_count = distinct_count;
+
+	return stats;
+}
+
 BaseStatistics BaseStatistics::DeserializeType(FieldReader &reader, LogicalType type) {
 	switch (GetStatsType(type)) {
 	case StatisticsType::NUMERIC_STATS:

package/src/duckdb/src/storage/statistics/column_statistics.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/storage/statistics/column_statistics.hpp"
 #include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
 
 namespace duckdb {
 
@@ -64,4 +66,18 @@ shared_ptr<ColumnStatistics> ColumnStatistics::Deserialize(Deserializer &source,
 	return make_shared<ColumnStatistics>(stats.Copy(), std::move(distinct_stats));
 }
 
+void ColumnStatistics::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "statistics", stats);
+	serializer.WritePropertyWithDefault(101, "distinct", distinct_stats, unique_ptr<DistinctStatistics>());
+}
+
+shared_ptr<ColumnStatistics> ColumnStatistics::FormatDeserialize(FormatDeserializer &deserializer) {
+	// TODO: do we read this as an property or into the object itself?
+	// we have this sort of pseudo inheritance going on here which is annoying
+	auto stats = BaseStatistics::FormatDeserialize(deserializer);
+	auto distinct_stats = deserializer.ReadPropertyWithDefault<unique_ptr<DistinctStatistics>>(
+	    101, "distinct", unique_ptr<DistinctStatistics>());
+	return make_shared<ColumnStatistics>(stats.Copy(), std::move(distinct_stats));
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/statistics/list_stats.cpp

@@ -4,6 +4,9 @@
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 void ListStats::Construct(BaseStatistics &stats) {
@@ -79,6 +82,24 @@ BaseStatistics ListStats::Deserialize(FieldReader &reader, LogicalType type) {
 	return result;
 }
 
+void ListStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto &child_stats = ListStats::GetChildStats(stats);
+	serializer.WriteProperty(200, "child_stats", child_stats);
+}
+
+BaseStatistics ListStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	D_ASSERT(type.InternalType() == PhysicalType::LIST);
+	auto &child_type = ListType::GetChildType(type);
+	BaseStatistics result(std::move(type));
+
+	// Push the logical type of the child type to the deserialization context
+	deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_type));
+	result.child_stats[0].Copy(deserializer.ReadProperty<BaseStatistics>(200, "child_stats"));
+	deserializer.Unset<LogicalType>();
+
+	return result;
+}
+
 string ListStats::ToString(const BaseStatistics &stats) {
 	auto &child_stats = ListStats::GetChildStats(stats);
 	return StringUtil::Format("[%s]", child_stats.ToString());

package/src/duckdb/src/storage/statistics/numeric_stats.cpp

@@ -4,6 +4,9 @@
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 template <>
@@ -403,7 +406,8 @@ Value NumericStats::MaxOrNull(const BaseStatistics &stats) {
 	return NumericStats::Max(stats);
 }
 
-void SerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value, FieldWriter &writer) {
+static void SerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value,
+                                       FieldWriter &writer) {
 	writer.WriteField<bool>(!has_value);
 	if (!has_value) {
 		return;
@@ -514,6 +518,127 @@ BaseStatistics NumericStats::Deserialize(FieldReader &reader, LogicalType type)
 	return result;
 }
 
+static void FormatSerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value,
+                                             FormatSerializer &serializer) {
+	serializer.WriteProperty(100, "has_value", has_value);
+	if (!has_value) {
+		return;
+	}
+	switch (type.InternalType()) {
+	case PhysicalType::BOOL:
+		serializer.WriteProperty(101, "value", val.value_.boolean);
+		break;
+	case PhysicalType::INT8:
+		serializer.WriteProperty(101, "value", val.value_.tinyint);
+		break;
+	case PhysicalType::INT16:
+		serializer.WriteProperty(101, "value", val.value_.smallint);
+		break;
+	case PhysicalType::INT32:
+		serializer.WriteProperty(101, "value", val.value_.integer);
+		break;
+	case PhysicalType::INT64:
+		serializer.WriteProperty(101, "value", val.value_.bigint);
+		break;
+	case PhysicalType::UINT8:
+		serializer.WriteProperty(101, "value", val.value_.utinyint);
+		break;
+	case PhysicalType::UINT16:
+		serializer.WriteProperty(101, "value", val.value_.usmallint);
+		break;
+	case PhysicalType::UINT32:
+		serializer.WriteProperty(101, "value", val.value_.uinteger);
+		break;
+	case PhysicalType::UINT64:
+		serializer.WriteProperty(101, "value", val.value_.ubigint);
+		break;
+	case PhysicalType::INT128:
+		serializer.WriteProperty(101, "value", val.value_.hugeint);
+		break;
+	case PhysicalType::FLOAT:
+		serializer.WriteProperty(101, "value", val.value_.float_);
+		break;
+	case PhysicalType::DOUBLE:
+		serializer.WriteProperty(101, "value", val.value_.double_);
+		break;
+	default:
+		throw InternalException("Unsupported type for serializing numeric statistics");
+	}
+}
+
+void NumericStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto &numeric_stats = NumericStats::GetDataUnsafe(stats);
+	serializer.WriteObject(200, "max", [&](FormatSerializer &object) {
+		FormatSerializeNumericStatsValue(stats.GetType(), numeric_stats.min, numeric_stats.has_min, object);
+	});
+	serializer.WriteObject(201, "min", [&](FormatSerializer &object) {
+		FormatSerializeNumericStatsValue(stats.GetType(), numeric_stats.max, numeric_stats.has_max, object);
+	});
+}
+
+static void FormatDeserializeNumericStatsValue(const LogicalType &type, NumericValueUnion &result, bool &has_stats,
+                                               FormatDeserializer &deserializer) {
+	auto has_value = deserializer.ReadProperty<bool>(100, "has_value");
+	if (!has_value) {
+		has_stats = false;
+		return;
+	}
+	has_stats = true;
+	switch (type.InternalType()) {
+	case PhysicalType::BOOL:
+		result.value_.boolean = deserializer.ReadProperty<bool>(101, "value");
+		break;
+	case PhysicalType::INT8:
+		result.value_.tinyint = deserializer.ReadProperty<int8_t>(101, "value");
+		break;
+	case PhysicalType::INT16:
+		result.value_.smallint = deserializer.ReadProperty<int16_t>(101, "value");
+		break;
+	case PhysicalType::INT32:
+		result.value_.integer = deserializer.ReadProperty<int32_t>(101, "value");
+		break;
+	case PhysicalType::INT64:
+		result.value_.bigint = deserializer.ReadProperty<int64_t>(101, "value");
+		break;
+	case PhysicalType::UINT8:
+		result.value_.utinyint = deserializer.ReadProperty<uint8_t>(101, "value");
+		break;
+	case PhysicalType::UINT16:
+		result.value_.usmallint = deserializer.ReadProperty<uint16_t>(101, "value");
+		break;
+	case PhysicalType::UINT32:
+		result.value_.uinteger = deserializer.ReadProperty<uint32_t>(101, "value");
+		break;
+	case PhysicalType::UINT64:
+		result.value_.ubigint = deserializer.ReadProperty<uint64_t>(101, "value");
+		break;
+	case PhysicalType::INT128:
+		result.value_.hugeint = deserializer.ReadProperty<hugeint_t>(101, "value");
+		break;
+	case PhysicalType::FLOAT:
+		result.value_.float_ = deserializer.ReadProperty<float>(101, "value");
+		break;
+	case PhysicalType::DOUBLE:
+		result.value_.double_ = deserializer.ReadProperty<double>(101, "value");
+		break;
+	default:
+		throw InternalException("Unsupported type for serializing numeric statistics");
+	}
+}
+
+BaseStatistics NumericStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	BaseStatistics result(std::move(type));
+	auto &numeric_stats = NumericStats::GetDataUnsafe(result);
+
+	deserializer.ReadObject(200, "max", [&](FormatDeserializer &object) {
+		FormatDeserializeNumericStatsValue(result.GetType(), numeric_stats.min, numeric_stats.has_min, object);
+	});
+	deserializer.ReadObject(201, "min", [&](FormatDeserializer &object) {
+		FormatDeserializeNumericStatsValue(result.GetType(), numeric_stats.max, numeric_stats.has_max, object);
+	});
+	return result;
+}
+
 string NumericStats::ToString(const BaseStatistics &stats) {
 	return StringUtil::Format("[Min: %s, Max: %s]", NumericStats::MinOrNull(stats).ToString(),
 	                          NumericStats::MaxOrNull(stats).ToString());

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -1,10 +1,14 @@
 #include "duckdb/storage/statistics/string_stats.hpp"
-#include "duckdb/storage/statistics/base_statistics.hpp"
+
 #include "duckdb/common/field_writer.hpp"
-#include "utf8proc_wrapper.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/main/error_manager.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "utf8proc_wrapper.hpp"
+
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -67,6 +71,24 @@ bool StringStats::CanContainUnicode(const BaseStatistics &stats) {
 	return StringStats::GetDataUnsafe(stats).has_unicode;
 }
 
+string GetStringMinMaxValue(const data_t data[]) {
+	idx_t len;
+	for (len = 0; len < StringStatsData::MAX_STRING_MINMAX_SIZE; len++) {
+		if (!data[len]) {
+			break;
+		}
+	}
+	return string(const_char_ptr_cast(data), len);
+}
+
+string StringStats::Min(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).min);
+}
+
+string StringStats::Max(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).max);
+}
+
 void StringStats::ResetMaxStringLength(BaseStatistics &stats) {
 	StringStats::GetDataUnsafe(stats).has_max_string_length = false;
 }
@@ -84,6 +106,15 @@ void StringStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
 	writer.WriteField<uint32_t>(string_data.max_string_length);
 }
 
+void StringStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	serializer.WriteProperty(200, "min", string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	serializer.WriteProperty(201, "max", string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	serializer.WriteProperty(202, "has_unicode", string_data.has_unicode);
+	serializer.WriteProperty(203, "has_max_string_length", string_data.has_max_string_length);
+	serializer.WriteProperty(204, "max_string_length", string_data.max_string_length);
+}
+
 BaseStatistics StringStats::Deserialize(FieldReader &reader, LogicalType type) {
 	BaseStatistics result(std::move(type));
 	auto &string_data = StringStats::GetDataUnsafe(result);
@@ -95,6 +126,17 @@ BaseStatistics StringStats::Deserialize(FieldReader &reader, LogicalType type) {
 	return result;
 }
 
+BaseStatistics StringStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	BaseStatistics result(std::move(type));
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	deserializer.ReadProperty(200, "min", string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	deserializer.ReadProperty(201, "max", string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	deserializer.ReadProperty(202, "has_unicode", string_data.has_unicode);
+	deserializer.ReadProperty(203, "has_max_string_length", string_data.has_max_string_length);
+	deserializer.ReadProperty(204, "max_string_length", string_data.max_string_length);
+	return result;
+}
+
 static int StringValueComparison(const_data_ptr_t data, idx_t len, const_data_ptr_t comparison) {
 	D_ASSERT(len <= StringStatsData::MAX_STRING_MINMAX_SIZE);
 	for (idx_t i = 0; i < len; i++) {

package/src/duckdb/src/storage/statistics/struct_stats.cpp

@@ -3,6 +3,9 @@
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/types/vector.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 void StructStats::Construct(BaseStatistics &stats) {
@@ -109,6 +112,30 @@ BaseStatistics StructStats::Deserialize(FieldReader &reader, LogicalType type) {
 	return result;
 }
 
+void StructStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto child_stats = StructStats::GetChildStats(stats);
+	auto child_count = StructType::GetChildCount(stats.GetType());
+
+	serializer.WriteList(200, "child_stats", child_count,
+	                     [&](FormatSerializer::List &list, idx_t i) { list.WriteElement(child_stats[i]); });
+}
+
+BaseStatistics StructStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
+
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+
+	deserializer.ReadList(200, "child_stats", [&](FormatDeserializer::List &list, idx_t i) {
+		deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_types[i].second));
+		auto stat = list.ReadElement<BaseStatistics>();
+		result.child_stats[i].Copy(stat);
+		deserializer.Unset<LogicalType>();
+	});
+
+	return result;
+}
+
 string StructStats::ToString(const BaseStatistics &stats) {
 	string result;
 	result += " {";

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,14 +2,15 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 51;
+const uint64_t VERSION_NUMBER = 57;
 
 struct StorageVersionInfo {
 	const char *version_name;
 	idx_t storage_version;
 };
 
-static StorageVersionInfo storage_version_info[] = {{"v0.7.0 or v0.7.1", 43},
+static StorageVersionInfo storage_version_info[] = {{"v0.8.0 or v0.8.1", 51},
+                                                    {"v0.7.0 or v0.7.1", 43},
                                                     {"v0.6.0 or v0.6.1", 39},
                                                     {"v0.5.0 or v0.5.1", 38},
                                                     {"v0.3.3, v0.3.4 or v0.4.0", 33},

package/src/duckdb/src/storage/storage_manager.cpp

@@ -82,6 +82,9 @@ public:
 	BlockManager &GetBlockManagerForRowData() override {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager() override {
+		return block_manager.GetMetadataManager();
+	}
 };
 
 SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only)
@@ -94,8 +97,13 @@ void SingleFileStorageManager::LoadDatabase() {
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 		return;
 	}
-
-	string wal_path = path + ".wal";
+	std::size_t question_mark_pos = path.find('?');
+	auto wal_path = path;
+	if (question_mark_pos != std::string::npos) {
+		wal_path.insert(question_mark_pos, ".wal");
+	} else {
+		wal_path += ".wal";
+	}
 	auto &fs = FileSystem::Get(db);
 	auto &config = DBConfig::Get(db);
 	bool truncate_wal = false;
@@ -135,8 +143,6 @@ void SingleFileStorageManager::LoadDatabase() {
 		//! Load from storage
 		auto checkpointer = SingleFileCheckpointReader(*this);
 		checkpointer.LoadFromStorage();
-		// finish load checkpoint, clear the cached handles of meta blocks
-		block_manager->ClearMetaBlockHandles();
 		// check if the WAL file exists
 		if (fs.FileExists(wal_path)) {
 			// replay the WAL
@@ -218,7 +224,7 @@ unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(T
 	return make_uniq<SingleFileStorageCommitState>(*this, checkpoint);
 }
 
-bool SingleFileStorageManager::IsCheckpointClean(block_id_t checkpoint_id) {
+bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id) {
 	return block_manager->IsRootBlock(checkpoint_id);
 }
 

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/storage/table/chunk_info.hpp"
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -42,6 +44,20 @@ unique_ptr<ChunkInfo> ChunkInfo::Deserialize(Deserializer &source) {
 	}
 }
 
+unique_ptr<ChunkInfo> ChunkInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<ChunkInfoType>(100, "type");
+	switch (type) {
+	case ChunkInfoType::EMPTY_INFO:
+		return nullptr;
+	case ChunkInfoType::CONSTANT_INFO:
+		return ChunkConstantInfo::FormatDeserialize(deserializer);
+	case ChunkInfoType::VECTOR_INFO:
+		return ChunkVectorInfo::FormatDeserialize(deserializer);
+	default:
+		throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type");
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Constant info
 //===--------------------------------------------------------------------===//
@@ -51,7 +67,7 @@ ChunkConstantInfo::ChunkConstantInfo(idx_t start)
 
 template <class OP>
 idx_t ChunkConstantInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id,
-                                               SelectionVector &sel_vector, idx_t max_count) {
+                                               SelectionVector &sel_vector, idx_t max_count) const {
 	if (OP::UseInsertedVersion(start_time, transaction_id, insert_id) &&
 	    OP::UseDeletedVersion(start_time, transaction_id, delete_id)) {
 		return max_count;
@@ -89,6 +105,10 @@ void ChunkConstantInfo::Serialize(Serializer &serializer) {
 	serializer.Write<idx_t>(start);
 }
 
+idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
+	return delete_id < TRANSACTION_ID_START ? max_count : 0;
+}
+
 unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
 	auto start = source.Read<idx_t>();
 
@@ -98,6 +118,24 @@ unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
 	return std::move(info);
 }
 
+void ChunkConstantInfo::FormatSerialize(FormatSerializer &serializer) const {
+	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
+	if (!is_deleted) {
+		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
+		return;
+	}
+	serializer.WriteProperty(100, "type", type);
+	serializer.WriteProperty(200, "start", start);
+}
+
+unique_ptr<ChunkInfo> ChunkConstantInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto start = deserializer.ReadProperty<idx_t>(200, "start");
+	auto info = make_uniq<ChunkConstantInfo>(start);
+	info->insert_id = 0;
+	info->delete_id = 0;
+	return std::move(info);
+}
+
 //===--------------------------------------------------------------------===//
 // Vector info
 //===--------------------------------------------------------------------===//
@@ -111,7 +149,7 @@ ChunkVectorInfo::ChunkVectorInfo(idx_t start)
 
 template <class OP>
 idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id,
-                                             SelectionVector &sel_vector, idx_t max_count) {
+                                             SelectionVector &sel_vector, idx_t max_count) const {
 	idx_t count = 0;
 	if (same_inserted_id && !any_deleted) {
 		// all tuples have the same inserted id: and no tuples were deleted
@@ -150,7 +188,7 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti
 }
 
 idx_t ChunkVectorInfo::GetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-                                    idx_t max_count) {
+                                    idx_t max_count) const {
 	return TemplatedGetSelVector<TransactionVersionOperator>(start_time, transaction_id, sel_vector, max_count);
 }
 
@@ -244,6 +282,19 @@ void ChunkVectorInfo::Serialize(Serializer &serializer) {
 	serializer.WriteData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
 }
 
+idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
+	if (!any_deleted) {
+		return 0;
+	}
+	idx_t delete_count = 0;
+	for (idx_t i = 0; i < max_count; i++) {
+		if (deleted[i] < TRANSACTION_ID_START) {
+			delete_count++;
+		}
+	}
+	return delete_count;
+}
+
 unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
 	auto start = source.Read<idx_t>();
 
@@ -259,4 +310,49 @@ unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
 	return std::move(result);
 }
 
+void ChunkVectorInfo::FormatSerialize(FormatSerializer &serializer) const {
+	SelectionVector sel(STANDARD_VECTOR_SIZE);
+	transaction_t start_time = TRANSACTION_ID_START - 1;
+	transaction_t transaction_id = DConstants::INVALID_INDEX;
+	idx_t count = GetSelVector(start_time, transaction_id, sel, STANDARD_VECTOR_SIZE);
+	if (count == STANDARD_VECTOR_SIZE) {
+		// nothing is deleted: skip writing anything
+		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
+		return;
+	}
+	if (count == 0) {
+		// everything is deleted: write a constant vector
+		serializer.WriteProperty(100, "type", ChunkInfoType::CONSTANT_INFO);
+		serializer.WriteProperty(200, "start", start);
+		return;
+	}
+	// write a boolean vector
+	serializer.WriteProperty(100, "type", ChunkInfoType::VECTOR_INFO);
+	serializer.WriteProperty(200, "start", start);
+	bool deleted_tuples[STANDARD_VECTOR_SIZE];
+	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
+		deleted_tuples[i] = true;
+	}
+	for (idx_t i = 0; i < count; i++) {
+		deleted_tuples[sel.get_index(i)] = false;
+	}
+	serializer.WriteProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
+}
+
+unique_ptr<ChunkInfo> ChunkVectorInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto start = deserializer.ReadProperty<idx_t>(200, "start");
+
+	auto result = make_uniq<ChunkVectorInfo>(start);
+	result->any_deleted = true;
+	bool deleted_tuples[STANDARD_VECTOR_SIZE];
+	deserializer.ReadProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples),
+	                          sizeof(bool) * STANDARD_VECTOR_SIZE);
+	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
+		if (deleted_tuples[i]) {
+			result->deleted[i] = 0;
+		}
+	}
+	return std::move(result);
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -242,9 +242,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 	}
 }
 
-void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes) {
-	D_ASSERT(!nodes.empty());
-	this->nodes = std::move(nodes);
+void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes_p) {
+	D_ASSERT(!nodes_p.empty());
+	this->nodes = std::move(nodes_p);
 	// first check if any of the segments have changes
 	if (!HasChanges()) {
 		// no changes: only need to write the metadata for this column

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -89,16 +89,19 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
-	auto data = FlatVector::GetData<uint64_t>(offset_vector);
-	auto last_entry = data[scan_count - 1];
+	UnifiedVectorFormat offsets;
+	offset_vector.ToUnifiedFormat(scan_count, offsets);
+	auto data = UnifiedVectorFormat::GetData<uint64_t>(offsets);
+	auto last_entry = data[offsets.sel->get_index(scan_count - 1)];
 
 	// shift all offsets so they are 0 at the first entry
 	auto result_data = FlatVector::GetData<list_entry_t>(result);
 	auto base_offset = state.last_offset;
 	idx_t current_offset = 0;
 	for (idx_t i = 0; i < scan_count; i++) {
+		auto offset_index = offsets.sel->get_index(i);
 		result_data[i].offset = current_offset;
-		result_data[i].length = data[i] - current_offset - base_offset;
+		result_data[i].length = data[offset_index] - current_offset - base_offset;
 		current_offset += result_data[i].length;
 	}