duckdb 0.8.2-dev3458.0 → 0.8.2-dev3949.0

package/src/duckdb/src/storage/serialization/serialize_storage.cpp

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_serialization.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/storage/block.hpp"
+#include "duckdb/storage/statistics/distinct_statistics.hpp"
+
+namespace duckdb {
+
+void DistinctStatistics::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "sample_count", sample_count);
+	serializer.WriteProperty(101, "total_count", total_count);
+	serializer.WriteProperty(102, "log", log);
+}
+
+unique_ptr<DistinctStatistics> DistinctStatistics::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto sample_count = deserializer.ReadProperty<idx_t>(100, "sample_count");
+	auto total_count = deserializer.ReadProperty<idx_t>(101, "total_count");
+	auto log = deserializer.ReadProperty<unique_ptr<HyperLogLog>>(102, "log");
+	auto result = duckdb::unique_ptr<DistinctStatistics>(new DistinctStatistics(std::move(log), sample_count, total_count));
+	return result;
+}
+
+void MetaBlockPointer::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "block_pointer", block_pointer);
+	serializer.WriteProperty(101, "offset", offset);
+}
+
+MetaBlockPointer MetaBlockPointer::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto block_pointer = deserializer.ReadProperty<idx_t>(100, "block_pointer");
+	auto offset = deserializer.ReadProperty<uint32_t>(101, "offset");
+	MetaBlockPointer result(block_pointer, offset);
+	return result;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/serialization/serialize_tableref.cpp

@@ -12,13 +12,13 @@ namespace duckdb {
 void TableRef::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty(100, "type", type);
 	serializer.WriteProperty(101, "alias", alias);
-	serializer.WriteOptionalProperty(102, "sample", sample);
+	serializer.WritePropertyWithDefault(102, "sample", sample, unique_ptr<SampleOptions>());
 }
 
 unique_ptr<TableRef> TableRef::FormatDeserialize(FormatDeserializer &deserializer) {
 	auto type = deserializer.ReadProperty<TableReferenceType>(100, "type");
 	auto alias = deserializer.ReadProperty<string>(101, "alias");
-	auto sample = deserializer.ReadOptionalProperty<unique_ptr<SampleOptions>>(102, "sample");
+	auto sample = deserializer.ReadPropertyWithDefault<unique_ptr<SampleOptions>>(102, "sample", unique_ptr<SampleOptions>());
 	unique_ptr<TableRef> result;
 	switch (type) {
 	case TableReferenceType::BASE_TABLE:
@@ -93,9 +93,9 @@ unique_ptr<TableRef> ExpressionListRef::FormatDeserialize(FormatDeserializer &de
 
 void JoinRef::FormatSerialize(FormatSerializer &serializer) const {
 	TableRef::FormatSerialize(serializer);
-	serializer.WriteProperty(200, "left", *left);
-	serializer.WriteProperty(201, "right", *right);
-	serializer.WriteOptionalProperty(202, "condition", condition);
+	serializer.WriteProperty(200, "left", left);
+	serializer.WriteProperty(201, "right", right);
+	serializer.WritePropertyWithDefault(202, "condition", condition, unique_ptr<ParsedExpression>());
 	serializer.WriteProperty(203, "join_type", type);
 	serializer.WriteProperty(204, "ref_type", ref_type);
 	serializer.WriteProperty(205, "using_columns", using_columns);
@@ -105,7 +105,7 @@ unique_ptr<TableRef> JoinRef::FormatDeserialize(FormatDeserializer &deserializer
 	auto result = duckdb::unique_ptr<JoinRef>(new JoinRef());
 	deserializer.ReadProperty(200, "left", result->left);
 	deserializer.ReadProperty(201, "right", result->right);
-	deserializer.ReadOptionalProperty(202, "condition", result->condition);
+	deserializer.ReadPropertyWithDefault(202, "condition", result->condition, unique_ptr<ParsedExpression>());
 	deserializer.ReadProperty(203, "join_type", result->type);
 	deserializer.ReadProperty(204, "ref_type", result->ref_type);
 	deserializer.ReadProperty(205, "using_columns", result->using_columns);
@@ -114,7 +114,7 @@ unique_ptr<TableRef> JoinRef::FormatDeserialize(FormatDeserializer &deserializer
 
 void PivotRef::FormatSerialize(FormatSerializer &serializer) const {
 	TableRef::FormatSerialize(serializer);
-	serializer.WriteProperty(200, "source", *source);
+	serializer.WriteProperty(200, "source", source);
 	serializer.WriteProperty(201, "aggregates", aggregates);
 	serializer.WriteProperty(202, "unpivot_names", unpivot_names);
 	serializer.WriteProperty(203, "pivots", pivots);
@@ -137,7 +137,7 @@ unique_ptr<TableRef> PivotRef::FormatDeserialize(FormatDeserializer &deserialize
 
 void SubqueryRef::FormatSerialize(FormatSerializer &serializer) const {
 	TableRef::FormatSerialize(serializer);
-	serializer.WriteProperty(200, "subquery", *subquery);
+	serializer.WriteProperty(200, "subquery", subquery);
 	serializer.WriteProperty(201, "column_name_alias", column_name_alias);
 }
 
@@ -150,7 +150,7 @@ unique_ptr<TableRef> SubqueryRef::FormatDeserialize(FormatDeserializer &deserial
 
 void TableFunctionRef::FormatSerialize(FormatSerializer &serializer) const {
 	TableRef::FormatSerialize(serializer);
-	serializer.WriteProperty(200, "function", *function);
+	serializer.WriteProperty(200, "function", function);
 	serializer.WriteProperty(201, "column_name_alias", column_name_alias);
 }
 

package/src/duckdb/src/storage/statistics/base_statistics.cpp

@@ -6,6 +6,9 @@
 #include "duckdb/storage/statistics/list_stats.hpp"
 #include "duckdb/storage/statistics/struct_stats.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 BaseStatistics::BaseStatistics() : type(LogicalType::INVALID) {
@@ -267,6 +270,10 @@ void BaseStatistics::CopyValidity(BaseStatistics &stats) {
 	has_no_null = stats.has_no_null;
 }
 
+void BaseStatistics::SetDistinctCount(idx_t count) {
+	this->distinct_count = count;
+}
+
 void BaseStatistics::Serialize(Serializer &serializer) const {
 	FieldWriter writer(serializer);
 	writer.WriteField<bool>(has_null);
@@ -276,10 +283,6 @@ void BaseStatistics::Serialize(Serializer &serializer) const {
 	writer.Finalize();
 }
 
-void BaseStatistics::SetDistinctCount(idx_t count) {
-	this->distinct_count = count;
-}
-
 void BaseStatistics::Serialize(FieldWriter &writer) const {
 	switch (GetStatsType()) {
 	case StatisticsType::NUMERIC_STATS:
@@ -298,6 +301,66 @@ void BaseStatistics::Serialize(FieldWriter &writer) const {
 		break;
 	}
 }
+
+void BaseStatistics::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "has_null", has_null);
+	serializer.WriteProperty(101, "has_no_null", has_no_null);
+	serializer.WriteProperty(102, "distinct_count", distinct_count);
+	serializer.WriteProperty(103, "stats_type", GetStatsType());
+	serializer.WriteObject(104, "stats", [this](FormatSerializer &serializer) {
+		switch (GetStatsType()) {
+		case StatisticsType::NUMERIC_STATS:
+			NumericStats::FormatSerialize(*this, serializer);
+			break;
+		case StatisticsType::STRING_STATS:
+			StringStats::FormatSerialize(*this, serializer);
+			break;
+		case StatisticsType::LIST_STATS:
+			ListStats::FormatSerialize(*this, serializer);
+			break;
+		case StatisticsType::STRUCT_STATS:
+			StructStats::FormatSerialize(*this, serializer);
+			break;
+		default:
+			throw NotImplementedException("Unrecognized StatisticsType for BaseStatistics::FormatSerialize");
+		}
+	});
+}
+
+BaseStatistics BaseStatistics::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto has_null = deserializer.ReadProperty<bool>(100, "has_null");
+	auto has_no_null = deserializer.ReadProperty<bool>(101, "has_no_null");
+	auto distinct_count = deserializer.ReadProperty<idx_t>(102, "distinct_count");
+
+	// Get the logical type from the deserializer context.
+	auto type = deserializer.Get<LogicalType &>();
+
+	BaseStatistics stats;
+	switch (GetStatsType(type)) {
+	case StatisticsType::NUMERIC_STATS:
+		stats = NumericStats::FormatDeserialize(deserializer, type);
+		break;
+	case StatisticsType::STRING_STATS:
+		stats = StringStats::FormatDeserialize(deserializer, type);
+		break;
+	case StatisticsType::LIST_STATS:
+		stats = ListStats::FormatDeserialize(deserializer, type);
+		break;
+	case StatisticsType::STRUCT_STATS:
+		stats = StructStats::FormatDeserialize(deserializer, type);
+		break;
+	default:
+		stats = BaseStatistics(std::move(type));
+		break;
+	}
+
+	stats.has_null = has_null;
+	stats.has_no_null = has_no_null;
+	stats.distinct_count = distinct_count;
+
+	return stats;
+}
+
 BaseStatistics BaseStatistics::DeserializeType(FieldReader &reader, LogicalType type) {
 	switch (GetStatsType(type)) {
 	case StatisticsType::NUMERIC_STATS:

package/src/duckdb/src/storage/statistics/column_statistics.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/storage/statistics/column_statistics.hpp"
 #include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
 
 namespace duckdb {
 
@@ -64,4 +66,18 @@ shared_ptr<ColumnStatistics> ColumnStatistics::Deserialize(Deserializer &source,
 	return make_shared<ColumnStatistics>(stats.Copy(), std::move(distinct_stats));
 }
 
+void ColumnStatistics::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "statistics", stats);
+	serializer.WritePropertyWithDefault(101, "distinct", distinct_stats, unique_ptr<DistinctStatistics>());
+}
+
+shared_ptr<ColumnStatistics> ColumnStatistics::FormatDeserialize(FormatDeserializer &deserializer) {
+	// TODO: do we read this as an property or into the object itself?
+	// we have this sort of pseudo inheritance going on here which is annoying
+	auto stats = BaseStatistics::FormatDeserialize(deserializer);
+	auto distinct_stats = deserializer.ReadPropertyWithDefault<unique_ptr<DistinctStatistics>>(
+	    101, "distinct", unique_ptr<DistinctStatistics>());
+	return make_shared<ColumnStatistics>(stats.Copy(), std::move(distinct_stats));
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/statistics/list_stats.cpp

@@ -4,6 +4,9 @@
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 void ListStats::Construct(BaseStatistics &stats) {
@@ -79,6 +82,24 @@ BaseStatistics ListStats::Deserialize(FieldReader &reader, LogicalType type) {
 	return result;
 }
 
+void ListStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto &child_stats = ListStats::GetChildStats(stats);
+	serializer.WriteProperty(200, "child_stats", child_stats);
+}
+
+BaseStatistics ListStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	D_ASSERT(type.InternalType() == PhysicalType::LIST);
+	auto &child_type = ListType::GetChildType(type);
+	BaseStatistics result(std::move(type));
+
+	// Push the logical type of the child type to the deserialization context
+	deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_type));
+	result.child_stats[0].Copy(deserializer.ReadProperty<BaseStatistics>(200, "child_stats"));
+	deserializer.Unset<LogicalType>();
+
+	return result;
+}
+
 string ListStats::ToString(const BaseStatistics &stats) {
 	auto &child_stats = ListStats::GetChildStats(stats);
 	return StringUtil::Format("[%s]", child_stats.ToString());

package/src/duckdb/src/storage/statistics/numeric_stats.cpp

@@ -4,6 +4,9 @@
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 template <>
@@ -403,7 +406,8 @@ Value NumericStats::MaxOrNull(const BaseStatistics &stats) {
 	return NumericStats::Max(stats);
 }
 
-void SerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value, FieldWriter &writer) {
+static void SerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value,
+                                       FieldWriter &writer) {
 	writer.WriteField<bool>(!has_value);
 	if (!has_value) {
 		return;
@@ -514,6 +518,127 @@ BaseStatistics NumericStats::Deserialize(FieldReader &reader, LogicalType type)
 	return result;
 }
 
+static void FormatSerializeNumericStatsValue(const LogicalType &type, NumericValueUnion val, bool has_value,
+                                             FormatSerializer &serializer) {
+	serializer.WriteProperty(100, "has_value", has_value);
+	if (!has_value) {
+		return;
+	}
+	switch (type.InternalType()) {
+	case PhysicalType::BOOL:
+		serializer.WriteProperty(101, "value", val.value_.boolean);
+		break;
+	case PhysicalType::INT8:
+		serializer.WriteProperty(101, "value", val.value_.tinyint);
+		break;
+	case PhysicalType::INT16:
+		serializer.WriteProperty(101, "value", val.value_.smallint);
+		break;
+	case PhysicalType::INT32:
+		serializer.WriteProperty(101, "value", val.value_.integer);
+		break;
+	case PhysicalType::INT64:
+		serializer.WriteProperty(101, "value", val.value_.bigint);
+		break;
+	case PhysicalType::UINT8:
+		serializer.WriteProperty(101, "value", val.value_.utinyint);
+		break;
+	case PhysicalType::UINT16:
+		serializer.WriteProperty(101, "value", val.value_.usmallint);
+		break;
+	case PhysicalType::UINT32:
+		serializer.WriteProperty(101, "value", val.value_.uinteger);
+		break;
+	case PhysicalType::UINT64:
+		serializer.WriteProperty(101, "value", val.value_.ubigint);
+		break;
+	case PhysicalType::INT128:
+		serializer.WriteProperty(101, "value", val.value_.hugeint);
+		break;
+	case PhysicalType::FLOAT:
+		serializer.WriteProperty(101, "value", val.value_.float_);
+		break;
+	case PhysicalType::DOUBLE:
+		serializer.WriteProperty(101, "value", val.value_.double_);
+		break;
+	default:
+		throw InternalException("Unsupported type for serializing numeric statistics");
+	}
+}
+
+void NumericStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto &numeric_stats = NumericStats::GetDataUnsafe(stats);
+	serializer.WriteObject(200, "max", [&](FormatSerializer &object) {
+		FormatSerializeNumericStatsValue(stats.GetType(), numeric_stats.min, numeric_stats.has_min, object);
+	});
+	serializer.WriteObject(201, "min", [&](FormatSerializer &object) {
+		FormatSerializeNumericStatsValue(stats.GetType(), numeric_stats.max, numeric_stats.has_max, object);
+	});
+}
+
+static void FormatDeserializeNumericStatsValue(const LogicalType &type, NumericValueUnion &result, bool &has_stats,
+                                               FormatDeserializer &deserializer) {
+	auto has_value = deserializer.ReadProperty<bool>(100, "has_value");
+	if (!has_value) {
+		has_stats = false;
+		return;
+	}
+	has_stats = true;
+	switch (type.InternalType()) {
+	case PhysicalType::BOOL:
+		result.value_.boolean = deserializer.ReadProperty<bool>(101, "value");
+		break;
+	case PhysicalType::INT8:
+		result.value_.tinyint = deserializer.ReadProperty<int8_t>(101, "value");
+		break;
+	case PhysicalType::INT16:
+		result.value_.smallint = deserializer.ReadProperty<int16_t>(101, "value");
+		break;
+	case PhysicalType::INT32:
+		result.value_.integer = deserializer.ReadProperty<int32_t>(101, "value");
+		break;
+	case PhysicalType::INT64:
+		result.value_.bigint = deserializer.ReadProperty<int64_t>(101, "value");
+		break;
+	case PhysicalType::UINT8:
+		result.value_.utinyint = deserializer.ReadProperty<uint8_t>(101, "value");
+		break;
+	case PhysicalType::UINT16:
+		result.value_.usmallint = deserializer.ReadProperty<uint16_t>(101, "value");
+		break;
+	case PhysicalType::UINT32:
+		result.value_.uinteger = deserializer.ReadProperty<uint32_t>(101, "value");
+		break;
+	case PhysicalType::UINT64:
+		result.value_.ubigint = deserializer.ReadProperty<uint64_t>(101, "value");
+		break;
+	case PhysicalType::INT128:
+		result.value_.hugeint = deserializer.ReadProperty<hugeint_t>(101, "value");
+		break;
+	case PhysicalType::FLOAT:
+		result.value_.float_ = deserializer.ReadProperty<float>(101, "value");
+		break;
+	case PhysicalType::DOUBLE:
+		result.value_.double_ = deserializer.ReadProperty<double>(101, "value");
+		break;
+	default:
+		throw InternalException("Unsupported type for serializing numeric statistics");
+	}
+}
+
+BaseStatistics NumericStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	BaseStatistics result(std::move(type));
+	auto &numeric_stats = NumericStats::GetDataUnsafe(result);
+
+	deserializer.ReadObject(200, "max", [&](FormatDeserializer &object) {
+		FormatDeserializeNumericStatsValue(result.GetType(), numeric_stats.min, numeric_stats.has_min, object);
+	});
+	deserializer.ReadObject(201, "min", [&](FormatDeserializer &object) {
+		FormatDeserializeNumericStatsValue(result.GetType(), numeric_stats.max, numeric_stats.has_max, object);
+	});
+	return result;
+}
+
 string NumericStats::ToString(const BaseStatistics &stats) {
 	return StringUtil::Format("[Min: %s, Max: %s]", NumericStats::MinOrNull(stats).ToString(),
 	                          NumericStats::MaxOrNull(stats).ToString());

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -7,6 +7,9 @@
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "utf8proc_wrapper.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 BaseStatistics StringStats::CreateUnknown(LogicalType type) {
@@ -103,6 +106,15 @@ void StringStats::Serialize(const BaseStatistics &stats, FieldWriter &writer) {
 	writer.WriteField<uint32_t>(string_data.max_string_length);
 }
 
+void StringStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto &string_data = StringStats::GetDataUnsafe(stats);
+	serializer.WriteProperty(200, "min", string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	serializer.WriteProperty(201, "max", string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	serializer.WriteProperty(202, "has_unicode", string_data.has_unicode);
+	serializer.WriteProperty(203, "has_max_string_length", string_data.has_max_string_length);
+	serializer.WriteProperty(204, "max_string_length", string_data.max_string_length);
+}
+
 BaseStatistics StringStats::Deserialize(FieldReader &reader, LogicalType type) {
 	BaseStatistics result(std::move(type));
 	auto &string_data = StringStats::GetDataUnsafe(result);
@@ -114,6 +126,17 @@ BaseStatistics StringStats::Deserialize(FieldReader &reader, LogicalType type) {
 	return result;
 }
 
+BaseStatistics StringStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	BaseStatistics result(std::move(type));
+	auto &string_data = StringStats::GetDataUnsafe(result);
+	deserializer.ReadProperty(200, "min", string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	deserializer.ReadProperty(201, "max", string_data.max, StringStatsData::MAX_STRING_MINMAX_SIZE);
+	deserializer.ReadProperty(202, "has_unicode", string_data.has_unicode);
+	deserializer.ReadProperty(203, "has_max_string_length", string_data.has_max_string_length);
+	deserializer.ReadProperty(204, "max_string_length", string_data.max_string_length);
+	return result;
+}
+
 static int StringValueComparison(const_data_ptr_t data, idx_t len, const_data_ptr_t comparison) {
 	D_ASSERT(len <= StringStatsData::MAX_STRING_MINMAX_SIZE);
 	for (idx_t i = 0; i < len; i++) {

package/src/duckdb/src/storage/statistics/struct_stats.cpp

@@ -3,6 +3,9 @@
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/types/vector.hpp"
 
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 namespace duckdb {
 
 void StructStats::Construct(BaseStatistics &stats) {
@@ -109,6 +112,30 @@ BaseStatistics StructStats::Deserialize(FieldReader &reader, LogicalType type) {
 	return result;
 }
 
+void StructStats::FormatSerialize(const BaseStatistics &stats, FormatSerializer &serializer) {
+	auto child_stats = StructStats::GetChildStats(stats);
+	auto child_count = StructType::GetChildCount(stats.GetType());
+
+	serializer.WriteList(200, "child_stats", child_count,
+	                     [&](FormatSerializer::List &list, idx_t i) { list.WriteElement(child_stats[i]); });
+}
+
+BaseStatistics StructStats::FormatDeserialize(FormatDeserializer &deserializer, LogicalType type) {
+	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
+
+	auto &child_types = StructType::GetChildTypes(type);
+	BaseStatistics result(std::move(type));
+
+	deserializer.ReadList(200, "child_stats", [&](FormatDeserializer::List &list, idx_t i) {
+		deserializer.Set<LogicalType &>(const_cast<LogicalType &>(child_types[i].second));
+		auto stat = list.ReadElement<BaseStatistics>();
+		result.child_stats[i].Copy(stat);
+		deserializer.Unset<LogicalType>();
+	});
+
+	return result;
+}
+
 string StructStats::ToString(const BaseStatistics &stats) {
 	string result;
 	result += " {";

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,7 +2,7 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 56;
+const uint64_t VERSION_NUMBER = 57;
 
 struct StorageVersionInfo {
 	const char *version_name;

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -1,6 +1,8 @@
 #include "duckdb/storage/table/chunk_info.hpp"
 #include "duckdb/transaction/transaction.hpp"
 #include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -42,6 +44,20 @@ unique_ptr<ChunkInfo> ChunkInfo::Deserialize(Deserializer &source) {
 	}
 }
 
+unique_ptr<ChunkInfo> ChunkInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<ChunkInfoType>(100, "type");
+	switch (type) {
+	case ChunkInfoType::EMPTY_INFO:
+		return nullptr;
+	case ChunkInfoType::CONSTANT_INFO:
+		return ChunkConstantInfo::FormatDeserialize(deserializer);
+	case ChunkInfoType::VECTOR_INFO:
+		return ChunkVectorInfo::FormatDeserialize(deserializer);
+	default:
+		throw SerializationException("Could not deserialize Chunk Info Type: unrecognized type");
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Constant info
 //===--------------------------------------------------------------------===//
@@ -51,7 +67,7 @@ ChunkConstantInfo::ChunkConstantInfo(idx_t start)
 
 template <class OP>
 idx_t ChunkConstantInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id,
-                                               SelectionVector &sel_vector, idx_t max_count) {
+                                               SelectionVector &sel_vector, idx_t max_count) const {
 	if (OP::UseInsertedVersion(start_time, transaction_id, insert_id) &&
 	    OP::UseDeletedVersion(start_time, transaction_id, delete_id)) {
 		return max_count;
@@ -102,6 +118,24 @@ unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
 	return std::move(info);
 }
 
+void ChunkConstantInfo::FormatSerialize(FormatSerializer &serializer) const {
+	bool is_deleted = insert_id >= TRANSACTION_ID_START || delete_id < TRANSACTION_ID_START;
+	if (!is_deleted) {
+		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
+		return;
+	}
+	serializer.WriteProperty(100, "type", type);
+	serializer.WriteProperty(200, "start", start);
+}
+
+unique_ptr<ChunkInfo> ChunkConstantInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto start = deserializer.ReadProperty<idx_t>(200, "start");
+	auto info = make_uniq<ChunkConstantInfo>(start);
+	info->insert_id = 0;
+	info->delete_id = 0;
+	return std::move(info);
+}
+
 //===--------------------------------------------------------------------===//
 // Vector info
 //===--------------------------------------------------------------------===//
@@ -115,7 +149,7 @@ ChunkVectorInfo::ChunkVectorInfo(idx_t start)
 
 template <class OP>
 idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id,
-                                             SelectionVector &sel_vector, idx_t max_count) {
+                                             SelectionVector &sel_vector, idx_t max_count) const {
 	idx_t count = 0;
 	if (same_inserted_id && !any_deleted) {
 		// all tuples have the same inserted id: and no tuples were deleted
@@ -154,7 +188,7 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti
 }
 
 idx_t ChunkVectorInfo::GetSelVector(transaction_t start_time, transaction_t transaction_id, SelectionVector &sel_vector,
-                                    idx_t max_count) {
+                                    idx_t max_count) const {
 	return TemplatedGetSelVector<TransactionVersionOperator>(start_time, transaction_id, sel_vector, max_count);
 }
 
@@ -276,4 +310,49 @@ unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
 	return std::move(result);
 }
 
+void ChunkVectorInfo::FormatSerialize(FormatSerializer &serializer) const {
+	SelectionVector sel(STANDARD_VECTOR_SIZE);
+	transaction_t start_time = TRANSACTION_ID_START - 1;
+	transaction_t transaction_id = DConstants::INVALID_INDEX;
+	idx_t count = GetSelVector(start_time, transaction_id, sel, STANDARD_VECTOR_SIZE);
+	if (count == STANDARD_VECTOR_SIZE) {
+		// nothing is deleted: skip writing anything
+		serializer.WriteProperty(100, "type", ChunkInfoType::EMPTY_INFO);
+		return;
+	}
+	if (count == 0) {
+		// everything is deleted: write a constant vector
+		serializer.WriteProperty(100, "type", ChunkInfoType::CONSTANT_INFO);
+		serializer.WriteProperty(200, "start", start);
+		return;
+	}
+	// write a boolean vector
+	serializer.WriteProperty(100, "type", ChunkInfoType::VECTOR_INFO);
+	serializer.WriteProperty(200, "start", start);
+	bool deleted_tuples[STANDARD_VECTOR_SIZE];
+	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
+		deleted_tuples[i] = true;
+	}
+	for (idx_t i = 0; i < count; i++) {
+		deleted_tuples[sel.get_index(i)] = false;
+	}
+	serializer.WriteProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
+}
+
+unique_ptr<ChunkInfo> ChunkVectorInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto start = deserializer.ReadProperty<idx_t>(200, "start");
+
+	auto result = make_uniq<ChunkVectorInfo>(start);
+	result->any_deleted = true;
+	bool deleted_tuples[STANDARD_VECTOR_SIZE];
+	deserializer.ReadProperty(201, "deleted_tuples", data_ptr_cast(deleted_tuples),
+	                          sizeof(bool) * STANDARD_VECTOR_SIZE);
+	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
+		if (deleted_tuples[i]) {
+			result->deleted[i] = 0;
+		}
+	}
+	return std::move(result);
+}
+
 } // namespace duckdb