duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/storage/serialization/serialize_tableref.cpp

@@ -0,0 +1,164 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_serialization.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/parser/tableref/list.hpp"
+
+namespace duckdb {
+
+void TableRef::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "type", type);
+	serializer.WriteProperty(101, "alias", alias);
+	serializer.WriteOptionalProperty(102, "sample", sample);
+}
+
+unique_ptr<TableRef> TableRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<TableReferenceType>(100, "type");
+	auto alias = deserializer.ReadProperty<string>(101, "alias");
+	auto sample = deserializer.ReadOptionalProperty<unique_ptr<SampleOptions>>(102, "sample");
+	unique_ptr<TableRef> result;
+	switch (type) {
+	case TableReferenceType::BASE_TABLE:
+		result = BaseTableRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::EMPTY:
+		result = EmptyTableRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::EXPRESSION_LIST:
+		result = ExpressionListRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::JOIN:
+		result = JoinRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::PIVOT:
+		result = PivotRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::SUBQUERY:
+		result = SubqueryRef::FormatDeserialize(deserializer);
+		break;
+	case TableReferenceType::TABLE_FUNCTION:
+		result = TableFunctionRef::FormatDeserialize(deserializer);
+		break;
+	default:
+		throw SerializationException("Unsupported type for deserialization of TableRef!");
+	}
+	result->alias = std::move(alias);
+	result->sample = std::move(sample);
+	return result;
+}
+
+void BaseTableRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "schema_name", schema_name);
+	serializer.WriteProperty(201, "table_name", table_name);
+	serializer.WriteProperty(202, "column_name_alias", column_name_alias);
+	serializer.WriteProperty(203, "catalog_name", catalog_name);
+}
+
+unique_ptr<TableRef> BaseTableRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<BaseTableRef>(new BaseTableRef());
+	deserializer.ReadProperty(200, "schema_name", result->schema_name);
+	deserializer.ReadProperty(201, "table_name", result->table_name);
+	deserializer.ReadProperty(202, "column_name_alias", result->column_name_alias);
+	deserializer.ReadProperty(203, "catalog_name", result->catalog_name);
+	return std::move(result);
+}
+
+void EmptyTableRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+}
+
+unique_ptr<TableRef> EmptyTableRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<EmptyTableRef>(new EmptyTableRef());
+	return std::move(result);
+}
+
+void ExpressionListRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "expected_names", expected_names);
+	serializer.WriteProperty(201, "expected_types", expected_types);
+	serializer.WriteProperty(202, "values", values);
+}
+
+unique_ptr<TableRef> ExpressionListRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<ExpressionListRef>(new ExpressionListRef());
+	deserializer.ReadProperty(200, "expected_names", result->expected_names);
+	deserializer.ReadProperty(201, "expected_types", result->expected_types);
+	deserializer.ReadProperty(202, "values", result->values);
+	return std::move(result);
+}
+
+void JoinRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "left", *left);
+	serializer.WriteProperty(201, "right", *right);
+	serializer.WriteOptionalProperty(202, "condition", condition);
+	serializer.WriteProperty(203, "join_type", type);
+	serializer.WriteProperty(204, "ref_type", ref_type);
+	serializer.WriteProperty(205, "using_columns", using_columns);
+}
+
+unique_ptr<TableRef> JoinRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<JoinRef>(new JoinRef());
+	deserializer.ReadProperty(200, "left", result->left);
+	deserializer.ReadProperty(201, "right", result->right);
+	deserializer.ReadOptionalProperty(202, "condition", result->condition);
+	deserializer.ReadProperty(203, "join_type", result->type);
+	deserializer.ReadProperty(204, "ref_type", result->ref_type);
+	deserializer.ReadProperty(205, "using_columns", result->using_columns);
+	return std::move(result);
+}
+
+void PivotRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "source", *source);
+	serializer.WriteProperty(201, "aggregates", aggregates);
+	serializer.WriteProperty(202, "unpivot_names", unpivot_names);
+	serializer.WriteProperty(203, "pivots", pivots);
+	serializer.WriteProperty(204, "groups", groups);
+	serializer.WriteProperty(205, "column_name_alias", column_name_alias);
+	serializer.WriteProperty(206, "include_nulls", include_nulls);
+}
+
+unique_ptr<TableRef> PivotRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<PivotRef>(new PivotRef());
+	deserializer.ReadProperty(200, "source", result->source);
+	deserializer.ReadProperty(201, "aggregates", result->aggregates);
+	deserializer.ReadProperty(202, "unpivot_names", result->unpivot_names);
+	deserializer.ReadProperty(203, "pivots", result->pivots);
+	deserializer.ReadProperty(204, "groups", result->groups);
+	deserializer.ReadProperty(205, "column_name_alias", result->column_name_alias);
+	deserializer.ReadProperty(206, "include_nulls", result->include_nulls);
+	return std::move(result);
+}
+
+void SubqueryRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "subquery", *subquery);
+	serializer.WriteProperty(201, "column_name_alias", column_name_alias);
+}
+
+unique_ptr<TableRef> SubqueryRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<SubqueryRef>(new SubqueryRef());
+	deserializer.ReadProperty(200, "subquery", result->subquery);
+	deserializer.ReadProperty(201, "column_name_alias", result->column_name_alias);
+	return std::move(result);
+}
+
+void TableFunctionRef::FormatSerialize(FormatSerializer &serializer) const {
+	TableRef::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "function", *function);
+	serializer.WriteProperty(201, "column_name_alias", column_name_alias);
+}
+
+unique_ptr<TableRef> TableFunctionRef::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::unique_ptr<TableFunctionRef>(new TableFunctionRef());
+	deserializer.ReadProperty(200, "function", result->function);
+	deserializer.ReadProperty(201, "column_name_alias", result->column_name_alias);
+	return std::move(result);
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/serialization/serialize_types.cpp

@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_serialization.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/extra_type_info.hpp"
+
+namespace duckdb {
+
+void ExtraTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "type", type);
+	serializer.WriteProperty(101, "alias", alias);
+}
+
+shared_ptr<ExtraTypeInfo> ExtraTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<ExtraTypeInfoType>(100, "type");
+	auto alias = deserializer.ReadProperty<string>(101, "alias");
+	shared_ptr<ExtraTypeInfo> result;
+	switch (type) {
+	case ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO:
+		result = AggregateStateTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::DECIMAL_TYPE_INFO:
+		result = DecimalTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::ENUM_TYPE_INFO:
+		result = EnumTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::GENERIC_TYPE_INFO:
+		result = make_shared<ExtraTypeInfo>(type);
+		break;
+	case ExtraTypeInfoType::INVALID_TYPE_INFO:
+		return nullptr;
+	case ExtraTypeInfoType::LIST_TYPE_INFO:
+		result = ListTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::STRING_TYPE_INFO:
+		result = StringTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::STRUCT_TYPE_INFO:
+		result = StructTypeInfo::FormatDeserialize(deserializer);
+		break;
+	case ExtraTypeInfoType::USER_TYPE_INFO:
+		result = UserTypeInfo::FormatDeserialize(deserializer);
+		break;
+	default:
+		throw SerializationException("Unsupported type for deserialization of ExtraTypeInfo!");
+	}
+	result->alias = std::move(alias);
+	return result;
+}
+
+void AggregateStateTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	ExtraTypeInfo::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "function_name", state_type.function_name);
+	serializer.WriteProperty(201, "return_type", state_type.return_type);
+	serializer.WriteProperty(202, "bound_argument_types", state_type.bound_argument_types);
+}
+
+shared_ptr<ExtraTypeInfo> AggregateStateTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::shared_ptr<AggregateStateTypeInfo>(new AggregateStateTypeInfo());
+	deserializer.ReadProperty(200, "function_name", result->state_type.function_name);
+	deserializer.ReadProperty(201, "return_type", result->state_type.return_type);
+	deserializer.ReadProperty(202, "bound_argument_types", result->state_type.bound_argument_types);
+	return std::move(result);
+}
+
+void DecimalTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	ExtraTypeInfo::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "width", width);
+	serializer.WriteProperty(201, "scale", scale);
+}
+
+shared_ptr<ExtraTypeInfo> DecimalTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::shared_ptr<DecimalTypeInfo>(new DecimalTypeInfo());
+	deserializer.ReadProperty(200, "width", result->width);
+	deserializer.ReadProperty(201, "scale", result->scale);
+	return std::move(result);
+}
+
+void ListTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	ExtraTypeInfo::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "child_type", child_type);
+}
+
+shared_ptr<ExtraTypeInfo> ListTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::shared_ptr<ListTypeInfo>(new ListTypeInfo());
+	deserializer.ReadProperty(200, "child_type", result->child_type);
+	return std::move(result);
+}
+
+void StringTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	ExtraTypeInfo::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "collation", collation);
+}
+
+shared_ptr<ExtraTypeInfo> StringTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::shared_ptr<StringTypeInfo>(new StringTypeInfo());
+	deserializer.ReadProperty(200, "collation", result->collation);
+	return std::move(result);
+}
+
+void StructTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	ExtraTypeInfo::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "child_types", child_types);
+}
+
+shared_ptr<ExtraTypeInfo> StructTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::shared_ptr<StructTypeInfo>(new StructTypeInfo());
+	deserializer.ReadProperty(200, "child_types", result->child_types);
+	return std::move(result);
+}
+
+void UserTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	ExtraTypeInfo::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "user_type_name", user_type_name);
+}
+
+shared_ptr<ExtraTypeInfo> UserTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = duckdb::shared_ptr<UserTypeInfo>(new UserTypeInfo());
+	deserializer.ReadProperty(200, "user_type_name", result->user_type_name);
+	return std::move(result);
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -6,8 +6,8 @@
 #include "duckdb/common/serializer/buffered_deserializer.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
 #include "duckdb/common/field_writer.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/main/config.hpp"
 
@@ -81,16 +81,16 @@ MainHeader MainHeader::Deserialize(Deserializer &source) {
 
 void DatabaseHeader::Serialize(Serializer &ser) {
 	ser.Write<uint64_t>(iteration);
-	ser.Write<block_id_t>(meta_block);
-	ser.Write<block_id_t>(free_list);
+	ser.Write<idx_t>(meta_block);
+	ser.Write<idx_t>(free_list);
 	ser.Write<uint64_t>(block_count);
 }
 
 DatabaseHeader DatabaseHeader::Deserialize(Deserializer &source) {
 	DatabaseHeader header;
 	header.iteration = source.Read<uint64_t>();
-	header.meta_block = source.Read<block_id_t>();
-	header.free_list = source.Read<block_id_t>();
+	header.meta_block = source.Read<idx_t>();
+	header.free_list = source.Read<idx_t>();
 	header.block_count = source.Read<uint64_t>();
 	return header;
 }
@@ -241,11 +241,12 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
 }
 
 void SingleFileBlockManager::LoadFreeList() {
-	if (free_list_id == INVALID_BLOCK) {
+	MetaBlockPointer free_pointer(free_list_id, 0);
+	if (!free_pointer.IsValid()) {
 		// no free list
 		return;
 	}
-	MetaBlockReader reader(*this, free_list_id);
+	MetadataReader reader(GetMetadataManager(), free_pointer, BlockReaderType::REGISTER_BLOCKS);
 	auto free_list_count = reader.Read<uint64_t>();
 	free_list.clear();
 	for (idx_t i = 0; i < free_list_count; i++) {
@@ -258,10 +259,12 @@ void SingleFileBlockManager::LoadFreeList() {
 		auto usage_count = reader.Read<uint32_t>();
 		multi_use_blocks[block_id] = usage_count;
 	}
+	GetMetadataManager().Deserialize(reader);
+	GetMetadataManager().MarkBlocksAsModified();
 }
 
-bool SingleFileBlockManager::IsRootBlock(block_id_t root) {
-	return root == meta_block;
+bool SingleFileBlockManager::IsRootBlock(MetaBlockPointer root) {
+	return root.block_pointer == meta_block;
 }
 
 block_id_t SingleFileBlockManager::GetFreeBlockId() {
@@ -327,7 +330,7 @@ void SingleFileBlockManager::IncreaseBlockReferenceCount(block_id_t block_id) {
 	}
 }
 
-block_id_t SingleFileBlockManager::GetMetaBlock() {
+idx_t SingleFileBlockManager::GetMetaBlock() {
 	return meta_block;
 }
 
@@ -368,50 +371,65 @@ void SingleFileBlockManager::Write(FileBuffer &buffer, block_id_t block_id) {
 	ChecksumAndWrite(buffer, BLOCK_START + block_id * Storage::BLOCK_ALLOC_SIZE);
 }
 
-vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
-	vector<block_id_t> free_list_blocks;
-
-	if (!free_list.empty() || !multi_use_blocks.empty() || !modified_blocks.empty()) {
-		// there are blocks in the free list or multi_use_blocks
-		// figure out how many blocks we need to write these to the file
-		auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
-		auto multi_use_blocks_size =
-		    sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
-		auto total_size = free_list_size + multi_use_blocks_size;
-		// because of potential alignment issues and needing to store a next pointer in a block we subtract
-		// a bit from the max block size
-		auto space_in_block = Storage::BLOCK_SIZE - 4 * sizeof(block_id_t);
-		auto total_blocks = (total_size + space_in_block - 1) / space_in_block;
-		D_ASSERT(total_size > 0);
-		D_ASSERT(total_blocks > 0);
-
-		// reserve the blocks that we are going to write
-		// since these blocks are no longer free we cannot just include them in the free list!
-		for (idx_t i = 0; i < total_blocks; i++) {
-			auto block_id = GetFreeBlockId();
-			free_list_blocks.push_back(block_id);
+void SingleFileBlockManager::Truncate() {
+	BlockManager::Truncate();
+	idx_t blocks_to_truncate = 0;
+	// reverse iterate over the free-list
+	for (auto entry = free_list.rbegin(); entry != free_list.rend(); entry++) {
+		auto block_id = *entry;
+		if (block_id + 1 != max_block) {
+			break;
 		}
+		blocks_to_truncate++;
+		max_block--;
+	}
+	if (blocks_to_truncate == 0) {
+		// nothing to truncate
+		return;
+	}
+	// truncate the file
+	for (idx_t i = 0; i < blocks_to_truncate; i++) {
+		free_list.erase(max_block + i);
+	}
+	handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
+}
+
+vector<MetadataHandle> SingleFileBlockManager::GetFreeListBlocks() {
+	vector<MetadataHandle> free_list_blocks;
+
+	auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
+	auto multi_use_blocks_size = sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
+	auto metadata_blocks = sizeof(uint64_t) + (sizeof(idx_t) * 2) * GetMetadataManager().BlockCount();
+	auto total_size = free_list_size + multi_use_blocks_size + metadata_blocks;
+
+	// reserve the blocks that we are going to write
+	// since these blocks are no longer free we cannot just include them in the free list!
+	auto block_size = MetadataManager::METADATA_BLOCK_SIZE - sizeof(idx_t);
+	while (total_size > 0) {
+		auto free_list_handle = GetMetadataManager().AllocateHandle();
+		free_list_blocks.push_back(std::move(free_list_handle));
+		total_size -= MinValue<idx_t>(total_size, block_size);
 	}
 
 	return free_list_blocks;
 }
 
-class FreeListBlockWriter : public MetaBlockWriter {
+class FreeListBlockWriter : public MetadataWriter {
 public:
-	FreeListBlockWriter(BlockManager &block_manager, vector<block_id_t> &free_list_blocks_p)
-	    : MetaBlockWriter(block_manager, free_list_blocks_p[0]), free_list_blocks(free_list_blocks_p), index(1) {
+	FreeListBlockWriter(MetadataManager &manager, vector<MetadataHandle> free_list_blocks_p)
+	    : MetadataWriter(manager), free_list_blocks(std::move(free_list_blocks_p)), index(0) {
 	}
 
-	vector<block_id_t> &free_list_blocks;
+	vector<MetadataHandle> free_list_blocks;
 	idx_t index;
 
 protected:
-	block_id_t GetNextBlockId() override {
+	MetadataHandle NextHandle() override {
 		if (index >= free_list_blocks.size()) {
 			throw InternalException(
 			    "Free List Block Writer ran out of blocks, this means not enough blocks were allocated up front");
 		}
-		return free_list_blocks[index++];
+		return std::move(free_list_blocks[index++]);
 	}
 };
 
@@ -419,7 +437,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	// set the iteration count
 	header.iteration = ++iteration_count;
 
-	vector<block_id_t> free_list_blocks = GetFreeListBlocks();
+	auto free_list_blocks = GetFreeListBlocks();
 
 	// now handle the free list
 	// add all modified blocks to the free list: they can now be written to again
@@ -428,20 +446,16 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	}
 	modified_blocks.clear();
 
+	auto &metadata_manager = GetMetadataManager();
 	if (!free_list_blocks.empty()) {
 		// there are blocks to write, either in the free_list or in the modified_blocks
 		// we write these blocks specifically to the free_list_blocks
-		// a normal MetaBlockWriter will fetch blocks to use from the free_list
+		// a normal MetadataWriter will fetch blocks to use from the free_list
 		// but since we are WRITING the free_list, this behavior is sub-optimal
+		FreeListBlockWriter writer(metadata_manager, std::move(free_list_blocks));
 
-		FreeListBlockWriter writer(*this, free_list_blocks);
-
-		auto ptr = writer.GetBlockPointer();
-		D_ASSERT(ptr.block_id == free_list_blocks[0]);
-		header.free_list = ptr.block_id;
-		for (auto &block_id : free_list_blocks) {
-			modified_blocks.insert(block_id);
-		}
+		auto ptr = writer.GetMetaBlockPointer();
+		header.free_list = ptr.block_pointer;
 
 		writer.Write<uint64_t>(free_list.size());
 		for (auto &block_id : free_list) {
@@ -452,11 +466,13 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 			writer.Write<block_id_t>(entry.first);
 			writer.Write<uint32_t>(entry.second);
 		}
+		GetMetadataManager().Serialize(writer);
 		writer.Flush();
 	} else {
 		// no blocks in the free list
-		header.free_list = INVALID_BLOCK;
+		header.free_list = DConstants::INVALID_INDEX;
 	}
+	metadata_manager.Flush();
 	header.block_count = max_block;
 
 	auto &config = DBConfig::Get(db);
@@ -471,7 +487,9 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
 	}
 	// set the header inside the buffer
 	header_buffer.Clear();
-	Store<DatabaseHeader>(header, header_buffer.buffer);
+	BufferedSerializer serializer;
+	header.Serialize(serializer);
+	memcpy(header_buffer.buffer, serializer.blob.data.get(), serializer.blob.size);
 	// now write the header to the file, active_header determines whether we write to h1 or h2
 	// note that if active_header is h1 we write to h2, and vice versa
 	ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -1,10 +1,11 @@
 #include "duckdb/storage/statistics/string_stats.hpp"
-#include "duckdb/storage/statistics/base_statistics.hpp"
+
 #include "duckdb/common/field_writer.hpp"
-#include "utf8proc_wrapper.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/main/error_manager.hpp"
+#include "duckdb/storage/statistics/base_statistics.hpp"
+#include "utf8proc_wrapper.hpp"
 
 namespace duckdb {
 
@@ -67,6 +68,24 @@ bool StringStats::CanContainUnicode(const BaseStatistics &stats) {
 	return StringStats::GetDataUnsafe(stats).has_unicode;
 }
 
+string GetStringMinMaxValue(const data_t data[]) {
+	idx_t len;
+	for (len = 0; len < StringStatsData::MAX_STRING_MINMAX_SIZE; len++) {
+		if (!data[len]) {
+			break;
+		}
+	}
+	return string(const_char_ptr_cast(data), len);
+}
+
+string StringStats::Min(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).min);
+}
+
+string StringStats::Max(const BaseStatistics &stats) {
+	return GetStringMinMaxValue(StringStats::GetDataUnsafe(stats).max);
+}
+
 void StringStats::ResetMaxStringLength(BaseStatistics &stats) {
 	StringStats::GetDataUnsafe(stats).has_max_string_length = false;
 }

package/src/duckdb/src/storage/storage_info.cpp

@@ -2,14 +2,15 @@
 
 namespace duckdb {
 
-const uint64_t VERSION_NUMBER = 51;
+const uint64_t VERSION_NUMBER = 56;
 
 struct StorageVersionInfo {
 	const char *version_name;
 	idx_t storage_version;
 };
 
-static StorageVersionInfo storage_version_info[] = {{"v0.7.0 or v0.7.1", 43},
+static StorageVersionInfo storage_version_info[] = {{"v0.8.0 or v0.8.1", 51},
+                                                    {"v0.7.0 or v0.7.1", 43},
                                                     {"v0.6.0 or v0.6.1", 39},
                                                     {"v0.5.0 or v0.5.1", 38},
                                                     {"v0.3.3, v0.3.4 or v0.4.0", 33},

package/src/duckdb/src/storage/storage_manager.cpp

@@ -82,6 +82,9 @@ public:
 	BlockManager &GetBlockManagerForRowData() override {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager() override {
+		return block_manager.GetMetadataManager();
+	}
 };
 
 SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only)
@@ -94,8 +97,13 @@ void SingleFileStorageManager::LoadDatabase() {
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 		return;
 	}
-
-	string wal_path = path + ".wal";
+	std::size_t question_mark_pos = path.find('?');
+	auto wal_path = path;
+	if (question_mark_pos != std::string::npos) {
+		wal_path.insert(question_mark_pos, ".wal");
+	} else {
+		wal_path += ".wal";
+	}
 	auto &fs = FileSystem::Get(db);
 	auto &config = DBConfig::Get(db);
 	bool truncate_wal = false;
@@ -135,8 +143,6 @@ void SingleFileStorageManager::LoadDatabase() {
 		//! Load from storage
 		auto checkpointer = SingleFileCheckpointReader(*this);
 		checkpointer.LoadFromStorage();
-		// finish load checkpoint, clear the cached handles of meta blocks
-		block_manager->ClearMetaBlockHandles();
 		// check if the WAL file exists
 		if (fs.FileExists(wal_path)) {
 			// replay the WAL
@@ -218,7 +224,7 @@ unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(T
 	return make_uniq<SingleFileStorageCommitState>(*this, checkpoint);
 }
 
-bool SingleFileStorageManager::IsCheckpointClean(block_id_t checkpoint_id) {
+bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id) {
 	return block_manager->IsRootBlock(checkpoint_id);
 }
 

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -89,6 +89,10 @@ void ChunkConstantInfo::Serialize(Serializer &serializer) {
 	serializer.Write<idx_t>(start);
 }
 
+idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
+	return delete_id < TRANSACTION_ID_START ? max_count : 0;
+}
+
 unique_ptr<ChunkInfo> ChunkConstantInfo::Deserialize(Deserializer &source) {
 	auto start = source.Read<idx_t>();
 
@@ -244,6 +248,19 @@ void ChunkVectorInfo::Serialize(Serializer &serializer) {
 	serializer.WriteData(data_ptr_cast(deleted_tuples), sizeof(bool) * STANDARD_VECTOR_SIZE);
 }
 
+idx_t ChunkVectorInfo::GetCommittedDeletedCount(idx_t max_count) {
+	if (!any_deleted) {
+		return 0;
+	}
+	idx_t delete_count = 0;
+	for (idx_t i = 0; i < max_count; i++) {
+		if (deleted[i] < TRANSACTION_ID_START) {
+			delete_count++;
+		}
+	}
+	return delete_count;
+}
+
 unique_ptr<ChunkInfo> ChunkVectorInfo::Deserialize(Deserializer &source) {
 	auto start = source.Read<idx_t>();
 

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -242,9 +242,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 	}
 }
 
-void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes) {
-	D_ASSERT(!nodes.empty());
-	this->nodes = std::move(nodes);
+void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes_p) {
+	D_ASSERT(!nodes_p.empty());
+	this->nodes = std::move(nodes_p);
 	// first check if any of the segments have changes
 	if (!HasChanges()) {
 		// no changes: only need to write the metadata for this column