duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -6,10 +6,11 @@
 #include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
 #include "duckdb/storage/table/row_group_segment_tree.hpp"
-#include "duckdb/storage/meta_block_reader.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
+#include "duckdb/common/serializer/binary_deserializer.hpp"
 
 namespace duckdb {
 
@@ -27,16 +28,19 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
 	current_row_group = 0;
 	max_row_group = data.row_group_count;
 	finished_loading = false;
-	reader = make_uniq<MetaBlockReader>(collection.GetBlockManager(), data.block_id);
-	reader->offset = data.offset;
+	reader = make_uniq<MetadataReader>(collection.GetMetadataManager(), data.block_pointer);
 }
 
 unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
 	if (current_row_group >= max_row_group) {
+		reader.reset();
 		finished_loading = true;
 		return nullptr;
 	}
-	auto row_group_pointer = RowGroup::Deserialize(*reader, collection.GetTypes());
+	BinaryDeserializer deserializer(*reader);
+	deserializer.Begin();
+	auto row_group_pointer = RowGroup::Deserialize(deserializer);
+	deserializer.End();
 	current_row_group++;
 	return make_uniq<RowGroup>(collection, std::move(row_group_pointer));
 }
@@ -67,8 +71,8 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
 	return GetTableInfo().db;
 }
 
-DatabaseInstance &RowGroupCollection::GetDatabase() {
-	return GetAttached().GetDatabase();
+MetadataManager &RowGroupCollection::GetMetadataManager() {
+	return GetBlockManager().GetMetadataManager();
 }
 
 //===--------------------------------------------------------------------===//
@@ -97,10 +101,6 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
 	return (RowGroup *)row_groups->GetSegmentByIndex(index);
 }
 
-idx_t RowGroupCollection::RowGroupCount() {
-	return row_groups->GetSegmentCount();
-}
-
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
@@ -340,7 +340,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 		auto current_row_group = state.row_group_append_state.row_group;
 		// check how much we can fit into the current row_group
 		idx_t append_count =
-		    MinValue<idx_t>(remaining, RowGroup::ROW_GROUP_SIZE - state.row_group_append_state.offset_in_row_group);
+		    MinValue<idx_t>(remaining, Storage::ROW_GROUP_SIZE - state.row_group_append_state.offset_in_row_group);
 		if (append_count > 0) {
 			current_row_group->Append(state.row_group_append_state, chunk, append_count);
 			// merge the stats
@@ -394,7 +394,7 @@ void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppend
 	auto remaining = state.total_append_count;
 	auto row_group = state.start_row_group;
 	while (remaining > 0) {
-		auto append_count = MinValue<idx_t>(remaining, RowGroup::ROW_GROUP_SIZE - row_group->count);
+		auto append_count = MinValue<idx_t>(remaining, Storage::ROW_GROUP_SIZE - row_group->count);
 		row_group->AppendVersionInfo(transaction, append_count);
 		remaining -= append_count;
 		row_group = row_groups->GetNextSegment(row_group);
@@ -427,9 +427,9 @@ void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start,
 	}
 }
 
-void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
-	if (total_rows != start_row + count) {
-		throw InternalException("Interleaved appends: this should no longer happen");
+void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
+	if (total_rows <= start_row) {
+		return;
 	}
 	total_rows = start_row;
 
@@ -600,11 +600,24 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 // Checkpoint
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &global_stats) {
-	for (auto &row_group : row_groups->Segments()) {
-		auto rowg_writer = writer.GetRowGroupWriter(row_group);
-		auto pointer = row_group.Checkpoint(*rowg_writer, global_stats);
-		writer.AddRowGroup(std::move(pointer), std::move(rowg_writer));
+	bool can_vacuum_deletes = info->indexes.Empty();
+	idx_t start = this->row_start;
+	auto segments = row_groups->MoveSegments();
+	auto l = row_groups->Lock();
+	for (auto &entry : segments) {
+		auto &row_group = *entry.node;
+		if (can_vacuum_deletes && row_group.AllDeleted()) {
+			row_group.CommitDrop();
+			continue;
+		}
+		row_group.MoveToCollection(*this, start);
+		auto row_group_writer = writer.GetRowGroupWriter(row_group);
+		auto pointer = row_group.Checkpoint(*row_group_writer, global_stats);
+		writer.AddRowGroup(std::move(pointer), std::move(row_group_writer));
+		row_groups->AppendSegment(l, std::move(entry.node));
+		start += row_group.count;
 	}
+	total_rows = start;
 }
 
 //===--------------------------------------------------------------------===//
@@ -637,7 +650,7 @@ vector<ColumnSegmentInfo> RowGroupCollection::GetColumnSegmentInfo() {
 // Alter
 //===--------------------------------------------------------------------===//
 shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &context, ColumnDefinition &new_column,
-                                                             Expression *default_value) {
+                                                             Expression &default_value) {
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
@@ -647,11 +660,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 	ExpressionExecutor executor(context);
 	DataChunk dummy_chunk;
 	Vector default_vector(new_column.GetType());
-	if (!default_value) {
-		FlatVector::Validity(default_vector).SetAllInvalid(STANDARD_VECTOR_SIZE);
-	} else {
-		executor.AddExpression(*default_value);
-	}
+	executor.AddExpression(default_value);
 
 	result->stats.InitializeAddColumn(stats, new_column.GetType());
 	auto &new_column_stats = result->stats.GetStats(new_column_idx);

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -0,0 +1,228 @@
+#include "duckdb/storage/table/row_version_manager.hpp"
+#include "duckdb/transaction/transaction_data.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
+#include "duckdb/common/pair.hpp"
+
+namespace duckdb {
+
+RowVersionManager::RowVersionManager(idx_t start) : start(start), has_changes(false) {
+}
+
+void RowVersionManager::SetStart(idx_t new_start) {
+	lock_guard<mutex> l(version_lock);
+	this->start = new_start;
+	idx_t current_start = start;
+	for (idx_t i = 0; i < Storage::ROW_GROUP_VECTOR_COUNT; i++) {
+		if (vector_info[i]) {
+			vector_info[i]->start = current_start;
+		}
+		current_start += STANDARD_VECTOR_SIZE;
+	}
+}
+
+idx_t RowVersionManager::GetCommittedDeletedCount(idx_t count) {
+	lock_guard<mutex> l(version_lock);
+	idx_t deleted_count = 0;
+	for (idx_t r = 0, i = 0; r < count; r += STANDARD_VECTOR_SIZE, i++) {
+		if (!vector_info[i]) {
+			continue;
+		}
+		idx_t max_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - r);
+		if (max_count == 0) {
+			break;
+		}
+		deleted_count += vector_info[i]->GetCommittedDeletedCount(max_count);
+	}
+	return deleted_count;
+}
+
+optional_ptr<ChunkInfo> RowVersionManager::GetChunkInfo(idx_t vector_idx) {
+	return vector_info[vector_idx].get();
+}
+
+idx_t RowVersionManager::GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector,
+                                      idx_t max_count) {
+	lock_guard<mutex> l(version_lock);
+	auto chunk_info = GetChunkInfo(vector_idx);
+	if (!chunk_info) {
+		return max_count;
+	}
+	return chunk_info->GetSelVector(transaction, sel_vector, max_count);
+}
+
+idx_t RowVersionManager::GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
+                                               SelectionVector &sel_vector, idx_t max_count) {
+	lock_guard<mutex> l(version_lock);
+	auto info = GetChunkInfo(vector_idx);
+	if (!info) {
+		return max_count;
+	}
+	return info->GetCommittedSelVector(start_time, transaction_id, sel_vector, max_count);
+}
+
+bool RowVersionManager::Fetch(TransactionData transaction, idx_t row) {
+	lock_guard<mutex> lock(version_lock);
+	idx_t vector_index = row / STANDARD_VECTOR_SIZE;
+	auto info = GetChunkInfo(vector_index);
+	if (!info) {
+		return true;
+	}
+	return info->Fetch(transaction, row - vector_index * STANDARD_VECTOR_SIZE);
+}
+
+void RowVersionManager::AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start,
+                                          idx_t row_group_end) {
+	lock_guard<mutex> lock(version_lock);
+	has_changes = true;
+	idx_t start_vector_idx = row_group_start / STANDARD_VECTOR_SIZE;
+	idx_t end_vector_idx = (row_group_end - 1) / STANDARD_VECTOR_SIZE;
+	for (idx_t vector_idx = start_vector_idx; vector_idx <= end_vector_idx; vector_idx++) {
+		idx_t vector_start =
+		    vector_idx == start_vector_idx ? row_group_start - start_vector_idx * STANDARD_VECTOR_SIZE : 0;
+		idx_t vector_end =
+		    vector_idx == end_vector_idx ? row_group_end - end_vector_idx * STANDARD_VECTOR_SIZE : STANDARD_VECTOR_SIZE;
+		if (vector_start == 0 && vector_end == STANDARD_VECTOR_SIZE) {
+			// entire vector is encapsulated by append: append a single constant
+			auto constant_info = make_uniq<ChunkConstantInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+			constant_info->insert_id = transaction.transaction_id;
+			constant_info->delete_id = NOT_DELETED_ID;
+			vector_info[vector_idx] = std::move(constant_info);
+		} else {
+			// part of a vector is encapsulated: append to that part
+			optional_ptr<ChunkVectorInfo> new_info;
+			if (!vector_info[vector_idx]) {
+				// first time appending to this vector: create new info
+				auto insert_info = make_uniq<ChunkVectorInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+				new_info = insert_info.get();
+				vector_info[vector_idx] = std::move(insert_info);
+			} else if (vector_info[vector_idx]->type == ChunkInfoType::VECTOR_INFO) {
+				// use existing vector
+				new_info = &vector_info[vector_idx]->Cast<ChunkVectorInfo>();
+			} else {
+				throw InternalException("Error in RowVersionManager::AppendVersionInfo - expected either a "
+				                        "ChunkVectorInfo or no version info");
+			}
+			new_info->Append(vector_start, vector_end, transaction.transaction_id);
+		}
+	}
+}
+
+void RowVersionManager::CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_t count) {
+	idx_t row_group_end = row_group_start + count;
+
+	lock_guard<mutex> lock(version_lock);
+	idx_t start_vector_idx = row_group_start / STANDARD_VECTOR_SIZE;
+	idx_t end_vector_idx = (row_group_end - 1) / STANDARD_VECTOR_SIZE;
+	for (idx_t vector_idx = start_vector_idx; vector_idx <= end_vector_idx; vector_idx++) {
+		idx_t vstart = vector_idx == start_vector_idx ? row_group_start - start_vector_idx * STANDARD_VECTOR_SIZE : 0;
+		idx_t vend =
+		    vector_idx == end_vector_idx ? row_group_end - end_vector_idx * STANDARD_VECTOR_SIZE : STANDARD_VECTOR_SIZE;
+
+		auto info = vector_info[vector_idx].get();
+		info->CommitAppend(commit_id, vstart, vend);
+	}
+}
+
+void RowVersionManager::RevertAppend(idx_t start_row) {
+	lock_guard<mutex> lock(version_lock);
+	idx_t start_vector_idx = (start_row + (STANDARD_VECTOR_SIZE - 1)) / STANDARD_VECTOR_SIZE;
+	for (idx_t vector_idx = start_vector_idx; vector_idx < Storage::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+		vector_info[vector_idx].reset();
+	}
+}
+
+ChunkVectorInfo &RowVersionManager::GetVectorInfo(idx_t vector_idx) {
+	if (!vector_info[vector_idx]) {
+		// no info yet: create it
+		vector_info[vector_idx] = make_uniq<ChunkVectorInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+	} else if (vector_info[vector_idx]->type == ChunkInfoType::CONSTANT_INFO) {
+		auto &constant = vector_info[vector_idx]->Cast<ChunkConstantInfo>();
+		// info exists but it's a constant info: convert to a vector info
+		auto new_info = make_uniq<ChunkVectorInfo>(start + vector_idx * STANDARD_VECTOR_SIZE);
+		new_info->insert_id = constant.insert_id;
+		for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
+			new_info->inserted[i] = constant.insert_id;
+		}
+		vector_info[vector_idx] = std::move(new_info);
+	}
+	D_ASSERT(vector_info[vector_idx]->type == ChunkInfoType::VECTOR_INFO);
+	return vector_info[vector_idx]->Cast<ChunkVectorInfo>();
+}
+
+idx_t RowVersionManager::DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count) {
+	lock_guard<mutex> lock(version_lock);
+	has_changes = true;
+	return GetVectorInfo(vector_idx).Delete(transaction_id, rows, count);
+}
+
+void RowVersionManager::CommitDelete(idx_t vector_idx, transaction_t commit_id, row_t rows[], idx_t count) {
+	lock_guard<mutex> lock(version_lock);
+	has_changes = true;
+	GetVectorInfo(vector_idx).CommitDelete(commit_id, rows, count);
+}
+
+vector<MetaBlockPointer> RowVersionManager::Checkpoint(MetadataManager &manager) {
+	if (!has_changes && !storage_pointers.empty()) {
+		// the row version manager already exists on disk and no changes were made
+		// we can write the current pointer as-is
+		// ensure the blocks we are pointing to are not marked as free
+		manager.ClearModifiedBlocks(storage_pointers);
+		// return the root pointer
+		return storage_pointers;
+	}
+	// first count how many ChunkInfo's we need to deserialize
+	vector<pair<idx_t, reference<ChunkInfo>>> to_serialize;
+	for (idx_t vector_idx = 0; vector_idx < Storage::ROW_GROUP_VECTOR_COUNT; vector_idx++) {
+		auto chunk_info = vector_info[vector_idx].get();
+		if (!chunk_info) {
+			continue;
+		}
+		if (!chunk_info->HasDeletes()) {
+			continue;
+		}
+		to_serialize.emplace_back(vector_idx, *chunk_info);
+	}
+	if (to_serialize.empty()) {
+		return vector<MetaBlockPointer>();
+	}
+
+	storage_pointers.clear();
+
+	MetadataWriter writer(manager, &storage_pointers);
+	// now serialize the actual version information
+	writer.Write<idx_t>(to_serialize.size());
+	for (auto &entry : to_serialize) {
+		auto &vector_idx = entry.first;
+		auto &chunk_info = entry.second.get();
+		writer.Write<idx_t>(vector_idx);
+		chunk_info.Write(writer);
+	}
+	writer.Flush();
+
+	has_changes = false;
+	return storage_pointers;
+}
+
+shared_ptr<RowVersionManager> RowVersionManager::Deserialize(MetaBlockPointer delete_pointer, MetadataManager &manager,
+                                                             idx_t start) {
+	if (!delete_pointer.IsValid()) {
+		return nullptr;
+	}
+	auto version_info = make_shared<RowVersionManager>(start);
+	MetadataReader source(manager, delete_pointer, &version_info->storage_pointers);
+	auto chunk_count = source.Read<idx_t>();
+	D_ASSERT(chunk_count > 0);
+	for (idx_t i = 0; i < chunk_count; i++) {
+		idx_t vector_index = source.Read<idx_t>();
+		if (vector_index >= Storage::ROW_GROUP_VECTOR_COUNT) {
+			throw Exception("In DeserializeDeletes, vector_index is out of range for the row group. Corrupted file?");
+		}
+		version_info->vector_info[vector_index] = ChunkInfo::Read(source);
+	}
+	version_info->has_changes = false;
+	return version_info;
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -4,8 +4,9 @@
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/planner/table_filter.hpp"
-#include "duckdb/transaction/transaction.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -173,9 +174,10 @@ public:
 		return std::move(global_stats);
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer) override {
-		ColumnCheckpointState::WriteDataPointers(writer);
-		validity_state->WriteDataPointers(writer);
+	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
+		ColumnCheckpointState::WriteDataPointers(writer, serializer);
+		serializer.WriteObject(101, "validity",
+		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
 	}
 };
 
@@ -202,9 +204,10 @@ void StandardColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState
 	validity.ScanCommittedRange(row_group_start, offset_in_row_group, count, scan_vector);
 }
 
-void StandardColumnData::DeserializeColumn(Deserializer &source) {
-	ColumnData::DeserializeColumn(source);
-	validity.DeserializeColumn(source);
+void StandardColumnData::DeserializeColumn(Deserializer &deserializer) {
+	ColumnData::DeserializeColumn(deserializer);
+	deserializer.ReadObject(101, "validity",
+	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
 }
 
 void StandardColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/table/struct_column_data.hpp"
 #include "duckdb/storage/statistics/struct_stats.hpp"
-#include "duckdb/transaction/transaction.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
@@ -14,6 +15,9 @@ StructColumnData::StructColumnData(BlockManager &block_manager, DataTableInfo &i
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
 	auto &child_types = StructType::GetChildTypes(type);
 	D_ASSERT(child_types.size() > 0);
+	if (type.id() != LogicalTypeId::UNION && StructType::IsUnnamed(type)) {
+		throw InvalidInputException("A table cannot be created from an unnamed struct");
+	}
 	// the sub column index, starting at 1 (0 is the validity mask)
 	idx_t sub_column_index = 1;
 	for (auto &child_type : child_types) {
@@ -127,6 +131,7 @@ void StructColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, V
 		sub_columns[i]->Append(StructStats::GetChildStats(stats, i), state.child_appends[i + 1], *child_entries[i],
 		                       count);
 	}
+	this->count += count;
 }
 
 void StructColumnData::RevertAppend(row_t start_row) {
@@ -134,6 +139,7 @@ void StructColumnData::RevertAppend(row_t start_row) {
 	for (auto &sub_column : sub_columns) {
 		sub_column->RevertAppend(start_row);
 	}
+	this->count = start_row - this->start;
 }
 
 idx_t StructColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
@@ -240,11 +246,13 @@ public:
 		return stats.ToUnique();
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer) override {
-		validity_state->WriteDataPointers(writer);
-		for (auto &state : child_states) {
-			state->WriteDataPointers(writer);
-		}
+	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
+		serializer.WriteObject(101, "validity",
+		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
+		serializer.WriteList(102, "sub_columns", child_states.size(), [&](Serializer::List &list, idx_t i) {
+			auto &state = child_states[i];
+			list.WriteObject([&](Serializer &serializer) { state->WriteDataPointers(writer, serializer); });
+		});
 	}
 };
 
@@ -265,11 +273,14 @@ unique_ptr<ColumnCheckpointState> StructColumnData::Checkpoint(RowGroup &row_gro
 	return std::move(checkpoint_state);
 }
 
-void StructColumnData::DeserializeColumn(Deserializer &source) {
-	validity.DeserializeColumn(source);
-	for (auto &sub_column : sub_columns) {
-		sub_column->DeserializeColumn(source);
-	}
+void StructColumnData::DeserializeColumn(Deserializer &deserializer) {
+	deserializer.ReadObject(101, "validity",
+	                        [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer); });
+
+	deserializer.ReadList(102, "sub_columns", [&](Deserializer::List &list, idx_t i) {
+		list.ReadObject([&](Deserializer &item) { sub_columns[i]->DeserializeColumn(item); });
+	});
+
 	this->count = validity.count;
 }
 

package/src/duckdb/src/storage/table/table_statistics.cpp

@@ -1,5 +1,7 @@
 #include "duckdb/storage/table/table_statistics.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -99,17 +101,25 @@ void TableStatistics::CopyStats(TableStatistics &other) {
 	}
 }
 
-void TableStatistics::Serialize(Serializer &serializer) {
-	for (auto &stats : column_stats) {
-		stats->Serialize(serializer);
-	}
+void TableStatistics::Serialize(Serializer &serializer) const {
+	serializer.WriteProperty(100, "column_stats", column_stats);
 }
 
-void TableStatistics::Deserialize(Deserializer &source, ColumnList &columns) {
-	for (auto &col : columns.Physical()) {
-		auto stats = ColumnStatistics::Deserialize(source, col.GetType());
-		column_stats.push_back(std::move(stats));
-	}
+void TableStatistics::Deserialize(Deserializer &deserializer, ColumnList &columns) {
+	auto physical_columns = columns.Physical();
+
+	auto iter = physical_columns.begin();
+	deserializer.ReadList(100, "column_stats", [&](Deserializer::List &list, idx_t i) {
+		auto &col = *iter;
+		iter.operator++();
+
+		auto type = col.GetType();
+		deserializer.Set<LogicalType &>(type);
+
+		column_stats.push_back(list.ReadElement<shared_ptr<ColumnStatistics>>());
+
+		deserializer.Unset<LogicalType>();
+	});
 }
 
 unique_ptr<TableStatisticsLock> TableStatistics::GetLock() {

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -328,7 +328,7 @@ void UpdateSegment::FetchCommittedRange(idx_t start_row, idx_t count, Vector &re
 	idx_t start_vector = start_row / STANDARD_VECTOR_SIZE;
 	idx_t end_vector = (end_row - 1) / STANDARD_VECTOR_SIZE;
 	D_ASSERT(start_vector <= end_vector);
-	D_ASSERT(end_vector < RowGroup::ROW_GROUP_VECTOR_COUNT);
+	D_ASSERT(end_vector < Storage::ROW_GROUP_VECTOR_COUNT);
 
 	for (idx_t vector_idx = start_vector; vector_idx <= end_vector; vector_idx++) {
 		if (!root->info[vector_idx]) {
@@ -427,7 +427,7 @@ void UpdateSegment::FetchRow(TransactionData transaction, idx_t row_id, Vector &
 	if (!root->info[vector_index]) {
 		return;
 	}
-	idx_t row_in_vector = row_id - vector_index * STANDARD_VECTOR_SIZE;
+	idx_t row_in_vector = (row_id - column_data.start) - vector_index * STANDARD_VECTOR_SIZE;
 	fetch_row_function(transaction.start_time, transaction.transaction_id, root->info[vector_index]->info.get(),
 	                   row_in_vector, result, result_idx);
 }
@@ -1089,7 +1089,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 	idx_t vector_offset = column_data.start + vector_index * STANDARD_VECTOR_SIZE;
 
 	D_ASSERT(idx_t(first_id) >= column_data.start);
-	D_ASSERT(vector_index < RowGroup::ROW_GROUP_VECTOR_COUNT);
+	D_ASSERT(vector_index < Storage::ROW_GROUP_VECTOR_COUNT);
 
 	// first check the version chain
 	UpdateInfo *node = nullptr;

package/src/duckdb/src/storage/table_index_list.cpp

@@ -54,7 +54,7 @@ void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, Data
 	                   ? ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE
 	                   : ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE;
 
-	// check whether or not the chunk can be inserted or deleted into the referenced table' storage
+	// check whether the chunk can be inserted or deleted into the referenced table storage
 	auto index = FindForeignKeyIndex(fk_keys, fk_type);
 	if (!index) {
 		throw InternalException("Internal Foreign Key error: could not find index to verify...");
@@ -79,7 +79,7 @@ vector<column_t> TableIndexList::GetRequiredColumns() {
 	return result;
 }
 
-vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetaBlockWriter &writer) {
+vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetadataWriter &writer) {
 	vector<BlockPointer> blocks_info;
 	for (auto &index : indexes) {
 		blocks_info.emplace_back(index->Serialize(writer));