duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -24,9 +24,23 @@ void ListColumnData::SetStart(idx_t new_start) {
 	validity.SetStart(new_start);
 }
 
-bool ListColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
+FilterPropagateResult ListColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
 	// table filters are not supported yet for list columns
-	return false;
+	return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+}
+
+void ListColumnData::InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) {
+	ColumnData::InitializePrefetch(prefetch_state, scan_state, rows);
+	validity.InitializePrefetch(prefetch_state, scan_state.child_states[0], rows);
+
+	// we can't know how many rows we need to prefetch for the child of this list without looking at the actual data
+	// we make an estimation by looking at how many rows the child column has versus this column
+	// e.g if the child column has 10K rows, and we have 1K rows, we estimate that each list has 10 elements
+	idx_t rows_per_list = 1;
+	if (child_column->count > this->count && this->count > 0) {
+		rows_per_list = child_column->count / this->count;
+	}
+	child_column->InitializePrefetch(prefetch_state, scan_state.child_states[1], rows * rows_per_list);
 }
 
 void ListColumnData::InitializeScan(ColumnScanState &state) {
@@ -326,12 +340,11 @@ public:
 		return stats.ToUnique();
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
-		ColumnCheckpointState::WriteDataPointers(writer, serializer);
-		serializer.WriteObject(101, "validity",
-		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
-		serializer.WriteObject(102, "child_column",
-		                       [&](Serializer &serializer) { child_state->WriteDataPointers(writer, serializer); });
+	PersistentColumnData ToPersistentData() override {
+		auto data = ColumnCheckpointState::ToPersistentData();
+		data.child_columns.push_back(validity_state->ToPersistentData());
+		data.child_columns.push_back(child_state->ToPersistentData());
+		return data;
 	}
 };
 
@@ -352,16 +365,22 @@ unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group
 	return base_state;
 }
 
-void ListColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
-	ColumnData::DeserializeColumn(deserializer, target_stats);
+bool ListColumnData::IsPersistent() {
+	return ColumnData::IsPersistent() && validity.IsPersistent() && child_column->IsPersistent();
+}
 
-	deserializer.ReadObject(
-	    101, "validity", [&](Deserializer &deserializer) { validity.DeserializeColumn(deserializer, target_stats); });
+PersistentColumnData ListColumnData::Serialize() {
+	auto persistent_data = ColumnData::Serialize();
+	persistent_data.child_columns.push_back(validity.Serialize());
+	persistent_data.child_columns.push_back(child_column->Serialize());
+	return persistent_data;
+}
 
+void ListColumnData::InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) {
+	ColumnData::InitializeColumn(column_data, target_stats);
+	validity.InitializeColumn(column_data.child_columns[0], target_stats);
 	auto &child_stats = ListStats::GetChildStats(target_stats);
-	deserializer.ReadObject(102, "child_column", [&](Deserializer &deserializer) {
-		child_column->DeserializeColumn(deserializer, child_stats);
-	});
+	child_column->InitializeColumn(column_data.child_columns[1], child_stats);
 }
 
 void ListColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,

package/src/duckdb/src/storage/table/row_group.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
-#include "duckdb/common/chrono.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
@@ -22,16 +21,18 @@
 #include "duckdb/common/serializer/binary_serializer.hpp"
 #include "duckdb/planner/filter/conjunction_filter.hpp"
 #include "duckdb/planner/filter/struct_filter.hpp"
+#include "duckdb/execution/adaptive_filter.hpp"
 
 namespace duckdb {
 
 RowGroup::RowGroup(RowGroupCollection &collection_p, idx_t start, idx_t count)
-    : SegmentBase<RowGroup>(start, count), collection(collection_p), allocation_size(0) {
+    : SegmentBase<RowGroup>(start, count), collection(collection_p), version_info(nullptr), allocation_size(0) {
 	Verify();
 }
 
 RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer)
-    : SegmentBase<RowGroup>(pointer.row_start, pointer.tuple_count), collection(collection_p), allocation_size(0) {
+    : SegmentBase<RowGroup>(pointer.row_start, pointer.tuple_count), collection(collection_p), version_info(nullptr),
+      allocation_size(0) {
 	// deserialize the columns
 	if (pointer.data_pointers.size() != collection_p.GetTypes().size()) {
 		throw IOException("Row group column count is unaligned with table column count. Corrupt file?");
@@ -48,6 +49,22 @@ RowGroup::RowGroup(RowGroupCollection &collection_p, RowGroupPointer pointer)
 	Verify();
 }
 
+RowGroup::RowGroup(RowGroupCollection &collection_p, PersistentRowGroupData &data)
+    : SegmentBase<RowGroup>(data.start, data.count), collection(collection_p), version_info(nullptr),
+      allocation_size(0) {
+	auto &block_manager = GetBlockManager();
+	auto &info = GetTableInfo();
+	auto &types = collection.get().GetTypes();
+	columns.reserve(types.size());
+	for (idx_t c = 0; c < types.size(); c++) {
+		auto entry = ColumnData::CreateColumn(block_manager, info, c, data.start, types[c], nullptr);
+		entry->InitializeColumn(data.column_data[c]);
+		columns.push_back(std::move(entry));
+	}
+
+	Verify();
+}
+
 void RowGroup::MoveToCollection(RowGroupCollection &collection_p, idx_t new_start) {
 	this->collection = collection_p;
 	this->start = new_start;
@@ -55,7 +72,7 @@ void RowGroup::MoveToCollection(RowGroupCollection &collection_p, idx_t new_star
 		column->SetStart(new_start);
 	}
 	if (!HasUnloadedDeletes()) {
-		auto &vinfo = GetVersionInfo();
+		auto vinfo = GetVersionInfo();
 		if (vinfo) {
 			vinfo->SetStart(new_start);
 		}
@@ -173,11 +190,9 @@ void CollectionScanState::Initialize(const vector<LogicalType> &types) {
 
 bool RowGroup::InitializeScanWithOffset(CollectionScanState &state, idx_t vector_offset) {
 	auto &column_ids = state.GetColumnIds();
-	auto filters = state.GetFilters();
-	if (filters) {
-		if (!CheckZonemap(*filters, column_ids)) {
-			return false;
-		}
+	auto &filters = state.GetFilterInfo();
+	if (!CheckZonemap(filters)) {
+		return false;
 	}
 
 	state.row_group = this;
@@ -205,11 +220,9 @@ bool RowGroup::InitializeScanWithOffset(CollectionScanState &state, idx_t vector
 
 bool RowGroup::InitializeScan(CollectionScanState &state) {
 	auto &column_ids = state.GetColumnIds();
-	auto filters = state.GetFilters();
-	if (filters) {
-		if (!CheckZonemap(*filters, column_ids)) {
-			return false;
-		}
+	auto &filters = state.GetFilterInfo();
+	if (!CheckZonemap(filters)) {
+		return false;
 	}
 	state.row_group = this;
 	state.vector_index = 0;
@@ -267,7 +280,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 
 	// set up the row_group based on this row_group
 	auto row_group = make_uniq<RowGroup>(new_collection, this->start, this->count);
-	row_group->version_info = GetOrCreateVersionInfoPtr();
+	row_group->SetVersionInfo(GetOrCreateVersionInfoPtr());
 	auto &cols = GetColumns();
 	for (idx_t i = 0; i < cols.size(); i++) {
 		if (i == changed_idx) {
@@ -307,7 +320,7 @@ unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, Col
 
 	// set up the row_group based on this row_group
 	auto row_group = make_uniq<RowGroup>(new_collection, this->start, this->count);
-	row_group->version_info = GetOrCreateVersionInfoPtr();
+	row_group->SetVersionInfo(GetOrCreateVersionInfoPtr());
 	row_group->columns = GetColumns();
 	// now add the new column
 	row_group->columns.push_back(std::move(added_column));
@@ -322,7 +335,7 @@ unique_ptr<RowGroup> RowGroup::RemoveColumn(RowGroupCollection &new_collection,
 	D_ASSERT(removed_column < columns.size());
 
 	auto row_group = make_uniq<RowGroup>(new_collection, this->start, this->count);
-	row_group->version_info = GetOrCreateVersionInfoPtr();
+	row_group->SetVersionInfo(GetOrCreateVersionInfoPtr());
 	// copy over all columns except for the removed one
 	auto &cols = GetColumns();
 	for (idx_t i = 0; i < cols.size(); i++) {
@@ -358,14 +371,23 @@ void RowGroup::NextVector(CollectionScanState &state) {
 	}
 }
 
-bool RowGroup::CheckZonemap(TableFilterSet &filters, const vector<storage_t> &column_ids) {
-	for (auto &entry : filters.filters) {
-		auto column_index = entry.first;
-		auto &filter = entry.second;
-		const auto &base_column_index = column_ids[column_index];
-		if (!GetColumn(base_column_index).CheckZonemap(*filter)) {
+bool RowGroup::CheckZonemap(ScanFilterInfo &filters) {
+	auto &filter_list = filters.GetFilterList();
+	// new row group - label all filters as up for grabs again
+	filters.CheckAllFilters();
+	for (idx_t i = 0; i < filter_list.size(); i++) {
+		auto &entry = filter_list[i];
+		auto &filter = entry.filter;
+		auto base_column_index = entry.table_column_index;
+		auto prune_result = GetColumn(base_column_index).CheckZonemap(filter);
+		if (prune_result == FilterPropagateResult::FILTER_ALWAYS_FALSE) {
 			return false;
 		}
+		if (prune_result == FilterPropagateResult::FILTER_ALWAYS_TRUE) {
+			// filter is always true - no need to check it
+			// label the filter as always true so we don't need to check it anymore
+			filters.SetFilterAlwaysTrue(i);
+		}
 	}
 	return true;
 }
@@ -405,40 +427,41 @@ static idx_t GetFilterScanCount(ColumnScanState &state, TableFilter &filter) {
 }
 
 bool RowGroup::CheckZonemapSegments(CollectionScanState &state) {
-	auto &column_ids = state.GetColumnIds();
-	auto filters = state.GetFilters();
-	if (!filters) {
-		return true;
-	}
-	for (auto &entry : filters->filters) {
-		D_ASSERT(entry.first < column_ids.size());
-		auto column_idx = entry.first;
-		const auto &base_column_idx = column_ids[column_idx];
-		bool read_segment = GetColumn(base_column_idx).CheckZonemap(state.column_scans[column_idx], *entry.second);
-		if (!read_segment) {
+	auto &filters = state.GetFilterInfo();
+	for (auto &entry : filters.GetFilterList()) {
+		if (entry.IsAlwaysTrue()) {
+			// filter is always true - avoid checking
+			continue;
+		}
+		auto column_idx = entry.scan_column_index;
+		auto base_column_idx = entry.table_column_index;
+		auto &filter = entry.filter;
 
-			idx_t target_row = GetFilterScanCount(state.column_scans[column_idx], *entry.second);
-			if (target_row >= state.max_row) {
-				target_row = state.max_row;
-			}
+		auto prune_result = GetColumn(base_column_idx).CheckZonemap(state.column_scans[column_idx], filter);
+		if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
+			continue;
+		}
+		idx_t target_row = GetFilterScanCount(state.column_scans[column_idx], filter);
+		if (target_row >= state.max_row) {
+			target_row = state.max_row;
+		}
 
-			D_ASSERT(target_row >= this->start);
-			D_ASSERT(target_row <= this->start + this->count);
-			idx_t target_vector_index = (target_row - this->start) / STANDARD_VECTOR_SIZE;
-			if (state.vector_index == target_vector_index) {
-				// we can't skip any full vectors because this segment contains less than a full vector
-				// for now we just bail-out
-				// FIXME: we could check if we can ALSO skip the next segments, in which case skipping a full vector
-				// might be possible
-				// we don't care that much though, since a single segment that fits less than a full vector is
-				// exceedingly rare
-				return true;
-			}
-			while (state.vector_index < target_vector_index) {
-				NextVector(state);
-			}
-			return false;
+		D_ASSERT(target_row >= this->start);
+		D_ASSERT(target_row <= this->start + this->count);
+		idx_t target_vector_index = (target_row - this->start) / STANDARD_VECTOR_SIZE;
+		if (state.vector_index == target_vector_index) {
+			// we can't skip any full vectors because this segment contains less than a full vector
+			// for now we just bail-out
+			// FIXME: we could check if we can ALSO skip the next segments, in which case skipping a full vector
+			// might be possible
+			// we don't care that much though, since a single segment that fits less than a full vector is
+			// exceedingly rare
+			return true;
 		}
+		while (state.vector_index < target_vector_index) {
+			NextVector(state);
+		}
+		return false;
 	}
 
 	return true;
@@ -448,9 +471,8 @@ template <TableScanType TYPE>
 void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result) {
 	const bool ALLOW_UPDATES = TYPE != TableScanType::TABLE_SCAN_COMMITTED_ROWS_DISALLOW_UPDATES &&
 	                           TYPE != TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED;
-	auto table_filters = state.GetFilters();
 	const auto &column_ids = state.GetColumnIds();
-	auto adaptive_filter = state.GetAdaptiveFilter();
+	auto &filter_info = state.GetFilterInfo();
 	while (true) {
 		if (state.vector_index * STANDARD_VECTOR_SIZE >= state.max_row_group_row) {
 			// exceeded the amount of rows to scan
@@ -463,11 +485,11 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s
 		if (!CheckZonemapSegments(state)) {
 			continue;
 		}
+
 		// second, scan the version chunk manager to figure out which tuples to load for this transaction
 		idx_t count;
-		SelectionVector valid_sel(STANDARD_VECTOR_SIZE);
 		if (TYPE == TableScanType::TABLE_SCAN_REGULAR) {
-			count = state.row_group->GetSelVector(transaction, state.vector_index, valid_sel, max_count);
+			count = state.row_group->GetSelVector(transaction, state.vector_index, state.valid_sel, max_count);
 			if (count == 0) {
 				// nothing to scan for this vector, skip the entire vector
 				NextVector(state);
@@ -475,7 +497,7 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s
 			}
 		} else if (TYPE == TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED) {
 			count = state.row_group->GetCommittedSelVector(transaction.start_time, transaction.transaction_id,
-			                                               state.vector_index, valid_sel, max_count);
+			                                               state.vector_index, state.valid_sel, max_count);
 			if (count == 0) {
 				// nothing to scan for this vector, skip the entire vector
 				NextVector(state);
@@ -484,7 +506,28 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s
 		} else {
 			count = max_count;
 		}
-		if (count == max_count && !table_filters) {
+		auto &block_manager = GetBlockManager();
+#ifndef DUCKDB_ALTERNATIVE_VERIFY
+		// // in regular operation we only prefetch from remote file systems
+		// // when alternative verify is set, we always prefetch for testing purposes
+		if (block_manager.IsRemote())
+#else
+		if (!block_manager.InMemory())
+#endif
+		{
+			PrefetchState prefetch_state;
+			for (idx_t i = 0; i < column_ids.size(); i++) {
+				const auto &column = column_ids[i];
+				if (column != COLUMN_IDENTIFIER_ROW_ID) {
+					GetColumn(column).InitializePrefetch(prefetch_state, state.column_scans[i], max_count);
+				}
+			}
+			auto &buffer_manager = block_manager.buffer_manager;
+			buffer_manager.Prefetch(prefetch_state.blocks);
+		}
+
+		bool has_filters = filter_info.HasFilters();
+		if (count == max_count && !has_filters) {
 			// scan all vectors completely: full scan without deletions or table filters
 			for (idx_t i = 0; i < column_ids.size(); i++) {
 				const auto &column = column_ids[i];
@@ -507,73 +550,83 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s
 			idx_t approved_tuple_count = count;
 			SelectionVector sel;
 			if (count != max_count) {
-				sel.Initialize(valid_sel);
+				sel.Initialize(state.valid_sel);
 			} else {
 				sel.Initialize(nullptr);
 			}
 			//! first, we scan the columns with filters, fetch their data and generate a selection vector.
 			//! get runtime statistics
-			auto start_time = high_resolution_clock::now();
-			if (table_filters) {
-				D_ASSERT(adaptive_filter);
+			auto adaptive_filter = filter_info.GetAdaptiveFilter();
+			auto filter_state = filter_info.BeginFilter();
+			if (has_filters) {
 				D_ASSERT(ALLOW_UPDATES);
-				for (idx_t i = 0; i < table_filters->filters.size(); i++) {
-					auto tf_idx = adaptive_filter->permutation[i];
-					auto col_idx = column_ids[tf_idx];
-					auto &col_data = GetColumn(col_idx);
-					col_data.Select(transaction, state.vector_index, state.column_scans[tf_idx], result.data[tf_idx],
-					                sel, approved_tuple_count, *table_filters->filters[tf_idx]);
+				auto &filter_list = filter_info.GetFilterList();
+				for (idx_t i = 0; i < filter_list.size(); i++) {
+					auto filter_idx = adaptive_filter->permutation[i];
+					auto &filter = filter_list[filter_idx];
+					if (filter.IsAlwaysTrue()) {
+						// this filter is always true - skip it
+						continue;
+					}
+					auto scan_idx = filter.scan_column_index;
+					auto &col_data = GetColumn(filter.table_column_index);
+					col_data.Select(transaction, state.vector_index, state.column_scans[scan_idx],
+					                result.data[scan_idx], sel, approved_tuple_count, filter.filter);
 				}
-				for (auto &table_filter : table_filters->filters) {
-					result.data[table_filter.first].Slice(sel, approved_tuple_count);
+				for (auto &table_filter : filter_list) {
+					if (table_filter.IsAlwaysTrue()) {
+						continue;
+					}
+					result.data[table_filter.scan_column_index].Slice(sel, approved_tuple_count);
 				}
 			}
 			if (approved_tuple_count == 0) {
 				// all rows were filtered out by the table filters
-				// skip this vector in all the scans that were not scanned yet
-				D_ASSERT(table_filters);
+				D_ASSERT(has_filters);
 				result.Reset();
+				// skip this vector in all the scans that were not scanned yet
 				for (idx_t i = 0; i < column_ids.size(); i++) {
 					auto col_idx = column_ids[i];
 					if (col_idx == COLUMN_IDENTIFIER_ROW_ID) {
 						continue;
 					}
-					if (table_filters->filters.find(i) == table_filters->filters.end()) {
-						auto &col_data = GetColumn(col_idx);
-						col_data.Skip(state.column_scans[i]);
+					if (has_filters && filter_info.ColumnHasFilters(i)) {
+						continue;
 					}
+					auto &col_data = GetColumn(col_idx);
+					col_data.Skip(state.column_scans[i]);
 				}
 				state.vector_index++;
 				continue;
 			}
 			//! Now we use the selection vector to fetch data for the other columns.
 			for (idx_t i = 0; i < column_ids.size(); i++) {
-				if (!table_filters || table_filters->filters.find(i) == table_filters->filters.end()) {
-					auto column = column_ids[i];
-					if (column == COLUMN_IDENTIFIER_ROW_ID) {
-						D_ASSERT(result.data[i].GetType().InternalType() == PhysicalType::INT64);
-						result.data[i].SetVectorType(VectorType::FLAT_VECTOR);
-						auto result_data = FlatVector::GetData<int64_t>(result.data[i]);
-						for (size_t sel_idx = 0; sel_idx < approved_tuple_count; sel_idx++) {
-							result_data[sel_idx] =
-							    UnsafeNumericCast<int64_t>(this->start + current_row + sel.get_index(sel_idx));
-						}
+				if (has_filters && filter_info.ColumnHasFilters(i)) {
+					// column has already been scanned as part of the filtering process
+					continue;
+				}
+				auto column = column_ids[i];
+				if (column == COLUMN_IDENTIFIER_ROW_ID) {
+					D_ASSERT(result.data[i].GetType().InternalType() == PhysicalType::INT64);
+					result.data[i].SetVectorType(VectorType::FLAT_VECTOR);
+					auto result_data = FlatVector::GetData<int64_t>(result.data[i]);
+					for (size_t sel_idx = 0; sel_idx < approved_tuple_count; sel_idx++) {
+						result_data[sel_idx] =
+						    UnsafeNumericCast<int64_t>(this->start + current_row + sel.get_index(sel_idx));
+					}
+				} else {
+					auto &col_data = GetColumn(column);
+					if (TYPE == TableScanType::TABLE_SCAN_REGULAR) {
+						col_data.FilterScan(transaction, state.vector_index, state.column_scans[i], result.data[i], sel,
+						                    approved_tuple_count);
 					} else {
-						auto &col_data = GetColumn(column);
-						if (TYPE == TableScanType::TABLE_SCAN_REGULAR) {
-							col_data.FilterScan(transaction, state.vector_index, state.column_scans[i], result.data[i],
-							                    sel, approved_tuple_count);
-						} else {
-							col_data.FilterScanCommitted(state.vector_index, state.column_scans[i], result.data[i], sel,
-							                             approved_tuple_count, ALLOW_UPDATES);
-						}
+						col_data.FilterScanCommitted(state.vector_index, state.column_scans[i], result.data[i], sel,
+						                             approved_tuple_count, ALLOW_UPDATES);
 					}
 				}
 			}
-			auto end_time = high_resolution_clock::now();
-			if (adaptive_filter && table_filters->filters.size() > 1) {
-				adaptive_filter->AdaptRuntimeStatistics(duration_cast<duration<double>>(end_time - start_time).count());
-			}
+			filter_info.EndFilter(filter_state);
+
 			D_ASSERT(approved_tuple_count > 0);
 			count = approved_tuple_count;
 		}
@@ -616,40 +669,60 @@ void RowGroup::ScanCommitted(CollectionScanState &state, DataChunk &result, Tabl
 	}
 }
 
-shared_ptr<RowVersionManager> &RowGroup::GetVersionInfo() {
+optional_ptr<RowVersionManager> RowGroup::GetVersionInfo() {
 	if (!HasUnloadedDeletes()) {
 		// deletes are loaded - return the version info
 		return version_info;
 	}
 	lock_guard<mutex> lock(row_group_lock);
 	// double-check after obtaining the lock whether or not deletes are still not loaded to avoid double load
-	if (HasUnloadedDeletes()) {
-		// deletes are not loaded - reload
-		auto root_delete = deletes_pointers[0];
-		version_info = RowVersionManager::Deserialize(root_delete, GetBlockManager().GetMetadataManager(), start);
-		deletes_is_loaded = true;
+	if (!HasUnloadedDeletes()) {
+		return version_info;
 	}
+	// deletes are not loaded - reload
+	auto root_delete = deletes_pointers[0];
+	auto loaded_info = RowVersionManager::Deserialize(root_delete, GetBlockManager().GetMetadataManager(), start);
+	SetVersionInfo(std::move(loaded_info));
+	deletes_is_loaded = true;
 	return version_info;
 }
 
-shared_ptr<RowVersionManager> &RowGroup::GetOrCreateVersionInfoPtr() {
+void RowGroup::SetVersionInfo(shared_ptr<RowVersionManager> version) {
+	owned_version_info = std::move(version);
+	version_info = owned_version_info.get();
+}
+
+shared_ptr<RowVersionManager> RowGroup::GetOrCreateVersionInfoInternal() {
+	// version info does not exist - need to create it
+	lock_guard<mutex> lock(row_group_lock);
+	if (!owned_version_info) {
+		auto new_info = make_shared_ptr<RowVersionManager>(start);
+		SetVersionInfo(std::move(new_info));
+	}
+	return owned_version_info;
+}
+
+shared_ptr<RowVersionManager> RowGroup::GetOrCreateVersionInfoPtr() {
 	auto vinfo = GetVersionInfo();
-	if (!vinfo) {
-		lock_guard<mutex> lock(row_group_lock);
-		if (!version_info) {
-			version_info = make_shared_ptr<RowVersionManager>(start);
-		}
+	if (vinfo) {
+		// version info exists - return it directly
+		return owned_version_info;
 	}
-	return version_info;
+	return GetOrCreateVersionInfoInternal();
 }
 
 RowVersionManager &RowGroup::GetOrCreateVersionInfo() {
-	return *GetOrCreateVersionInfoPtr();
+	auto vinfo = GetVersionInfo();
+	if (vinfo) {
+		// version info exists - return it directly
+		return *vinfo;
+	}
+	return *GetOrCreateVersionInfoInternal();
 }
 
 idx_t RowGroup::GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector,
                              idx_t max_count) {
-	auto &vinfo = GetVersionInfo();
+	auto vinfo = GetVersionInfo();
 	if (!vinfo) {
 		return max_count;
 	}
@@ -658,7 +731,7 @@ idx_t RowGroup::GetSelVector(TransactionData transaction, idx_t vector_idx, Sele
 
 idx_t RowGroup::GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
                                       SelectionVector &sel_vector, idx_t max_count) {
-	auto &vinfo = GetVersionInfo();
+	auto vinfo = GetVersionInfo();
 	if (!vinfo) {
 		return max_count;
 	}
@@ -667,7 +740,7 @@ idx_t RowGroup::GetCommittedSelVector(transaction_t start_time, transaction_t tr
 
 bool RowGroup::Fetch(TransactionData transaction, idx_t row) {
 	D_ASSERT(row < this->count);
-	auto &vinfo = GetVersionInfo();
+	auto vinfo = GetVersionInfo();
 	if (!vinfo) {
 		return true;
 	}
@@ -745,6 +818,11 @@ void RowGroup::Append(RowGroupAppendState &state, DataChunk &chunk, idx_t append
 	state.offset_in_row_group += append_count;
 }
 
+void RowGroup::CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count) {
+	auto &vinfo = GetOrCreateVersionInfo();
+	vinfo.CleanupAppend(lowest_transaction, start, count);
+}
+
 void RowGroup::Update(TransactionData transaction, DataChunk &update_chunk, row_t *ids, idx_t offset, idx_t count,
                       const vector<PhysicalIndex> &column_ids) {
 #ifdef DEBUG
@@ -796,6 +874,13 @@ void RowGroup::MergeIntoStatistics(idx_t column_idx, BaseStatistics &other) {
 	col_data.MergeIntoStatistics(other);
 }
 
+void RowGroup::MergeIntoStatistics(TableStatistics &other) {
+	auto stats_lock = other.GetLock();
+	for (idx_t i = 0; i < columns.size(); i++) {
+		MergeIntoStatistics(i, other.GetStats(*stats_lock, i).Statistics());
+	}
+}
+
 CompressionType ColumnCheckpointInfo::GetCompressionType() {
 	return info.compression_types[column_idx];
 }
@@ -830,7 +915,7 @@ RowGroupWriteData RowGroup::WriteToDisk(RowGroupWriteInfo &info) {
 }
 
 idx_t RowGroup::GetCommittedRowCount() {
-	auto &vinfo = GetVersionInfo();
+	auto vinfo = GetVersionInfo();
 	if (!vinfo) {
 		return count;
 	}
@@ -888,9 +973,10 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite
 		//
 		// Just as above, the state can refer to many other states, so this
 		// can cascade recursively into more pointer writes.
+		auto persistent_data = state->ToPersistentData();
 		BinarySerializer serializer(data_writer);
 		serializer.Begin();
-		state->WriteDataPointers(writer, serializer);
+		persistent_data.Serialize(serializer);
 		serializer.End();
 	}
 	row_group_pointer.deletes_pointers = CheckpointDeletes(writer.GetPayloadWriter().GetManager());
@@ -898,6 +984,27 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite
 	return row_group_pointer;
 }
 
+bool RowGroup::IsPersistent() const {
+	for (auto &column : columns) {
+		if (!column->IsPersistent()) {
+			// column is not persistent
+			return false;
+		}
+	}
+	return true;
+}
+
+PersistentRowGroupData RowGroup::SerializeRowGroupInfo() const {
+	// all columns are persistent - serialize
+	PersistentRowGroupData result;
+	for (auto &col : columns) {
+		result.column_data.push_back(col->Serialize());
+	}
+	result.start = start;
+	result.count = count;
+	return result;
+}
+
 vector<MetaBlockPointer> RowGroup::CheckpointDeletes(MetadataManager &manager) {
 	if (HasUnloadedDeletes()) {
 		// deletes were not loaded so they cannot be changed
@@ -905,11 +1012,12 @@ vector<MetaBlockPointer> RowGroup::CheckpointDeletes(MetadataManager &manager) {
 		manager.ClearModifiedBlocks(deletes_pointers);
 		return deletes_pointers;
 	}
-	if (!version_info) {
+	auto vinfo = GetVersionInfo();
+	if (!vinfo) {
 		// no version information: write nothing
 		return vector<MetaBlockPointer>();
 	}
-	return version_info->Checkpoint(manager);
+	return vinfo->Checkpoint(manager);
 }
 
 void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &serializer) {