duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/storage/table/row_group_collection.hpp"
+
 #include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/execution/index/bound_index.hpp"
@@ -15,6 +16,7 @@
 #include "duckdb/storage/table/row_group_segment_tree.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
+#include "duckdb/main/settings.hpp"
 
 namespace duckdb {
 
@@ -33,6 +35,7 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
 	max_row_group = data.row_group_count;
 	finished_loading = false;
 	reader = make_uniq<MetadataReader>(collection.GetMetadataManager(), data.block_pointer);
+	root_pointer = data.block_pointer;
 }
 
 unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
@@ -62,7 +65,7 @@ RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, BlockMa
                                        vector<LogicalType> types_p, idx_t row_start_p, idx_t total_rows_p,
                                        idx_t row_group_size_p)
     : block_manager(block_manager), row_group_size(row_group_size_p), total_rows(total_rows_p), info(std::move(info_p)),
-      types(std::move(types_p)), row_start(row_start_p), allocation_size(0) {
+      types(std::move(types_p)), row_start(row_start_p), allocation_size(0), requires_new_row_group(false) {
 	row_groups = make_shared_ptr<RowGroupSegmentTree>(*this);
 }
 
@@ -95,6 +98,11 @@ void RowGroupCollection::Initialize(PersistentTableData &data) {
 	this->total_rows = data.total_rows;
 	row_groups->Initialize(data);
 	stats.Initialize(types, data);
+	metadata_pointer = data.base_table_pointer;
+}
+
+void RowGroupCollection::FinalizeCheckpoint(MetaBlockPointer pointer) {
+	metadata_pointer = pointer;
 }
 
 void RowGroupCollection::Initialize(PersistentCollectionData &data) {
@@ -108,6 +116,10 @@ void RowGroupCollection::Initialize(PersistentCollectionData &data) {
 	}
 }
 
+void RowGroupCollection::SetAppendRequiresNewRowGroup() {
+	requires_new_row_group = true;
+}
+
 void RowGroupCollection::InitializeEmpty() {
 	stats.InitializeEmpty(types);
 }
@@ -117,6 +129,7 @@ void RowGroupCollection::AppendRowGroup(SegmentLock &l, idx_t start_row) {
 	auto new_row_group = make_uniq<RowGroup>(*this, start_row, 0U);
 	new_row_group->InitializeEmpty(types);
 	row_groups->AppendSegment(l, std::move(new_row_group));
+	requires_new_row_group = false;
 }
 
 RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
@@ -305,6 +318,19 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 	result.SetCardinality(count);
 }
 
+bool RowGroupCollection::CanFetch(TransactionData transaction, const row_t row_id) {
+	RowGroup *row_group;
+	{
+		idx_t segment_index;
+		auto l = row_groups->Lock();
+		if (!row_groups->TryGetSegmentIndex(l, UnsafeNumericCast<idx_t>(row_id), segment_index)) {
+			return false;
+		}
+		row_group = row_groups->GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment_index));
+	}
+	return row_group->Fetch(transaction, UnsafeNumericCast<idx_t>(row_id) - row_group->start);
+}
+
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
@@ -332,9 +358,9 @@ void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppe
 
 	// start writing to the row_groups
 	auto l = row_groups->Lock();
-	if (IsEmpty(l)) {
+	if (IsEmpty(l) || requires_new_row_group) {
 		// empty row group collection: empty first row group
-		AppendRowGroup(l, row_start);
+		AppendRowGroup(l, row_start + total_rows);
 	}
 	state.start_row_group = row_groups->GetLastSegment(l);
 	D_ASSERT(this->row_start + total_rows == state.start_row_group->start + state.start_row_group->count);
@@ -476,12 +502,17 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
 		segment_index = segment_count - 1;
 	}
 	auto &segment = *row_groups->GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment_index));
-
-	// remove any segments AFTER this segment: they should be deleted entirely
-	row_groups->EraseSegments(l, segment_index);
-
-	segment.next = nullptr;
-	segment.RevertAppend(start_row);
+	if (segment.start == start_row) {
+		// we are truncating exactly this row group - erase it entirely
+		row_groups->EraseSegments(l, segment_index);
+	} else {
+		// we need to truncate within a row group
+		// remove any segments AFTER this segment: they should be deleted entirely
+		row_groups->EraseSegments(l, segment_index + 1);
+
+		segment.next = nullptr;
+		segment.RevertAppend(start_row);
+	}
 }
 
 void RowGroupCollection::CleanupAppend(transaction_t lowest_transaction, idx_t start, idx_t count) {
@@ -591,31 +622,37 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable &table,
 //===--------------------------------------------------------------------===//
 // Update
 //===--------------------------------------------------------------------===//
-void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const vector<PhysicalIndex> &column_ids,
-                                DataChunk &updates) {
+optional_ptr<RowGroup> RowGroupCollection::NextUpdateRowGroup(row_t *ids, idx_t &pos, idx_t count) const {
+	auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(ids[pos]));
+
+	row_t base_id =
+	    UnsafeNumericCast<row_t>(row_group->start + ((UnsafeNumericCast<idx_t>(ids[pos]) - row_group->start) /
+	                                                 STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE));
+	auto max_id =
+	    MinValue<row_t>(base_id + STANDARD_VECTOR_SIZE, UnsafeNumericCast<row_t>(row_group->start + row_group->count));
+	for (pos++; pos < count; pos++) {
+		D_ASSERT(ids[pos] >= 0);
+		// check if this id still belongs to this vector in this row group
+		if (ids[pos] < base_id) {
+			// id is before vector start -> it does not
+			break;
+		}
+		if (ids[pos] >= max_id) {
+			// id is after the maximum id in this vector -> it does not
+			break;
+		}
+	}
+	return row_group;
+}
+
+void RowGroupCollection::Update(TransactionData transaction, DataTable &data_table, row_t *ids,
+                                const vector<PhysicalIndex> &column_ids, DataChunk &updates) {
 	D_ASSERT(updates.size() >= 1);
 	idx_t pos = 0;
 	do {
 		idx_t start = pos;
-		auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(ids[pos]));
-		row_t base_id =
-		    UnsafeNumericCast<row_t>(row_group->start + ((UnsafeNumericCast<idx_t>(ids[pos]) - row_group->start) /
-		                                                 STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE));
-		auto max_id = MinValue<row_t>(base_id + STANDARD_VECTOR_SIZE,
-		                              UnsafeNumericCast<row_t>(row_group->start + row_group->count));
-		for (pos++; pos < updates.size(); pos++) {
-			D_ASSERT(ids[pos] >= 0);
-			// check if this id still belongs to this vector in this row group
-			if (ids[pos] < base_id) {
-				// id is before vector start -> it does not
-				break;
-			}
-			if (ids[pos] >= max_id) {
-				// id is after the maximum id in this vector -> it does not
-				break;
-			}
-		}
-		row_group->Update(transaction, updates, ids, start, pos - start, column_ids);
+		auto row_group = NextUpdateRowGroup(ids, pos, updates.size());
+		row_group->Update(transaction, data_table, updates, ids, start, pos - start, column_ids);
 
 		auto l = stats.GetLock();
 		for (idx_t i = 0; i < column_ids.size(); i++) {
@@ -727,19 +764,20 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_
 	}
 }
 
-void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_ids, const vector<column_t> &column_path,
-                                      DataChunk &updates) {
-	auto first_id = FlatVector::GetValue<row_t>(row_ids, 0);
-	if (first_id >= MAX_ROW_ID) {
-		throw NotImplementedException("Cannot update a column-path on transaction local data");
-	}
-	// find the row_group this id belongs to
-	auto primary_column_idx = column_path[0];
-	auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(first_id));
-	row_group->UpdateColumn(transaction, updates, row_ids, column_path);
+void RowGroupCollection::UpdateColumn(TransactionData transaction, DataTable &data_table, Vector &row_ids,
+                                      const vector<column_t> &column_path, DataChunk &updates) {
+	D_ASSERT(updates.size() >= 1);
+	auto ids = FlatVector::GetData<row_t>(row_ids);
+	idx_t pos = 0;
+	do {
+		idx_t start = pos;
+		auto row_group = NextUpdateRowGroup(ids, pos, updates.size());
+		row_group->UpdateColumn(transaction, data_table, updates, row_ids, start, pos - start, column_path);
 
-	auto lock = stats.GetLock();
-	row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(*lock, primary_column_idx).Statistics());
+		auto lock = stats.GetLock();
+		auto primary_column_idx = column_path[0];
+		row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(*lock, primary_column_idx).Statistics());
+	} while (pos < updates.size());
 }
 
 //===--------------------------------------------------------------------===//
@@ -927,9 +965,10 @@ private:
 
 void RowGroupCollection::InitializeVacuumState(CollectionCheckpointState &checkpoint_state, VacuumState &state,
                                                vector<SegmentNode<RowGroup>> &segments) {
-	bool is_full_checkpoint = checkpoint_state.writer.GetCheckpointType() == CheckpointType::FULL_CHECKPOINT;
+	auto checkpoint_type = checkpoint_state.writer.GetCheckpointType();
+	bool vacuum_is_allowed = checkpoint_type != CheckpointType::CONCURRENT_CHECKPOINT;
 	// currently we can only vacuum deletes if we are doing a full checkpoint and there are no indexes
-	state.can_vacuum_deletes = info->GetIndexes().Empty() && is_full_checkpoint;
+	state.can_vacuum_deletes = info->GetIndexes().Empty() && vacuum_is_allowed;
 	if (!state.can_vacuum_deletes) {
 		return;
 	}
@@ -969,13 +1008,13 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 	}
 	idx_t merge_rows;
 	idx_t next_idx = 0;
-	idx_t merge_count;
-	idx_t target_count;
+	idx_t merge_count = 0;
+	idx_t target_count = 0;
 	bool perform_merge = false;
 	// check if we can merge row groups adjacent to the current segment_idx
 	// we try merging row groups into batches of 1-3 row groups
 	// our goal is to reduce the amount of row groups
-	// hence we target_count should be less than merge_count for a marge to be worth it
+	// hence we target_count should be less than merge_count for a merge to be worth it
 	// we greedily prefer to merge to the lowest target_count
 	// i.e. we prefer to merge 2 row groups into 1, than 3 row groups into 2
 	const idx_t row_group_size = GetRowGroupSize();
@@ -995,6 +1034,22 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 			merge_rows += state.row_group_counts[next_idx];
 			merge_count++;
 		}
+		if (next_idx == checkpoint_state.segments.size()) {
+			// in order to prevent poor performance when performing small appends, we only merge row groups at the end
+			// if we can reach a "target" size of twice the current size, or the max row group size
+			// this is to prevent repeated expensive checkpoints where:
+			// we have a row group with 100K rows
+			// merge it with a row group with 1 row, creating a row group with 100K+1 rows
+			// merge it with a row group with 1 row, creating a row group with 100K+2 rows
+			// etc. This leads to constant rewriting of the original 100K rows.
+			idx_t minimum_target =
+			    MinValue<idx_t>(state.row_group_counts[segment_idx] * 2, row_group_size) * target_count;
+			if (merge_rows >= STANDARD_VECTOR_SIZE && merge_rows < minimum_target) {
+				// we haven't reached the minimum target - don't do this vacuum
+				next_idx = segment_idx + 1;
+				continue;
+			}
+		}
 		if (target_count < merge_count) {
 			// we can reduce "merge_count" row groups to "target_count"
 			// perform the merge at this level
@@ -1006,6 +1061,8 @@ bool RowGroupCollection::ScheduleVacuumTasks(CollectionCheckpointState &checkpoi
 		return false;
 	}
 	// schedule the vacuum task
+	DUCKDB_LOG(checkpoint_state.writer.GetDatabase(), CheckpointLogType, GetAttached(), *info, segment_idx, merge_count,
+	           target_count, merge_rows, state.row_start);
 	auto vacuum_task = make_uniq<VacuumTask>(checkpoint_state, state, segment_idx, merge_count, target_count,
 	                                         merge_rows, state.row_start);
 	checkpoint_state.executor->ScheduleTask(std::move(vacuum_task));
@@ -1035,12 +1092,11 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 	try {
 		// schedule tasks
 		idx_t total_vacuum_tasks = 0;
-		auto &config = DBConfig::GetConfig(writer.GetDatabase());
-
+		auto max_vacuum_tasks = DBConfig::GetSetting<MaxVacuumTasksSetting>(writer.GetDatabase());
 		for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
 			auto &entry = segments[segment_idx];
-			auto vacuum_tasks = ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx,
-			                                        total_vacuum_tasks < config.options.max_vacuum_tasks);
+			auto vacuum_tasks =
+			    ScheduleVacuumTasks(checkpoint_state, vacuum_state, segment_idx, total_vacuum_tasks < max_vacuum_tasks);
 			if (vacuum_tasks) {
 				// vacuum tasks were scheduled - don't schedule a checkpoint task yet
 				total_vacuum_tasks++;
@@ -1052,8 +1108,12 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 			}
 			// schedule a checkpoint task for this row group
 			entry.node->MoveToCollection(*this, vacuum_state.row_start);
-			auto checkpoint_task = GetCheckpointTask(checkpoint_state, segment_idx);
-			checkpoint_state.executor->ScheduleTask(std::move(checkpoint_task));
+			if (writer.GetCheckpointType() != CheckpointType::VACUUM_ONLY) {
+				DUCKDB_LOG(checkpoint_state.writer.GetDatabase(), CheckpointLogType, GetAttached(), *info, segment_idx,
+				           *entry.node);
+				auto checkpoint_task = GetCheckpointTask(checkpoint_state, segment_idx);
+				checkpoint_state.executor->ScheduleTask(std::move(checkpoint_task));
+			}
 			vacuum_state.row_start += entry.node->count;
 		}
 	} catch (const std::exception &e) {
@@ -1066,6 +1126,39 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 	checkpoint_state.executor->WorkOnTasks();
 
 	// no errors - finalize the row groups
+	// if the table already exists on disk - check if all row groups have stayed the same
+	if (DBConfig::GetSetting<ExperimentalMetadataReuseSetting>(writer.GetDatabase()) && metadata_pointer.IsValid()) {
+		bool table_has_changes = false;
+		for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
+			auto &entry = segments[segment_idx];
+			if (!entry.node) {
+				table_has_changes = true;
+				break;
+			}
+			auto &write_state = checkpoint_state.write_data[segment_idx];
+			if (write_state.existing_pointers.empty()) {
+				table_has_changes = true;
+				break;
+			}
+		}
+		if (!table_has_changes) {
+			// table is unmodified and already exists on disk
+			// we can directly re-use the metadata pointer
+			// mark all blocks associated with row groups as still being in-use
+			auto &metadata_manager = writer.GetMetadataManager();
+			for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
+				auto &entry = segments[segment_idx];
+				auto &row_group = *entry.node;
+				auto &write_state = checkpoint_state.write_data[segment_idx];
+				metadata_manager.ClearModifiedBlocks(write_state.existing_pointers);
+				metadata_manager.ClearModifiedBlocks(row_group.GetDeletesPointers());
+				row_groups->AppendSegment(l, std::move(entry.node));
+			}
+			writer.WriteUnchangedTable(metadata_pointer, total_rows.load());
+			return;
+		}
+	}
+
 	idx_t new_total_rows = 0;
 	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
 		auto &entry = segments[segment_idx];
@@ -1074,6 +1167,13 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 			continue;
 		}
 		auto &row_group = *entry.node;
+		if (!checkpoint_state.writers[segment_idx]) {
+			// row group was not checkpointed - this can happen if compressing is disabled for in-memory tables
+			D_ASSERT(writer.GetCheckpointType() == CheckpointType::VACUUM_ONLY);
+			row_groups->AppendSegment(l, std::move(entry.node));
+			new_total_rows += row_group.count;
+			continue;
+		}
 		auto row_group_writer = std::move(checkpoint_state.writers[segment_idx]);
 		if (!row_group_writer) {
 			throw InternalException("Missing row group writer for index %llu", segment_idx);
@@ -1085,6 +1185,8 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 		new_total_rows += row_group.count;
 	}
 	total_rows = new_total_rows;
+	l.Release();
+	Verify();
 }
 
 //===--------------------------------------------------------------------===//
@@ -1118,7 +1220,8 @@ vector<PartitionStatistics> RowGroupCollection::GetPartitionStats() const {
 //===--------------------------------------------------------------------===//
 vector<ColumnSegmentInfo> RowGroupCollection::GetColumnSegmentInfo() {
 	vector<ColumnSegmentInfo> result;
-	for (auto &row_group : row_groups->Segments()) {
+	auto lock = row_groups->Lock();
+	for (auto &row_group : row_groups->Segments(lock)) {
 		row_group.GetColumnSegmentInfo(row_group.index, result);
 	}
 	return result;

package/src/duckdb/src/storage/table/row_id_column_data.cpp

@@ -0,0 +1,174 @@
+#include "duckdb/storage/table/row_id_column_data.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/storage/table/update_segment.hpp"
+
+namespace duckdb {
+
+RowIdColumnData::RowIdColumnData(BlockManager &block_manager, DataTableInfo &info, idx_t start_row)
+    : ColumnData(block_manager, info, COLUMN_IDENTIFIER_ROW_ID, start_row, LogicalType(LogicalTypeId::BIGINT),
+                 nullptr) {
+}
+
+FilterPropagateResult RowIdColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
+	return RowGroup::CheckRowIdFilter(filter, start, start + count);
+	;
+}
+
+void RowIdColumnData::InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) {
+}
+
+void RowIdColumnData::InitializeScan(ColumnScanState &state) {
+	InitializeScanWithOffset(state, start);
+}
+
+void RowIdColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx) {
+	state.current = nullptr;
+	state.segment_tree = nullptr;
+	state.row_index = row_idx;
+	state.internal_index = state.row_index;
+	state.initialized = true;
+	state.scan_state.reset();
+	state.last_offset = 0;
+}
+
+idx_t RowIdColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                            idx_t scan_count) {
+	return ScanCommitted(vector_index, state, result, true, scan_count);
+}
+
+idx_t RowIdColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+                                     idx_t scan_count) {
+	return ScanCount(state, result, scan_count, 0);
+}
+
+void RowIdColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t count,
+                                         Vector &result) {
+	D_ASSERT(this->start == row_group_start);
+	result.Sequence(UnsafeNumericCast<int64_t>(this->start + offset_in_row_group), 1, count);
+}
+
+idx_t RowIdColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count, idx_t result_offset) {
+	if (result_offset != 0) {
+		throw InternalException("RowIdColumnData result_offset must be 0");
+	}
+	ScanCommittedRange(start, state.row_index - start, count, result);
+	state.row_index += count;
+	return count;
+}
+
+void RowIdColumnData::Filter(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                             SelectionVector &sel, idx_t &count, const TableFilter &filter,
+                             TableFilterState &filter_state) {
+	auto current_row = state.row_index;
+	auto max_count = GetVectorCount(vector_index);
+	state.row_index += max_count;
+	// We do another quick statistics scan for row ids here
+	const auto rowid_start = current_row;
+	const auto rowid_end = current_row + max_count;
+	const auto prune_result = RowGroup::CheckRowIdFilter(filter, rowid_start, rowid_end);
+	if (prune_result == FilterPropagateResult::FILTER_ALWAYS_FALSE) {
+		// We can just break out of the loop here.
+		count = 0;
+		return;
+	}
+
+	// Generate row ids
+	// Create sequence for row ids
+	result.SetVectorType(VectorType::FLAT_VECTOR);
+	auto result_data = FlatVector::GetData<row_t>(result);
+	for (size_t sel_idx = 0; sel_idx < count; sel_idx++) {
+		result_data[sel.get_index(sel_idx)] = UnsafeNumericCast<int64_t>(current_row + sel.get_index(sel_idx));
+	}
+
+	// Was this filter always true? If so, we dont need to apply it
+	if (prune_result == FilterPropagateResult::FILTER_ALWAYS_TRUE) {
+		return;
+	}
+
+	// Now apply the filter
+	UnifiedVectorFormat vdata;
+	result.ToUnifiedFormat(count, vdata);
+	ColumnSegment::FilterSelection(sel, result, vdata, filter, filter_state, count, count);
+}
+
+void RowIdColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                             SelectionVector &sel, idx_t count) {
+	SelectCommitted(vector_index, state, result, sel, count, true);
+}
+
+void RowIdColumnData::SelectCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, SelectionVector &sel,
+                                      idx_t count, bool allow_updates) {
+	result.SetVectorType(VectorType::FLAT_VECTOR);
+	auto result_data = FlatVector::GetData<row_t>(result);
+	for (size_t sel_idx = 0; sel_idx < count; sel_idx++) {
+		result_data[sel_idx] = UnsafeNumericCast<row_t>(state.row_index + sel.get_index(sel_idx));
+	}
+	state.row_index += GetVectorCount(vector_index);
+}
+
+idx_t RowIdColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
+	throw InternalException("Fetch is not supported for row id columns");
+}
+
+void RowIdColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
+                               idx_t result_idx) {
+	result.SetVectorType(VectorType::FLAT_VECTOR);
+	auto data = FlatVector::GetData<row_t>(result);
+	data[result_idx] = row_id;
+}
+void RowIdColumnData::Skip(ColumnScanState &state, idx_t count) {
+	state.row_index += count;
+	state.internal_index = state.row_index;
+}
+
+void RowIdColumnData::InitializeAppend(ColumnAppendState &state) {
+	throw InternalException("RowIdColumnData cannot be appended to");
+}
+
+void RowIdColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector &vector, idx_t count) {
+	throw InternalException("RowIdColumnData cannot be appended to");
+}
+
+void RowIdColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata,
+                                 idx_t count) {
+	throw InternalException("RowIdColumnData cannot be appended to");
+}
+
+void RowIdColumnData::RevertAppend(row_t start_row) {
+	throw InternalException("RowIdColumnData cannot be appended to");
+}
+
+void RowIdColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index,
+                             Vector &update_vector, row_t *row_ids, idx_t update_count) {
+	throw InternalException("RowIdColumnData cannot be updated");
+}
+
+void RowIdColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table,
+                                   const vector<column_t> &column_path, Vector &update_vector, row_t *row_ids,
+                                   idx_t update_count, idx_t depth) {
+	throw InternalException("RowIdColumnData cannot be updated");
+}
+
+void RowIdColumnData::CommitDropColumn() {
+	throw InternalException("RowIdColumnData cannot be dropped");
+}
+
+unique_ptr<ColumnCheckpointState> RowIdColumnData::CreateCheckpointState(RowGroup &row_group,
+                                                                         PartialBlockManager &partial_block_manager) {
+	throw InternalException("RowIdColumnData cannot be checkpointed");
+}
+
+unique_ptr<ColumnCheckpointState> RowIdColumnData::Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &info) {
+	throw InternalException("RowIdColumnData cannot be checkpointed");
+}
+
+void RowIdColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
+                                     Vector &scan_vector) {
+	throw InternalException("RowIdColumnData cannot be checkpointed");
+}
+
+bool RowIdColumnData::IsPersistent() {
+	throw InternalException("RowIdColumnData cannot be persisted");
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -221,7 +221,7 @@ vector<MetaBlockPointer> RowVersionManager::Checkpoint(MetadataManager &manager)
 		// we can write the current pointer as-is
 		// ensure the blocks we are pointing to are not marked as free
 		manager.ClearModifiedBlocks(storage_pointers);
-		// return the root pointer
+		// return the current set of pointers
 		return storage_pointers;
 	}
 	// first count how many ChunkInfo's we need to deserialize

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -152,20 +152,29 @@ idx_t StandardColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &re
 	return scan_count;
 }
 
-void StandardColumnData::Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
-                                idx_t update_count) {
-	ColumnData::Update(transaction, column_index, update_vector, row_ids, update_count);
-	validity.Update(transaction, column_index, update_vector, row_ids, update_count);
+void StandardColumnData::Update(TransactionData transaction, DataTable &data_table, idx_t column_index,
+                                Vector &update_vector, row_t *row_ids, idx_t update_count) {
+	ColumnScanState standard_state, validity_state;
+	Vector base_vector(type);
+	auto standard_fetch = FetchUpdateData(standard_state, row_ids, base_vector);
+	auto validity_fetch = validity.FetchUpdateData(validity_state, row_ids, base_vector);
+	if (standard_fetch != validity_fetch) {
+		throw InternalException("Unaligned fetch in validity and main column data for update");
+	}
+
+	UpdateInternal(transaction, data_table, column_index, update_vector, row_ids, update_count, base_vector);
+	validity.UpdateInternal(transaction, data_table, column_index, update_vector, row_ids, update_count, base_vector);
 }
 
-void StandardColumnData::UpdateColumn(TransactionData transaction, const vector<column_t> &column_path,
-                                      Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) {
+void StandardColumnData::UpdateColumn(TransactionData transaction, DataTable &data_table,
+                                      const vector<column_t> &column_path, Vector &update_vector, row_t *row_ids,
+                                      idx_t update_count, idx_t depth) {
 	if (depth >= column_path.size()) {
 		// update this column
-		ColumnData::Update(transaction, column_path[0], update_vector, row_ids, update_count);
+		ColumnData::Update(transaction, data_table, column_path[0], update_vector, row_ids, update_count);
 	} else {
 		// update the child column (i.e. the validity column)
-		validity.UpdateColumn(transaction, column_path, update_vector, row_ids, update_count, depth + 1);
+		validity.UpdateColumn(transaction, data_table, column_path, update_vector, row_ids, update_count, depth + 1);
 	}
 }
 
@@ -271,6 +280,10 @@ bool StandardColumnData::IsPersistent() {
 	return ColumnData::IsPersistent() && validity.IsPersistent();
 }
 
+bool StandardColumnData::HasAnyChanges() const {
+	return ColumnData::HasAnyChanges() || validity.HasAnyChanges();
+}
+
 PersistentColumnData StandardColumnData::Serialize() {
 	auto persistent_data = ColumnData::Serialize();
 	persistent_data.child_columns.push_back(validity.Serialize());