duckdb 1.4.3-dev0.0 → 1.4.4-dev0.0

package/src/duckdb/src/storage/table/row_group.cpp

@@ -1053,7 +1053,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite
 		row_group_pointer.data_pointers = column_pointers;
 		row_group_pointer.has_metadata_blocks = true;
 		row_group_pointer.extra_metadata_blocks = write_data.existing_extra_metadata_blocks;
-		row_group_pointer.deletes_pointers = deletes_pointers;
+		row_group_pointer.deletes_pointers = CheckpointDeletes(*metadata_manager);
 		vector<MetaBlockPointer> extra_metadata_block_pointers;
 		extra_metadata_block_pointers.reserve(write_data.existing_extra_metadata_blocks.size());
 		for (auto &block_pointer : write_data.existing_extra_metadata_blocks) {
@@ -1061,7 +1061,6 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite
 		}
 		metadata_manager->ClearModifiedBlocks(column_pointers);
 		metadata_manager->ClearModifiedBlocks(extra_metadata_block_pointers);
-		metadata_manager->ClearModifiedBlocks(deletes_pointers);
 		// remember metadata_blocks to avoid loading them on future checkpoints
 		has_metadata_blocks = true;
 		extra_metadata_blocks = row_group_pointer.extra_metadata_blocks;
@@ -1109,14 +1108,13 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWrite
 		row_group_pointer.extra_metadata_blocks.push_back(column_pointer.block_pointer);
 		metadata_blocks.insert(column_pointer.block_pointer);
 	}
+	if (metadata_manager) {
+		row_group_pointer.deletes_pointers = CheckpointDeletes(*metadata_manager);
+	}
 	// set up the pointers correctly within this row group for future operations
 	column_pointers = row_group_pointer.data_pointers;
 	has_metadata_blocks = true;
 	extra_metadata_blocks = row_group_pointer.extra_metadata_blocks;
-
-	if (metadata_manager) {
-		row_group_pointer.deletes_pointers = CheckpointDeletes(*metadata_manager);
-	}
 	Verify();
 	return row_group_pointer;
 }
@@ -1125,11 +1123,11 @@ bool RowGroup::HasChanges() const {
 	if (has_changes) {
 		return true;
 	}
-	if (version_info.load()) {
+	auto version_info_loaded = version_info.load();
+	if (version_info_loaded && version_info_loaded->HasUnserializedChanges()) {
 		// we have deletes
 		return true;
 	}
-	D_ASSERT(!deletes_is_loaded.load());
 	// check if any of the columns have changes
 	// avoid loading unloaded columns - unloaded columns can never have changes
 	for (idx_t c = 0; c < columns.size(); c++) {

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -14,6 +14,7 @@
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/persistent_table_data.hpp"
 #include "duckdb/storage/table/row_group_segment_tree.hpp"
+#include "duckdb/storage/table/row_version_manager.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
 #include "duckdb/main/settings.hpp"
@@ -505,6 +506,11 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
 	if (segment.start == start_row) {
 		// we are truncating exactly this row group - erase it entirely
 		row_groups->EraseSegments(l, segment_index);
+		if (segment_index > 0) {
+			// if we have a previous segment, we need to update the next pointer
+			auto previous_segment = row_groups->GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment_index - 1));
+			previous_segment->next = nullptr;
+		}
 	} else {
 		// we need to truncate within a row group
 		// remove any segments AFTER this segment: they should be deleted entirely
@@ -1167,7 +1173,7 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 					extra_metadata_block_pointers.emplace_back(block_pointer, 0);
 				}
 				metadata_manager.ClearModifiedBlocks(extra_metadata_block_pointers);
-				metadata_manager.ClearModifiedBlocks(row_group.GetDeletesPointers());
+				row_group.CheckpointDeletes(metadata_manager);
 				row_groups->AppendSegment(l, std::move(entry.node));
 			}
 			writer.WriteUnchangedTable(metadata_pointer, total_rows.load());
@@ -1285,6 +1291,40 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 
 				throw InternalException("Reloading blocks just written does not yield same blocks: " + oss.str());
 			}
+
+			vector<MetaBlockPointer> read_deletes_pointers;
+			if (!pointer_copy.deletes_pointers.empty()) {
+				auto root_delete = pointer_copy.deletes_pointers[0];
+				auto vm = RowVersionManager::Deserialize(root_delete, GetBlockManager().GetMetadataManager(),
+				                                         row_group.start);
+				read_deletes_pointers = vm->GetStoragePointers();
+			}
+
+			set<idx_t> all_written_deletes_block_ids;
+			for (auto &ptr : pointer_copy.deletes_pointers) {
+				all_written_deletes_block_ids.insert(ptr.block_pointer);
+			}
+			set<idx_t> all_read_deletes_block_ids;
+			for (auto &ptr : read_deletes_pointers) {
+				all_read_deletes_block_ids.insert(ptr.block_pointer);
+			}
+
+			if (all_written_deletes_block_ids != all_read_deletes_block_ids) {
+				std::stringstream oss;
+				oss << "Written: ";
+				for (auto &block : all_written_deletes_block_ids) {
+					oss << block << ", ";
+				}
+				oss << "\n";
+				oss << "Read: ";
+				for (auto &block : all_read_deletes_block_ids) {
+					oss << block << ", ";
+				}
+				oss << "\n";
+
+				throw InternalException("Reloading deletes blocks just written does not yield same blocks: " +
+				                        oss.str());
+			}
 		}
 	}
 	total_rows = new_total_rows;

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -7,7 +7,7 @@
 
 namespace duckdb {
 
-RowVersionManager::RowVersionManager(idx_t start) noexcept : start(start), has_changes(false) {
+RowVersionManager::RowVersionManager(idx_t start) noexcept : start(start), has_unserialized_changes(false) {
 }
 
 void RowVersionManager::SetStart(idx_t new_start) {
@@ -88,7 +88,7 @@ void RowVersionManager::FillVectorInfo(idx_t vector_idx) {
 void RowVersionManager::AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start,
                                           idx_t row_group_end) {
 	lock_guard<mutex> lock(version_lock);
-	has_changes = true;
+	has_unserialized_changes = true;
 	idx_t start_vector_idx = row_group_start / STANDARD_VECTOR_SIZE;
 	idx_t end_vector_idx = (row_group_end - 1) / STANDARD_VECTOR_SIZE;
 
@@ -141,6 +141,7 @@ void RowVersionManager::CommitAppend(transaction_t commit_id, idx_t row_group_st
 		idx_t vend =
 		    vector_idx == end_vector_idx ? row_group_end - end_vector_idx * STANDARD_VECTOR_SIZE : STANDARD_VECTOR_SIZE;
 		auto &info = *vector_info[vector_idx];
+		D_ASSERT(has_unserialized_changes);
 		info.CommitAppend(commit_id, vstart, vend);
 	}
 }
@@ -167,10 +168,12 @@ void RowVersionManager::CleanupAppend(transaction_t lowest_active_transaction, i
 		}
 		auto &info = *vector_info[vector_idx];
 		// if we wrote the entire chunk info try to compress it
-		unique_ptr<ChunkInfo> new_info;
-		auto cleanup = info.Cleanup(lowest_active_transaction, new_info);
+		auto cleanup = info.Cleanup(lowest_active_transaction);
 		if (cleanup) {
-			vector_info[vector_idx] = std::move(new_info);
+			if (info.HasDeletes()) {
+				has_unserialized_changes = true;
+			}
+			vector_info[vector_idx].reset();
 		}
 	}
 }
@@ -179,6 +182,7 @@ void RowVersionManager::RevertAppend(idx_t start_row) {
 	lock_guard<mutex> lock(version_lock);
 	idx_t start_vector_idx = (start_row + (STANDARD_VECTOR_SIZE - 1)) / STANDARD_VECTOR_SIZE;
 	for (idx_t vector_idx = start_vector_idx; vector_idx < vector_info.size(); vector_idx++) {
+		D_ASSERT(has_unserialized_changes);
 		vector_info[vector_idx].reset();
 	}
 }
@@ -205,19 +209,19 @@ ChunkVectorInfo &RowVersionManager::GetVectorInfo(idx_t vector_idx) {
 
 idx_t RowVersionManager::DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count) {
 	lock_guard<mutex> lock(version_lock);
-	has_changes = true;
+	has_unserialized_changes = true;
 	return GetVectorInfo(vector_idx).Delete(transaction_id, rows, count);
 }
 
 void RowVersionManager::CommitDelete(idx_t vector_idx, transaction_t commit_id, const DeleteInfo &info) {
 	lock_guard<mutex> lock(version_lock);
-	has_changes = true;
+	has_unserialized_changes = true;
 	GetVectorInfo(vector_idx).CommitDelete(commit_id, info);
 }
 
 vector<MetaBlockPointer> RowVersionManager::Checkpoint(MetadataManager &manager) {
-	if (!has_changes && !storage_pointers.empty()) {
-		// the row version manager already exists on disk and no changes were made
+	lock_guard<mutex> lock(version_lock);
+	if (!has_unserialized_changes) {
 		// we can write the current pointer as-is
 		// ensure the blocks we are pointing to are not marked as free
 		manager.ClearModifiedBlocks(storage_pointers);
@@ -236,24 +240,23 @@ vector<MetaBlockPointer> RowVersionManager::Checkpoint(MetadataManager &manager)
 		}
 		to_serialize.emplace_back(vector_idx, *chunk_info);
 	}
-	if (to_serialize.empty()) {
-		return vector<MetaBlockPointer>();
-	}
 
 	storage_pointers.clear();
 
-	MetadataWriter writer(manager, &storage_pointers);
-	// now serialize the actual version information
-	writer.Write<idx_t>(to_serialize.size());
-	for (auto &entry : to_serialize) {
-		auto &vector_idx = entry.first;
-		auto &chunk_info = entry.second.get();
-		writer.Write<idx_t>(vector_idx);
-		chunk_info.Write(writer);
+	if (!to_serialize.empty()) {
+		MetadataWriter writer(manager, &storage_pointers);
+		// now serialize the actual version information
+		writer.Write<idx_t>(to_serialize.size());
+		for (auto &entry : to_serialize) {
+			auto &vector_idx = entry.first;
+			auto &chunk_info = entry.second.get();
+			writer.Write<idx_t>(vector_idx);
+			chunk_info.Write(writer);
+		}
+		writer.Flush();
 	}
-	writer.Flush();
 
-	has_changes = false;
+	has_unserialized_changes = false;
 	return storage_pointers;
 }
 
@@ -277,8 +280,19 @@ shared_ptr<RowVersionManager> RowVersionManager::Deserialize(MetaBlockPointer de
 		version_info->FillVectorInfo(vector_index);
 		version_info->vector_info[vector_index] = ChunkInfo::Read(source);
 	}
-	version_info->has_changes = false;
+	version_info->has_unserialized_changes = false;
 	return version_info;
 }
 
+bool RowVersionManager::HasUnserializedChanges() {
+	lock_guard<mutex> lock(version_lock);
+	return has_unserialized_changes;
+}
+
+vector<MetaBlockPointer> RowVersionManager::GetStoragePointers() {
+	lock_guard<mutex> lock(version_lock);
+	D_ASSERT(!has_unserialized_changes);
+	return storage_pointers;
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -170,12 +170,12 @@ void StandardColumnData::UpdateColumn(TransactionData transaction, DataTable &da
                                       const vector<column_t> &column_path, Vector &update_vector, row_t *row_ids,
                                       idx_t update_count, idx_t depth) {
 	if (depth >= column_path.size()) {
-		// update this column
+		// Update the column.
 		ColumnData::Update(transaction, data_table, column_path[0], update_vector, row_ids, update_count);
-	} else {
-		// update the child column (i.e. the validity column)
-		validity.UpdateColumn(transaction, data_table, column_path, update_vector, row_ids, update_count, depth + 1);
+		return;
 	}
+	// Update the child column, which is the validity column.
+	validity.UpdateWithBase(transaction, data_table, column_path[0], update_vector, row_ids, update_count, *this);
 }
 
 unique_ptr<BaseStatistics> StandardColumnData::GetUpdateStatistics() {
@@ -200,8 +200,8 @@ void StandardColumnData::FetchRow(TransactionData transaction, ColumnFetchState
 		auto child_state = make_uniq<ColumnFetchState>();
 		state.child_states.push_back(std::move(child_state));
 	}
-	validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx);
 	ColumnData::FetchRow(transaction, state, row_id, result, result_idx);
+	validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx);
 }
 
 void StandardColumnData::CommitDropColumn() {

package/src/duckdb/src/storage/table/validity_column_data.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/storage/table/validity_column_data.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table/update_segment.hpp"
+#include "duckdb/storage/table/standard_column_data.hpp"
 
 namespace duckdb {
 
@@ -13,6 +14,22 @@ FilterPropagateResult ValidityColumnData::CheckZonemap(ColumnScanState &state, T
 	return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 }
 
+void ValidityColumnData::UpdateWithBase(TransactionData transaction, DataTable &data_table, idx_t column_index,
+                                        Vector &update_vector, row_t *row_ids, idx_t update_count, ColumnData &base) {
+	Vector base_vector(base.type);
+	ColumnScanState validity_scan_state;
+	FetchUpdateData(validity_scan_state, row_ids, base_vector);
+
+	if (validity_scan_state.current->GetCompressionFunction().type == CompressionType::COMPRESSION_EMPTY) {
+		// The validity is actually covered by the data, so we read it to get the validity for UpdateInternal.
+		ColumnScanState data_scan_state;
+		auto fetch_count = base.Fetch(data_scan_state, row_ids[0], base_vector);
+		base_vector.Flatten(fetch_count);
+	}
+
+	UpdateInternal(transaction, data_table, column_index, update_vector, row_ids, update_count, base_vector);
+}
+
 void ValidityColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata,
                                     idx_t count) {
 	lock_guard<mutex> l(stats_lock);