duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -14,6 +14,7 @@
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/execution/task_error_manager.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
+#include "duckdb/execution/index/bound_index.hpp"
 
 namespace duckdb {
 
@@ -55,7 +56,7 @@ RowGroupCollection::RowGroupCollection(shared_ptr<DataTableInfo> info_p, BlockMa
                                        vector<LogicalType> types_p, idx_t row_start_p, idx_t total_rows_p)
     : block_manager(block_manager), total_rows(total_rows_p), info(std::move(info_p)), types(std::move(types_p)),
       row_start(row_start_p), allocation_size(0) {
-	row_groups = make_shared<RowGroupSegmentTree>(*this);
+	row_groups = make_shared_ptr<RowGroupSegmentTree>(*this);
 }
 
 idx_t RowGroupCollection::GetTotalRows() const {
@@ -67,11 +68,11 @@ const vector<LogicalType> &RowGroupCollection::GetTypes() const {
 }
 
 Allocator &RowGroupCollection::GetAllocator() const {
-	return Allocator::Get(info->db);
+	return Allocator::Get(info->GetDB());
 }
 
 AttachedDatabase &RowGroupCollection::GetAttached() {
-	return GetTableInfo().db;
+	return GetTableInfo().GetDB();
 }
 
 MetadataManager &RowGroupCollection::GetMetadataManager() {
@@ -95,7 +96,7 @@ void RowGroupCollection::InitializeEmpty() {
 
 void RowGroupCollection::AppendRowGroup(SegmentLock &l, idx_t start_row) {
 	D_ASSERT(start_row >= row_start);
-	auto new_row_group = make_uniq<RowGroup>(*this, start_row, 0);
+	auto new_row_group = make_uniq<RowGroup>(*this, start_row, 0U);
 	new_row_group->InitializeEmpty(types);
 	row_groups->AppendSegment(l, std::move(new_row_group));
 }
@@ -271,13 +272,13 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 		{
 			idx_t segment_index;
 			auto l = row_groups->Lock();
-			if (!row_groups->TryGetSegmentIndex(l, row_id, segment_index)) {
+			if (!row_groups->TryGetSegmentIndex(l, UnsafeNumericCast<idx_t>(row_id), segment_index)) {
 				// in parallel append scenarios it is possible for the row_id
 				continue;
 			}
-			row_group = row_groups->GetSegmentByIndex(l, segment_index);
+			row_group = row_groups->GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment_index));
 		}
-		if (!row_group->Fetch(transaction, row_id - row_group->start)) {
+		if (!row_group->Fetch(transaction, UnsafeNumericCast<idx_t>(row_id) - row_group->start)) {
 			continue;
 		}
 		row_group->FetchRow(transaction, state, column_ids, row_id, result, count);
@@ -306,7 +307,7 @@ bool RowGroupCollection::IsEmpty(SegmentLock &l) const {
 }
 
 void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppendState &state) {
-	state.row_start = total_rows;
+	state.row_start = UnsafeNumericCast<row_t>(total_rows.load());
 	state.current_row = state.row_start;
 	state.total_append_count = 0;
 
@@ -347,7 +348,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 			// merge the stats
 			auto stats_lock = stats.GetLock();
 			for (idx_t i = 0; i < types.size(); i++) {
-				current_row_group->MergeIntoStatistics(i, stats.GetStats(i).Statistics());
+				current_row_group->MergeIntoStatistics(i, stats.GetStats(*stats_lock, i).Statistics());
 			}
 		}
 		remaining -= append_count;
@@ -376,7 +377,7 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 	state.current_row += row_t(total_append_count);
 	auto stats_lock = stats.GetLock();
 	for (idx_t col_idx = 0; col_idx < types.size(); col_idx++) {
-		stats.GetStats(col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
+		stats.GetStats(*stats_lock, col_idx).UpdateDistinctStatistics(chunk.data[col_idx], chunk.size());
 	}
 	return new_row_group;
 }
@@ -433,7 +434,7 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
 		// revert from the last segment
 		segment_index = segment_count - 1;
 	}
-	auto &segment = *row_groups->GetSegmentByIndex(l, segment_index);
+	auto &segment = *row_groups->GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment_index));
 
 	// remove any segments AFTER this segment: they should be deleted entirely
 	row_groups->EraseSegments(l, segment_index);
@@ -468,7 +469,7 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable &table,
 	idx_t pos = 0;
 	do {
 		idx_t start = pos;
-		auto row_group = row_groups->GetSegment(ids[start]);
+		auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(ids[start]));
 		for (pos++; pos < count; pos++) {
 			D_ASSERT(ids[pos] >= 0);
 			// check if this id still belongs to this row group
@@ -494,10 +495,12 @@ void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const v
 	idx_t pos = 0;
 	do {
 		idx_t start = pos;
-		auto row_group = row_groups->GetSegment(ids[pos]);
+		auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(ids[pos]));
 		row_t base_id =
-		    row_group->start + ((ids[pos] - row_group->start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
-		row_t max_id = MinValue<row_t>(base_id + STANDARD_VECTOR_SIZE, row_group->start + row_group->count);
+		    UnsafeNumericCast<row_t>(row_group->start + ((UnsafeNumericCast<idx_t>(ids[pos]) - row_group->start) /
+		                                                 STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE));
+		auto max_id = MinValue<row_t>(base_id + STANDARD_VECTOR_SIZE,
+		                              UnsafeNumericCast<row_t>(row_group->start + row_group->count));
 		for (pos++; pos < updates.size(); pos++) {
 			D_ASSERT(ids[pos] >= 0);
 			// check if this id still belongs to this vector in this row group
@@ -544,8 +547,8 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_
 		result.Reset();
 		// figure out which row_group to fetch from
 		auto row_id = row_ids[r];
-		auto row_group = row_groups->GetSegment(row_id);
-		auto row_group_vector_idx = (row_id - row_group->start) / STANDARD_VECTOR_SIZE;
+		auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(row_id));
+		auto row_group_vector_idx = (UnsafeNumericCast<idx_t>(row_id) - row_group->start) / STANDARD_VECTOR_SIZE;
 		auto base_row_id = row_group_vector_idx * STANDARD_VECTOR_SIZE + row_group->start;
 
 		// fetch the current vector
@@ -572,7 +575,14 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_
 		result.Slice(sel, sel_count);
 
 		indexes.Scan([&](Index &index) {
-			index.Delete(result, row_identifiers);
+			if (index.IsBound()) {
+				index.Cast<BoundIndex>().Delete(result, row_identifiers);
+			} else {
+				throw MissingExtensionException(
+				    "Cannot delete from index '%s', unknown index type '%s'. You need to load the "
+				    "extension that provides this index type before table '%s' can be modified.",
+				    index.GetIndexName(), index.GetIndexType(), info->GetTableName());
+			}
 			return false;
 		});
 	}
@@ -586,10 +596,11 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 	}
 	// find the row_group this id belongs to
 	auto primary_column_idx = column_path[0];
-	auto row_group = row_groups->GetSegment(first_id);
+	auto row_group = row_groups->GetSegment(UnsafeNumericCast<idx_t>(first_id));
 	row_group->UpdateColumn(transaction, updates, row_ids, column_path);
 
-	row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(primary_column_idx).Statistics());
+	auto lock = stats.GetLock();
+	row_group->MergeIntoStatistics(primary_column_idx, stats.GetStats(*lock, primary_column_idx).Statistics());
 }
 
 //===--------------------------------------------------------------------===//
@@ -789,7 +800,7 @@ public:
 				scan_chunk.Reset();
 
 				current_row_group.ScanCommitted(scan_state.table_state, scan_chunk,
-				                                TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
+				                                TableScanType::TABLE_SCAN_LATEST_COMMITTED_ROWS);
 				if (scan_chunk.size() == 0) {
 					break;
 				}
@@ -843,8 +854,11 @@ private:
 	idx_t row_start;
 };
 
-void RowGroupCollection::InitializeVacuumState(VacuumState &state, vector<SegmentNode<RowGroup>> &segments) {
-	state.can_vacuum_deletes = info->indexes.Empty();
+void RowGroupCollection::InitializeVacuumState(CollectionCheckpointState &checkpoint_state, VacuumState &state,
+                                               vector<SegmentNode<RowGroup>> &segments) {
+	bool is_full_checkpoint = checkpoint_state.writer.GetCheckpointType() == CheckpointType::FULL_CHECKPOINT;
+	// currently we can only vacuum deletes if we are doing a full checkpoint and there are no indexes
+	state.can_vacuum_deletes = info->GetIndexes().Empty() && is_full_checkpoint;
 	if (!state.can_vacuum_deletes) {
 		return;
 	}
@@ -941,7 +955,7 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl
 	CollectionCheckpointState checkpoint_state(*this, writer, segments, global_stats);
 
 	VacuumState vacuum_state;
-	InitializeVacuumState(vacuum_state, segments);
+	InitializeVacuumState(checkpoint_state, vacuum_state, segments);
 	// schedule tasks
 	for (idx_t segment_idx = 0; segment_idx < segments.size(); segment_idx++) {
 		auto &entry = segments[segment_idx];
@@ -1026,25 +1040,24 @@ vector<ColumnSegmentInfo> RowGroupCollection::GetColumnSegmentInfo() {
 // Alter
 //===--------------------------------------------------------------------===//
 shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &context, ColumnDefinition &new_column,
-                                                             Expression &default_value) {
+                                                             ExpressionExecutor &default_executor) {
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
 	auto result =
-	    make_shared<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
+	    make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
 
-	ExpressionExecutor executor(context);
 	DataChunk dummy_chunk;
 	Vector default_vector(new_column.GetType());
-	executor.AddExpression(default_value);
 
 	result->stats.InitializeAddColumn(stats, new_column.GetType());
-	auto &new_column_stats = result->stats.GetStats(new_column_idx);
+	auto lock = result->stats.GetLock();
+	auto &new_column_stats = result->stats.GetStats(*lock, new_column_idx);
 
 	// fill the column with its DEFAULT value, or NULL if none is specified
 	auto new_stats = make_uniq<SegmentStatistics>(new_column.GetType());
 	for (auto &current_row_group : row_groups->Segments()) {
-		auto new_row_group = current_row_group.AddColumn(*result, new_column, executor, default_value, default_vector);
+		auto new_row_group = current_row_group.AddColumn(*result, new_column, default_executor, default_vector);
 		// merge in the statistics
 		new_row_group->MergeIntoStatistics(new_column_idx, new_column_stats.Statistics());
 
@@ -1056,10 +1069,10 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ClientContext &cont
 shared_ptr<RowGroupCollection> RowGroupCollection::RemoveColumn(idx_t col_idx) {
 	D_ASSERT(col_idx < types.size());
 	auto new_types = types;
-	new_types.erase(new_types.begin() + col_idx);
+	new_types.erase_at(col_idx);
 
 	auto result =
-	    make_shared<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
+	    make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
 	result->stats.InitializeRemoveColumn(stats, col_idx);
 
 	for (auto &current_row_group : row_groups->Segments()) {
@@ -1077,7 +1090,7 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(ClientContext &cont
 	new_types[changed_idx] = target_type;
 
 	auto result =
-	    make_shared<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
+	    make_shared_ptr<RowGroupCollection>(info, block_manager, std::move(new_types), row_start, total_rows.load());
 	result->stats.InitializeAlterType(stats, changed_idx, target_type);
 
 	vector<LogicalType> scan_types;
@@ -1099,7 +1112,8 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(ClientContext &cont
 	scan_state.table_state.max_row = row_start + total_rows;
 
 	// now alter the type of the column within all of the row_groups individually
-	auto &changed_stats = result->stats.GetStats(changed_idx);
+	auto lock = result->stats.GetLock();
+	auto &changed_stats = result->stats.GetStats(*lock, changed_idx);
 	for (auto &current_row_group : row_groups->Segments()) {
 		auto new_row_group = current_row_group.AlterType(*result, target_type, changed_idx, executor,
 		                                                 scan_state.table_state, scan_chunk);
@@ -1139,8 +1153,8 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst
 		}
 		// Check constraint
 		if (VectorOperations::HasNull(scan_chunk.data[0], scan_chunk.size())) {
-			throw ConstraintException("NOT NULL constraint failed: %s.%s", info->table,
-			                          parent.column_definitions[physical_index].GetName());
+			throw ConstraintException("NOT NULL constraint failed: %s.%s", info->GetTableName(),
+			                          parent.Columns()[physical_index].GetName());
 		}
 	}
 }
@@ -1158,8 +1172,8 @@ unique_ptr<BaseStatistics> RowGroupCollection::CopyStats(column_t column_id) {
 
 void RowGroupCollection::SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats) {
 	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
-	auto stats_guard = stats.GetLock();
-	stats.GetStats(column_id).SetDistinct(std::move(distinct_stats));
+	auto stats_lock = stats.GetLock();
+	stats.GetStats(*stats_lock, column_id).SetDistinct(std::move(distinct_stats));
 }
 
 } // namespace duckdb

package/src/duckdb/src/storage/table/row_version_manager.cpp

@@ -69,7 +69,7 @@ bool RowVersionManager::Fetch(TransactionData transaction, idx_t row) {
 	if (!info) {
 		return true;
 	}
-	return info->Fetch(transaction, row - vector_index * STANDARD_VECTOR_SIZE);
+	return info->Fetch(transaction, UnsafeNumericCast<row_t>(row - vector_index * STANDARD_VECTOR_SIZE));
 }
 
 void RowVersionManager::AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start,
@@ -212,7 +212,7 @@ shared_ptr<RowVersionManager> RowVersionManager::Deserialize(MetaBlockPointer de
 	if (!delete_pointer.IsValid()) {
 		return nullptr;
 	}
-	auto version_info = make_shared<RowVersionManager>(start);
+	auto version_info = make_shared_ptr<RowVersionManager>(start);
 	MetadataReader source(manager, delete_pointer, &version_info->storage_pointers);
 	auto chunk_count = source.Read<idx_t>();
 	D_ASSERT(chunk_count > 0);

package/src/duckdb/src/storage/table/standard_column_data.cpp

@@ -21,8 +21,16 @@ void StandardColumnData::SetStart(idx_t new_start) {
 	validity.SetStart(new_start);
 }
 
-bool StandardColumnData::HasUpdates() const {
-	return ColumnData::HasUpdates() || validity.HasUpdates();
+ScanVectorType StandardColumnData::GetVectorScanType(ColumnScanState &state, idx_t scan_count) {
+	// if either the current column data, or the validity column data requires flat vectors, we scan flat vectors
+	auto scan_type = ColumnData::GetVectorScanType(state, scan_count);
+	if (scan_type == ScanVectorType::SCAN_FLAT_VECTOR) {
+		return ScanVectorType::SCAN_FLAT_VECTOR;
+	}
+	if (state.child_states.empty()) {
+		return scan_type;
+	}
+	return validity.GetVectorScanType(state.child_states[0], scan_count);
 }
 
 bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
@@ -31,10 +39,15 @@ bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filte
 			return true;
 		}
 		state.segment_checked = true;
-		auto prune_result = filter.CheckStatistics(state.current->stats.statistics);
-		if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
-			return true;
+		FilterPropagateResult prune_result;
+		{
+			lock_guard<mutex> l(stats_lock);
+			prune_result = filter.CheckStatistics(state.current->stats.statistics);
+			if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
+				return true;
+			}
 		}
+		lock_guard<mutex> l(update_lock);
 		if (updates) {
 			auto update_stats = updates->GetStatistics();
 			prune_result = filter.CheckStatistics(*update_stats);
@@ -63,19 +76,19 @@ void StandardColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t
 	validity.InitializeScanWithOffset(state.child_states[0], row_idx);
 }
 
-idx_t StandardColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state,
-                               Vector &result) {
+idx_t StandardColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                               idx_t target_count) {
 	D_ASSERT(state.row_index == state.child_states[0].row_index);
-	auto scan_count = ColumnData::Scan(transaction, vector_index, state, result);
-	validity.Scan(transaction, vector_index, state.child_states[0], result);
+	auto scan_count = ColumnData::Scan(transaction, vector_index, state, result, target_count);
+	validity.Scan(transaction, vector_index, state.child_states[0], result, target_count);
 	return scan_count;
 }
 
-idx_t StandardColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result,
-                                        bool allow_updates) {
+idx_t StandardColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+                                        idx_t target_count) {
 	D_ASSERT(state.row_index == state.child_states[0].row_index);
-	auto scan_count = ColumnData::ScanCommitted(vector_index, state, result, allow_updates);
-	validity.ScanCommitted(vector_index, state.child_states[0], result, allow_updates);
+	auto scan_count = ColumnData::ScanCommitted(vector_index, state, result, allow_updates, target_count);
+	validity.ScanCommitted(vector_index, state.child_states[0], result, allow_updates, target_count);
 	return scan_count;
 }
 
@@ -192,15 +205,14 @@ StandardColumnData::CreateCheckpointState(RowGroup &row_group, PartialBlockManag
 }
 
 unique_ptr<ColumnCheckpointState> StandardColumnData::Checkpoint(RowGroup &row_group,
-                                                                 PartialBlockManager &partial_block_manager,
                                                                  ColumnCheckpointInfo &checkpoint_info) {
 	// we need to checkpoint the main column data first
 	// that is because the checkpointing of the main column data ALSO scans the validity data
 	// to prevent reading the validity data immediately after it is checkpointed we first checkpoint the main column
 	// this is necessary for concurrent checkpointing as due to the partial block manager checkpointed data might be
 	// flushed to disk by a different thread than the one that wrote it, causing a data race
-	auto base_state = ColumnData::Checkpoint(row_group, partial_block_manager, checkpoint_info);
-	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto base_state = ColumnData::Checkpoint(row_group, checkpoint_info);
+	auto validity_state = validity.Checkpoint(row_group, checkpoint_info);
 	auto &checkpoint_state = base_state->Cast<StandardColumnCheckpointState>();
 	checkpoint_state.validity_state = std::move(validity_state);
 	return base_state;

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -42,10 +42,16 @@ bool StructColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter)
 			return true;
 		}
 		state.segment_checked = true;
-		auto prune_result = filter.CheckStatistics(state.current->stats.statistics);
-		if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
-			return true;
+
+		FilterPropagateResult prune_result;
+		{
+			lock_guard<mutex> l(stats_lock);
+			prune_result = filter.CheckStatistics(state.current->stats.statistics);
+			if (prune_result != FilterPropagateResult::FILTER_ALWAYS_FALSE) {
+				return true;
+			}
 		}
+		lock_guard<mutex> l(update_lock);
 		if (updates) {
 			auto update_stats = updates->GetStatistics();
 			prune_result = filter.CheckStatistics(*update_stats);
@@ -90,20 +96,23 @@ void StructColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t ro
 	}
 }
 
-idx_t StructColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	auto scan_count = validity.Scan(transaction, vector_index, state.child_states[0], result);
+idx_t StructColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                             idx_t target_count) {
+	auto scan_count = validity.Scan(transaction, vector_index, state.child_states[0], result, target_count);
 	auto &child_entries = StructVector::GetEntries(result);
 	for (idx_t i = 0; i < sub_columns.size(); i++) {
-		sub_columns[i]->Scan(transaction, vector_index, state.child_states[i + 1], *child_entries[i]);
+		sub_columns[i]->Scan(transaction, vector_index, state.child_states[i + 1], *child_entries[i], target_count);
 	}
 	return scan_count;
 }
 
-idx_t StructColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) {
-	auto scan_count = validity.ScanCommitted(vector_index, state.child_states[0], result, allow_updates);
+idx_t StructColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+                                      idx_t target_count) {
+	auto scan_count = validity.ScanCommitted(vector_index, state.child_states[0], result, allow_updates, target_count);
 	auto &child_entries = StructVector::GetEntries(result);
 	for (idx_t i = 0; i < sub_columns.size(); i++) {
-		sub_columns[i]->ScanCommitted(vector_index, state.child_states[i + 1], *child_entries[i], allow_updates);
+		sub_columns[i]->ScanCommitted(vector_index, state.child_states[i + 1], *child_entries[i], allow_updates,
+		                              target_count);
 	}
 	return scan_count;
 }
@@ -157,7 +166,7 @@ void StructColumnData::RevertAppend(row_t start_row) {
 	for (auto &sub_column : sub_columns) {
 		sub_column->RevertAppend(start_row);
 	}
-	this->count = start_row - this->start;
+	this->count = UnsafeNumericCast<idx_t>(start_row) - this->start;
 }
 
 idx_t StructColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
@@ -281,13 +290,11 @@ unique_ptr<ColumnCheckpointState> StructColumnData::CreateCheckpointState(RowGro
 }
 
 unique_ptr<ColumnCheckpointState> StructColumnData::Checkpoint(RowGroup &row_group,
-                                                               PartialBlockManager &partial_block_manager,
                                                                ColumnCheckpointInfo &checkpoint_info) {
-	auto checkpoint_state = make_uniq<StructColumnCheckpointState>(row_group, *this, partial_block_manager);
-	checkpoint_state->validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto checkpoint_state = make_uniq<StructColumnCheckpointState>(row_group, *this, checkpoint_info.info.manager);
+	checkpoint_state->validity_state = validity.Checkpoint(row_group, checkpoint_info);
 	for (auto &sub_column : sub_columns) {
-		checkpoint_state->child_states.push_back(
-		    sub_column->Checkpoint(row_group, partial_block_manager, checkpoint_info));
+		checkpoint_state->child_states.push_back(sub_column->Checkpoint(row_group, checkpoint_info));
 	}
 	return std::move(checkpoint_state);
 }
@@ -301,7 +308,7 @@ void StructColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatist
 		list.ReadObject([&](Deserializer &item) { sub_columns[i]->DeserializeColumn(item, child_stats); });
 	});
 
-	this->count = validity.count;
+	this->count = validity.count.load();
 }
 
 void StructColumnData::GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,

package/src/duckdb/src/storage/table/table_statistics.cpp

@@ -9,6 +9,7 @@ namespace duckdb {
 void TableStatistics::Initialize(const vector<LogicalType> &types, PersistentTableData &data) {
 	D_ASSERT(Empty());
 
+	stats_lock = make_shared_ptr<mutex>();
 	column_stats = std::move(data.table_stats.column_stats);
 	if (column_stats.size() != types.size()) { // LCOV_EXCL_START
 		throw IOException("Table statistics column count is not aligned with table column count. Corrupt file?");
@@ -18,6 +19,7 @@ void TableStatistics::Initialize(const vector<LogicalType> &types, PersistentTab
 void TableStatistics::InitializeEmpty(const vector<LogicalType> &types) {
 	D_ASSERT(Empty());
 
+	stats_lock = make_shared_ptr<mutex>();
 	for (auto &type : types) {
 		column_stats.push_back(ColumnStatistics::CreateEmptyStats(type));
 	}
@@ -25,8 +27,10 @@ void TableStatistics::InitializeEmpty(const vector<LogicalType> &types) {
 
 void TableStatistics::InitializeAddColumn(TableStatistics &parent, const LogicalType &new_column_type) {
 	D_ASSERT(Empty());
+	D_ASSERT(parent.stats_lock);
 
-	lock_guard<mutex> stats_lock(parent.stats_lock);
+	stats_lock = parent.stats_lock;
+	lock_guard<mutex> lock(*stats_lock);
 	for (idx_t i = 0; i < parent.column_stats.size(); i++) {
 		column_stats.push_back(parent.column_stats[i]);
 	}
@@ -35,8 +39,10 @@ void TableStatistics::InitializeAddColumn(TableStatistics &parent, const Logical
 
 void TableStatistics::InitializeRemoveColumn(TableStatistics &parent, idx_t removed_column) {
 	D_ASSERT(Empty());
+	D_ASSERT(parent.stats_lock);
 
-	lock_guard<mutex> stats_lock(parent.stats_lock);
+	stats_lock = parent.stats_lock;
+	lock_guard<mutex> lock(*stats_lock);
 	for (idx_t i = 0; i < parent.column_stats.size(); i++) {
 		if (i != removed_column) {
 			column_stats.push_back(parent.column_stats[i]);
@@ -46,8 +52,10 @@ void TableStatistics::InitializeRemoveColumn(TableStatistics &parent, idx_t remo
 
 void TableStatistics::InitializeAlterType(TableStatistics &parent, idx_t changed_idx, const LogicalType &new_type) {
 	D_ASSERT(Empty());
+	D_ASSERT(parent.stats_lock);
 
-	lock_guard<mutex> stats_lock(parent.stats_lock);
+	stats_lock = parent.stats_lock;
+	lock_guard<mutex> lock(*stats_lock);
 	for (idx_t i = 0; i < parent.column_stats.size(); i++) {
 		if (i == changed_idx) {
 			column_stats.push_back(ColumnStatistics::CreateEmptyStats(new_type));
@@ -59,8 +67,10 @@ void TableStatistics::InitializeAlterType(TableStatistics &parent, idx_t changed
 
 void TableStatistics::InitializeAddConstraint(TableStatistics &parent) {
 	D_ASSERT(Empty());
+	D_ASSERT(parent.stats_lock);
 
-	lock_guard<mutex> stats_lock(parent.stats_lock);
+	stats_lock = parent.stats_lock;
+	lock_guard<mutex> lock(*stats_lock);
 	for (idx_t i = 0; i < parent.column_stats.size(); i++) {
 		column_stats.push_back(parent.column_stats[i]);
 	}
@@ -86,12 +96,12 @@ void TableStatistics::MergeStats(TableStatisticsLock &lock, idx_t i, BaseStatist
 	column_stats[i]->Statistics().Merge(stats);
 }
 
-ColumnStatistics &TableStatistics::GetStats(idx_t i) {
+ColumnStatistics &TableStatistics::GetStats(TableStatisticsLock &lock, idx_t i) {
 	return *column_stats[i];
 }
 
 unique_ptr<BaseStatistics> TableStatistics::CopyStats(idx_t i) {
-	lock_guard<mutex> l(stats_lock);
+	lock_guard<mutex> l(*stats_lock);
 	auto result = column_stats[i]->Statistics().Copy();
 	if (column_stats[i]->HasDistinctStats()) {
 		result.SetDistinctCount(column_stats[i]->DistinctStats().GetCount());
@@ -100,6 +110,13 @@ unique_ptr<BaseStatistics> TableStatistics::CopyStats(idx_t i) {
 }
 
 void TableStatistics::CopyStats(TableStatistics &other) {
+	TableStatisticsLock lock(*stats_lock);
+	CopyStats(lock, other);
+}
+
+void TableStatistics::CopyStats(TableStatisticsLock &lock, TableStatistics &other) {
+	D_ASSERT(other.Empty());
+	other.stats_lock = make_shared_ptr<mutex>();
 	for (auto &stats : column_stats) {
 		other.column_stats.push_back(stats->Copy());
 	}
@@ -125,14 +142,16 @@ void TableStatistics::Deserialize(Deserializer &deserializer, ColumnList &column
 
 		deserializer.Unset<LogicalType>();
 	});
-	table_sample = deserializer.ReadPropertyWithDefault<unique_ptr<BlockingSample>>(101, "sample", nullptr);
+	table_sample = deserializer.ReadPropertyWithDefault<unique_ptr<BlockingSample>>(101, "table_sample", nullptr);
 }
 
 unique_ptr<TableStatisticsLock> TableStatistics::GetLock() {
-	return make_uniq<TableStatisticsLock>(stats_lock);
+	D_ASSERT(stats_lock);
+	return make_uniq<TableStatisticsLock>(*stats_lock);
 }
 
 bool TableStatistics::Empty() {
+	D_ASSERT(column_stats.empty() == (stats_lock.get() == nullptr));
 	return column_stats.empty();
 }
 

package/src/duckdb/src/storage/table/update_segment.cpp

@@ -581,7 +581,7 @@ void UpdateSegment::InitializeUpdateInfo(UpdateInfo &info, row_t *ids, const Sel
 		auto idx = sel.get_index(i);
 		auto id = ids[idx];
 		D_ASSERT(idx_t(id) >= vector_offset && idx_t(id) < vector_offset + STANDARD_VECTOR_SIZE);
-		info.tuples[i] = NumericCast<sel_t>(id - vector_offset);
+		info.tuples[i] = NumericCast<sel_t>(NumericCast<idx_t>(id) - vector_offset);
 	};
 }
 
@@ -697,7 +697,7 @@ static idx_t MergeLoop(row_t a[], sel_t b[], idx_t acount, idx_t bcount, idx_t a
 	idx_t count = 0;
 	while (aidx < acount && bidx < bcount) {
 		auto a_index = asel.get_index(aidx);
-		auto a_id = a[a_index] - aoffset;
+		auto a_id = UnsafeNumericCast<idx_t>(a[a_index]) - aoffset;
 		auto b_id = b[bidx];
 		if (a_id == b_id) {
 			merge(a_id, a_index, bidx, count);
@@ -716,7 +716,7 @@ static idx_t MergeLoop(row_t a[], sel_t b[], idx_t acount, idx_t bcount, idx_t a
 	}
 	for (; aidx < acount; aidx++) {
 		auto a_index = asel.get_index(aidx);
-		pick_a(a[a_index] - aoffset, a_index, count);
+		pick_a(UnsafeNumericCast<idx_t>(a[a_index]) - aoffset, a_index, count);
 		count++;
 	}
 	for (; bidx < bcount; bidx++) {
@@ -777,7 +777,7 @@ static void MergeUpdateLoopInternal(UpdateInfo *base_info, V *base_table_data, U
 	for (idx_t i = 0; i < count; i++) {
 		auto idx = sel.get_index(i);
 		// we have to merge the info for "ids[i]"
-		auto update_id = ids[idx] - base_id;
+		auto update_id = UnsafeNumericCast<idx_t>(ids[idx]) - base_id;
 
 		while (update_info_offset < update_info->N && update_info->tuples[update_info_offset] < update_id) {
 			// old id comes before the current id: write it
@@ -1103,7 +1103,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 	// get the vector index based on the first id
 	// we assert that all updates must be part of the same vector
 	auto first_id = ids[sel.get_index(0)];
-	idx_t vector_index = (first_id - column_data.start) / STANDARD_VECTOR_SIZE;
+	idx_t vector_index = (UnsafeNumericCast<idx_t>(first_id) - column_data.start) / STANDARD_VECTOR_SIZE;
 	idx_t vector_offset = column_data.start + vector_index * STANDARD_VECTOR_SIZE;
 
 	D_ASSERT(idx_t(first_id) >= column_data.start);
@@ -1116,7 +1116,7 @@ void UpdateSegment::Update(TransactionData transaction, idx_t column_index, Vect
 		// there is already a version here, check if there are any conflicts and search for the node that belongs to
 		// this transaction in the version chain
 		auto base_info = root->info[vector_index]->info.get();
-		CheckForConflicts(base_info->next, transaction, ids, sel, count, vector_offset, node);
+		CheckForConflicts(base_info->next, transaction, ids, sel, count, UnsafeNumericCast<row_t>(vector_offset), node);
 
 		// there are no conflicts
 		// first, check if this thread has already done any updates

package/src/duckdb/src/storage/table/validity_column_data.cpp

@@ -13,4 +13,9 @@ bool ValidityColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filte
 	return true;
 }
 
+void ValidityColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata,
+                                    idx_t count) {
+	lock_guard<mutex> l(stats_lock);
+	ColumnData::AppendData(stats, state, vdata, count);
+}
 } // namespace duckdb