duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/storage/table/column_data.cpp

@@ -43,7 +43,7 @@ void ColumnData::SetStart(idx_t new_start) {
 }
 
 DatabaseInstance &ColumnData::GetDatabase() const {
-	return info.db.GetDatabase();
+	return info.GetDB().GetDatabase();
 }
 
 DataTableInfo &ColumnData::GetTableInfo() const {
@@ -91,7 +91,26 @@ void ColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_idx)
 	state.last_offset = 0;
 }
 
-idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, bool has_updates) {
+ScanVectorType ColumnData::GetVectorScanType(ColumnScanState &state, idx_t scan_count) {
+	if (HasUpdates()) {
+		// if we have updates we need to merge in the updates
+		// always need to scan flat vectors
+		return ScanVectorType::SCAN_FLAT_VECTOR;
+	}
+	// check if the current segment has enough data remaining
+	idx_t remaining_in_segment = state.current->start + state.current->count - state.row_index;
+	if (remaining_in_segment < scan_count) {
+		// there is not enough data remaining in the current segment so we need to scan across segments
+		// we need flat vectors here
+		return ScanVectorType::SCAN_FLAT_VECTOR;
+	}
+	return ScanVectorType::SCAN_ENTIRE_VECTOR;
+}
+
+idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remaining, ScanVectorType scan_type) {
+	if (scan_type == ScanVectorType::SCAN_FLAT_VECTOR && result.GetVectorType() != VectorType::FLAT_VECTOR) {
+		throw InternalException("ScanVector called with SCAN_FLAT_VECTOR but result is not a flat vector");
+	}
 	state.previous_states.clear();
 	if (!state.initialized) {
 		D_ASSERT(state.current);
@@ -115,11 +134,11 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 			if (state.scan_options && state.scan_options->force_fetch_row) {
 				for (idx_t i = 0; i < scan_count; i++) {
 					ColumnFetchState fetch_state;
-					state.current->FetchRow(fetch_state, state.row_index + i, result, result_offset + i);
+					state.current->FetchRow(fetch_state, UnsafeNumericCast<row_t>(state.row_index + i), result,
+					                        result_offset + i);
 				}
 			} else {
-				state.current->Scan(state, scan_count, result, result_offset,
-				                    !has_updates && scan_count == initial_remaining);
+				state.current->Scan(state, scan_count, result, result_offset, scan_type);
 			}
 
 			state.row_index += scan_count;
@@ -170,7 +189,7 @@ void ColumnData::FetchUpdateRow(TransactionData transaction, row_t row_id, Vecto
 	if (!updates) {
 		return;
 	}
-	updates->FetchRow(transaction, row_id, result, result_idx);
+	updates->FetchRow(transaction, NumericCast<idx_t>(row_id), result, result_idx);
 }
 
 void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
@@ -183,42 +202,58 @@ void ColumnData::UpdateInternal(TransactionData transaction, idx_t column_index,
 }
 
 template <bool SCAN_COMMITTED, bool ALLOW_UPDATES>
-idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	idx_t current_row = vector_index * STANDARD_VECTOR_SIZE;
-	auto vector_count = MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - current_row);
-
-	auto scan_count = ScanVector(state, result, vector_count, HasUpdates());
+idx_t ColumnData::ScanVector(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                             idx_t target_scan) {
+	auto scan_count = ScanVector(state, result, target_scan, GetVectorScanType(state, target_scan));
 	FetchUpdates(transaction, vector_index, result, scan_count, ALLOW_UPDATES, SCAN_COMMITTED);
 	return scan_count;
 }
 
 template idx_t ColumnData::ScanVector<false, false>(TransactionData transaction, idx_t vector_index,
-                                                    ColumnScanState &state, Vector &result);
+                                                    ColumnScanState &state, Vector &result, idx_t target_scan);
 template idx_t ColumnData::ScanVector<true, false>(TransactionData transaction, idx_t vector_index,
-                                                   ColumnScanState &state, Vector &result);
+                                                   ColumnScanState &state, Vector &result, idx_t target_scan);
 template idx_t ColumnData::ScanVector<false, true>(TransactionData transaction, idx_t vector_index,
-                                                   ColumnScanState &state, Vector &result);
+                                                   ColumnScanState &state, Vector &result, idx_t target_scan);
 template idx_t ColumnData::ScanVector<true, true>(TransactionData transaction, idx_t vector_index,
-                                                  ColumnScanState &state, Vector &result);
+                                                  ColumnScanState &state, Vector &result, idx_t target_scan);
 
 idx_t ColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	return ScanVector<false, true>(transaction, vector_index, state, result);
+	auto target_count = GetVectorCount(vector_index);
+	return Scan(transaction, vector_index, state, result, target_count);
 }
 
 idx_t ColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) {
+	auto target_count = GetVectorCount(vector_index);
+	return ScanCommitted(vector_index, state, result, allow_updates, target_count);
+}
+
+idx_t ColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                       idx_t scan_count) {
+	return ScanVector<false, true>(transaction, vector_index, state, result, scan_count);
+}
+
+idx_t ColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+                                idx_t scan_count) {
 	if (allow_updates) {
-		return ScanVector<true, true>(TransactionData(0, 0), vector_index, state, result);
+		return ScanVector<true, true>(TransactionData(0, 0), vector_index, state, result, scan_count);
 	} else {
-		return ScanVector<true, false>(TransactionData(0, 0), vector_index, state, result);
+		return ScanVector<true, false>(TransactionData(0, 0), vector_index, state, result, scan_count);
 	}
 }
 
+idx_t ColumnData::GetVectorCount(idx_t vector_index) const {
+	idx_t current_row = vector_index * STANDARD_VECTOR_SIZE;
+	return MinValue<idx_t>(STANDARD_VECTOR_SIZE, count - current_row);
+}
+
 void ColumnData::ScanCommittedRange(idx_t row_group_start, idx_t offset_in_row_group, idx_t s_count, Vector &result) {
 	ColumnScanState child_state;
 	InitializeScanWithOffset(child_state, row_group_start + offset_in_row_group);
 	bool has_updates = HasUpdates();
-	auto scan_count = ScanVector(child_state, result, s_count, has_updates);
+	auto scan_count = ScanVector(child_state, result, s_count, ScanVectorType::SCAN_FLAT_VECTOR);
 	if (has_updates) {
+		D_ASSERT(result.GetVectorType() == VectorType::FLAT_VECTOR);
 		result.Flatten(scan_count);
 		updates->FetchCommittedRange(offset_in_row_group, s_count, result);
 	}
@@ -230,7 +265,7 @@ idx_t ColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t scan_c
 	}
 	// ScanCount can only be used if there are no updates
 	D_ASSERT(!HasUpdates());
-	return ScanVector(state, result, scan_count, false);
+	return ScanVector(state, result, scan_count, ScanVectorType::SCAN_FLAT_VECTOR);
 }
 
 void ColumnData::Select(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
@@ -260,7 +295,7 @@ void ColumnData::Skip(ColumnScanState &state, idx_t s_count) {
 
 void ColumnData::Append(BaseStatistics &append_stats, ColumnAppendState &state, Vector &vector, idx_t append_count) {
 	UnifiedVectorFormat vdata;
-	vector.ToUnifiedFormat(count, vdata);
+	vector.ToUnifiedFormat(append_count, vdata);
 	AppendData(append_stats, state, vdata, append_count);
 }
 
@@ -268,6 +303,7 @@ void ColumnData::Append(ColumnAppendState &state, Vector &vector, idx_t append_c
 	if (parent || !stats) {
 		throw InternalException("ColumnData::Append called on a column with a parent or without stats");
 	}
+	lock_guard<mutex> l(stats_lock);
 	Append(stats->statistics, state, vector, append_count);
 }
 
@@ -275,6 +311,7 @@ bool ColumnData::CheckZonemap(TableFilter &filter) {
 	if (!stats) {
 		throw InternalException("ColumnData::CheckZonemap called on a column without stats");
 	}
+	lock_guard<mutex> l(stats_lock);
 	auto propagate_result = filter.CheckStatistics(stats->statistics);
 	if (propagate_result == FilterPropagateResult::FILTER_ALWAYS_FALSE ||
 	    propagate_result == FilterPropagateResult::FILTER_FALSE_OR_NULL) {
@@ -287,6 +324,7 @@ unique_ptr<BaseStatistics> ColumnData::GetStatistics() {
 	if (!stats) {
 		throw InternalException("ColumnData::GetStatistics called on a column without stats");
 	}
+	lock_guard<mutex> l(stats_lock);
 	return stats->statistics.ToUnique();
 }
 
@@ -294,6 +332,7 @@ void ColumnData::MergeStatistics(const BaseStatistics &other) {
 	if (!stats) {
 		throw InternalException("ColumnData::MergeStatistics called on a column without stats");
 	}
+	lock_guard<mutex> l(stats_lock);
 	return stats->statistics.Merge(other);
 }
 
@@ -301,6 +340,7 @@ void ColumnData::MergeIntoStatistics(BaseStatistics &other) {
 	if (!stats) {
 		throw InternalException("ColumnData::MergeIntoStatistics called on a column without stats");
 	}
+	lock_guard<mutex> l(stats_lock);
 	return other.Merge(stats->statistics);
 }
 
@@ -360,36 +400,38 @@ void ColumnData::RevertAppend(row_t start_row) {
 		return;
 	}
 	// find the segment index that the current row belongs to
-	idx_t segment_index = data.GetSegmentIndex(l, start_row);
-	auto segment = data.GetSegmentByIndex(l, segment_index);
+	idx_t segment_index = data.GetSegmentIndex(l, UnsafeNumericCast<idx_t>(start_row));
+	auto segment = data.GetSegmentByIndex(l, UnsafeNumericCast<int64_t>(segment_index));
 	auto &transient = *segment;
 	D_ASSERT(transient.segment_type == ColumnSegmentType::TRANSIENT);
 
 	// remove any segments AFTER this segment: they should be deleted entirely
 	data.EraseSegments(l, segment_index);
 
-	this->count = start_row - this->start;
+	this->count = UnsafeNumericCast<idx_t>(start_row) - this->start;
 	segment->next = nullptr;
-	transient.RevertAppend(start_row);
+	transient.RevertAppend(UnsafeNumericCast<idx_t>(start_row));
 }
 
 idx_t ColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {
 	D_ASSERT(row_id >= 0);
 	D_ASSERT(idx_t(row_id) >= start);
 	// perform the fetch within the segment
-	state.row_index = start + ((row_id - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
+	state.row_index =
+	    start + ((UnsafeNumericCast<idx_t>(row_id) - start) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE);
 	state.current = data.GetSegment(state.row_index);
 	state.internal_index = state.current->start;
-	return ScanVector(state, result, STANDARD_VECTOR_SIZE, false);
+	return ScanVector(state, result, STANDARD_VECTOR_SIZE, ScanVectorType::SCAN_FLAT_VECTOR);
 }
 
 void ColumnData::FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
                           idx_t result_idx) {
-	auto segment = data.GetSegment(row_id);
+	auto segment = data.GetSegment(UnsafeNumericCast<idx_t>(row_id));
 
 	// now perform the fetch within the segment
 	segment->FetchRow(state, row_id, result, result_idx);
 	// merge any updates made to this row
+
 	FetchUpdateRow(transaction, row_id, result, result_idx);
 }
 
@@ -445,10 +487,10 @@ void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state,
 	if (state.scan_options && state.scan_options->force_fetch_row) {
 		for (idx_t i = 0; i < count; i++) {
 			ColumnFetchState fetch_state;
-			segment.FetchRow(fetch_state, state.row_index + i, scan_vector, i);
+			segment.FetchRow(fetch_state, UnsafeNumericCast<row_t>(state.row_index + i), scan_vector, i);
 		}
 	} else {
-		segment.Scan(state, count, scan_vector, 0, !HasUpdates());
+		segment.Scan(state, count, scan_vector, 0, ScanVectorType::SCAN_FLAT_VECTOR);
 	}
 
 	if (updates) {
@@ -457,12 +499,10 @@ void ColumnData::CheckpointScan(ColumnSegment &segment, ColumnScanState &state,
 	}
 }
 
-unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
-                                                         PartialBlockManager &partial_block_manager,
-                                                         ColumnCheckpointInfo &checkpoint_info) {
+unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group, ColumnCheckpointInfo &checkpoint_info) {
 	// scan the segments of the column data
 	// set up the checkpoint state
-	auto checkpoint_state = CreateCheckpointState(row_group, partial_block_manager);
+	auto checkpoint_state = CreateCheckpointState(row_group, checkpoint_info.info.manager);
 	checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type).ToUnique();
 
 	auto l = data.Lock();
@@ -484,7 +524,7 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 
 void ColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
 	// load the data pointers for the column
-	deserializer.Set<DatabaseInstance &>(info.db.GetDatabase());
+	deserializer.Set<DatabaseInstance &>(info.GetDB().GetDatabase());
 	deserializer.Set<LogicalType &>(type);
 
 	vector<DataPointer> data_pointers;
@@ -540,7 +580,7 @@ void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_p
 
 	// iterate over the segments
 	idx_t segment_idx = 0;
-	auto segment = (ColumnSegment *)data.GetRootSegment();
+	auto segment = data.GetRootSegment();
 	while (segment) {
 		ColumnSegmentInfo column_info;
 		column_info.row_group_index = row_group_index;
@@ -551,7 +591,10 @@ void ColumnData::GetColumnSegmentInfo(idx_t row_group_index, vector<idx_t> col_p
 		column_info.segment_start = segment->start;
 		column_info.segment_count = segment->count;
 		column_info.compression_type = CompressionTypeToString(segment->function.get().type);
-		column_info.segment_stats = segment->stats.statistics.ToString();
+		{
+			lock_guard<mutex> l(stats_lock);
+			column_info.segment_stats = segment->stats.statistics.ToString();
+		}
 		column_info.has_updates = ColumnData::HasUpdates();
 		// persistent
 		// block_id

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -100,7 +100,7 @@ unique_ptr<AnalyzeState> ColumnDataCheckpointer::DetectBestCompressionMethod(idx
 	auto &config = DBConfig::GetConfig(GetDatabase());
 	CompressionType forced_method = CompressionType::COMPRESSION_AUTO;
 
-	auto compression_type = checkpoint_info.compression_type;
+	auto compression_type = checkpoint_info.GetCompressionType();
 	if (compression_type != CompressionType::COMPRESSION_AUTO) {
 		forced_method = ForceCompression(compression_functions, compression_type);
 	}

package/src/duckdb/src/storage/table/column_segment.cpp

@@ -60,8 +60,8 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance
 	} else {
 		buffer_manager.Allocate(MemoryTag::IN_MEMORY_TABLE, segment_size, false, &block);
 	}
-	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0, *function,
-	                                BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0, segment_size);
+	return make_uniq<ColumnSegment>(db, std::move(block), type, ColumnSegmentType::TRANSIENT, start, 0U, *function,
+	                                BaseStatistics::CreateEmpty(type), INVALID_BLOCK, 0U, segment_size);
 }
 
 //===--------------------------------------------------------------------===//
@@ -104,8 +104,8 @@ void ColumnSegment::InitializeScan(ColumnScanState &state) {
 }
 
 void ColumnSegment::Scan(ColumnScanState &state, idx_t scan_count, Vector &result, idx_t result_offset,
-                         bool entire_vector) {
-	if (entire_vector) {
+                         ScanVectorType scan_type) {
+	if (scan_type == ScanVectorType::SCAN_ENTIRE_VECTOR) {
 		D_ASSERT(result_offset == 0);
 		Scan(state, scan_count, result);
 	} else {
@@ -132,7 +132,8 @@ void ColumnSegment::ScanPartial(ColumnScanState &state, idx_t scan_count, Vector
 // Fetch
 //===--------------------------------------------------------------------===//
 void ColumnSegment::FetchRow(ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) {
-	function.get().fetch_row(*this, state, row_id - this->start, result, result_idx);
+	function.get().fetch_row(*this, state, UnsafeNumericCast<int64_t>(UnsafeNumericCast<idx_t>(row_id) - this->start),
+	                         result, result_idx);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/table/list_column_data.cpp

@@ -44,7 +44,7 @@ uint64_t ListColumnData::FetchListOffset(idx_t row_idx) {
 	auto segment = data.GetSegment(row_idx);
 	ColumnFetchState fetch_state;
 	Vector result(type, 1);
-	segment->FetchRow(fetch_state, row_idx, result, 0);
+	segment->FetchRow(fetch_state, UnsafeNumericCast<row_t>(row_idx), result, 0U);
 
 	// initialize the child scan with the required offset
 	return FlatVector::GetData<uint64_t>(result)[0];
@@ -70,12 +70,14 @@ void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_
 	state.last_offset = child_offset;
 }
 
-idx_t ListColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result) {
-	return ScanCount(state, result, STANDARD_VECTOR_SIZE);
+idx_t ListColumnData::Scan(TransactionData transaction, idx_t vector_index, ColumnScanState &state, Vector &result,
+                           idx_t scan_count) {
+	return ScanCount(state, result, scan_count);
 }
 
-idx_t ListColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates) {
-	return ScanCount(state, result, STANDARD_VECTOR_SIZE);
+idx_t ListColumnData::ScanCommitted(idx_t vector_index, ColumnScanState &state, Vector &result, bool allow_updates,
+                                    idx_t scan_count) {
+	return ScanCount(state, result, scan_count);
 }
 
 idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t count) {
@@ -86,7 +88,7 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	D_ASSERT(!updates);
 
 	Vector offset_vector(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, offset_vector, count, false);
+	idx_t scan_count = ScanVector(state, offset_vector, count, ScanVectorType::SCAN_FLAT_VECTOR);
 	D_ASSERT(scan_count > 0);
 	validity.ScanCount(state.child_states[0], result, count);
 
@@ -133,7 +135,7 @@ void ListColumnData::Skip(ColumnScanState &state, idx_t count) {
 	// note that we only need to read the first and last entry
 	// however, let's just read all "count" entries for now
 	Vector offset_vector(LogicalType::UBIGINT, count);
-	idx_t scan_count = ScanVector(state, offset_vector, count, false);
+	idx_t scan_count = ScanVector(state, offset_vector, count, ScanVectorType::SCAN_FLAT_VECTOR);
 	D_ASSERT(scan_count > 0);
 
 	UnifiedVectorFormat offsets;
@@ -235,7 +237,7 @@ void ListColumnData::RevertAppend(row_t start_row) {
 	if (column_count > start) {
 		// revert append in the child column
 		auto list_offset = FetchListOffset(column_count - 1);
-		child_column->RevertAppend(list_offset);
+		child_column->RevertAppend(UnsafeNumericCast<row_t>(list_offset));
 	}
 }
 
@@ -269,8 +271,8 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 	}
 
 	// now perform the fetch within the segment
-	auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(row_id - 1);
-	auto end_offset = FetchListOffset(row_id);
+	auto start_offset = idx_t(row_id) == this->start ? 0 : FetchListOffset(UnsafeNumericCast<idx_t>(row_id - 1));
+	auto end_offset = FetchListOffset(UnsafeNumericCast<idx_t>(row_id));
 	validity.FetchRow(transaction, *state.child_states[0], row_id, result, result_idx);
 
 	auto &validity = FlatVector::Validity(result);
@@ -339,11 +341,10 @@ unique_ptr<ColumnCheckpointState> ListColumnData::CreateCheckpointState(RowGroup
 }
 
 unique_ptr<ColumnCheckpointState> ListColumnData::Checkpoint(RowGroup &row_group,
-                                                             PartialBlockManager &partial_block_manager,
                                                              ColumnCheckpointInfo &checkpoint_info) {
-	auto base_state = ColumnData::Checkpoint(row_group, partial_block_manager, checkpoint_info);
-	auto validity_state = validity.Checkpoint(row_group, partial_block_manager, checkpoint_info);
-	auto child_state = child_column->Checkpoint(row_group, partial_block_manager, checkpoint_info);
+	auto base_state = ColumnData::Checkpoint(row_group, checkpoint_info);
+	auto validity_state = validity.Checkpoint(row_group, checkpoint_info);
+	auto child_state = child_column->Checkpoint(row_group, checkpoint_info);
 
 	auto &checkpoint_state = base_state->Cast<ListColumnCheckpointState>();
 	checkpoint_state.validity_state = std::move(validity_state);

package/src/duckdb/src/storage/table/row_group.cpp

@@ -106,7 +106,7 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
 	if (this->columns[c]->count != this->count) {
 		throw InternalException("Corrupted database - loaded column with index %llu at row start %llu, count %llu did "
 		                        "not match count of row group %llu",
-		                        c, start, this->columns[c]->count, this->count.load());
+		                        c, start, this->columns[c]->count.load(), this->count.load());
 	}
 	return *columns[c];
 }
@@ -273,6 +273,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 		if (i == changed_idx) {
 			// this is the altered column: use the new column
 			row_group->columns.push_back(std::move(column_data));
+			column_data.reset();
 		} else {
 			// this column was not altered: use the data directly
 			row_group->columns.push_back(cols[i]);
@@ -283,7 +284,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(RowGroupCollection &new_collection, con
 }
 
 unique_ptr<RowGroup> RowGroup::AddColumn(RowGroupCollection &new_collection, ColumnDefinition &new_column,
-                                         ExpressionExecutor &executor, Expression &default_value, Vector &result) {
+                                         ExpressionExecutor &executor, Vector &result) {
 	Verify();
 
 	// construct a new column data for the new column
@@ -417,6 +418,9 @@ bool RowGroup::CheckZonemapSegments(CollectionScanState &state) {
 		if (!read_segment) {
 
 			idx_t target_row = GetFilterScanCount(state.column_scans[column_idx], *entry.second);
+			if (target_row >= state.max_row) {
+				target_row = state.max_row;
+			}
 
 			D_ASSERT(target_row >= this->start);
 			D_ASSERT(target_row <= this->start + this->count);
@@ -487,7 +491,7 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s
 				if (column == COLUMN_IDENTIFIER_ROW_ID) {
 					// scan row id
 					D_ASSERT(result.data[i].GetType().InternalType() == ROW_TYPE);
-					result.data[i].Sequence(this->start + current_row, 1, count);
+					result.data[i].Sequence(UnsafeNumericCast<int64_t>(this->start + current_row), 1, count);
 				} else {
 					auto &col_data = GetColumn(column);
 					if (TYPE != TableScanType::TABLE_SCAN_REGULAR) {
@@ -551,7 +555,8 @@ void RowGroup::TemplatedScan(TransactionData transaction, CollectionScanState &s
 						result.data[i].SetVectorType(VectorType::FLAT_VECTOR);
 						auto result_data = FlatVector::GetData<int64_t>(result.data[i]);
 						for (size_t sel_idx = 0; sel_idx < approved_tuple_count; sel_idx++) {
-							result_data[sel_idx] = this->start + current_row + sel.get_index(sel_idx);
+							result_data[sel_idx] =
+							    UnsafeNumericCast<int64_t>(this->start + current_row + sel.get_index(sel_idx));
 						}
 					} else {
 						auto &col_data = GetColumn(column);
@@ -585,9 +590,16 @@ void RowGroup::Scan(TransactionData transaction, CollectionScanState &state, Dat
 void RowGroup::ScanCommitted(CollectionScanState &state, DataChunk &result, TableScanType type) {
 	auto &transaction_manager = DuckTransactionManager::Get(GetCollection().GetAttached());
 
-	auto lowest_active_start = transaction_manager.LowestActiveStart();
-	auto lowest_active_id = transaction_manager.LowestActiveId();
-	TransactionData data(lowest_active_id, lowest_active_start);
+	transaction_t start_ts;
+	transaction_t transaction_id;
+	if (type == TableScanType::TABLE_SCAN_LATEST_COMMITTED_ROWS) {
+		start_ts = transaction_manager.GetLastCommit() + 1;
+		transaction_id = MAX_TRANSACTION_ID;
+	} else {
+		start_ts = transaction_manager.LowestActiveStart();
+		transaction_id = transaction_manager.LowestActiveId();
+	}
+	TransactionData data(transaction_id, start_ts);
 	switch (type) {
 	case TableScanType::TABLE_SCAN_COMMITTED_ROWS:
 		TemplatedScan<TableScanType::TABLE_SCAN_COMMITTED_ROWS>(data, state, result);
@@ -596,6 +608,7 @@ void RowGroup::ScanCommitted(CollectionScanState &state, DataChunk &result, Tabl
 		TemplatedScan<TableScanType::TABLE_SCAN_COMMITTED_ROWS_DISALLOW_UPDATES>(data, state, result);
 		break;
 	case TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED:
+	case TableScanType::TABLE_SCAN_LATEST_COMMITTED_ROWS:
 		TemplatedScan<TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED>(data, state, result);
 		break;
 	default:
@@ -624,7 +637,7 @@ shared_ptr<RowVersionManager> &RowGroup::GetOrCreateVersionInfoPtr() {
 	if (!vinfo) {
 		lock_guard<mutex> lock(row_group_lock);
 		if (!version_info) {
-			version_info = make_shared<RowVersionManager>(start);
+			version_info = make_shared_ptr<RowVersionManager>(start);
 		}
 	}
 	return version_info;
@@ -703,7 +716,7 @@ void RowGroup::RevertAppend(idx_t row_group_start) {
 	auto &vinfo = GetOrCreateVersionInfo();
 	vinfo.RevertAppend(row_group_start - this->start);
 	for (auto &column : columns) {
-		column->RevertAppend(row_group_start);
+		column->RevertAppend(UnsafeNumericCast<row_t>(row_group_start));
 	}
 	this->count = MinValue<idx_t>(row_group_start - this->start, this->count);
 	Verify();
@@ -770,24 +783,24 @@ void RowGroup::UpdateColumn(TransactionData transaction, DataChunk &updates, Vec
 
 unique_ptr<BaseStatistics> RowGroup::GetStatistics(idx_t column_idx) {
 	auto &col_data = GetColumn(column_idx);
-	lock_guard<mutex> slock(stats_lock);
 	return col_data.GetStatistics();
 }
 
 void RowGroup::MergeStatistics(idx_t column_idx, const BaseStatistics &other) {
 	auto &col_data = GetColumn(column_idx);
-	lock_guard<mutex> slock(stats_lock);
 	col_data.MergeStatistics(other);
 }
 
 void RowGroup::MergeIntoStatistics(idx_t column_idx, BaseStatistics &other) {
 	auto &col_data = GetColumn(column_idx);
-	lock_guard<mutex> slock(stats_lock);
 	col_data.MergeIntoStatistics(other);
 }
 
-RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
-                                        const vector<CompressionType> &compression_types) {
+CompressionType ColumnCheckpointInfo::GetCompressionType() {
+	return info.compression_types[column_idx];
+}
+
+RowGroupWriteData RowGroup::WriteToDisk(RowGroupWriteInfo &info) {
 	RowGroupWriteData result;
 	result.states.reserve(columns.size());
 	result.statistics.reserve(columns.size());
@@ -802,8 +815,8 @@ RowGroupWriteData RowGroup::WriteToDisk(PartialBlockManager &manager,
 	// pointers all end up densely packed, and thus more cache-friendly.
 	for (idx_t column_idx = 0; column_idx < GetColumnCount(); column_idx++) {
 		auto &column = GetColumn(column_idx);
-		ColumnCheckpointInfo checkpoint_info {compression_types[column_idx]};
-		auto checkpoint_state = column.Checkpoint(*this, manager, checkpoint_info);
+		ColumnCheckpointInfo checkpoint_info(info, column_idx);
+		auto checkpoint_state = column.Checkpoint(*this, checkpoint_info);
 		D_ASSERT(checkpoint_state);
 
 		auto stats = checkpoint_state->GetStatistics();
@@ -841,20 +854,22 @@ RowGroupWriteData RowGroup::WriteToDisk(RowGroupWriter &writer) {
 		if (column.count != this->count) {
 			throw InternalException("Corrupted in-memory column - column with index %llu has misaligned count (row "
 			                        "group has %llu rows, column has %llu)",
-			                        column_idx, this->count.load(), column.count);
+			                        column_idx, this->count.load(), column.count.load());
 		}
 		compression_types.push_back(writer.GetColumnCompressionType(column_idx));
 	}
 
-	return WriteToDisk(writer.GetPartialBlockManager(), compression_types);
+	RowGroupWriteInfo info(writer.GetPartialBlockManager(), compression_types, writer.GetCheckpointType());
+	return WriteToDisk(info);
 }
 
 RowGroupPointer RowGroup::Checkpoint(RowGroupWriteData write_data, RowGroupWriter &writer,
                                      TableStatistics &global_stats) {
 	RowGroupPointer row_group_pointer;
 
+	auto lock = global_stats.GetLock();
 	for (idx_t column_idx = 0; column_idx < GetColumnCount(); column_idx++) {
-		global_stats.GetStats(column_idx).Statistics().Merge(write_data.statistics[column_idx]);
+		global_stats.GetStats(*lock, column_idx).Statistics().Merge(write_data.statistics[column_idx]);
 	}
 
 	// construct the row group pointer and write the column meta data to disk
@@ -955,7 +970,7 @@ idx_t RowGroup::Delete(TransactionData transaction, DataTable &table, row_t *ids
 	for (idx_t i = 0; i < count; i++) {
 		D_ASSERT(ids[i] >= 0);
 		D_ASSERT(idx_t(ids[i]) >= this->start && idx_t(ids[i]) < this->start + this->count);
-		del_state.Delete(ids[i] - this->start);
+		del_state.Delete(ids[i] - UnsafeNumericCast<row_t>(this->start));
 	}
 	del_state.Flush();
 	return del_state.delete_count;
@@ -975,15 +990,15 @@ idx_t RowGroup::DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t
 
 void VersionDeleteState::Delete(row_t row_id) {
 	D_ASSERT(row_id >= 0);
-	idx_t vector_idx = row_id / STANDARD_VECTOR_SIZE;
-	idx_t idx_in_vector = row_id - vector_idx * STANDARD_VECTOR_SIZE;
+	idx_t vector_idx = UnsafeNumericCast<idx_t>(row_id) / STANDARD_VECTOR_SIZE;
+	idx_t idx_in_vector = UnsafeNumericCast<idx_t>(row_id) - vector_idx * STANDARD_VECTOR_SIZE;
 	if (current_chunk != vector_idx) {
 		Flush();
 
 		current_chunk = vector_idx;
 		chunk_row = vector_idx * STANDARD_VECTOR_SIZE;
 	}
-	rows[count++] = idx_in_vector;
+	rows[count++] = UnsafeNumericCast<row_t>(idx_in_vector);
 }
 
 void VersionDeleteState::Flush() {